scripts/last-dotplot
author Martin C. Frith
Thu Mar 02 10:10:51 2017 +0900 (2017-03-02)
changeset 839 bbc6f00e683b
parent 838 7b972338fe04
child 840 85a72978fb7d
permissions -rwxr-xr-x
Added trim options to last-dotplot.
     1 #! /usr/bin/env python
     2 
     3 # Read pair-wise alignments in MAF or LAST tabular format: write an
     4 # "Oxford grid", a.k.a. dotplot.
     5 
     6 # TODO: Currently, pixels with zero aligned nt-pairs are white, and
     7 # pixels with one or more aligned nt-pairs are black.  This can look
     8 # too crowded for large genome alignments.  I tried shading each pixel
     9 # according to the number of aligned nt-pairs within it, but the
    10 # result is too faint.  How can this be done better?
    11 
    12 import fileinput, fnmatch, itertools, optparse, os, re, sys
    13 
    14 # Try to make PIL/PILLOW work:
    15 try: from PIL import Image, ImageDraw, ImageFont, ImageColor
    16 except ImportError: import Image, ImageDraw, ImageFont, ImageColor
    17 
    18 def warn(message):
    19     prog = os.path.basename(sys.argv[0])
    20     sys.stderr.write(prog + ": " + message + "\n")
    21 
    22 def tabBlocks(beg1, beg2, blocks):
    23     '''Get the gapless blocks of an alignment, from LAST tabular format.'''
    24     for i in blocks.split(","):
    25         if ":" in i:
    26             x, y = i.split(":")
    27             beg1 += int(x)
    28             beg2 += int(y)
    29         else:
    30             size = int(i)
    31             yield beg1, beg2, size
    32             beg1 += size
    33             beg2 += size
    34 
    35 def mafBlocks(beg1, beg2, seq1, seq2):
    36     '''Get the gapless blocks of an alignment, from MAF format.'''
    37     size = 0
    38     for x, y in itertools.izip(seq1, seq2):
    39         if x == "-":
    40             if size:
    41                 yield beg1, beg2, size
    42                 beg1 += size
    43                 beg2 += size
    44                 size = 0
    45             beg2 += 1
    46         elif y == "-":
    47             if size:
    48                 yield beg1, beg2, size
    49                 beg1 += size
    50                 beg2 += size
    51                 size = 0
    52             beg1 += 1
    53         else:
    54             size += 1
    55     if size: yield beg1, beg2, size
    56 
    57 def alignmentInput(lines):
    58     '''Get alignments and sequence lengths, from MAF or tabular format.'''
    59     mafCount = 0
    60     for line in lines:
    61         w = line.split()
    62         if line[0].isdigit():  # tabular format
    63             chr1, beg1, seqlen1 = w[1], int(w[2]), int(w[5])
    64             if w[4] == "-": beg1 -= seqlen1
    65             chr2, beg2, seqlen2 = w[6], int(w[7]), int(w[10])
    66             if w[9] == "-": beg2 -= seqlen2
    67             blocks = list(tabBlocks(beg1, beg2, w[11]))
    68             yield chr1, seqlen1, chr2, seqlen2, blocks
    69         elif line[0] == "s":  # MAF format
    70             if mafCount == 0:
    71                 chr1, beg1, seqlen1, seq1 = w[1], int(w[2]), int(w[5]), w[6]
    72                 if w[4] == "-": beg1 -= seqlen1
    73                 mafCount = 1
    74             else:
    75                 chr2, beg2, seqlen2, seq2 = w[1], int(w[2]), int(w[5]), w[6]
    76                 if w[4] == "-": beg2 -= seqlen2
    77                 blocks = list(mafBlocks(beg1, beg2, seq1, seq2))
    78                 yield chr1, seqlen1, chr2, seqlen2, blocks
    79                 mafCount = 0
    80 
    81 def isWantedSequenceName(name, patterns):
    82     if not patterns: return True
    83     base = name.split(".")[-1]  # allow for names like hg19.chr7
    84     for i in patterns:
    85         if fnmatch.fnmatchcase(name, i): return True
    86         if fnmatch.fnmatchcase(base, i): return True
    87     return False
    88 
    89 def updateSeqLimits(isTrim, seqLimits, seqName, seqLen, blocks, index):
    90     if isTrim:
    91         beg = blocks[0][index]
    92         end = blocks[-1][index] + blocks[-1][2]
    93         if beg < 0: beg, end = -end, -beg
    94         if seqName in seqLimits:
    95             b, e = seqLimits[seqName]
    96             seqLimits[seqName] = min(b, beg), max(e, end)
    97         else:
    98             seqLimits[seqName] = beg, end
    99     else:
   100         seqLimits[seqName] = 0, seqLen
   101 
   102 def readAlignments(fileName, opts):
   103     '''Get alignments and sequence limits, from MAF or tabular format.'''
   104     alignments = []
   105     seqLimits1 = {}
   106     seqLimits2 = {}
   107     lines = fileinput.input(fileName)
   108     for seqName1, seqLen1, seqName2, seqLen2, blocks in alignmentInput(lines):
   109         if not isWantedSequenceName(seqName1, opts.seq1): continue
   110         if not isWantedSequenceName(seqName2, opts.seq2): continue
   111         aln = seqName1, seqName2, blocks
   112         alignments.append(aln)
   113         updateSeqLimits(opts.trim1, seqLimits1, seqName1, seqLen1, blocks, 0)
   114         updateSeqLimits(opts.trim2, seqLimits2, seqName2, seqLen2, blocks, 1)
   115     return alignments, seqLimits1, seqLimits2
   116 
   117 def natural_sort_key(my_string):
   118     '''Return a sort key for "natural" ordering, e.g. chr9 < chr10.'''
   119     parts = re.split(r'(\d+)', my_string)
   120     parts[1::2] = map(int, parts[1::2])
   121     return parts
   122 
   123 def get_text_sizes(my_strings, font, fontsize, image_mode):
   124     '''Get widths & heights, in pixels, of some strings.'''
   125     if fontsize == 0: return [(0, 0) for i in my_strings]
   126     image_size = 1, 1
   127     im = Image.new(image_mode, image_size)
   128     draw = ImageDraw.Draw(im)
   129     return [draw.textsize(i, font=font) for i in my_strings]
   130 
   131 def get_seq_info(seqLimits, font, fontsize, image_mode):
   132     '''Return miscellaneous information about the sequences.'''
   133     seqNames = seqLimits.keys()
   134     seqNames.sort(key=natural_sort_key)
   135     seq_sizes = [seqLimits[i][1] - seqLimits[i][0] for i in seqNames]
   136     name_sizes = get_text_sizes(seqNames, font, fontsize, image_mode)
   137     margin = max(zip(*name_sizes)[1])  # maximum text height
   138     return seqNames, seq_sizes, name_sizes, margin
   139 
   140 def div_ceil(x, y):
   141     '''Return x / y rounded up.'''
   142     q, r = divmod(x, y)
   143     return q + (r != 0)
   144 
   145 def tot_seq_pix(seq_sizes, bp_per_pix):
   146     '''Return the total pixels needed for sequences of the given sizes.'''
   147     return sum([div_ceil(i, bp_per_pix) for i in seq_sizes])
   148 
   149 def get_bp_per_pix(seq_sizes, pix_tween_seqs, pix_limit):
   150     '''Get the minimum bp-per-pixel that fits in the size limit.'''
   151     seq_num = len(seq_sizes)
   152     seq_pix_limit = pix_limit - pix_tween_seqs * (seq_num - 1)
   153     if seq_pix_limit < seq_num:
   154         raise Exception("can't fit the image: too many sequences?")
   155     lower_bound = div_ceil(sum(seq_sizes), seq_pix_limit)
   156     for bp_per_pix in itertools.count(lower_bound):  # slow linear search
   157         if tot_seq_pix(seq_sizes, bp_per_pix) <= seq_pix_limit: break
   158     return bp_per_pix
   159 
   160 def get_seq_starts(seq_pix, pix_tween_seqs, margin):
   161     '''Get the start pixel for each sequence.'''
   162     seq_starts = []
   163     pix_tot = margin - pix_tween_seqs
   164     for i in seq_pix:
   165         pix_tot += pix_tween_seqs
   166         seq_starts.append(pix_tot)
   167         pix_tot += i
   168     return seq_starts
   169 
   170 def get_pix_info(seq_sizes, bp_per_pix, pix_tween_seqs, margin):
   171     '''Return pixel information about the sequences.'''
   172     seq_pix = [div_ceil(i, bp_per_pix) for i in seq_sizes]
   173     seq_starts = get_seq_starts(seq_pix, pix_tween_seqs, margin)
   174     tot_pix = seq_starts[-1] + seq_pix[-1]
   175     return seq_pix, seq_starts, tot_pix
   176 
   177 def drawLineForward(hits, width, bp_per_pix, beg1, beg2, size):
   178     while True:
   179         q1, r1 = divmod(beg1, bp_per_pix)
   180         q2, r2 = divmod(beg2, bp_per_pix)
   181         hits[q2 * width + q1] |= 1
   182         next_pix = min(bp_per_pix - r1, bp_per_pix - r2)
   183         if next_pix >= size: break
   184         beg1 += next_pix
   185         beg2 += next_pix
   186         size -= next_pix
   187 
   188 def drawLineReverse(hits, width, bp_per_pix, beg1, beg2, size):
   189     beg2 = -1 - beg2
   190     while True:
   191         q1, r1 = divmod(beg1, bp_per_pix)
   192         q2, r2 = divmod(beg2, bp_per_pix)
   193         hits[q2 * width + q1] |= 2
   194         next_pix = min(bp_per_pix - r1, r2 + 1)
   195         if next_pix >= size: break
   196         beg1 += next_pix
   197         beg2 -= next_pix
   198         size -= next_pix
   199 
   200 def alignmentPixels(width, height, alignments, bp_per_pix, origins1, origins2):
   201     hits = [0] * (width * height)  # the image data
   202     for seq1, seq2, blocks in alignments:
   203         ori1 = origins1[seq1]
   204         ori2 = origins2[seq2]
   205         for beg1, beg2, size in blocks:
   206             if beg1 < 0:
   207                 beg1 = -(beg1 + size)
   208                 beg2 = -(beg2 + size)
   209             if beg2 >= 0:
   210                 drawLineForward(hits, width, bp_per_pix,
   211                                 beg1 + ori1, beg2 + ori2, size)
   212             else:
   213                 drawLineReverse(hits, width, bp_per_pix,
   214                                 beg1 + ori1, beg2 - ori2, size)
   215     return hits
   216 
   217 def expandedSeqDict(seqDict):
   218     '''Allow lookup by short sequence names, e.g. chr7 as well as hg19.chr7.'''
   219     newDict = {}
   220     for name, x in seqDict.items():
   221         base = name.split(".")[-1]
   222         newDict[name] = x
   223         newDict[base] = x
   224     return newDict
   225 
   226 def isExtraFirstGapField(fields):
   227     return fields[4].isdigit()
   228 
   229 def readGaps(fileName, seqLimits):
   230     '''Read locations of unsequenced gaps, from an agp or gap file.'''
   231     if not fileName: return
   232     seqLimits = expandedSeqDict(seqLimits)
   233     for line in fileinput.input(fileName):
   234         w = line.split()
   235         if not w or w[0][0] == "#": continue
   236         if isExtraFirstGapField(w): w = w[1:]
   237         if w[4] not in "NU": continue
   238         seqName = w[0]
   239         if seqName not in seqLimits: continue
   240         cropBeg, cropEnd = seqLimits[seqName]
   241         end = int(w[2])
   242         beg = end - int(w[5])  # zero-based coordinate
   243         b = max(beg, cropBeg)
   244         e = min(end, cropEnd)
   245         if b >= e: continue
   246         bridgedText = w[7]
   247         yield seqName, b, e, bridgedText
   248 
   249 def drawUnsequencedGaps(im, gaps, origins, margin, limit, isTop, bridgedText,
   250                         bp_per_pix, color):
   251     '''Draw rectangles representing unsequenced gaps.'''
   252     for seqName, beg, end, b in gaps:
   253         if b != bridgedText: continue
   254         ori = origins[seqName]
   255         b = div_ceil(ori + beg, bp_per_pix)  # use fully-covered pixels only
   256         e = (ori + end) // bp_per_pix
   257         if e <= b: continue
   258         if isTop: box = b, margin, e, limit
   259         else:     box = margin, b, limit, e
   260         im.paste(color, box)
   261 
   262 def make_label(text, text_size, range_start, range_size):
   263     '''Return an axis label with endpoint & sort-order information.'''
   264     text_width  = text_size[0]
   265     label_start = range_start + (range_size - text_width) // 2
   266     label_end   = label_start + text_width
   267     sort_key    = text_width - range_size
   268     return sort_key, label_start, label_end, text
   269 
   270 def get_nonoverlapping_labels(labels, label_space):
   271     '''Get a subset of non-overlapping axis labels, greedily.'''
   272     nonoverlapping_labels = []
   273     for i in labels:
   274         if True not in [i[1] < j[2] + label_space and j[1] < i[2] + label_space
   275                         for j in nonoverlapping_labels]:
   276             nonoverlapping_labels.append(i)
   277     return nonoverlapping_labels
   278 
   279 def get_axis_image(seqNames, name_sizes, seq_starts, seq_pix,
   280                    font, image_mode, opts):
   281     '''Make an image of axis labels.'''
   282     min_pos = seq_starts[0]
   283     max_pos = seq_starts[-1] + seq_pix[-1]
   284     height = max(zip(*name_sizes)[1])
   285     labels = [make_label(i, j, k, l) for i, j, k, l in
   286               zip(seqNames, name_sizes, seq_starts, seq_pix)]
   287     labels = [i for i in labels if i[1] >= min_pos and i[2] <= max_pos]
   288     labels.sort()
   289     labels = get_nonoverlapping_labels(labels, opts.label_space)
   290     image_size = max_pos, height
   291     im = Image.new(image_mode, image_size, opts.border_shade)
   292     draw = ImageDraw.Draw(im)
   293     for i in labels:
   294         position = i[1], 0
   295         draw.text(position, i[3], font=font, fill=opts.text_color)
   296     return im
   297 
   298 def seqOrigins(seqNames, seq_starts, seqLimits, bp_per_pix):
   299     for i, j in zip(seqNames, seq_starts):
   300         yield i, bp_per_pix * j - seqLimits[i][0]
   301 
   302 def lastDotplot(opts, args):
   303     if opts.fontfile:  font = ImageFont.truetype(opts.fontfile, opts.fontsize)
   304     else:              font = ImageFont.load_default()
   305 
   306     image_mode = 'RGB'
   307     forward_color = ImageColor.getcolor(opts.forwardcolor, image_mode)
   308     reverse_color = ImageColor.getcolor(opts.reversecolor, image_mode)
   309     zipped_colors = zip(forward_color, reverse_color)
   310     overlap_color = tuple([(i + j) // 2 for i, j in zipped_colors])
   311 
   312     warn("reading alignments...")
   313     alignments, seqLimits1, seqLimits2 = readAlignments(args[0], opts)
   314     warn("done")
   315 
   316     if not alignments: raise Exception("there are no alignments")
   317 
   318     seq_info1 = get_seq_info(seqLimits1, font, opts.fontsize, image_mode)
   319     seq_info2 = get_seq_info(seqLimits2, font, opts.fontsize, image_mode)
   320     seqNames1, seq_sizes1, name_sizes1, margin1 = seq_info1
   321     seqNames2, seq_sizes2, name_sizes2, margin2 = seq_info2
   322 
   323     warn("choosing bp per pixel...")
   324     pix_limit1 = opts.width  - margin1
   325     pix_limit2 = opts.height - margin2
   326     bp_per_pix1 = get_bp_per_pix(seq_sizes1, opts.pix_tween_seqs, pix_limit1)
   327     bp_per_pix2 = get_bp_per_pix(seq_sizes2, opts.pix_tween_seqs, pix_limit2)
   328     bp_per_pix = max(bp_per_pix1, bp_per_pix2)
   329     warn("bp per pixel = " + str(bp_per_pix))
   330 
   331     seq_pix1, seq_starts1, width  = get_pix_info(seq_sizes1, bp_per_pix,
   332                                                  opts.pix_tween_seqs, margin1)
   333     seq_pix2, seq_starts2, height = get_pix_info(seq_sizes2, bp_per_pix,
   334                                                  opts.pix_tween_seqs, margin2)
   335 
   336     origins1 = dict(seqOrigins(seqNames1, seq_starts1, seqLimits1, bp_per_pix))
   337     origins2 = dict(seqOrigins(seqNames2, seq_starts2, seqLimits2, bp_per_pix))
   338 
   339     warn("processing alignments...")
   340     hits = alignmentPixels(width, height, alignments, bp_per_pix,
   341                            origins1, origins2)
   342     warn("done")
   343 
   344     image_size = width, height
   345     im = Image.new(image_mode, image_size, opts.background_color)
   346 
   347     origins1 = expandedSeqDict(origins1)
   348     origins2 = expandedSeqDict(origins2)
   349     gaps1 = list(readGaps(opts.gap1, seqLimits1))
   350     gaps2 = list(readGaps(opts.gap2, seqLimits2))
   351     # draw bridged gaps first, then unbridged gaps on top:
   352     drawUnsequencedGaps(im, gaps1, origins1, margin2, height, True, "yes",
   353                         bp_per_pix, opts.bridged_color)
   354     drawUnsequencedGaps(im, gaps2, origins2, margin1, width, False, "yes",
   355                         bp_per_pix, opts.bridged_color)
   356     drawUnsequencedGaps(im, gaps1, origins1, margin2, height, True, "no",
   357                         bp_per_pix, opts.unbridged_color)
   358     drawUnsequencedGaps(im, gaps2, origins2, margin1, width, False, "no",
   359                         bp_per_pix, opts.unbridged_color)
   360 
   361     for i in range(height):
   362         for j in range(width):
   363             store_value = hits[i * width + j]
   364             xy = j, i
   365             if   store_value == 1: im.putpixel(xy, forward_color)
   366             elif store_value == 2: im.putpixel(xy, reverse_color)
   367             elif store_value == 3: im.putpixel(xy, overlap_color)
   368 
   369     if opts.fontsize != 0:
   370         axis1 = get_axis_image(seqNames1, name_sizes1, seq_starts1, seq_pix1,
   371                                font, image_mode, opts)
   372         axis2 = get_axis_image(seqNames2, name_sizes2, seq_starts2, seq_pix2,
   373                                font, image_mode, opts)
   374         axis2 = axis2.transpose(Image.ROTATE_270)  # !!! bug hotspot
   375         im.paste(axis1, (0, 0))
   376         im.paste(axis2, (0, 0))
   377 
   378     for i in seq_starts1[1:]:
   379         box = i - opts.pix_tween_seqs, margin2, i, height
   380         im.paste(opts.border_shade, box)
   381 
   382     for i in seq_starts2[1:]:
   383         box = margin1, i - opts.pix_tween_seqs, width, i
   384         im.paste(opts.border_shade, box)
   385 
   386     im.save(args[1])
   387 
   388 if __name__ == "__main__":
   389     usage = """%prog --help
   390    or: %prog [options] maf-or-tab-alignments dotplot.png
   391    or: %prog [options] maf-or-tab-alignments dotplot.gif
   392    or: ..."""
   393     description = "Draw a dotplot of pair-wise sequence alignments in MAF or tabular format."
   394     op = optparse.OptionParser(usage=usage, description=description)
   395     op.add_option("-1", "--seq1", metavar="PATTERN", action="append",
   396                   help="which sequences to show from the 1st genome")
   397     op.add_option("-2", "--seq2", metavar="PATTERN", action="append",
   398                   help="which sequences to show from the 2nd genome")
   399     # Replace "width" & "height" with a single "length" option?
   400     op.add_option("-x", "--width", type="int", default=1000,
   401                   help="maximum width in pixels (default: %default)")
   402     op.add_option("-y", "--height", type="int", default=1000,
   403                   help="maximum height in pixels (default: %default)")
   404     op.add_option("-f", "--fontfile", metavar="FILE",
   405                   help="TrueType or OpenType font file")
   406     op.add_option("-s", "--fontsize", metavar="SIZE", type="int", default=11,
   407                   help="TrueType or OpenType font size (default: %default)")
   408     op.add_option("-c", "--forwardcolor", metavar="COLOR", default="red",
   409                   help="color for forward alignments (default: %default)")
   410     op.add_option("-r", "--reversecolor", metavar="COLOR", default="blue",
   411                   help="color for reverse alignments (default: %default)")
   412     op.add_option("--trim1", action="store_true",
   413                   help="trim unaligned sequence flanks from the 1st genome")
   414     op.add_option("--trim2", action="store_true",
   415                   help="trim unaligned sequence flanks from the 2nd genome")
   416     og = optparse.OptionGroup(op, "Unsequenced gap options")
   417     og.add_option("--gap1", metavar="FILE",
   418                   help="read genome1 unsequenced gaps from agp or gap file")
   419     og.add_option("--gap2", metavar="FILE",
   420                   help="read genome2 unsequenced gaps from agp or gap file")
   421     og.add_option("--bridged-color", metavar="COLOR", default="yellow",
   422                   help="color for bridged gaps (default: %default)")
   423     og.add_option("--unbridged-color", metavar="COLOR", default="pink",
   424                   help="color for unbridged gaps (default: %default)")
   425     op.add_option_group(og)
   426     (opts, args) = op.parse_args()
   427     if len(args) != 2: op.error("2 arguments needed")
   428 
   429     opts.text_color = "black"
   430     opts.background_color = "white"
   431     opts.pix_tween_seqs = 2  # number of border pixels between sequences
   432     opts.border_shade = 239, 239, 239  # the shade of grey for border pixels
   433     opts.label_space = 5     # minimum number of pixels between axis labels
   434 
   435     try: lastDotplot(opts, args)
   436     except KeyboardInterrupt: pass  # avoid silly error message
   437     except Exception, e:
   438         prog = os.path.basename(sys.argv[0])
   439         sys.exit(prog + ": error: " + str(e))