3 # Read pair-wise alignments in MAF or LAST tabular format: write an
4 # "Oxford grid", a.k.a. dotplot.
6 # TODO: Currently, pixels with zero aligned nt-pairs are white, and
7 # pixels with one or more aligned nt-pairs are black. This can look
8 # too crowded for large genome alignments. I tried shading each pixel
9 # according to the number of aligned nt-pairs within it, but the
10 # result is too faint. How can this be done better?
12 import fnmatch, itertools, optparse, os, re, sys
14 # Try to make PIL/PILLOW work:
15 try: from PIL import Image, ImageDraw, ImageFont, ImageColor
16 except ImportError: import Image, ImageDraw, ImageFont, ImageColor
18 def myOpen(fileName): # faster than fileinput
24 prog = os.path.basename(sys.argv[0])
25 sys.stderr.write(prog + ": " + message + "\n")
27 def croppedBlocks(blocks, range1, range2):
28 cropBeg1, cropEnd1 = range1
29 cropBeg2, cropEnd2 = range2
30 if blocks[0][0] < 0: cropBeg1, cropEnd1 = -cropEnd1, -cropBeg1
31 if blocks[0][1] < 0: cropBeg2, cropEnd2 = -cropEnd2, -cropBeg2
32 for beg1, beg2, size in blocks:
33 b1 = max(cropBeg1, beg1)
34 e1 = min(cropEnd1, beg1 + size)
37 b2 = max(cropBeg2, b1 + offset)
38 e2 = min(cropEnd2, e1 + offset)
40 yield b2 - offset, b2, e2 - b2
42 def tabBlocks(beg1, beg2, blocks):
43 '''Get the gapless blocks of an alignment, from LAST tabular format.'''
44 for i in blocks.split(","):
51 yield beg1, beg2, size
55 def mafBlocks(beg1, beg2, seq1, seq2):
56 '''Get the gapless blocks of an alignment, from MAF format.'''
58 for x, y in itertools.izip(seq1, seq2):
61 yield beg1, beg2, size
68 yield beg1, beg2, size
75 if size: yield beg1, beg2, size
77 def alignmentInput(lines):
78 '''Get alignments and sequence lengths, from MAF or tabular format.'''
82 if line[0].isdigit(): # tabular format
83 chr1, beg1, seqlen1 = w[1], int(w[2]), int(w[5])
84 if w[4] == "-": beg1 -= seqlen1
85 chr2, beg2, seqlen2 = w[6], int(w[7]), int(w[10])
86 if w[9] == "-": beg2 -= seqlen2
87 blocks = tabBlocks(beg1, beg2, w[11])
88 yield chr1, seqlen1, chr2, seqlen2, blocks
89 elif line[0] == "s": # MAF format
91 chr1, beg1, seqlen1, seq1 = w[1], int(w[2]), int(w[5]), w[6]
92 if w[4] == "-": beg1 -= seqlen1
95 chr2, beg2, seqlen2, seq2 = w[1], int(w[2]), int(w[5]), w[6]
96 if w[4] == "-": beg2 -= seqlen2
97 blocks = mafBlocks(beg1, beg2, seq1, seq2)
98 yield chr1, seqlen1, chr2, seqlen2, blocks
101 def seqRangeFromText(text):
103 pattern, interval = text.rsplit(":", 1)
105 beg, end = interval.rsplit("-", 1)
106 return pattern, int(beg), int(end) # beg may be negative
107 return text, 0, sys.maxsize
109 def rangeFromSeqName(seqRanges, name, seqLen):
110 if not seqRanges: return 0, seqLen
111 base = name.split(".")[-1] # allow for names like hg19.chr7
112 for pat, beg, end in seqRanges:
113 if fnmatch.fnmatchcase(name, pat) or fnmatch.fnmatchcase(base, pat):
114 return max(beg, 0), min(end, seqLen)
117 def updateSeqs(isTrim, seqNames, seqLimits, seqName, seqRange, blocks, index):
118 if seqName not in seqLimits:
119 seqNames.append(seqName)
121 beg = blocks[0][index]
122 end = blocks[-1][index] + blocks[-1][2]
123 if beg < 0: beg, end = -end, -beg
124 if seqName in seqLimits:
125 b, e = seqLimits[seqName]
126 seqLimits[seqName] = min(b, beg), max(e, end)
128 seqLimits[seqName] = beg, end
130 seqLimits[seqName] = seqRange
132 def readAlignments(fileName, opts):
133 '''Get alignments and sequence limits, from MAF or tabular format.'''
134 seqRanges1 = map(seqRangeFromText, opts.seq1)
135 seqRanges2 = map(seqRangeFromText, opts.seq2)
142 lines = myOpen(fileName)
143 for seqName1, seqLen1, seqName2, seqLen2, blocks in alignmentInput(lines):
144 range1 = rangeFromSeqName(seqRanges1, seqName1, seqLen1)
145 if not range1: continue
146 range2 = rangeFromSeqName(seqRanges2, seqName2, seqLen2)
147 if not range2: continue
148 b = list(croppedBlocks(list(blocks), range1, range2))
150 aln = seqName1, seqName2, b
151 alignments.append(aln)
152 updateSeqs(opts.trim1, seqNames1, seqLimits1, seqName1, range1, b, 0)
153 updateSeqs(opts.trim2, seqNames2, seqLimits2, seqName2, range2, b, 1)
154 return alignments, seqNames1, seqNames2, seqLimits1, seqLimits2
156 def natural_sort_key(my_string):
157 '''Return a sort key for "natural" ordering, e.g. chr9 < chr10.'''
158 parts = re.split(r'(\d+)', my_string)
159 parts[1::2] = map(int, parts[1::2])
162 def get_text_sizes(my_strings, font, fontsize, image_mode):
163 '''Get widths & heights, in pixels, of some strings.'''
164 if fontsize == 0: return [(0, 0) for i in my_strings]
166 im = Image.new(image_mode, image_size)
167 draw = ImageDraw.Draw(im)
168 return [draw.textsize(i, font=font) for i in my_strings]
171 suffixes = "bp", "kb", "Mb", "Gb"
172 for i, x in enumerate(suffixes):
175 return "%.2g" % (1.0 * size / j) + x
176 if size < j * 1000 or i == len(suffixes) - 1:
177 return "%.0f" % (1.0 * size / j) + x
179 def seqNameAndSizeText(seqName, seqSize):
180 return seqName + ": " + sizeText(seqSize)
182 def getSeqInfo(sortOpt, seqNames, seqLimits,
183 font, fontsize, image_mode, isShowSize):
184 '''Return miscellaneous information about the sequences.'''
186 seqNames.sort(key=natural_sort_key)
187 seqSizes = [seqLimits[i][1] - seqLimits[i][0] for i in seqNames]
189 seqRecords = sorted(zip(seqSizes, seqNames), reverse=True)
190 seqSizes = [i[0] for i in seqRecords]
191 seqNames = [i[1] for i in seqRecords]
193 seqLabels = map(seqNameAndSizeText, seqNames, seqSizes)
196 labelSizes = get_text_sizes(seqLabels, font, fontsize, image_mode)
197 margin = max(zip(*labelSizes)[1]) # maximum text height
198 return seqNames, seqSizes, seqLabels, labelSizes, margin
201 '''Return x / y rounded up.'''
205 def get_bp_per_pix(seq_sizes, pix_tween_seqs, pix_limit):
206 '''Get the minimum bp-per-pixel that fits in the size limit.'''
207 seq_num = len(seq_sizes)
208 seq_pix_limit = pix_limit - pix_tween_seqs * (seq_num - 1)
209 if seq_pix_limit < seq_num:
210 raise Exception("can't fit the image: too many sequences?")
211 negLimit = -seq_pix_limit
212 negBpPerPix = sum(seq_sizes) // negLimit
214 if sum(i // negBpPerPix for i in seq_sizes) >= negLimit:
218 def get_seq_starts(seq_pix, pix_tween_seqs, margin):
219 '''Get the start pixel for each sequence.'''
221 pix_tot = margin - pix_tween_seqs
223 pix_tot += pix_tween_seqs
224 seq_starts.append(pix_tot)
228 def get_pix_info(seq_sizes, bp_per_pix, pix_tween_seqs, margin):
229 '''Return pixel information about the sequences.'''
230 seq_pix = [div_ceil(i, bp_per_pix) for i in seq_sizes]
231 seq_starts = get_seq_starts(seq_pix, pix_tween_seqs, margin)
232 tot_pix = seq_starts[-1] + seq_pix[-1]
233 return seq_pix, seq_starts, tot_pix
235 def drawLineForward(hits, width, bp_per_pix, beg1, beg2, size):
237 q1, r1 = divmod(beg1, bp_per_pix)
238 q2, r2 = divmod(beg2, bp_per_pix)
239 hits[q2 * width + q1] |= 1
240 next_pix = min(bp_per_pix - r1, bp_per_pix - r2)
241 if next_pix >= size: break
246 def drawLineReverse(hits, width, bp_per_pix, beg1, beg2, size):
249 q1, r1 = divmod(beg1, bp_per_pix)
250 q2, r2 = divmod(beg2, bp_per_pix)
251 hits[q2 * width + q1] |= 2
252 next_pix = min(bp_per_pix - r1, r2 + 1)
253 if next_pix >= size: break
258 def alignmentPixels(width, height, alignments, bp_per_pix, origins1, origins2):
259 hits = [0] * (width * height) # the image data
260 for seq1, seq2, blocks in alignments:
261 ori1 = origins1[seq1]
262 ori2 = origins2[seq2]
263 for beg1, beg2, size in blocks:
265 beg1 = -(beg1 + size)
266 beg2 = -(beg2 + size)
268 drawLineForward(hits, width, bp_per_pix,
269 beg1 + ori1, beg2 + ori2, size)
271 drawLineReverse(hits, width, bp_per_pix,
272 beg1 + ori1, beg2 - ori2, size)
275 def expandedSeqDict(seqDict):
276 '''Allow lookup by short sequence names, e.g. chr7 as well as hg19.chr7.'''
277 newDict = seqDict.copy()
278 for name, x in seqDict.items():
280 base = name.split(".")[-1]
281 if base in newDict: # an ambiguous case was found:
282 return seqDict # so give up completely
286 def readBed(fileName, seqLimits):
287 if not fileName: return
288 for line in myOpen(fileName):
292 if seqName not in seqLimits: continue
301 if len(w) > 8 and w[8].count(",") == 2:
302 color = "rgb(" + w[8] + ")"
307 yield layer, color, seqName, beg, end
309 def commaSeparatedInts(text):
310 return map(int, text.rstrip(",").split(","))
312 def readGenePred(opts, fileName, seqLimits):
313 if not fileName: return
314 for line in myOpen(fileName):
315 fields = line.split()
316 if not fields: continue
317 if fields[2] not in "+-": fields = fields[1:]
319 if seqName not in seqLimits: continue
321 cdsBeg = int(fields[5])
322 cdsEnd = int(fields[6])
323 exonBegs = commaSeparatedInts(fields[8])
324 exonEnds = commaSeparatedInts(fields[9])
325 for beg, end in zip(exonBegs, exonEnds):
326 yield 300, opts.exon_color, seqName, beg, end
329 if b < e: yield 400, opts.cds_color, seqName, b, e
331 def readRmsk(fileName, seqLimits):
332 if not fileName: return
333 for line in myOpen(fileName):
334 fields = line.split()
335 if len(fields) == 17: # rmsk.txt
337 if seqName not in seqLimits: continue # do this ASAP for speed
341 repeatClass = fields[11]
342 elif len(fields) == 15: # .out
344 if seqName not in seqLimits: continue
345 beg = int(fields[5]) - 1
348 repeatClass = fields[10]
351 if repeatClass in ("Low_complexity", "Simple_repeat"):
352 yield 200, "#ffe4ff", seqName, beg, end
354 yield 100, "#fff4f4", seqName, beg, end
356 yield 100, "#f4f4ff", seqName, beg, end
358 def isExtraFirstGapField(fields):
359 return fields[4].isdigit()
361 def readGaps(opts, fileName, seqLimits):
362 '''Read locations of unsequenced gaps, from an agp or gap file.'''
363 if not fileName: return
364 for line in myOpen(fileName):
366 if not w or w[0][0] == "#": continue
367 if isExtraFirstGapField(w): w = w[1:]
368 if w[4] not in "NU": continue
370 if seqName not in seqLimits: continue
372 beg = end - int(w[5]) # zero-based coordinate
374 yield 3000, opts.bridged_color, seqName, beg, end
376 yield 2000, opts.unbridged_color, seqName, beg, end
378 def bedBoxes(beds, seqLimits, origins, margin, edge, isTop, bpPerPix):
379 for layer, color, seqName, beg, end in beds:
380 cropBeg, cropEnd = seqLimits[seqName]
381 beg = max(beg, cropBeg)
382 end = min(end, cropEnd)
383 if beg >= end: continue
384 ori = origins[seqName]
386 # include partly-covered pixels
387 b = (ori + beg) // bpPerPix
388 e = div_ceil(ori + end, bpPerPix)
390 # exclude partly-covered pixels
391 b = div_ceil(ori + beg, bpPerPix)
392 e = (ori + end) // bpPerPix
395 box = b, margin, e, edge
397 box = margin, b, edge, e
398 yield layer, color, box
400 def drawAnnotations(im, boxes):
401 # xxx use partial transparency for different-color overlaps?
402 for layer, color, box in boxes:
405 def make_label(text, text_size, range_start, range_size):
406 '''Return an axis label with endpoint & sort-order information.'''
407 text_width = text_size[0]
408 label_start = range_start + (range_size - text_width) // 2
409 label_end = label_start + text_width
410 sort_key = text_width - range_size
411 return sort_key, label_start, label_end, text
413 def get_nonoverlapping_labels(labels, label_space):
414 '''Get a subset of non-overlapping axis labels, greedily.'''
415 nonoverlapping_labels = []
417 if True not in [i[1] < j[2] + label_space and j[1] < i[2] + label_space
418 for j in nonoverlapping_labels]:
419 nonoverlapping_labels.append(i)
420 return nonoverlapping_labels
422 def get_axis_image(seqNames, name_sizes, seq_starts, seq_pix,
423 font, image_mode, opts):
424 '''Make an image of axis labels.'''
425 min_pos = seq_starts[0]
426 max_pos = seq_starts[-1] + seq_pix[-1]
427 height = max(zip(*name_sizes)[1])
428 labels = map(make_label, seqNames, name_sizes, seq_starts, seq_pix)
429 labels = [i for i in labels if i[1] >= min_pos and i[2] <= max_pos]
431 labels = get_nonoverlapping_labels(labels, opts.label_space)
432 image_size = max_pos, height
433 im = Image.new(image_mode, image_size, opts.border_color)
434 draw = ImageDraw.Draw(im)
437 draw.text(position, i[3], font=font, fill=opts.text_color)
440 def seqOrigins(seqNames, seq_starts, seqLimits, bp_per_pix):
441 for i, j in zip(seqNames, seq_starts):
442 yield i, bp_per_pix * j - seqLimits[i][0]
444 def lastDotplot(opts, args):
445 if opts.fontfile: font = ImageFont.truetype(opts.fontfile, opts.fontsize)
446 else: font = ImageFont.load_default()
449 forward_color = ImageColor.getcolor(opts.forwardcolor, image_mode)
450 reverse_color = ImageColor.getcolor(opts.reversecolor, image_mode)
451 zipped_colors = zip(forward_color, reverse_color)
452 overlap_color = tuple([(i + j) // 2 for i, j in zipped_colors])
454 warn("reading alignments...")
455 alignmentInfo = readAlignments(args[0], opts)
456 alignments, seqNames1, seqNames2, seqLimits1, seqLimits2 = alignmentInfo
458 if not alignments: raise Exception("there are no alignments")
460 i1 = getSeqInfo(opts.sort1, seqNames1, seqLimits1,
461 font, opts.fontsize, image_mode, opts.lengths1)
462 seqNames1, seqSizes1, seqLabels1, labelSizes1, margin1 = i1
464 i2 = getSeqInfo(opts.sort2, seqNames2, seqLimits2,
465 font, opts.fontsize, image_mode, opts.lengths2)
466 seqNames2, seqSizes2, seqLabels2, labelSizes2, margin2 = i2
468 warn("choosing bp per pixel...")
469 pix_limit1 = opts.width - margin1
470 pix_limit2 = opts.height - margin2
471 bpPerPix1 = get_bp_per_pix(seqSizes1, opts.border_pixels, pix_limit1)
472 bpPerPix2 = get_bp_per_pix(seqSizes2, opts.border_pixels, pix_limit2)
473 bpPerPix = max(bpPerPix1, bpPerPix2)
474 warn("bp per pixel = " + str(bpPerPix))
476 seq_pix1, seq_starts1, width = get_pix_info(seqSizes1, bpPerPix,
477 opts.border_pixels, margin1)
478 seq_pix2, seq_starts2, height = get_pix_info(seqSizes2, bpPerPix,
479 opts.border_pixels, margin2)
480 warn("width: " + str(width))
481 warn("height: " + str(height))
483 origins1 = dict(seqOrigins(seqNames1, seq_starts1, seqLimits1, bpPerPix))
484 origins2 = dict(seqOrigins(seqNames2, seq_starts2, seqLimits2, bpPerPix))
486 warn("processing alignments...")
487 hits = alignmentPixels(width, height, alignments, bpPerPix,
491 image_size = width, height
492 im = Image.new(image_mode, image_size, opts.background_color)
494 seqLimits1 = expandedSeqDict(seqLimits1)
495 seqLimits2 = expandedSeqDict(seqLimits2)
496 origins1 = expandedSeqDict(origins1)
497 origins2 = expandedSeqDict(origins2)
499 beds1 = itertools.chain(readBed(opts.bed1, seqLimits1),
500 readRmsk(opts.rmsk1, seqLimits1),
501 readGenePred(opts, opts.genePred1, seqLimits1),
502 readGaps(opts, opts.gap1, seqLimits1))
503 b1 = bedBoxes(beds1, seqLimits1, origins1, margin2, height, True, bpPerPix)
505 beds2 = itertools.chain(readBed(opts.bed2, seqLimits2),
506 readRmsk(opts.rmsk2, seqLimits2),
507 readGenePred(opts, opts.genePred2, seqLimits2),
508 readGaps(opts, opts.gap2, seqLimits2))
509 b2 = bedBoxes(beds2, seqLimits2, origins2, margin1, width, False, bpPerPix)
511 boxes = sorted(itertools.chain(b1, b2))
512 drawAnnotations(im, boxes)
514 for i in range(height):
515 for j in range(width):
516 store_value = hits[i * width + j]
518 if store_value == 1: im.putpixel(xy, forward_color)
519 elif store_value == 2: im.putpixel(xy, reverse_color)
520 elif store_value == 3: im.putpixel(xy, overlap_color)
522 if opts.fontsize != 0:
523 axis1 = get_axis_image(seqLabels1, labelSizes1, seq_starts1, seq_pix1,
524 font, image_mode, opts)
525 axis2 = get_axis_image(seqLabels2, labelSizes2, seq_starts2, seq_pix2,
526 font, image_mode, opts)
527 axis2 = axis2.transpose(Image.ROTATE_270) # !!! bug hotspot
528 im.paste(axis1, (0, 0))
529 im.paste(axis2, (0, 0))
531 for i in seq_starts1[1:]:
532 box = i - opts.border_pixels, margin2, i, height
533 im.paste(opts.border_color, box)
535 for i in seq_starts2[1:]:
536 box = margin1, i - opts.border_pixels, width, i
537 im.paste(opts.border_color, box)
541 if __name__ == "__main__":
542 usage = """%prog --help
543 or: %prog [options] maf-or-tab-alignments dotplot.png
544 or: %prog [options] maf-or-tab-alignments dotplot.gif
546 description = "Draw a dotplot of pair-wise sequence alignments in MAF or tabular format."
547 op = optparse.OptionParser(usage=usage, description=description)
548 op.add_option("-1", "--seq1", metavar="PATTERN", action="append",
550 help="which sequences to show from the 1st genome")
551 op.add_option("-2", "--seq2", metavar="PATTERN", action="append",
553 help="which sequences to show from the 2nd genome")
554 # Replace "width" & "height" with a single "length" option?
555 op.add_option("-x", "--width", type="int", default=1000,
556 help="maximum width in pixels (default: %default)")
557 op.add_option("-y", "--height", type="int", default=1000,
558 help="maximum height in pixels (default: %default)")
559 op.add_option("-c", "--forwardcolor", metavar="COLOR", default="red",
560 help="color for forward alignments (default: %default)")
561 op.add_option("-r", "--reversecolor", metavar="COLOR", default="blue",
562 help="color for reverse alignments (default: %default)")
563 op.add_option("--sort1", type="int", default=1, metavar="N",
564 help="genome1 sequence order: 0=input order, 1=name order, "
565 "2=length order (default=%default)")
566 op.add_option("--sort2", type="int", default=1, metavar="N",
567 help="genome2 sequence order: 0=input order, 1=name order, "
568 "2=length order (default=%default)")
569 op.add_option("--trim1", action="store_true",
570 help="trim unaligned sequence flanks from the 1st genome")
571 op.add_option("--trim2", action="store_true",
572 help="trim unaligned sequence flanks from the 2nd genome")
573 op.add_option("--border-pixels", metavar="INT", type="int", default=1,
574 help="number of pixels between sequences (default=%default)")
575 op.add_option("--border-color", metavar="COLOR", default="#dcdcdc",
576 help="color for pixels between sequences (default=%default)")
577 # xxx --margin-color?
579 og = optparse.OptionGroup(op, "Text options")
580 og.add_option("-f", "--fontfile", metavar="FILE",
581 help="TrueType or OpenType font file")
582 og.add_option("-s", "--fontsize", metavar="SIZE", type="int", default=11,
583 help="TrueType or OpenType font size (default: %default)")
584 og.add_option("--lengths1", action="store_true",
585 help="show sequence lengths for the 1st (horizontal) genome")
586 og.add_option("--lengths2", action="store_true",
587 help="show sequence lengths for the 2nd (vertical) genome")
588 op.add_option_group(og)
590 og = optparse.OptionGroup(op, "Annotation options")
591 og.add_option("--bed1", metavar="FILE",
592 help="read genome1 annotations from BED file")
593 og.add_option("--bed2", metavar="FILE",
594 help="read genome2 annotations from BED file")
595 og.add_option("--rmsk1", metavar="FILE", help="read genome1 repeats from "
596 "RepeatMasker .out or rmsk.txt file")
597 og.add_option("--rmsk2", metavar="FILE", help="read genome2 repeats from "
598 "RepeatMasker .out or rmsk.txt file")
599 op.add_option_group(og)
601 og = optparse.OptionGroup(op, "Gene options")
602 og.add_option("--genePred1", metavar="FILE",
603 help="read genome1 genes from genePred file")
604 og.add_option("--genePred2", metavar="FILE",
605 help="read genome2 genes from genePred file")
606 og.add_option("--exon-color", metavar="COLOR", default="#dfd",
607 help="color for exons (default=%default)")
608 og.add_option("--cds-color", metavar="COLOR", default="#bdb",
609 help="color for protein-coding regions (default=%default)")
610 op.add_option_group(og)
612 og = optparse.OptionGroup(op, "Unsequenced gap options")
613 og.add_option("--gap1", metavar="FILE",
614 help="read genome1 unsequenced gaps from agp or gap file")
615 og.add_option("--gap2", metavar="FILE",
616 help="read genome2 unsequenced gaps from agp or gap file")
617 og.add_option("--bridged-color", metavar="COLOR", default="yellow",
618 help="color for bridged gaps (default: %default)")
619 og.add_option("--unbridged-color", metavar="COLOR", default="pink",
620 help="color for unbridged gaps (default: %default)")
621 op.add_option_group(og)
622 (opts, args) = op.parse_args()
623 if len(args) != 2: op.error("2 arguments needed")
625 opts.text_color = "black"
626 opts.background_color = "white"
627 opts.label_space = 5 # minimum number of pixels between axis labels
629 try: lastDotplot(opts, args)
630 except KeyboardInterrupt: pass # avoid silly error message
632 prog = os.path.basename(sys.argv[0])
633 sys.exit(prog + ": error: " + str(e))