Add maf-convert to gff (with help from C. Plessy)
2 # Author: Martin C. Frith 2008
3 # SPDX-License-Identifier: GPL-3.0-or-later
5 # Read pair-wise alignments in MAF or LAST tabular format: write an
6 # "Oxford grid", a.k.a. dotplot.
8 # TODO: Currently, pixels with zero aligned nt-pairs are white, and
9 # pixels with one or more aligned nt-pairs are black. This can look
10 # too crowded for large genome alignments. I tried shading each pixel
11 # according to the number of aligned nt-pairs within it, but the
12 # result is too faint. How can this be done better?
17 from fnmatch import fnmatchcase
19 from operator import itemgetter
21 import itertools, optparse, os, re, sys
23 # Try to make PIL/PILLOW work:
25 from PIL import Image, ImageDraw, ImageFont, ImageColor
27 import Image, ImageDraw, ImageFont, ImageColor
30 from future_builtins import zip
34 def myOpen(fileName): # faster than fileinput
39 if fileName.endswith(".gz"):
40 return gzip.open(fileName, "rt") # xxx dubious for Python2
43 def groupByFirstItem(things):
44 for k, v in itertools.groupby(things, itemgetter(0)):
45 yield k, [i[1:] for i in v]
47 def croppedBlocks(blocks, ranges1, ranges2):
48 headBeg1, headBeg2, headSize = blocks[0]
51 cropBeg1, cropEnd1 = r1
53 cropBeg1, cropEnd1 = -cropEnd1, -cropBeg1
54 cropBeg2, cropEnd2 = r2
56 cropBeg2, cropEnd2 = -cropEnd2, -cropBeg2
57 for beg1, beg2, size in blocks:
58 b1 = max(cropBeg1, beg1)
59 e1 = min(cropEnd1, beg1 + size)
62 b2 = max(cropBeg2, b1 + offset)
63 e2 = min(cropEnd2, e1 + offset)
65 yield b2 - offset, b2, e2 - b2
67 def tabBlocks(beg1, beg2, blocks):
68 '''Get the gapless blocks of an alignment, from LAST tabular format.'''
69 for i in blocks.split(","):
76 yield beg1, beg2, size
80 def mafBlocks(beg1, beg2, seq1, seq2):
81 '''Get the gapless blocks of an alignment, from MAF format.'''
83 for x, y in zip(seq1, seq2):
86 yield beg1, beg2, size
93 yield beg1, beg2, size
100 if size: yield beg1, beg2, size
102 def alignmentFromSegment(qrySeqName, qrySeqLen, segment):
103 refSeqLen = sys.maxsize # XXX
104 refSeqName, refSeqBeg, qrySeqBeg, size = segment
105 block = refSeqBeg, qrySeqBeg, size
106 return refSeqName, refSeqLen, qrySeqName, qrySeqLen, [block]
108 def alignmentInput(lines):
109 '''Get alignments and sequence lengths, from MAF or tabular format.'''
119 yield alignmentFromSegment(qrySeqName, qrySeqLen, i)
123 elif len(w) == 2 and qrySeqName and w[1].isdigit():
124 qrySeqLen += int(w[1])
125 elif len(w) == 4 and qrySeqName and w[1].isdigit() and w[3].isdigit():
126 refSeqName, refSeqBeg, refSeqEnd = w[0], int(w[1]), int(w[3])
127 size = abs(refSeqEnd - refSeqBeg)
128 if refSeqBeg > refSeqEnd:
129 refSeqBeg = -refSeqBeg
130 segments.append((refSeqName, refSeqBeg, qrySeqLen, size))
132 elif line[0].isdigit(): # tabular format
133 chr1, beg1, seqlen1 = w[1], int(w[2]), int(w[5])
134 if w[4] == "-": beg1 -= seqlen1
135 chr2, beg2, seqlen2 = w[6], int(w[7]), int(w[10])
136 if w[9] == "-": beg2 -= seqlen2
137 blocks = tabBlocks(beg1, beg2, w[11])
138 yield chr1, seqlen1, chr2, seqlen2, blocks
139 elif line[0] == "s": # MAF format
141 chr1, beg1, seqlen1, seq1 = w[1], int(w[2]), int(w[5]), w[6]
142 if w[4] == "-": beg1 -= seqlen1
145 chr2, beg2, seqlen2, seq2 = w[1], int(w[2]), int(w[5]), w[6]
146 if w[4] == "-": beg2 -= seqlen2
147 blocks = mafBlocks(beg1, beg2, seq1, seq2)
148 yield chr1, seqlen1, chr2, seqlen2, blocks
151 yield alignmentFromSegment(qrySeqName, qrySeqLen, i)
153 def seqRequestFromText(text):
155 pattern, interval = text.rsplit(":", 1)
157 beg, end = interval.rsplit("-", 1)
158 return pattern, int(beg), int(end) # beg may be negative
159 return text, 0, sys.maxsize
161 def rangesFromSeqName(seqRequests, name, seqLen):
163 base = name.split(".")[-1] # allow for names like hg19.chr7
164 for pat, beg, end in seqRequests:
165 if fnmatchcase(name, pat) or fnmatchcase(base, pat):
166 yield max(beg, 0), min(end, seqLen)
170 def updateSeqs(coverDict, seqRanges, seqName, ranges, coveredRange):
171 beg, end = coveredRange
173 coveredRange = -end, -beg
174 if seqName in coverDict:
175 coverDict[seqName].append(coveredRange)
177 coverDict[seqName] = [coveredRange]
178 for beg, end in ranges:
179 r = seqName, beg, end
182 def readAlignments(fileName, opts):
183 '''Get alignments and sequence limits, from MAF or tabular format.'''
184 seqRequests1 = [seqRequestFromText(i) for i in opts.seq1]
185 seqRequests2 = [seqRequestFromText(i) for i in opts.seq2]
192 lines = myOpen(fileName)
193 for seqName1, seqLen1, seqName2, seqLen2, blocks in alignmentInput(lines):
194 ranges1 = sorted(rangesFromSeqName(seqRequests1, seqName1, seqLen1))
195 if not ranges1: continue
196 ranges2 = sorted(rangesFromSeqName(seqRequests2, seqName2, seqLen2))
197 if not ranges2: continue
198 b = list(croppedBlocks(list(blocks), ranges1, ranges2))
200 aln = seqName1, seqName2, b
201 alignments.append(aln)
202 coveredRange1 = b[0][0], b[-1][0] + b[-1][2]
203 updateSeqs(coverDict1, seqRanges1, seqName1, ranges1, coveredRange1)
204 coveredRange2 = b[0][1], b[-1][1] + b[-1][2]
205 updateSeqs(coverDict2, seqRanges2, seqName2, ranges2, coveredRange2)
206 return alignments, seqRanges1, coverDict1, seqRanges2, coverDict2
208 def nameAndRangesFromDict(cropDict, seqName):
209 if seqName in cropDict:
210 return seqName, cropDict[seqName]
211 n = seqName.split(".")[-1]
213 return n, cropDict[n]
216 def rangesForSecondaryAlignments(primaryRanges, seqLen):
221 def readSecondaryAlignments(opts, cropRanges1, cropRanges2):
222 cropDict1 = dict(groupByFirstItem(cropRanges1))
223 cropDict2 = dict(groupByFirstItem(cropRanges2))
230 lines = myOpen(opts.alignments)
231 for seqName1, seqLen1, seqName2, seqLen2, blocks in alignmentInput(lines):
232 seqName1, ranges1 = nameAndRangesFromDict(cropDict1, seqName1)
233 seqName2, ranges2 = nameAndRangesFromDict(cropDict2, seqName2)
234 if not ranges1 and not ranges2:
236 r1 = rangesForSecondaryAlignments(ranges1, seqLen1)
237 r2 = rangesForSecondaryAlignments(ranges2, seqLen2)
238 b = list(croppedBlocks(list(blocks), r1, r2))
240 aln = seqName1, seqName2, b
241 alignments.append(aln)
243 coveredRange1 = b[0][0], b[-1][0] + b[-1][2]
244 updateSeqs(coverDict1, seqRanges1, seqName1, r1, coveredRange1)
246 coveredRange2 = b[0][1], b[-1][1] + b[-1][2]
247 updateSeqs(coverDict2, seqRanges2, seqName2, r2, coveredRange2)
248 return alignments, seqRanges1, coverDict1, seqRanges2, coverDict2
250 def twoValuesFromOption(text, separator):
251 if separator in text:
252 return text.split(separator)
255 def mergedRanges(ranges):
256 oldBeg, maxEnd = ranges[0]
257 for beg, end in ranges:
266 def mergedRangesPerSeq(coverDict):
267 for k, v in coverDict.items():
269 yield k, list(mergedRanges(v))
271 def coveredLength(mergedCoverDict):
272 return sum(sum(e - b for b, e in v) for v in mergedCoverDict.values())
274 def trimmed(seqRanges, coverDict, minAlignedBases, maxGapFrac, endPad, midPad):
275 maxEndGapFrac, maxMidGapFrac = twoValuesFromOption(maxGapFrac, ",")
276 maxEndGap = max(float(maxEndGapFrac) * minAlignedBases, endPad * 1.0)
277 maxMidGap = max(float(maxMidGapFrac) * minAlignedBases, midPad * 2.0)
279 for seqName, rangeBeg, rangeEnd in seqRanges:
280 seqBlocks = coverDict[seqName]
281 blocks = [i for i in seqBlocks if i[0] < rangeEnd and i[1] > rangeBeg]
282 if blocks[0][0] - rangeBeg > maxEndGap:
283 rangeBeg = blocks[0][0] - endPad
284 for j, y in enumerate(blocks):
287 if y[0] - x[1] > maxMidGap:
288 yield seqName, rangeBeg, x[1] + midPad
289 rangeBeg = y[0] - midPad
290 if rangeEnd - blocks[-1][1] > maxEndGap:
291 rangeEnd = blocks[-1][1] + endPad
292 yield seqName, rangeBeg, rangeEnd
294 def rangesWithStrandInfo(seqRanges, strandOpt, alignments, seqIndex):
296 forwardMinusReverse = collections.defaultdict(int)
299 beg1, beg2, size = blocks[0]
300 numOfAlignedLetterPairs = sum(i[2] for i in blocks)
301 if (beg1 < 0) != (beg2 < 0): # opposite-strand alignment
302 numOfAlignedLetterPairs *= -1
303 forwardMinusReverse[i[seqIndex]] += numOfAlignedLetterPairs
305 for seqName, beg, end in seqRanges:
307 strandNum = 1 if forwardMinusReverse[seqName] >= 0 else 2
308 yield seqName, beg, end, strandNum
310 def natural_sort_key(my_string):
311 '''Return a sort key for "natural" ordering, e.g. chr9 < chr10.'''
312 parts = re.split(r'(\d+)', my_string)
313 parts[1::2] = map(int, parts[1::2])
316 def nameKey(oneSeqRanges):
317 return natural_sort_key(oneSeqRanges[0][0])
319 def sizeKey(oneSeqRanges):
320 return sum(b - e for n, b, e, s in oneSeqRanges), nameKey(oneSeqRanges)
322 def alignmentKey(seqNamesToLists, oneSeqRanges):
323 seqName = oneSeqRanges[0][0]
324 alignmentsOfThisSequence = seqNamesToLists[seqName]
325 numOfAlignedLetterPairs = sum(i[3] for i in alignmentsOfThisSequence)
326 toMiddle = numOfAlignedLetterPairs // 2
327 for i in alignmentsOfThisSequence:
330 return i[1:3] # sequence-rank and "position" of this alignment
332 def rankAndFlipPerSeq(seqRanges):
333 rangesGroupedBySeqName = itertools.groupby(seqRanges, itemgetter(0))
334 for rank, group in enumerate(rangesGroupedBySeqName):
335 seqName, ranges = group
336 strandNum = next(ranges)[3]
337 flip = 1 if strandNum < 2 else -1
338 yield seqName, (rank, flip)
340 def alignmentSortData(alignments, seqIndex, otherNamesToRanksAndFlips):
341 otherIndex = 1 - seqIndex
344 otherRank, otherFlip = otherNamesToRanksAndFlips[i[otherIndex]]
345 otherPos = otherFlip * abs(blocks[0][otherIndex] +
346 blocks[-1][otherIndex] + blocks[-1][2])
347 numOfAlignedLetterPairs = sum(i[2] for i in blocks)
348 yield i[seqIndex], otherRank, otherPos, numOfAlignedLetterPairs
350 def mySortedRanges(seqRanges, sortOpt, seqIndex, alignments, otherRanges):
351 rangesGroupedBySeqName = itertools.groupby(seqRanges, itemgetter(0))
352 g = [list(ranges) for seqName, ranges in rangesGroupedBySeqName]
361 otherNamesToRanksAndFlips = dict(rankAndFlipPerSeq(otherRanges))
362 alns = sorted(alignmentSortData(alignments, seqIndex,
363 otherNamesToRanksAndFlips))
364 alnsGroupedBySeqName = itertools.groupby(alns, itemgetter(0))
365 seqNamesToLists = dict((k, list(v)) for k, v in alnsGroupedBySeqName)
366 g.sort(key=functools.partial(alignmentKey, seqNamesToLists))
367 return [j for i in g for j in i]
369 def allSortedRanges(opts, alignments, alignmentsB,
370 seqRanges1, seqRangesB1, seqRanges2, seqRangesB2):
371 o1, oB1 = twoValuesFromOption(opts.strands1, ":")
372 o2, oB2 = twoValuesFromOption(opts.strands2, ":")
373 if o1 == "1" and o2 == "1":
374 raise Exception("the strand options have circular dependency")
375 seqRanges1 = list(rangesWithStrandInfo(seqRanges1, o1, alignments, 0))
376 seqRanges2 = list(rangesWithStrandInfo(seqRanges2, o2, alignments, 1))
377 seqRangesB1 = list(rangesWithStrandInfo(seqRangesB1, oB1, alignmentsB, 0))
378 seqRangesB2 = list(rangesWithStrandInfo(seqRangesB2, oB2, alignmentsB, 1))
380 o1, oB1 = twoValuesFromOption(opts.sort1, ":")
381 o2, oB2 = twoValuesFromOption(opts.sort2, ":")
382 if o1 == "3" and o2 == "3":
383 raise Exception("the sort options have circular dependency")
385 s1 = mySortedRanges(seqRanges1, o1, None, None, None)
387 s2 = mySortedRanges(seqRanges2, o2, None, None, None)
389 s1 = mySortedRanges(seqRanges1, o1, 0, alignments, s2)
391 s2 = mySortedRanges(seqRanges2, o2, 1, alignments, s1)
392 t1 = mySortedRanges(seqRangesB1, oB1, 0, alignmentsB, s2)
393 t2 = mySortedRanges(seqRangesB2, oB2, 1, alignmentsB, s1)
394 return s1 + t1, s2 + t2
400 groups.append(t[-3:])
402 return ",".join(reversed(groups))
405 suffixes = "bp", "kb", "Mb", "Gb"
406 for i, x in enumerate(suffixes):
409 return "%.2g" % (1.0 * size / j) + x
410 if size < j * 1000 or i == len(suffixes) - 1:
411 return "%.0f" % (1.0 * size / j) + x
413 def labelText(seqRange, labelOpt):
414 seqName, beg, end, strandNum = seqRange
416 return seqName + ": " + sizeText(end - beg)
418 return seqName + ":" + prettyNum(beg) + ": " + sizeText(end - beg)
420 return seqName + ":" + prettyNum(beg) + "-" + prettyNum(end)
423 def rangeLabels(seqRanges, labelOpt, font, fontsize, image_mode, textRot):
426 im = Image.new(image_mode, image_size)
427 draw = ImageDraw.Draw(im)
430 text = labelText(r, labelOpt)
432 x, y = draw.textsize(text, font=font)
435 yield text, x, y, r[3]
437 def dataFromRanges(sortedRanges, font, fontSize, imageMode, labelOpt, textRot):
438 for seqName, rangeBeg, rangeEnd, strandNum in sortedRanges:
439 out = [seqName, str(rangeBeg), str(rangeEnd)]
441 out.append(".+-"[strandNum])
442 logging.info("\t".join(out))
444 rangeSizes = [e - b for n, b, e, s in sortedRanges]
445 labs = list(rangeLabels(sortedRanges, labelOpt, font, fontSize,
447 margin = max(i[2] for i in labs)
448 # xxx the margin may be too big, because some labels may get omitted
449 return rangeSizes, labs, margin
452 '''Return x / y rounded up.'''
456 def get_bp_per_pix(rangeSizes, pixTweenRanges, maxPixels):
457 '''Get the minimum bp-per-pixel that fits in the size limit.'''
458 logging.info("choosing bp per pixel...")
459 numOfRanges = len(rangeSizes)
460 maxPixelsInRanges = maxPixels - pixTweenRanges * (numOfRanges - 1)
461 if maxPixelsInRanges < numOfRanges:
462 raise Exception("can't fit the image: too many sequences?")
463 negLimit = -maxPixelsInRanges
464 negBpPerPix = sum(rangeSizes) // negLimit
466 if sum(i // negBpPerPix for i in rangeSizes) >= negLimit:
470 def getRangePixBegs(rangePixLens, pixTweenRanges, margin):
471 '''Get the start pixel for each range.'''
473 pix_tot = margin - pixTweenRanges
474 for i in rangePixLens:
475 pix_tot += pixTweenRanges
476 rangePixBegs.append(pix_tot)
480 def pixelData(rangeSizes, bp_per_pix, pixTweenRanges, margin):
481 '''Return pixel information about the ranges.'''
482 rangePixLens = [div_ceil(i, bp_per_pix) for i in rangeSizes]
483 rangePixBegs = getRangePixBegs(rangePixLens, pixTweenRanges, margin)
484 tot_pix = rangePixBegs[-1] + rangePixLens[-1]
485 return rangePixBegs, rangePixLens, tot_pix
487 def drawLineForward(hits, width, bp_per_pix, beg1, beg2, size):
489 q1, r1 = divmod(beg1, bp_per_pix)
490 q2, r2 = divmod(beg2, bp_per_pix)
491 hits[q2 * width + q1] |= 1
492 next_pix = min(bp_per_pix - r1, bp_per_pix - r2)
493 if next_pix >= size: break
498 def drawLineReverse(hits, width, bp_per_pix, beg1, beg2, size):
500 q1, r1 = divmod(beg1, bp_per_pix)
501 q2, r2 = divmod(beg2, bp_per_pix)
502 hits[q2 * width + q1] |= 2
503 next_pix = min(bp_per_pix - r1, r2 + 1)
504 if next_pix >= size: break
509 def strandAndOrigin(ranges, beg, size):
510 isReverseStrand = (beg < 0)
513 for rangeBeg, rangeEnd, isReverseRange, origin in ranges:
514 if rangeEnd > beg: # assumes the ranges are sorted
515 return (isReverseStrand != isReverseRange), origin
517 def alignmentPixels(width, height, alignments, bp_per_pix,
518 rangeDict1, rangeDict2):
519 hits = [0] * (width * height) # the image data
520 for seq1, seq2, blocks in alignments:
521 beg1, beg2, size = blocks[0]
522 isReverse1, ori1 = strandAndOrigin(rangeDict1[seq1], beg1, size)
523 isReverse2, ori2 = strandAndOrigin(rangeDict2[seq2], beg2, size)
524 for beg1, beg2, size in blocks:
526 beg1 = -(beg1 + size)
527 beg2 = -(beg2 + size)
528 if isReverse1 == isReverse2:
529 drawLineForward(hits, width, bp_per_pix,
530 ori1 + beg1, ori2 + beg2, size)
532 drawLineReverse(hits, width, bp_per_pix,
533 ori1 + beg1, ori2 - beg2 - 1, size)
536 def orientedBlocks(alignments, seqIndex):
537 otherIndex = 1 - seqIndex
539 seq1, seq2, blocks = a
543 b = -(beg1 + size), -(beg2 + size), size
544 yield a[seqIndex], b[seqIndex], a[otherIndex], b[otherIndex], size
546 def drawJoins(im, alignments, bpPerPix, seqIndex, rangeDict1, rangeDict2):
547 blocks = orientedBlocks(alignments, seqIndex)
549 for seq1, beg1, seq2, beg2, size in sorted(blocks):
550 isReverse1, ori1 = strandAndOrigin(rangeDict1[seq1], beg1, size)
551 isReverse2, ori2 = strandAndOrigin(rangeDict2[seq2], beg2, size)
552 end1 = beg1 + size - 1
553 end2 = beg2 + size - 1
560 newPix1 = (ori1 + beg1) // bpPerPix
561 newPix2 = (ori2 + beg2) // bpPerPix
563 lowerPix2 = min(oldPix2, newPix2)
564 upperPix2 = max(oldPix2, newPix2)
565 midPix1 = (oldPix1 + newPix1) // 2
567 midPix1 = (oldPix1 + newPix1 + 1) // 2
568 oldPix1, newPix1 = newPix1, oldPix1
569 if upperPix2 - lowerPix2 > 1 and oldPix1 <= newPix1 <= oldPix1 + 1:
571 box = midPix1, lowerPix2, midPix1 + 1, upperPix2 + 1
573 box = lowerPix2, midPix1, upperPix2 + 1, midPix1 + 1
574 im.paste("lightgray", box)
575 oldPix1 = (ori1 + end1) // bpPerPix
576 oldPix2 = (ori2 + end2) // bpPerPix
579 def expandedSeqDict(seqDict):
580 '''Allow lookup by short sequence names, e.g. chr7 as well as hg19.chr7.'''
581 newDict = seqDict.copy()
582 for name, x in seqDict.items():
584 base = name.split(".")[-1]
585 if base in newDict: # an ambiguous case was found:
586 return seqDict # so give up completely
590 def readBed(fileName, rangeDict):
591 for line in myOpen(fileName):
595 if seqName not in rangeDict: continue
604 if len(w) > 8 and w[8].count(",") == 2:
605 color = "rgb(" + w[8] + ")"
608 isRev = rangeDict[seqName][0][2]
609 if strand == "+" and not isRev or strand == "-" and isRev:
611 if strand == "-" and not isRev or strand == "+" and isRev:
613 yield layer, color, seqName, beg, end
615 def commaSeparatedInts(text):
616 return map(int, text.rstrip(",").split(","))
618 def readGenePred(opts, fileName, rangeDict):
619 for line in myOpen(fileName):
620 fields = line.split()
621 if not fields: continue
622 if fields[2] not in "+-": fields = fields[1:]
624 if seqName not in rangeDict: continue
626 cdsBeg = int(fields[5])
627 cdsEnd = int(fields[6])
628 exonBegs = commaSeparatedInts(fields[8])
629 exonEnds = commaSeparatedInts(fields[9])
630 for beg, end in zip(exonBegs, exonEnds):
631 yield 300, opts.exon_color, seqName, beg, end
634 if b < e: yield 400, opts.cds_color, seqName, b, e
636 def readRmsk(fileName, rangeDict):
637 for line in myOpen(fileName):
638 fields = line.split()
639 if len(fields) == 17: # rmsk.txt
641 if seqName not in rangeDict: continue # do this ASAP for speed
645 repeatClass = fields[11]
646 elif len(fields) == 15: # .out
648 if seqName not in rangeDict: continue
649 beg = int(fields[5]) - 1
652 repeatClass = fields[10].split("/")[0]
655 if repeatClass in ("Low_complexity", "Simple_repeat", "Satellite"):
656 yield 200, "#fbf", seqName, beg, end
657 elif (strand == "+") != rangeDict[seqName][0][2]:
658 yield 100, "#ffe8e8", seqName, beg, end
660 yield 100, "#e8e8ff", seqName, beg, end
662 def isExtraFirstGapField(fields):
663 return fields[4].isdigit()
665 def readGaps(opts, fileName, rangeDict):
666 '''Read locations of unsequenced gaps, from an agp or gap file.'''
667 for line in myOpen(fileName):
669 if not w or w[0][0] == "#": continue
670 if isExtraFirstGapField(w): w = w[1:]
671 if w[4] not in "NU": continue
673 if seqName not in rangeDict: continue
675 beg = end - int(w[5]) # zero-based coordinate
677 yield 3000, opts.bridged_color, seqName, beg, end
679 yield 2000, opts.unbridged_color, seqName, beg, end
681 def bedBoxes(beds, rangeDict, margin, edge, isTop, bpPerPix):
682 for layer, color, seqName, bedBeg, bedEnd in beds:
683 for rangeBeg, rangeEnd, isReverseRange, origin in rangeDict[seqName]:
684 beg = max(bedBeg, rangeBeg)
685 end = min(bedEnd, rangeEnd)
686 if beg >= end: continue
688 beg, end = -end, -beg
690 # include partly-covered pixels
691 b = (origin + beg) // bpPerPix
692 e = div_ceil(origin + end, bpPerPix)
694 # exclude partly-covered pixels
695 b = div_ceil(origin + beg, bpPerPix)
696 e = (origin + end) // bpPerPix
698 if bedEnd >= rangeEnd: # include partly-covered end pixels
700 b = (origin + beg) // bpPerPix
702 e = div_ceil(origin + end, bpPerPix)
704 box = b, margin, e, edge
706 box = margin, b, edge, e
707 yield layer, color, box
709 def drawAnnotations(im, boxes):
710 # xxx use partial transparency for different-color overlaps?
711 for layer, color, box in boxes:
714 def placedLabels(labels, rangePixBegs, rangePixLens, beg, end):
715 '''Return axis labels with endpoint & sort-order information.'''
717 for i, j, k in zip(labels, rangePixBegs, rangePixLens):
718 text, textWidth, textHeight, strandNum = i
719 if textWidth > maxWidth:
721 labelBeg = j + (k - textWidth) // 2
722 labelEnd = labelBeg + textWidth
723 sortKey = textWidth - k
725 sortKey += maxWidth * (beg - labelBeg)
727 labelEnd = beg + textWidth
729 sortKey += maxWidth * (labelEnd - end)
731 labelBeg = end - textWidth
732 yield sortKey, labelBeg, labelEnd, text, textHeight, strandNum
734 def nonoverlappingLabels(labels, minPixTweenLabels):
735 '''Get a subset of non-overlapping axis labels, greedily.'''
738 beg = i[1] - minPixTweenLabels
739 end = i[2] + minPixTweenLabels
740 if all(j[2] <= beg or j[1] >= end for j in out):
744 def axisImage(labels, rangePixBegs, rangePixLens, textRot,
745 textAln, font, image_mode, opts):
746 '''Make an image of axis labels.'''
747 beg = rangePixBegs[0]
748 end = rangePixBegs[-1] + rangePixLens[-1]
749 margin = max(i[2] for i in labels)
750 labels = sorted(placedLabels(labels, rangePixBegs, rangePixLens, beg, end))
751 minPixTweenLabels = 0 if textRot else opts.label_space
752 labels = nonoverlappingLabels(labels, minPixTweenLabels)
753 image_size = (margin, end) if textRot else (end, margin)
754 im = Image.new(image_mode, image_size, opts.margin_color)
755 draw = ImageDraw.Draw(im)
756 for sortKey, labelBeg, labelEnd, text, textHeight, strandNum in labels:
757 base = margin - textHeight if textAln else 0
758 position = (base, labelBeg) if textRot else (labelBeg, base)
759 fill = ("black", opts.forwardcolor, opts.reversecolor)[strandNum]
760 draw.text(position, text, font=font, fill=fill)
763 def rangesWithOrigins(sortedRanges, rangePixBegs, rangePixLens, bpPerPix):
764 for i, j, k in zip(sortedRanges, rangePixBegs, rangePixLens):
765 seqName, rangeBeg, rangeEnd, strandNum = i
766 isReverseRange = (strandNum > 1)
768 origin = bpPerPix * (j + k) + rangeBeg
770 origin = bpPerPix * j - rangeBeg
771 yield seqName, (rangeBeg, rangeEnd, isReverseRange, origin)
773 def rangesPerSeq(sortedRanges, rangePixBegs, rangePixLens, bpPerPix):
774 a = rangesWithOrigins(sortedRanges, rangePixBegs, rangePixLens, bpPerPix)
775 for k, v in itertools.groupby(a, itemgetter(0)):
776 yield k, sorted(i[1] for i in v)
780 return ImageFont.truetype(opts.fontfile, opts.fontsize)
783 x = ["fc-match", "-f%{file}", "arial"]
784 p = subprocess.Popen(x, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
785 universal_newlines=True)
786 out, err = p.communicate()
787 fileNames.append(out)
789 logging.info("fc-match error: " + str(e))
790 fileNames.append("/Library/Fonts/Arial.ttf") # for Mac
793 font = ImageFont.truetype(i, opts.fontsize)
794 logging.info("font: " + i)
797 logging.info("font load error: " + str(e))
798 return ImageFont.load_default()
800 def sequenceSizesAndNames(seqRanges):
801 for seqName, ranges in itertools.groupby(seqRanges, itemgetter(0)):
802 size = sum(e - b for n, b, e in ranges)
805 def biggestSequences(seqRanges, maxNumOfSequences):
806 s = sorted(sequenceSizesAndNames(seqRanges), reverse=True)
807 if len(s) > maxNumOfSequences:
808 logging.warning("too many sequences - discarding the smallest ones")
809 s = s[:maxNumOfSequences]
810 return set(i[1] for i in s)
812 def remainingSequenceRanges(seqRanges, alignments, seqIndex):
813 remainingSequences = set(i[seqIndex] for i in alignments)
814 return [i for i in seqRanges if i[0] in remainingSequences]
816 def lastDotplot(opts, args):
817 logLevel = logging.INFO if opts.verbose else logging.WARNING
818 logging.basicConfig(format="%(filename)s: %(message)s", level=logLevel)
822 forward_color = ImageColor.getcolor(opts.forwardcolor, image_mode)
823 reverse_color = ImageColor.getcolor(opts.reversecolor, image_mode)
824 zipped_colors = zip(forward_color, reverse_color)
825 overlap_color = tuple([(i + j) // 2 for i, j in zipped_colors])
827 maxGap1, maxGapB1 = twoValuesFromOption(opts.max_gap1, ":")
828 maxGap2, maxGapB2 = twoValuesFromOption(opts.max_gap2, ":")
830 logging.info("reading alignments...")
831 alnData = readAlignments(args[0], opts)
832 alignments, seqRanges1, coverDict1, seqRanges2, coverDict2 = alnData
833 if not alignments: raise Exception("there are no alignments")
834 logging.info("cutting...")
835 coverDict1 = dict(mergedRangesPerSeq(coverDict1))
836 coverDict2 = dict(mergedRangesPerSeq(coverDict2))
837 minAlignedBases = min(coveredLength(coverDict1), coveredLength(coverDict2))
838 pad = int(opts.pad * minAlignedBases)
839 cutRanges1 = list(trimmed(seqRanges1, coverDict1, minAlignedBases,
841 cutRanges2 = list(trimmed(seqRanges2, coverDict2, minAlignedBases,
844 biggestSeqs1 = biggestSequences(cutRanges1, opts.maxseqs)
845 cutRanges1 = [i for i in cutRanges1 if i[0] in biggestSeqs1]
846 alignments = [i for i in alignments if i[0] in biggestSeqs1]
847 cutRanges2 = remainingSequenceRanges(cutRanges2, alignments, 1)
849 biggestSeqs2 = biggestSequences(cutRanges2, opts.maxseqs)
850 cutRanges2 = [i for i in cutRanges2 if i[0] in biggestSeqs2]
851 alignments = [i for i in alignments if i[1] in biggestSeqs2]
852 cutRanges1 = remainingSequenceRanges(cutRanges1, alignments, 0)
854 logging.info("reading secondary alignments...")
855 alnDataB = readSecondaryAlignments(opts, cutRanges1, cutRanges2)
856 alignmentsB, seqRangesB1, coverDictB1, seqRangesB2, coverDictB2 = alnDataB
857 logging.info("cutting...")
858 coverDictB1 = dict(mergedRangesPerSeq(coverDictB1))
859 coverDictB2 = dict(mergedRangesPerSeq(coverDictB2))
860 cutRangesB1 = trimmed(seqRangesB1, coverDictB1, minAlignedBases,
862 cutRangesB2 = trimmed(seqRangesB2, coverDictB2, minAlignedBases,
865 logging.info("sorting...")
866 sortOut = allSortedRanges(opts, alignments, alignmentsB,
867 cutRanges1, cutRangesB1, cutRanges2, cutRangesB2)
868 sortedRanges1, sortedRanges2 = sortOut
870 textRot1 = "vertical".startswith(opts.rot1)
871 i1 = dataFromRanges(sortedRanges1, font,
872 opts.fontsize, image_mode, opts.labels1, textRot1)
873 rangeSizes1, labelData1, tMargin = i1
875 textRot2 = "horizontal".startswith(opts.rot2)
876 i2 = dataFromRanges(sortedRanges2, font,
877 opts.fontsize, image_mode, opts.labels2, textRot2)
878 rangeSizes2, labelData2, lMargin = i2
880 maxPixels1 = opts.width - lMargin
881 maxPixels2 = opts.height - tMargin
882 bpPerPix1 = get_bp_per_pix(rangeSizes1, opts.border_pixels, maxPixels1)
883 bpPerPix2 = get_bp_per_pix(rangeSizes2, opts.border_pixels, maxPixels2)
884 bpPerPix = max(bpPerPix1, bpPerPix2)
885 logging.info("bp per pixel = " + str(bpPerPix))
887 p1 = pixelData(rangeSizes1, bpPerPix, opts.border_pixels, lMargin)
888 rangePixBegs1, rangePixLens1, width = p1
889 rangeDict1 = dict(rangesPerSeq(sortedRanges1, rangePixBegs1,
890 rangePixLens1, bpPerPix))
892 p2 = pixelData(rangeSizes2, bpPerPix, opts.border_pixels, tMargin)
893 rangePixBegs2, rangePixLens2, height = p2
894 rangeDict2 = dict(rangesPerSeq(sortedRanges2, rangePixBegs2,
895 rangePixLens2, bpPerPix))
897 logging.info("width: " + str(width))
898 logging.info("height: " + str(height))
900 logging.info("processing alignments...")
901 allAlignments = alignments + alignmentsB
902 hits = alignmentPixels(width, height, allAlignments, bpPerPix,
903 rangeDict1, rangeDict2)
905 logging.info("reading annotations...")
907 rangeDict1 = expandedSeqDict(rangeDict1)
908 beds1 = itertools.chain(readBed(opts.bed1, rangeDict1),
909 readRmsk(opts.rmsk1, rangeDict1),
910 readGenePred(opts, opts.genePred1, rangeDict1),
911 readGaps(opts, opts.gap1, rangeDict1))
912 b1 = bedBoxes(beds1, rangeDict1, tMargin, height, True, bpPerPix)
914 rangeDict2 = expandedSeqDict(rangeDict2)
915 beds2 = itertools.chain(readBed(opts.bed2, rangeDict2),
916 readRmsk(opts.rmsk2, rangeDict2),
917 readGenePred(opts, opts.genePred2, rangeDict2),
918 readGaps(opts, opts.gap2, rangeDict2))
919 b2 = bedBoxes(beds2, rangeDict2, lMargin, width, False, bpPerPix)
921 boxes = sorted(itertools.chain(b1, b2))
923 logging.info("drawing...")
925 image_size = width, height
926 im = Image.new(image_mode, image_size, opts.background_color)
928 drawAnnotations(im, boxes)
930 joinA, joinB = twoValuesFromOption(opts.join, ":")
932 drawJoins(im, alignments, bpPerPix, 0, rangeDict1, rangeDict2)
934 drawJoins(im, alignmentsB, bpPerPix, 0, rangeDict1, rangeDict2)
936 drawJoins(im, alignments, bpPerPix, 1, rangeDict2, rangeDict1)
938 drawJoins(im, alignmentsB, bpPerPix, 1, rangeDict2, rangeDict1)
940 for i in range(height):
941 for j in range(width):
942 store_value = hits[i * width + j]
944 if store_value == 1: im.putpixel(xy, forward_color)
945 elif store_value == 2: im.putpixel(xy, reverse_color)
946 elif store_value == 3: im.putpixel(xy, overlap_color)
948 if opts.fontsize != 0:
949 axis1 = axisImage(labelData1, rangePixBegs1, rangePixLens1,
950 textRot1, False, font, image_mode, opts)
952 axis1 = axis1.transpose(Image.ROTATE_90)
953 axis2 = axisImage(labelData2, rangePixBegs2, rangePixLens2,
954 textRot2, textRot2, font, image_mode, opts)
956 axis2 = axis2.transpose(Image.ROTATE_270)
957 im.paste(axis1, (0, 0))
958 im.paste(axis2, (0, 0))
960 for i in rangePixBegs1[1:]:
961 box = i - opts.border_pixels, tMargin, i, height
962 im.paste(opts.border_color, box)
964 for i in rangePixBegs2[1:]:
965 box = lMargin, i - opts.border_pixels, width, i
966 im.paste(opts.border_color, box)
970 if __name__ == "__main__":
971 usage = """%prog --help
972 or: %prog [options] maf-or-tab-alignments dotplot.png
973 or: %prog [options] maf-or-tab-alignments dotplot.gif
975 description = "Draw a dotplot of pair-wise sequence alignments in MAF or tabular format."
976 op = optparse.OptionParser(usage=usage, description=description)
977 op.add_option("-v", "--verbose", action="count",
978 help="show progress messages & data about the plot")
979 # Replace "width" & "height" with a single "length" option?
980 op.add_option("-x", "--width", metavar="INT", type="int", default=1000,
981 help="maximum width in pixels (default: %default)")
982 op.add_option("-y", "--height", metavar="INT", type="int", default=1000,
983 help="maximum height in pixels (default: %default)")
984 op.add_option("-m", "--maxseqs", type="int", default=100, metavar="M",
985 help="maximum number of horizontal or vertical sequences "
986 "(default=%default)")
987 op.add_option("-1", "--seq1", metavar="PATTERN", action="append",
989 help="which sequences to show from the 1st genome")
990 op.add_option("-2", "--seq2", metavar="PATTERN", action="append",
992 help="which sequences to show from the 2nd genome")
993 op.add_option("-c", "--forwardcolor", metavar="COLOR", default="red",
994 help="color for forward alignments (default: %default)")
995 op.add_option("-r", "--reversecolor", metavar="COLOR", default="blue",
996 help="color for reverse alignments (default: %default)")
997 op.add_option("--alignments", metavar="FILE", help="secondary alignments")
998 op.add_option("--sort1", default="1", metavar="N",
999 help="genome1 sequence order: 0=input order, 1=name order, "
1000 "2=length order, 3=alignment order (default=%default)")
1001 op.add_option("--sort2", default="1", metavar="N",
1002 help="genome2 sequence order: 0=input order, 1=name order, "
1003 "2=length order, 3=alignment order (default=%default)")
1004 op.add_option("--strands1", default="0", metavar="N", help=
1005 "genome1 sequence orientation: 0=forward orientation, "
1006 "1=alignment orientation (default=%default)")
1007 op.add_option("--strands2", default="0", metavar="N", help=
1008 "genome2 sequence orientation: 0=forward orientation, "
1009 "1=alignment orientation (default=%default)")
1010 op.add_option("--max-gap1", metavar="FRAC", default="0.5,2", help=
1011 "maximum unaligned (end,mid) gap in genome1: "
1012 "fraction of aligned length (default=%default)")
1013 op.add_option("--max-gap2", metavar="FRAC", default="0.5,2", help=
1014 "maximum unaligned (end,mid) gap in genome2: "
1015 "fraction of aligned length (default=%default)")
1016 op.add_option("--pad", metavar="FRAC", type="float", default=0.04, help=
1017 "pad length when cutting unaligned gaps: "
1018 "fraction of aligned length (default=%default)")
1019 op.add_option("-j", "--join", default="0", metavar="N", help=
1020 "join: 0=nothing, 1=alignments adjacent in genome1, "
1021 "2=alignments adjacent in genome2 (default=%default)")
1022 op.add_option("--border-pixels", metavar="INT", type="int", default=1,
1023 help="number of pixels between sequences (default=%default)")
1024 op.add_option("--border-color", metavar="COLOR", default="black",
1025 help="color for pixels between sequences (default=%default)")
1026 # --break-color and/or --break-pixels for intra-sequence breaks?
1027 op.add_option("--margin-color", metavar="COLOR", default="#dcdcdc",
1028 help="margin color")
1030 og = optparse.OptionGroup(op, "Text options")
1031 og.add_option("-f", "--fontfile", metavar="FILE",
1032 help="TrueType or OpenType font file")
1033 og.add_option("-s", "--fontsize", metavar="SIZE", type="int", default=14,
1034 help="TrueType or OpenType font size (default: %default)")
1035 og.add_option("--labels1", type="int", default=0, metavar="N", help=
1036 "genome1 labels: 0=name, 1=name:length, "
1037 "2=name:start:length, 3=name:start-end (default=%default)")
1038 og.add_option("--labels2", type="int", default=0, metavar="N", help=
1039 "genome2 labels: 0=name, 1=name:length, "
1040 "2=name:start:length, 3=name:start-end (default=%default)")
1041 og.add_option("--rot1", metavar="ROT", default="h",
1042 help="text rotation for the 1st genome (default=%default)")
1043 og.add_option("--rot2", metavar="ROT", default="v",
1044 help="text rotation for the 2nd genome (default=%default)")
1045 op.add_option_group(og)
1047 og = optparse.OptionGroup(op, "Annotation options")
1048 og.add_option("--bed1", metavar="FILE",
1049 help="read genome1 annotations from BED file")
1050 og.add_option("--bed2", metavar="FILE",
1051 help="read genome2 annotations from BED file")
1052 og.add_option("--rmsk1", metavar="FILE", help="read genome1 repeats from "
1053 "RepeatMasker .out or rmsk.txt file")
1054 og.add_option("--rmsk2", metavar="FILE", help="read genome2 repeats from "
1055 "RepeatMasker .out or rmsk.txt file")
1056 op.add_option_group(og)
1058 og = optparse.OptionGroup(op, "Gene options")
1059 og.add_option("--genePred1", metavar="FILE",
1060 help="read genome1 genes from genePred file")
1061 og.add_option("--genePred2", metavar="FILE",
1062 help="read genome2 genes from genePred file")
1063 og.add_option("--exon-color", metavar="COLOR", default="PaleGreen",
1064 help="color for exons (default=%default)")
1065 og.add_option("--cds-color", metavar="COLOR", default="LimeGreen",
1066 help="color for protein-coding regions (default=%default)")
1067 op.add_option_group(og)
1069 og = optparse.OptionGroup(op, "Unsequenced gap options")
1070 og.add_option("--gap1", metavar="FILE",
1071 help="read genome1 unsequenced gaps from agp or gap file")
1072 og.add_option("--gap2", metavar="FILE",
1073 help="read genome2 unsequenced gaps from agp or gap file")
1074 og.add_option("--bridged-color", metavar="COLOR", default="yellow",
1075 help="color for bridged gaps (default: %default)")
1076 og.add_option("--unbridged-color", metavar="COLOR", default="orange",
1077 help="color for unbridged gaps (default: %default)")
1078 op.add_option_group(og)
1079 (opts, args) = op.parse_args()
1080 if len(args) != 2: op.error("2 arguments needed")
1082 opts.background_color = "white"
1083 opts.label_space = 5 # minimum number of pixels between axis labels
1085 try: lastDotplot(opts, args)
1086 except KeyboardInterrupt: pass # avoid silly error message
1087 except Exception as e:
1088 prog = os.path.basename(sys.argv[0])
1089 sys.exit(prog + ": error: " + str(e))