scripts/last-dotplot
author Martin C. Frith
Wed Nov 08 15:45:08 2017 +0900 (2017-11-08)
changeset 908 6840398323e1
parent 907 87991b57ed04
child 909 92dfad507286
permissions -rwxr-xr-x
Refactor last-dotplot
Martin@1
     1
#! /usr/bin/env python
Martin@1
     2
Martin@272
     3
# Read pair-wise alignments in MAF or LAST tabular format: write an
Martin@272
     4
# "Oxford grid", a.k.a. dotplot.
Martin@1
     5
Martin@1
     6
# TODO: Currently, pixels with zero aligned nt-pairs are white, and
Martin@1
     7
# pixels with one or more aligned nt-pairs are black.  This can look
Martin@1
     8
# too crowded for large genome alignments.  I tried shading each pixel
Martin@1
     9
# according to the number of aligned nt-pairs within it, but the
Martin@1
    10
# result is too faint.  How can this be done better?
Martin@1
    11
Martin@875
    12
import gzip
Martin@896
    13
from fnmatch import fnmatchcase
Martin@906
    14
from operator import itemgetter
Martin@903
    15
import subprocess
Martin@896
    16
import itertools, optparse, os, re, sys
Martin@475
    17
Martin@475
    18
# Try to make PIL/PILLOW work:
Martin@475
    19
try: from PIL import Image, ImageDraw, ImageFont, ImageColor
Martin@475
    20
except ImportError: import Image, ImageDraw, ImageFont, ImageColor
Martin@1
    21
Martin@844
    22
def myOpen(fileName):  # faster than fileinput
Martin@904
    23
    if fileName is None:
Martin@904
    24
        return []
Martin@844
    25
    if fileName == "-":
Martin@844
    26
        return sys.stdin
Martin@875
    27
    if fileName.endswith(".gz"):
Martin@875
    28
        return gzip.open(fileName)
Martin@844
    29
    return open(fileName)
Martin@844
    30
Martin@644
    31
def warn(message):
Martin@866
    32
    if opts.verbose:
Martin@866
    33
        prog = os.path.basename(sys.argv[0])
Martin@866
    34
        sys.stderr.write(prog + ": " + message + "\n")
Martin@644
    35
Martin@908
    36
def croppedBlocks(blocks, ranges1, ranges2):
Martin@908
    37
    headBeg1, headBeg2, headSize = blocks[0]
Martin@908
    38
    for r1 in ranges1:
Martin@908
    39
        for r2 in ranges2:
Martin@908
    40
            cropBeg1, cropEnd1 = r1
Martin@908
    41
            if headBeg1 < 0:
Martin@908
    42
                cropBeg1, cropEnd1 = -cropEnd1, -cropBeg1
Martin@908
    43
            cropBeg2, cropEnd2 = r2
Martin@908
    44
            if headBeg2 < 0:
Martin@908
    45
                cropBeg2, cropEnd2 = -cropEnd2, -cropBeg2
Martin@908
    46
            for beg1, beg2, size in blocks:
Martin@908
    47
                b1 = max(cropBeg1, beg1)
Martin@908
    48
                e1 = min(cropEnd1, beg1 + size)
Martin@908
    49
                if b1 >= e1: continue
Martin@908
    50
                offset = beg2 - beg1
Martin@908
    51
                b2 = max(cropBeg2, b1 + offset)
Martin@908
    52
                e2 = min(cropEnd2, e1 + offset)
Martin@908
    53
                if b2 >= e2: continue
Martin@908
    54
                yield b2 - offset, b2, e2 - b2
Martin@840
    55
Martin@482
    56
def tabBlocks(beg1, beg2, blocks):
Martin@482
    57
    '''Get the gapless blocks of an alignment, from LAST tabular format.'''
Martin@482
    58
    for i in blocks.split(","):
Martin@482
    59
        if ":" in i:
Martin@482
    60
            x, y = i.split(":")
Martin@482
    61
            beg1 += int(x)
Martin@482
    62
            beg2 += int(y)
Martin@482
    63
        else:
Martin@482
    64
            size = int(i)
Martin@482
    65
            yield beg1, beg2, size
Martin@482
    66
            beg1 += size
Martin@482
    67
            beg2 += size
Martin@272
    68
Martin@482
    69
def mafBlocks(beg1, beg2, seq1, seq2):
Martin@482
    70
    '''Get the gapless blocks of an alignment, from MAF format.'''
Martin@482
    71
    size = 0
Martin@482
    72
    for x, y in itertools.izip(seq1, seq2):
Martin@482
    73
        if x == "-":
Martin@482
    74
            if size:
Martin@482
    75
                yield beg1, beg2, size
Martin@482
    76
                beg1 += size
Martin@482
    77
                beg2 += size
Martin@482
    78
                size = 0
Martin@482
    79
            beg2 += 1
Martin@482
    80
        elif y == "-":
Martin@482
    81
            if size:
Martin@482
    82
                yield beg1, beg2, size
Martin@482
    83
                beg1 += size
Martin@482
    84
                beg2 += size
Martin@482
    85
                size = 0
Martin@482
    86
            beg1 += 1
Martin@272
    87
        else:
Martin@482
    88
            size += 1
Martin@482
    89
    if size: yield beg1, beg2, size
Martin@272
    90
Martin@482
    91
def alignmentInput(lines):
Martin@482
    92
    '''Get alignments and sequence lengths, from MAF or tabular format.'''
Martin@482
    93
    mafCount = 0
Martin@272
    94
    for line in lines:
Martin@272
    95
        w = line.split()
Martin@272
    96
        if line[0].isdigit():  # tabular format
Martin@482
    97
            chr1, beg1, seqlen1 = w[1], int(w[2]), int(w[5])
Martin@482
    98
            if w[4] == "-": beg1 -= seqlen1
Martin@482
    99
            chr2, beg2, seqlen2 = w[6], int(w[7]), int(w[10])
Martin@482
   100
            if w[9] == "-": beg2 -= seqlen2
Martin@847
   101
            blocks = tabBlocks(beg1, beg2, w[11])
Martin@482
   102
            yield chr1, seqlen1, chr2, seqlen2, blocks
Martin@272
   103
        elif line[0] == "s":  # MAF format
Martin@482
   104
            if mafCount == 0:
Martin@482
   105
                chr1, beg1, seqlen1, seq1 = w[1], int(w[2]), int(w[5]), w[6]
Martin@482
   106
                if w[4] == "-": beg1 -= seqlen1
Martin@482
   107
                mafCount = 1
Martin@482
   108
            else:
Martin@482
   109
                chr2, beg2, seqlen2, seq2 = w[1], int(w[2]), int(w[5]), w[6]
Martin@482
   110
                if w[4] == "-": beg2 -= seqlen2
Martin@847
   111
                blocks = mafBlocks(beg1, beg2, seq1, seq2)
Martin@482
   112
                yield chr1, seqlen1, chr2, seqlen2, blocks
Martin@482
   113
                mafCount = 0
Martin@272
   114
Martin@897
   115
def seqRequestFromText(text):
Martin@840
   116
    if ":" in text:
Martin@840
   117
        pattern, interval = text.rsplit(":", 1)
Martin@840
   118
        if "-" in interval:
Martin@840
   119
            beg, end = interval.rsplit("-", 1)
Martin@840
   120
            return pattern, int(beg), int(end)  # beg may be negative
Martin@840
   121
    return text, 0, sys.maxsize
Martin@840
   122
Martin@897
   123
def rangeFromSeqName(seqRequests, name, seqLen):
Martin@908
   124
    if seqRequests:
Martin@908
   125
        base = name.split(".")[-1]  # allow for names like hg19.chr7
Martin@908
   126
        for pat, beg, end in seqRequests:
Martin@908
   127
            if fnmatchcase(name, pat) or fnmatchcase(base, pat):
Martin@908
   128
                return max(beg, 0), min(end, seqLen)
Martin@908
   129
        return None
Martin@908
   130
    else:
Martin@908
   131
        return 0, seqLen
Martin@651
   132
Martin@851
   133
def updateSeqs(isTrim, seqNames, seqLimits, seqName, seqRange, blocks, index):
Martin@851
   134
    if seqName not in seqLimits:
Martin@851
   135
        seqNames.append(seqName)
Martin@839
   136
    if isTrim:
Martin@839
   137
        beg = blocks[0][index]
Martin@839
   138
        end = blocks[-1][index] + blocks[-1][2]
Martin@839
   139
        if beg < 0: beg, end = -end, -beg
Martin@839
   140
        if seqName in seqLimits:
Martin@839
   141
            b, e = seqLimits[seqName]
Martin@839
   142
            seqLimits[seqName] = min(b, beg), max(e, end)
Martin@839
   143
        else:
Martin@839
   144
            seqLimits[seqName] = beg, end
Martin@839
   145
    else:
Martin@840
   146
        seqLimits[seqName] = seqRange
Martin@839
   147
Martin@651
   148
def readAlignments(fileName, opts):
Martin@839
   149
    '''Get alignments and sequence limits, from MAF or tabular format.'''
Martin@897
   150
    seqRequests1 = map(seqRequestFromText, opts.seq1)
Martin@897
   151
    seqRequests2 = map(seqRequestFromText, opts.seq2)
Martin@840
   152
Martin@482
   153
    alignments = []
Martin@851
   154
    seqNames1 = []
Martin@851
   155
    seqNames2 = []
Martin@839
   156
    seqLimits1 = {}
Martin@839
   157
    seqLimits2 = {}
Martin@844
   158
    lines = myOpen(fileName)
Martin@838
   159
    for seqName1, seqLen1, seqName2, seqLen2, blocks in alignmentInput(lines):
Martin@897
   160
        range1 = rangeFromSeqName(seqRequests1, seqName1, seqLen1)
Martin@844
   161
        if not range1: continue
Martin@897
   162
        range2 = rangeFromSeqName(seqRequests2, seqName2, seqLen2)
Martin@844
   163
        if not range2: continue
Martin@908
   164
        b = list(croppedBlocks(list(blocks), [range1], [range2]))
Martin@840
   165
        if not b: continue
Martin@840
   166
        aln = seqName1, seqName2, b
Martin@482
   167
        alignments.append(aln)
Martin@851
   168
        updateSeqs(opts.trim1, seqNames1, seqLimits1, seqName1, range1, b, 0)
Martin@851
   169
        updateSeqs(opts.trim2, seqNames2, seqLimits2, seqName2, range2, b, 1)
Martin@908
   170
    seqRanges1 = [(i, seqLimits1[i][0], seqLimits1[i][1]) for i in seqNames1]
Martin@908
   171
    seqRanges2 = [(i, seqLimits2[i][0], seqLimits2[i][1]) for i in seqNames2]
Martin@908
   172
    return alignments, seqRanges1, seqRanges2
Martin@1
   173
Martin@1
   174
def natural_sort_key(my_string):
Martin@1
   175
    '''Return a sort key for "natural" ordering, e.g. chr9 < chr10.'''
Martin@1
   176
    parts = re.split(r'(\d+)', my_string)
Martin@1
   177
    parts[1::2] = map(int, parts[1::2])
Martin@1
   178
    return parts
Martin@1
   179
Martin@907
   180
def nameKey(oneSeqRanges):
Martin@907
   181
    return natural_sort_key(oneSeqRanges[0][0])
Martin@907
   182
Martin@907
   183
def sizeKey(oneSeqRanges):
Martin@907
   184
    return sum(b - e for n, b, e in oneSeqRanges), nameKey(oneSeqRanges)
Martin@907
   185
Martin@907
   186
def getSortedRanges(seqRanges, sortOpt):
Martin@907
   187
    g = [list(v) for k, v in itertools.groupby(seqRanges, itemgetter(0))]
Martin@907
   188
    if sortOpt == "1":
Martin@907
   189
        g.sort(key=nameKey)
Martin@907
   190
    if sortOpt == "2":
Martin@907
   191
        g.sort(key=sizeKey)
Martin@907
   192
    return [j for i in g for j in i]
Martin@907
   193
Martin@898
   194
def prettyNum(n):
Martin@898
   195
    t = str(n)
Martin@898
   196
    groups = []
Martin@898
   197
    while t:
Martin@898
   198
        groups.append(t[-3:])
Martin@898
   199
        t = t[:-3]
Martin@904
   200
    return ",".join(reversed(groups))
Martin@898
   201
Martin@846
   202
def sizeText(size):
Martin@846
   203
    suffixes = "bp", "kb", "Mb", "Gb"
Martin@846
   204
    for i, x in enumerate(suffixes):
Martin@846
   205
        j = 10 ** (i * 3)
Martin@846
   206
        if size < j * 10:
Martin@846
   207
            return "%.2g" % (1.0 * size / j) + x
Martin@846
   208
        if size < j * 1000 or i == len(suffixes) - 1:
Martin@846
   209
            return "%.0f" % (1.0 * size / j) + x
Martin@846
   210
Martin@898
   211
def labelText(seqRange, labelOpt):
Martin@898
   212
    seqName, beg, end = seqRange
Martin@898
   213
    if labelOpt == 1:
Martin@898
   214
        return seqName + ": " + sizeText(end - beg)
Martin@898
   215
    if labelOpt == 2:
Martin@904
   216
        return seqName + ":" + prettyNum(beg) + ": " + sizeText(end - beg)
Martin@898
   217
    if labelOpt == 3:
Martin@904
   218
        return seqName + ":" + prettyNum(beg) + "-" + prettyNum(end)
Martin@898
   219
    return seqName
Martin@846
   220
Martin@899
   221
def rangeLabels(seqRanges, labelOpt, font, fontsize, image_mode, textRot):
Martin@899
   222
    if fontsize:
Martin@899
   223
        image_size = 1, 1
Martin@899
   224
        im = Image.new(image_mode, image_size)
Martin@899
   225
        draw = ImageDraw.Draw(im)
Martin@899
   226
    x = y = 0
Martin@899
   227
    for r in seqRanges:
Martin@899
   228
        text = labelText(r, labelOpt)
Martin@899
   229
        if fontsize:
Martin@899
   230
            x, y = draw.textsize(text, font=font)
Martin@899
   231
            if textRot:
Martin@899
   232
                x, y = y, x
Martin@899
   233
        yield text, x, y
Martin@899
   234
Martin@908
   235
def dataFromRanges(seqRanges, sortOpt, font,
Martin@908
   236
                   fontsize, image_mode, labelOpt, textRot):
Martin@907
   237
    s = getSortedRanges(seqRanges, sortOpt)
Martin@907
   238
    for i in s:
Martin@907
   239
        warn("\t".join(map(str, i)))
Martin@866
   240
    warn("")
Martin@907
   241
    rangeSizes = [e - b for n, b, e in s]
Martin@907
   242
    labs = list(rangeLabels(s, labelOpt, font, fontsize, image_mode, textRot))
Martin@907
   243
    margin = max(i[2] for i in labs)
Martin@878
   244
    # xxx the margin may be too big, because some labels may get omitted
Martin@907
   245
    return s, rangeSizes, labs, margin
Martin@1
   246
Martin@1
   247
def div_ceil(x, y):
Martin@1
   248
    '''Return x / y rounded up.'''
Martin@1
   249
    q, r = divmod(x, y)
Martin@1
   250
    return q + (r != 0)
Martin@1
   251
Martin@896
   252
def get_bp_per_pix(rangeSizes, pixTweenRanges, maxPixels):
Martin@1
   253
    '''Get the minimum bp-per-pixel that fits in the size limit.'''
Martin@904
   254
    warn("choosing bp per pixel...")
Martin@896
   255
    numOfRanges = len(rangeSizes)
Martin@896
   256
    maxPixelsInRanges = maxPixels - pixTweenRanges * (numOfRanges - 1)
Martin@896
   257
    if maxPixelsInRanges < numOfRanges:
Martin@649
   258
        raise Exception("can't fit the image: too many sequences?")
Martin@896
   259
    negLimit = -maxPixelsInRanges
Martin@896
   260
    negBpPerPix = sum(rangeSizes) // negLimit
Martin@863
   261
    while True:
Martin@896
   262
        if sum(i // negBpPerPix for i in rangeSizes) >= negLimit:
Martin@863
   263
            return -negBpPerPix
Martin@863
   264
        negBpPerPix -= 1
Martin@1
   265
Martin@900
   266
def getRangePixBegs(rangePixLens, pixTweenRanges, margin):
Martin@900
   267
    '''Get the start pixel for each range.'''
Martin@900
   268
    rangePixBegs = []
Martin@896
   269
    pix_tot = margin - pixTweenRanges
Martin@900
   270
    for i in rangePixLens:
Martin@896
   271
        pix_tot += pixTweenRanges
Martin@900
   272
        rangePixBegs.append(pix_tot)
Martin@1
   273
        pix_tot += i
Martin@900
   274
    return rangePixBegs
Martin@1
   275
Martin@896
   276
def pixelData(rangeSizes, bp_per_pix, pixTweenRanges, margin):
Martin@900
   277
    '''Return pixel information about the ranges.'''
Martin@900
   278
    rangePixLens = [div_ceil(i, bp_per_pix) for i in rangeSizes]
Martin@900
   279
    rangePixBegs = getRangePixBegs(rangePixLens, pixTweenRanges, margin)
Martin@900
   280
    tot_pix = rangePixBegs[-1] + rangePixLens[-1]
Martin@900
   281
    return rangePixBegs, rangePixLens, tot_pix
Martin@1
   282
Martin@835
   283
def drawLineForward(hits, width, bp_per_pix, beg1, beg2, size):
Martin@639
   284
    while True:
Martin@639
   285
        q1, r1 = divmod(beg1, bp_per_pix)
Martin@639
   286
        q2, r2 = divmod(beg2, bp_per_pix)
Martin@835
   287
        hits[q2 * width + q1] |= 1
Martin@639
   288
        next_pix = min(bp_per_pix - r1, bp_per_pix - r2)
Martin@639
   289
        if next_pix >= size: break
Martin@639
   290
        beg1 += next_pix
Martin@639
   291
        beg2 += next_pix
Martin@639
   292
        size -= next_pix
Martin@639
   293
Martin@835
   294
def drawLineReverse(hits, width, bp_per_pix, beg1, beg2, size):
Martin@639
   295
    beg2 = -1 - beg2
Martin@639
   296
    while True:
Martin@639
   297
        q1, r1 = divmod(beg1, bp_per_pix)
Martin@639
   298
        q2, r2 = divmod(beg2, bp_per_pix)
Martin@835
   299
        hits[q2 * width + q1] |= 2
Martin@639
   300
        next_pix = min(bp_per_pix - r1, r2 + 1)
Martin@639
   301
        if next_pix >= size: break
Martin@639
   302
        beg1 += next_pix
Martin@639
   303
        beg2 -= next_pix
Martin@639
   304
        size -= next_pix
Martin@639
   305
Martin@905
   306
def findOrigin(ranges, beg, size):
Martin@905
   307
    if beg < 0:
Martin@905
   308
        beg = -(beg + size)
Martin@905
   309
    for rangeBeg, rangeEnd, origin in ranges:
Martin@905
   310
        if rangeEnd > beg:
Martin@905
   311
            return origin
Martin@905
   312
Martin@905
   313
def alignmentPixels(width, height, alignments, bp_per_pix,
Martin@905
   314
                    rangeDict1, rangeDict2):
Martin@640
   315
    hits = [0] * (width * height)  # the image data
Martin@640
   316
    for seq1, seq2, blocks in alignments:
Martin@905
   317
        beg1, beg2, size = blocks[0]
Martin@905
   318
        ori1 = findOrigin(rangeDict1[seq1], beg1, size)
Martin@905
   319
        ori2 = findOrigin(rangeDict2[seq2], beg2, size)
Martin@640
   320
        for beg1, beg2, size in blocks:
Martin@640
   321
            if beg1 < 0:
Martin@640
   322
                beg1 = -(beg1 + size)
Martin@640
   323
                beg2 = -(beg2 + size)
Martin@640
   324
            if beg2 >= 0:
Martin@835
   325
                drawLineForward(hits, width, bp_per_pix,
Martin@835
   326
                                beg1 + ori1, beg2 + ori2, size)
Martin@640
   327
            else:
Martin@835
   328
                drawLineReverse(hits, width, bp_per_pix,
Martin@835
   329
                                beg1 + ori1, beg2 - ori2, size)
Martin@640
   330
    return hits
Martin@1
   331
Martin@650
   332
def expandedSeqDict(seqDict):
Martin@650
   333
    '''Allow lookup by short sequence names, e.g. chr7 as well as hg19.chr7.'''
Martin@861
   334
    newDict = seqDict.copy()
Martin@650
   335
    for name, x in seqDict.items():
Martin@861
   336
        if "." in name:
Martin@861
   337
            base = name.split(".")[-1]
Martin@861
   338
            if base in newDict:  # an ambiguous case was found:
Martin@861
   339
                return seqDict   # so give up completely
Martin@861
   340
            newDict[base] = x
Martin@650
   341
    return newDict
Martin@650
   342
Martin@905
   343
def readBed(fileName, rangeDict):
Martin@845
   344
    for line in myOpen(fileName):
Martin@845
   345
        w = line.split()
Martin@847
   346
        if not w: continue
Martin@845
   347
        seqName = w[0]
Martin@905
   348
        if seqName not in rangeDict: continue
Martin@845
   349
        beg = int(w[1])
Martin@845
   350
        end = int(w[2])
Martin@857
   351
        layer = 900
Martin@895
   352
        color = "#fbf"
Martin@858
   353
        if len(w) > 4:
Martin@858
   354
            if w[4] != ".":
Martin@858
   355
                layer = float(w[4])
Martin@858
   356
            if len(w) > 5:
Martin@858
   357
                if len(w) > 8 and w[8].count(",") == 2:
Martin@858
   358
                    color = "rgb(" + w[8] + ")"
Martin@858
   359
                elif w[5] == "+":
Martin@895
   360
                    color = "#ffe8e8"
Martin@858
   361
                elif w[5] == "-":
Martin@895
   362
                    color = "#e8e8ff"
Martin@859
   363
        yield layer, color, seqName, beg, end
Martin@845
   364
Martin@860
   365
def commaSeparatedInts(text):
Martin@860
   366
    return map(int, text.rstrip(",").split(","))
Martin@860
   367
Martin@905
   368
def readGenePred(opts, fileName, rangeDict):
Martin@860
   369
    for line in myOpen(fileName):
Martin@860
   370
        fields = line.split()
Martin@860
   371
        if not fields: continue
Martin@860
   372
        if fields[2] not in "+-": fields = fields[1:]
Martin@860
   373
        seqName = fields[1]
Martin@905
   374
        if seqName not in rangeDict: continue
Martin@860
   375
        #strand = fields[2]
Martin@860
   376
        cdsBeg = int(fields[5])
Martin@860
   377
        cdsEnd = int(fields[6])
Martin@860
   378
        exonBegs = commaSeparatedInts(fields[8])
Martin@860
   379
        exonEnds = commaSeparatedInts(fields[9])
Martin@860
   380
        for beg, end in zip(exonBegs, exonEnds):
Martin@860
   381
            yield 300, opts.exon_color, seqName, beg, end
Martin@860
   382
            b = max(beg, cdsBeg)
Martin@860
   383
            e = min(end, cdsEnd)
Martin@860
   384
            if b < e: yield 400, opts.cds_color, seqName, b, e
Martin@860
   385
Martin@905
   386
def readRmsk(fileName, rangeDict):
Martin@860
   387
    for line in myOpen(fileName):
Martin@860
   388
        fields = line.split()
Martin@860
   389
        if len(fields) == 17:  # rmsk.txt
Martin@860
   390
            seqName = fields[5]
Martin@905
   391
            if seqName not in rangeDict: continue  # do this ASAP for speed
Martin@860
   392
            beg = int(fields[6])
Martin@860
   393
            end = int(fields[7])
Martin@860
   394
            strand = fields[9]
Martin@860
   395
            repeatClass = fields[11]
Martin@860
   396
        elif len(fields) == 15:  # .out
Martin@860
   397
            seqName = fields[4]
Martin@905
   398
            if seqName not in rangeDict: continue
Martin@860
   399
            beg = int(fields[5]) - 1
Martin@860
   400
            end = int(fields[6])
Martin@860
   401
            strand = fields[8]
Martin@860
   402
            repeatClass = fields[10]
Martin@860
   403
        else:
Martin@860
   404
            continue
Martin@860
   405
        if repeatClass in ("Low_complexity", "Simple_repeat"):
Martin@895
   406
            yield 200, "#fbf", seqName, beg, end
Martin@860
   407
        elif strand == "+":
Martin@895
   408
            yield 100, "#ffe8e8", seqName, beg, end
Martin@860
   409
        else:
Martin@895
   410
            yield 100, "#e8e8ff", seqName, beg, end
Martin@860
   411
Martin@650
   412
def isExtraFirstGapField(fields):
Martin@650
   413
    return fields[4].isdigit()
Martin@650
   414
Martin@905
   415
def readGaps(opts, fileName, rangeDict):
Martin@650
   416
    '''Read locations of unsequenced gaps, from an agp or gap file.'''
Martin@844
   417
    for line in myOpen(fileName):
Martin@650
   418
        w = line.split()
Martin@650
   419
        if not w or w[0][0] == "#": continue
Martin@650
   420
        if isExtraFirstGapField(w): w = w[1:]
Martin@650
   421
        if w[4] not in "NU": continue
Martin@650
   422
        seqName = w[0]
Martin@905
   423
        if seqName not in rangeDict: continue
Martin@650
   424
        end = int(w[2])
Martin@650
   425
        beg = end - int(w[5])  # zero-based coordinate
Martin@857
   426
        if w[7] == "yes":
Martin@859
   427
            yield 3000, opts.bridged_color, seqName, beg, end
Martin@857
   428
        else:
Martin@859
   429
            yield 2000, opts.unbridged_color, seqName, beg, end
Martin@650
   430
Martin@905
   431
def bedBoxes(beds, rangeDict, margin, edge, isTop, bpPerPix):
Martin@897
   432
    for layer, color, seqName, bedBeg, bedEnd in beds:
Martin@905
   433
        for rangeBeg, rangeEnd, origin in rangeDict[seqName]:
Martin@905
   434
            beg = max(bedBeg, rangeBeg)
Martin@905
   435
            end = min(bedEnd, rangeEnd)
Martin@905
   436
            if beg >= end: continue
Martin@905
   437
            if layer <= 1000:
Martin@905
   438
                # include partly-covered pixels
Martin@905
   439
                b = (origin + beg) // bpPerPix
Martin@905
   440
                e = div_ceil(origin + end, bpPerPix)
Martin@905
   441
            else:
Martin@905
   442
                # exclude partly-covered pixels
Martin@905
   443
                b = div_ceil(origin + beg, bpPerPix)
Martin@905
   444
                e = (origin + end) // bpPerPix
Martin@905
   445
                if e <= b: continue
Martin@905
   446
            if isTop:
Martin@905
   447
                box = b, margin, e, edge
Martin@905
   448
            else:
Martin@905
   449
                box = margin, b, edge, e
Martin@905
   450
            yield layer, color, box
Martin@845
   451
Martin@857
   452
def drawAnnotations(im, boxes):
Martin@857
   453
    # xxx use partial transparency for different-color overlaps?
Martin@857
   454
    for layer, color, box in boxes:
Martin@650
   455
        im.paste(color, box)
Martin@650
   456
Martin@901
   457
def placedLabels(labels, rangePixBegs, rangePixLens, beg, end):
Martin@901
   458
    '''Return axis labels with endpoint & sort-order information.'''
Martin@901
   459
    maxWidth = end - beg
Martin@901
   460
    for i, j, k in zip(labels, rangePixBegs, rangePixLens):
Martin@901
   461
        text, textWidth, textHeight = i
Martin@901
   462
        if textWidth > maxWidth:
Martin@901
   463
            continue
Martin@901
   464
        labelBeg = j + (k - textWidth) // 2
Martin@901
   465
        labelEnd = labelBeg + textWidth
Martin@901
   466
        sortKey = textWidth - k
Martin@901
   467
        if labelBeg < beg:
Martin@904
   468
            sortKey += maxWidth * (beg - labelBeg)
Martin@901
   469
            labelBeg = beg
Martin@901
   470
            labelEnd = beg + textWidth
Martin@901
   471
        if labelEnd > end:
Martin@904
   472
            sortKey += maxWidth * (labelEnd - end)
Martin@901
   473
            labelEnd = end
Martin@901
   474
            labelBeg = end - textWidth
Martin@901
   475
        yield sortKey, labelBeg, labelEnd, text, textHeight
Martin@1
   476
Martin@897
   477
def nonoverlappingLabels(labels, minPixTweenLabels):
Martin@1
   478
    '''Get a subset of non-overlapping axis labels, greedily.'''
Martin@897
   479
    out = []
Martin@1
   480
    for i in labels:
Martin@897
   481
        beg = i[1] - minPixTweenLabels
Martin@897
   482
        end = i[2] + minPixTweenLabels
Martin@897
   483
        if all(j[2] <= beg or j[1] >= end for j in out):
Martin@897
   484
            out.append(i)
Martin@897
   485
    return out
Martin@1
   486
Martin@901
   487
def axisImage(labels, rangePixBegs, rangePixLens, textRot,
Martin@895
   488
              textAln, font, image_mode, opts):
Martin@1
   489
    '''Make an image of axis labels.'''
Martin@900
   490
    beg = rangePixBegs[0]
Martin@900
   491
    end = rangePixBegs[-1] + rangePixLens[-1]
Martin@901
   492
    margin = max(i[2] for i in labels)
Martin@901
   493
    labels = sorted(placedLabels(labels, rangePixBegs, rangePixLens, beg, end))
Martin@878
   494
    minPixTweenLabels = 0 if textRot else opts.label_space
Martin@897
   495
    labels = nonoverlappingLabels(labels, minPixTweenLabels)
Martin@897
   496
    image_size = (margin, end) if textRot else (end, margin)
Martin@895
   497
    im = Image.new(image_mode, image_size, opts.margin_color)
Martin@1
   498
    draw = ImageDraw.Draw(im)
Martin@1
   499
    for i in labels:
Martin@878
   500
        base = margin - i[4] if textAln else 0
Martin@878
   501
        position = (base, i[1]) if textRot else (i[1], base)
Martin@646
   502
        draw.text(position, i[3], font=font, fill=opts.text_color)
Martin@1
   503
    return im
Martin@1
   504
Martin@906
   505
def rangesWithOrigins(sortedRanges, rangePixBegs, bpPerPix):
Martin@906
   506
    for i, j in zip(sortedRanges, rangePixBegs):
Martin@906
   507
        seqName, rangeBeg, rangeEnd = i
Martin@906
   508
        origin = bpPerPix * j - rangeBeg
Martin@906
   509
        yield seqName, (rangeBeg, rangeEnd, origin)
Martin@906
   510
Martin@906
   511
def rangesPerSeq(sortedRanges, rangePixBegs, bpPerPix):
Martin@906
   512
    a = rangesWithOrigins(sortedRanges, rangePixBegs, bpPerPix)
Martin@906
   513
    for k, v in itertools.groupby(a, itemgetter(0)):
Martin@906
   514
        yield k, [i[1] for i in v]
Martin@836
   515
Martin@903
   516
def getFont(opts):
Martin@903
   517
    if opts.fontfile:
Martin@903
   518
        return ImageFont.truetype(opts.fontfile, opts.fontsize)
Martin@903
   519
    fileNames = []
Martin@903
   520
    try:
Martin@903
   521
        x = ["fc-match", "-f%{file}", "arial"]
Martin@903
   522
        p = subprocess.Popen(x, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
Martin@903
   523
        out, err = p.communicate()
Martin@903
   524
        fileNames.append(out)
Martin@903
   525
    except OSError as e:
Martin@903
   526
        warn("fc-match error: " + str(e))
Martin@903
   527
    fileNames.append("/Library/Fonts/Arial.ttf")  # for Mac
Martin@903
   528
    for i in fileNames:
Martin@903
   529
        try:
Martin@903
   530
            font = ImageFont.truetype(i, opts.fontsize)
Martin@903
   531
            warn("font: " + i)
Martin@903
   532
            return font
Martin@903
   533
        except IOError as e:
Martin@903
   534
            warn("font load error: " + str(e))
Martin@903
   535
    return ImageFont.load_default()
Martin@903
   536
Martin@648
   537
def lastDotplot(opts, args):
Martin@903
   538
    font = getFont(opts)
Martin@643
   539
    image_mode = 'RGB'
Martin@643
   540
    forward_color = ImageColor.getcolor(opts.forwardcolor, image_mode)
Martin@643
   541
    reverse_color = ImageColor.getcolor(opts.reversecolor, image_mode)
Martin@643
   542
    zipped_colors = zip(forward_color, reverse_color)
Martin@643
   543
    overlap_color = tuple([(i + j) // 2 for i, j in zipped_colors])
Martin@641
   544
Martin@644
   545
    warn("reading alignments...")
Martin@904
   546
    alnData = readAlignments(args[0], opts)
Martin@908
   547
    alignments, seqRanges1, seqRanges2 = alnData
Martin@644
   548
    warn("done")
Martin@649
   549
    if not alignments: raise Exception("there are no alignments")
Martin@641
   550
Martin@878
   551
    textRot1 = "vertical".startswith(opts.rot1)
Martin@908
   552
    i1 = dataFromRanges(seqRanges1, opts.sort1, font,
Martin@908
   553
                        opts.fontsize, image_mode, opts.labels1, textRot1)
Martin@906
   554
    sortedRanges1, rangeSizes1, labelData1, tMargin = i1
Martin@846
   555
Martin@878
   556
    textRot2 = "horizontal".startswith(opts.rot2)
Martin@908
   557
    i2 = dataFromRanges(seqRanges2, opts.sort2, font,
Martin@908
   558
                        opts.fontsize, image_mode, opts.labels2, textRot2)
Martin@906
   559
    sortedRanges2, rangeSizes2, labelData2, lMargin = i2
Martin@641
   560
Martin@896
   561
    maxPixels1 = opts.width  - lMargin
Martin@896
   562
    maxPixels2 = opts.height - tMargin
Martin@897
   563
    bpPerPix1 = get_bp_per_pix(rangeSizes1, opts.border_pixels, maxPixels1)
Martin@897
   564
    bpPerPix2 = get_bp_per_pix(rangeSizes2, opts.border_pixels, maxPixels2)
Martin@855
   565
    bpPerPix = max(bpPerPix1, bpPerPix2)
Martin@855
   566
    warn("bp per pixel = " + str(bpPerPix))
Martin@641
   567
Martin@900
   568
    p1 = pixelData(rangeSizes1, bpPerPix, opts.border_pixels, lMargin)
Martin@900
   569
    rangePixBegs1, rangePixLens1, width = p1
Martin@906
   570
    rangeDict1 = dict(rangesPerSeq(sortedRanges1, rangePixBegs1, bpPerPix))
Martin@900
   571
Martin@900
   572
    p2 = pixelData(rangeSizes2, bpPerPix, opts.border_pixels, tMargin)
Martin@900
   573
    rangePixBegs2, rangePixLens2, height = p2
Martin@906
   574
    rangeDict2 = dict(rangesPerSeq(sortedRanges2, rangePixBegs2, bpPerPix))
Martin@900
   575
Martin@847
   576
    warn("width:  " + str(width))
Martin@847
   577
    warn("height: " + str(height))
Martin@839
   578
Martin@644
   579
    warn("processing alignments...")
Martin@855
   580
    hits = alignmentPixels(width, height, alignments, bpPerPix,
Martin@905
   581
                           rangeDict1, rangeDict2)
Martin@641
   582
Martin@904
   583
    warn("reading annotations...")
Martin@134
   584
Martin@905
   585
    rangeDict1 = expandedSeqDict(rangeDict1)
Martin@905
   586
    beds1 = itertools.chain(readBed(opts.bed1, rangeDict1),
Martin@905
   587
                            readRmsk(opts.rmsk1, rangeDict1),
Martin@905
   588
                            readGenePred(opts, opts.genePred1, rangeDict1),
Martin@905
   589
                            readGaps(opts, opts.gap1, rangeDict1))
Martin@905
   590
    b1 = bedBoxes(beds1, rangeDict1, tMargin, height, True, bpPerPix)
Martin@845
   591
Martin@905
   592
    rangeDict2 = expandedSeqDict(rangeDict2)
Martin@905
   593
    beds2 = itertools.chain(readBed(opts.bed2, rangeDict2),
Martin@905
   594
                            readRmsk(opts.rmsk2, rangeDict2),
Martin@905
   595
                            readGenePred(opts, opts.genePred2, rangeDict2),
Martin@905
   596
                            readGaps(opts, opts.gap2, rangeDict2))
Martin@905
   597
    b2 = bedBoxes(beds2, rangeDict2, lMargin, width, False, bpPerPix)
Martin@857
   598
Martin@857
   599
    boxes = sorted(itertools.chain(b1, b2))
Martin@904
   600
Martin@904
   601
    warn("drawing...")
Martin@904
   602
Martin@904
   603
    image_size = width, height
Martin@904
   604
    im = Image.new(image_mode, image_size, opts.background_color)
Martin@904
   605
Martin@857
   606
    drawAnnotations(im, boxes)
Martin@650
   607
Martin@643
   608
    for i in range(height):
Martin@643
   609
        for j in range(width):
Martin@643
   610
            store_value = hits[i * width + j]
Martin@643
   611
            xy = j, i
Martin@643
   612
            if   store_value == 1: im.putpixel(xy, forward_color)
Martin@643
   613
            elif store_value == 2: im.putpixel(xy, reverse_color)
Martin@643
   614
            elif store_value == 3: im.putpixel(xy, overlap_color)
Martin@95
   615
Martin@643
   616
    if opts.fontsize != 0:
Martin@900
   617
        axis1 = axisImage(labelData1, rangePixBegs1, rangePixLens1,
Martin@895
   618
                          textRot1, False, font, image_mode, opts)
Martin@878
   619
        if textRot1:
Martin@878
   620
            axis1 = axis1.transpose(Image.ROTATE_90)
Martin@900
   621
        axis2 = axisImage(labelData2, rangePixBegs2, rangePixLens2,
Martin@895
   622
                          textRot2, textRot2, font, image_mode, opts)
Martin@878
   623
        if not textRot2:
Martin@878
   624
            axis2 = axis2.transpose(Image.ROTATE_270)
Martin@643
   625
        im.paste(axis1, (0, 0))
Martin@643
   626
        im.paste(axis2, (0, 0))
Martin@1
   627
Martin@900
   628
    for i in rangePixBegs1[1:]:
Martin@877
   629
        box = i - opts.border_pixels, tMargin, i, height
Martin@852
   630
        im.paste(opts.border_color, box)
Martin@1
   631
Martin@900
   632
    for i in rangePixBegs2[1:]:
Martin@877
   633
        box = lMargin, i - opts.border_pixels, width, i
Martin@852
   634
        im.paste(opts.border_color, box)
Martin@1
   635
Martin@643
   636
    im.save(args[1])
Martin@648
   637
Martin@648
   638
if __name__ == "__main__":
Martin@649
   639
    usage = """%prog --help
Martin@649
   640
   or: %prog [options] maf-or-tab-alignments dotplot.png
Martin@649
   641
   or: %prog [options] maf-or-tab-alignments dotplot.gif
Martin@649
   642
   or: ..."""
Martin@649
   643
    description = "Draw a dotplot of pair-wise sequence alignments in MAF or tabular format."
Martin@649
   644
    op = optparse.OptionParser(usage=usage, description=description)
Martin@866
   645
    op.add_option("-v", "--verbose", action="count",
Martin@866
   646
                  help="show progress messages & data about the plot")
Martin@651
   647
    op.add_option("-1", "--seq1", metavar="PATTERN", action="append",
Martin@840
   648
                  default=[],
Martin@651
   649
                  help="which sequences to show from the 1st genome")
Martin@651
   650
    op.add_option("-2", "--seq2", metavar="PATTERN", action="append",
Martin@840
   651
                  default=[],
Martin@651
   652
                  help="which sequences to show from the 2nd genome")
Martin@648
   653
    # Replace "width" & "height" with a single "length" option?
Martin@648
   654
    op.add_option("-x", "--width", type="int", default=1000,
Martin@648
   655
                  help="maximum width in pixels (default: %default)")
Martin@648
   656
    op.add_option("-y", "--height", type="int", default=1000,
Martin@648
   657
                  help="maximum height in pixels (default: %default)")
Martin@649
   658
    op.add_option("-c", "--forwardcolor", metavar="COLOR", default="red",
Martin@649
   659
                  help="color for forward alignments (default: %default)")
Martin@649
   660
    op.add_option("-r", "--reversecolor", metavar="COLOR", default="blue",
Martin@649
   661
                  help="color for reverse alignments (default: %default)")
Martin@907
   662
    op.add_option("--sort1", default="1", metavar="N",
Martin@851
   663
                  help="genome1 sequence order: 0=input order, 1=name order, "
Martin@851
   664
                  "2=length order (default=%default)")
Martin@907
   665
    op.add_option("--sort2", default="1", metavar="N",
Martin@851
   666
                  help="genome2 sequence order: 0=input order, 1=name order, "
Martin@851
   667
                  "2=length order (default=%default)")
Martin@839
   668
    op.add_option("--trim1", action="store_true",
Martin@839
   669
                  help="trim unaligned sequence flanks from the 1st genome")
Martin@839
   670
    op.add_option("--trim2", action="store_true",
Martin@839
   671
                  help="trim unaligned sequence flanks from the 2nd genome")
Martin@852
   672
    op.add_option("--border-pixels", metavar="INT", type="int", default=1,
Martin@852
   673
                  help="number of pixels between sequences (default=%default)")
Martin@895
   674
    op.add_option("--border-color", metavar="COLOR", default="black",
Martin@852
   675
                  help="color for pixels between sequences (default=%default)")
Martin@895
   676
    op.add_option("--margin-color", metavar="COLOR", default="#dcdcdc",
Martin@895
   677
                  help="margin color")
Martin@846
   678
Martin@850
   679
    og = optparse.OptionGroup(op, "Text options")
Martin@850
   680
    og.add_option("-f", "--fontfile", metavar="FILE",
Martin@850
   681
                  help="TrueType or OpenType font file")
Martin@903
   682
    og.add_option("-s", "--fontsize", metavar="SIZE", type="int", default=14,
Martin@850
   683
                  help="TrueType or OpenType font size (default: %default)")
Martin@898
   684
    og.add_option("--labels1", type="int", default=0, metavar="N", help=
Martin@898
   685
                  "genome1 labels: 0=name, 1=name:length, "
Martin@898
   686
                  "2=name:start:length, 3=name:start-end (default=%default)")
Martin@898
   687
    og.add_option("--labels2", type="int", default=0, metavar="N", help=
Martin@898
   688
                  "genome2 labels: 0=name, 1=name:length, "
Martin@898
   689
                  "2=name:start:length, 3=name:start-end (default=%default)")
Martin@878
   690
    og.add_option("--rot1", metavar="ROT", default="h",
Martin@878
   691
                  help="text rotation for the 1st genome (default=%default)")
Martin@878
   692
    og.add_option("--rot2", metavar="ROT", default="v",
Martin@878
   693
                  help="text rotation for the 2nd genome (default=%default)")
Martin@850
   694
    op.add_option_group(og)
Martin@850
   695
Martin@860
   696
    og = optparse.OptionGroup(op, "Annotation options")
Martin@860
   697
    og.add_option("--bed1", metavar="FILE",
Martin@860
   698
                  help="read genome1 annotations from BED file")
Martin@860
   699
    og.add_option("--bed2", metavar="FILE",
Martin@860
   700
                  help="read genome2 annotations from BED file")
Martin@860
   701
    og.add_option("--rmsk1", metavar="FILE", help="read genome1 repeats from "
Martin@860
   702
                  "RepeatMasker .out or rmsk.txt file")
Martin@860
   703
    og.add_option("--rmsk2", metavar="FILE", help="read genome2 repeats from "
Martin@860
   704
                  "RepeatMasker .out or rmsk.txt file")
Martin@860
   705
    op.add_option_group(og)
Martin@860
   706
Martin@860
   707
    og = optparse.OptionGroup(op, "Gene options")
Martin@860
   708
    og.add_option("--genePred1", metavar="FILE",
Martin@860
   709
                  help="read genome1 genes from genePred file")
Martin@860
   710
    og.add_option("--genePred2", metavar="FILE",
Martin@860
   711
                  help="read genome2 genes from genePred file")
Martin@895
   712
    og.add_option("--exon-color", metavar="COLOR", default="PaleGreen",
Martin@860
   713
                  help="color for exons (default=%default)")
Martin@895
   714
    og.add_option("--cds-color", metavar="COLOR", default="LimeGreen",
Martin@860
   715
                  help="color for protein-coding regions (default=%default)")
Martin@860
   716
    op.add_option_group(og)
Martin@860
   717
Martin@650
   718
    og = optparse.OptionGroup(op, "Unsequenced gap options")
Martin@650
   719
    og.add_option("--gap1", metavar="FILE",
Martin@650
   720
                  help="read genome1 unsequenced gaps from agp or gap file")
Martin@650
   721
    og.add_option("--gap2", metavar="FILE",
Martin@650
   722
                  help="read genome2 unsequenced gaps from agp or gap file")
Martin@650
   723
    og.add_option("--bridged-color", metavar="COLOR", default="yellow",
Martin@650
   724
                  help="color for bridged gaps (default: %default)")
Martin@895
   725
    og.add_option("--unbridged-color", metavar="COLOR", default="orange",
Martin@650
   726
                  help="color for unbridged gaps (default: %default)")
Martin@650
   727
    op.add_option_group(og)
Martin@648
   728
    (opts, args) = op.parse_args()
Martin@648
   729
    if len(args) != 2: op.error("2 arguments needed")
Martin@648
   730
Martin@648
   731
    opts.text_color = "black"
Martin@648
   732
    opts.background_color = "white"
Martin@648
   733
    opts.label_space = 5     # minimum number of pixels between axis labels
Martin@648
   734
Martin@649
   735
    try: lastDotplot(opts, args)
Martin@649
   736
    except KeyboardInterrupt: pass  # avoid silly error message
Martin@903
   737
    except Exception as e:
Martin@649
   738
        prog = os.path.basename(sys.argv[0])
Martin@649
   739
        sys.exit(prog + ": error: " + str(e))