i18n/posplit
author FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
Thu, 11 Feb 2016 23:15:34 +0900
changeset 28074 a1924bc6e267
parent 20363 e3ee7ec85a15
child 29152 c5057b7780dc
permissions -rwxr-xr-x
i18n: calculate correct line number in source of messages to be translated Before this patch, line number in source of the message to be translated is wrong in hg.pot, if corresponded message is placed after ".. DIRECTIVE::", because number of lines related to such directive isn't added to variable "delta", which holds number of untranslated lines in given text. This patch always adds "2" to "delta", because text block is split into translation units by "\n\n".
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
11389
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
     1
#!/usr/bin/env python
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
     2
#
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
     3
# posplit - split messages in paragraphs on .po/.pot files
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
     4
#
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
     5
# license: MIT/X11/Expat
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
     6
#
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
     7
20359
ff6ab0b2ebf7 i18n: posplit writes a warning for translators before rst directives
Simon Heimberg <simohe@besonet.ch>
parents: 11389
diff changeset
     8
import re
11389
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
     9
import sys
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    10
import polib
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    11
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    12
def addentry(po, entry, cache):
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    13
    e = cache.get(entry.msgid)
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    14
    if e:
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    15
        e.occurrences.extend(entry.occurrences)
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    16
    else:
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    17
        po.append(entry)
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    18
        cache[entry.msgid] = entry
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    19
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    20
def mkentry(orig, delta, msgid, msgstr):
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    21
    entry = polib.POEntry()
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    22
    entry.merge(orig)
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    23
    entry.msgid = msgid or orig.msgid
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    24
    entry.msgstr = msgstr or orig.msgstr
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    25
    entry.occurrences = [(p, int(l) + delta) for (p, l) in orig.occurrences]
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    26
    return entry
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    27
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    28
if __name__ == "__main__":
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    29
    po = polib.pofile(sys.argv[1])
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    30
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    31
    cache = {}
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    32
    entries = po[:]
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    33
    po[:] = []
20359
ff6ab0b2ebf7 i18n: posplit writes a warning for translators before rst directives
Simon Heimberg <simohe@besonet.ch>
parents: 11389
diff changeset
    34
    findd = re.compile(r' *\.\. (\w+)::') # for finding directives
11389
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    35
    for entry in entries:
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    36
        msgids = entry.msgid.split(u'\n\n')
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    37
        if entry.msgstr:
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    38
            msgstrs = entry.msgstr.split(u'\n\n')
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    39
        else:
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    40
            msgstrs = [u''] * len(msgids)
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    41
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    42
        if len(msgids) != len(msgstrs):
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    43
            # places the whole existing translation as a fuzzy
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    44
            # translation for each paragraph, to give the
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    45
            # translator a chance to recover part of the old
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    46
            # translation - erasing extra paragraphs is
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    47
            # probably better than retranslating all from start
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    48
            if 'fuzzy' not in entry.flags:
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    49
                entry.flags.append('fuzzy')
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    50
            msgstrs = [entry.msgstr] * len(msgids)
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    51
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    52
        delta = 0
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    53
        for msgid, msgstr in zip(msgids, msgstrs):
20361
3fe079d3a2b4 i18n: posplit removes the entry "::" from the pot file
Simon Heimberg <simohe@besonet.ch>
parents: 20359
diff changeset
    54
            if msgid and msgid != '::':
11389
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    55
                newentry = mkentry(entry, delta, msgid, msgstr)
20359
ff6ab0b2ebf7 i18n: posplit writes a warning for translators before rst directives
Simon Heimberg <simohe@besonet.ch>
parents: 11389
diff changeset
    56
                mdirective = findd.match(msgid)
ff6ab0b2ebf7 i18n: posplit writes a warning for translators before rst directives
Simon Heimberg <simohe@besonet.ch>
parents: 11389
diff changeset
    57
                if mdirective:
20362
1bce1078501d i18n: leave out entries which contain only a rst directive
Simon Heimberg <simohe@besonet.ch>
parents: 20361
diff changeset
    58
                    if not msgid[mdirective.end():].rstrip():
1bce1078501d i18n: leave out entries which contain only a rst directive
Simon Heimberg <simohe@besonet.ch>
parents: 20361
diff changeset
    59
                        # only directive, nothing to translate here
28074
a1924bc6e267 i18n: calculate correct line number in source of messages to be translated
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20363
diff changeset
    60
                        delta += 2
20362
1bce1078501d i18n: leave out entries which contain only a rst directive
Simon Heimberg <simohe@besonet.ch>
parents: 20361
diff changeset
    61
                        continue
20359
ff6ab0b2ebf7 i18n: posplit writes a warning for translators before rst directives
Simon Heimberg <simohe@besonet.ch>
parents: 11389
diff changeset
    62
                    directive = mdirective.group(1)
20363
e3ee7ec85a15 i18n: leave out entries which contain only rst syntax
Simon Heimberg <simohe@besonet.ch>
parents: 20362
diff changeset
    63
                    if directive in ('container', 'include'):
e3ee7ec85a15 i18n: leave out entries which contain only rst syntax
Simon Heimberg <simohe@besonet.ch>
parents: 20362
diff changeset
    64
                        if msgid.rstrip('\n').count('\n') == 0:
e3ee7ec85a15 i18n: leave out entries which contain only rst syntax
Simon Heimberg <simohe@besonet.ch>
parents: 20362
diff changeset
    65
                            # only rst syntax, nothing to translate
28074
a1924bc6e267 i18n: calculate correct line number in source of messages to be translated
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20363
diff changeset
    66
                            delta += 2
20363
e3ee7ec85a15 i18n: leave out entries which contain only rst syntax
Simon Heimberg <simohe@besonet.ch>
parents: 20362
diff changeset
    67
                            continue
e3ee7ec85a15 i18n: leave out entries which contain only rst syntax
Simon Heimberg <simohe@besonet.ch>
parents: 20362
diff changeset
    68
                        else:
e3ee7ec85a15 i18n: leave out entries which contain only rst syntax
Simon Heimberg <simohe@besonet.ch>
parents: 20362
diff changeset
    69
                            # lines following directly, unexpected
e3ee7ec85a15 i18n: leave out entries which contain only rst syntax
Simon Heimberg <simohe@besonet.ch>
parents: 20362
diff changeset
    70
                            print 'Warning: text follows line with directive' \
e3ee7ec85a15 i18n: leave out entries which contain only rst syntax
Simon Heimberg <simohe@besonet.ch>
parents: 20362
diff changeset
    71
                                  ' %s' % directive
20359
ff6ab0b2ebf7 i18n: posplit writes a warning for translators before rst directives
Simon Heimberg <simohe@besonet.ch>
parents: 11389
diff changeset
    72
                    comment = 'do not translate: .. %s::' % directive
ff6ab0b2ebf7 i18n: posplit writes a warning for translators before rst directives
Simon Heimberg <simohe@besonet.ch>
parents: 11389
diff changeset
    73
                    if not newentry.comment:
ff6ab0b2ebf7 i18n: posplit writes a warning for translators before rst directives
Simon Heimberg <simohe@besonet.ch>
parents: 11389
diff changeset
    74
                        newentry.comment = comment
ff6ab0b2ebf7 i18n: posplit writes a warning for translators before rst directives
Simon Heimberg <simohe@besonet.ch>
parents: 11389
diff changeset
    75
                    elif comment not in newentry.comment:
ff6ab0b2ebf7 i18n: posplit writes a warning for translators before rst directives
Simon Heimberg <simohe@besonet.ch>
parents: 11389
diff changeset
    76
                        newentry.comment += '\n' + comment
11389
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    77
                addentry(po, newentry, cache)
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    78
            delta += 2 + msgid.count('\n')
4fd49329a1b5 i18n: script for splitting large messages on .po/.pot files
Wagner Bruna <wbruna@yahoo.com>
parents:
diff changeset
    79
    po.save()