Ren'Py Translator ToolKit  Check-in [a919022399]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Move auxiliary code in a package and tidy naming
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: a919022399f007058c60e9f76f7846dcbefbd94c43b6808ed94dbfb7b9295727
User & Date: Beuc 2019-05-24 08:09:10
Context
2019-05-30
16:56
Fix ImportError by committing missing __init__.py check-in: fe3f2ca5bd user: Beuc tags: trunk
2019-05-24
08:09
Move auxiliary code in a package and tidy naming check-in: a919022399 user: Beuc tags: trunk
07:39
Add git export script check-in: c85c8b9576 user: Beuc tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to mo2tl.py.

24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import print_function
import sys, os, fnmatch
import re
import subprocess, shutil
import tempfile
import tlparser, tlrun
import gettext

# Doc: manual .mo test:
# mkdir -p $LANG/LC_MESSAGES/
# msgfmt xxx.po -o $LANG/LC_MESSAGES/game.mo
# TEXTDOMAINDIR=. gettext -s -d game "Start"
# TEXTDOMAINDIR=. gettext -s -d game "script_abcd1234"$'\x4'"You've created a new Ren'Py game."







|







24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import print_function
import sys, os, fnmatch
import re
import subprocess, shutil
import tempfile
import rttk.run, rttk.tlparser
import gettext

# Doc: manual .mo test:
# mkdir -p $LANG/LC_MESSAGES/
# msgfmt xxx.po -o $LANG/LC_MESSAGES/game.mo
# TEXTDOMAINDIR=. gettext -s -d game "Start"
# TEXTDOMAINDIR=. gettext -s -d game "script_abcd1234"$'\x4'"You've created a new Ren'Py game."
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    print("Calling Ren'Py translate to get untranslated strings")
    try:
        # Ensure Ren'Py keeps the strings order (rather than append new strings)
        shutil.rmtree(os.path.join(projectpath,'game','tl','pot'))
    except OSError:
        pass
    # using --compile otherwise Ren'Py sometimes skips half of the files
    tlrun.renpy([projectpath, 'translate', 'pot', '--compile'])
    
    # Prepare msgid:untranslated_string index
    originals = []
    for curdir, subdirs, filenames in os.walk(os.path.join(projectpath,'game','tl','pot')):
        for filename in fnmatch.filter(filenames, '*.rpy'):
            print("Parsing  " + os.path.join(curdir,filename))
            f = open(os.path.join(curdir,filename), 'r')
            lines = f.readlines()
            if lines[0].startswith('\xef\xbb\xbf'):
                lines[0] = lines[0][3:]  # BOM

            lines.reverse()
            while len(lines) > 0:
                originals.extend(tlparser.parse_next_block(lines))

    o_blocks_index = {}
    o_basestr_index = {}
    for s in originals:
        if s['id']:
            o_blocks_index[s['id']] = s['text']
        else:
            o_basestr_index[s['text']] = s['translation']

    print("Calling Ren'Py translate to refresh " + renpy_target_language)
    tlrun.renpy([projectpath, 'translate', renpy_target_language])

    # Setup gettext directory structure
    localedir = tempfile.mkdtemp()
    if not os.environ.has_key('LANG'):
        os.environ['LANG'] = 'en_US.UTF-8'
    msgdir = os.path.join(localedir,
                          os.environ['LANG'],







|













|










|







67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    print("Calling Ren'Py translate to get untranslated strings")
    try:
        # Ensure Ren'Py keeps the strings order (rather than append new strings)
        shutil.rmtree(os.path.join(projectpath,'game','tl','pot'))
    except OSError:
        pass
    # using --compile otherwise Ren'Py sometimes skips half of the files
    rttk.run.renpy([projectpath, 'translate', 'pot', '--compile'])
    
    # Prepare msgid:untranslated_string index
    originals = []
    for curdir, subdirs, filenames in os.walk(os.path.join(projectpath,'game','tl','pot')):
        for filename in fnmatch.filter(filenames, '*.rpy'):
            print("Parsing  " + os.path.join(curdir,filename))
            f = open(os.path.join(curdir,filename), 'r')
            lines = f.readlines()
            if lines[0].startswith('\xef\xbb\xbf'):
                lines[0] = lines[0][3:]  # BOM

            lines.reverse()
            while len(lines) > 0:
                originals.extend(rttk.tlparser.parse_next_block(lines))

    o_blocks_index = {}
    o_basestr_index = {}
    for s in originals:
        if s['id']:
            o_blocks_index[s['id']] = s['text']
        else:
            o_basestr_index[s['text']] = s['translation']

    print("Calling Ren'Py translate to refresh " + renpy_target_language)
    rttk.run.renpy([projectpath, 'translate', renpy_target_language])

    # Setup gettext directory structure
    localedir = tempfile.mkdtemp()
    if not os.environ.has_key('LANG'):
        os.environ['LANG'] = 'en_US.UTF-8'
    msgdir = os.path.join(localedir,
                          os.environ['LANG'],
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
            lines.reverse()  # reverse so we can pop/append efficiently
            f_in.close()
        
            out = open(scriptpath, 'w')
            out.write('\xef\xbb\xbf')  # BOM, just in case
            while len(lines) > 0:
                line = lines.pop()
                if tlparser.is_empty(line):
                    out.write(line)
                elif tlparser.is_comment(line):
                    out.write(line)
                elif tlparser.is_block_start(line):
                    msgid = line.strip(':\n').split()[2]
                    if msgid == 'strings':
                        # basic strings block
                        out.write(line)
                        s = None
                        translation = ''
                        msgctxt = ''
                        while len(lines) > 0:
                            line = lines.pop()
                            if tlparser.is_empty(line):
                                pass
                            elif tlparser.is_comment(line):
                                msgctxt = line.lstrip().lstrip('#').strip()
                            elif not line.startswith(' '):
                                # end of block
                                lines.append(line)
                                break
                            elif line.lstrip().startswith('old '):
                                msgstr = tlparser.extract_base_string(line)['text']
                                lookup = msgstr.decode('string_escape')
                                lookup = msgctxt+'\x04'+lookup
                                translation = gettext.dgettext('game', lookup)
                                if translation == lookup:
                                        # no match with context, try without
                                        lookup = msgstr.decode('string_escape')
                                        translation = gettext.dgettext('game', lookup)
                                translation = c_escape(translation)
                                msgctxt = ''
                            elif line.lstrip().startswith('new '):
                                if translation is not None:
                                    s = tlparser.extract_base_string(line)
                                    line = line[:s['start']]+translation+line[s['end']:]
                                translation = None
                            else:
                                # unknown
                                pass
                            out.write(line)
                    else:
                        # dialog block
                        out.write(line)
                        while len(lines) > 0:
                            line = lines.pop()
                            if tlparser.is_empty(line):
                                pass
                            elif not line.startswith(' '):
                                # end of block
                                lines.append(line)
                                break
                            elif tlparser.is_comment(line):
                                # untranslated original
                                pass
                            else:
                                # dialog line
                                s = tlparser.extract_dialog_string(line)
                                if s is None:
                                    pass  # not a dialog line
                                elif o_blocks_index.get(msgid, None) is None:
                                    pass  # obsolete string
                                else:
                                    msgstr = o_blocks_index[msgid]
                                    msgctxt = msgid







|

|

|









|

|






|











|











|





|




|







127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
            lines.reverse()  # reverse so we can pop/append efficiently
            f_in.close()
        
            out = open(scriptpath, 'w')
            out.write('\xef\xbb\xbf')  # BOM, just in case
            while len(lines) > 0:
                line = lines.pop()
                if rttk.tlparser.is_empty(line):
                    out.write(line)
                elif rttk.tlparser.is_comment(line):
                    out.write(line)
                elif rttk.tlparser.is_block_start(line):
                    msgid = line.strip(':\n').split()[2]
                    if msgid == 'strings':
                        # basic strings block
                        out.write(line)
                        s = None
                        translation = ''
                        msgctxt = ''
                        while len(lines) > 0:
                            line = lines.pop()
                            if rttk.tlparser.is_empty(line):
                                pass
                            elif rttk.tlparser.is_comment(line):
                                msgctxt = line.lstrip().lstrip('#').strip()
                            elif not line.startswith(' '):
                                # end of block
                                lines.append(line)
                                break
                            elif line.lstrip().startswith('old '):
                                msgstr = rttk.tlparser.extract_base_string(line)['text']
                                lookup = msgstr.decode('string_escape')
                                lookup = msgctxt+'\x04'+lookup
                                translation = gettext.dgettext('game', lookup)
                                if translation == lookup:
                                        # no match with context, try without
                                        lookup = msgstr.decode('string_escape')
                                        translation = gettext.dgettext('game', lookup)
                                translation = c_escape(translation)
                                msgctxt = ''
                            elif line.lstrip().startswith('new '):
                                if translation is not None:
                                    s = rttk.tlparser.extract_base_string(line)
                                    line = line[:s['start']]+translation+line[s['end']:]
                                translation = None
                            else:
                                # unknown
                                pass
                            out.write(line)
                    else:
                        # dialog block
                        out.write(line)
                        while len(lines) > 0:
                            line = lines.pop()
                            if rttk.tlparser.is_empty(line):
                                pass
                            elif not line.startswith(' '):
                                # end of block
                                lines.append(line)
                                break
                            elif rttk.tlparser.is_comment(line):
                                # untranslated original
                                pass
                            else:
                                # dialog line
                                s = rttk.tlparser.extract_dialog_string(line)
                                if s is None:
                                    pass  # not a dialog line
                                elif o_blocks_index.get(msgid, None) is None:
                                    pass  # obsolete string
                                else:
                                    msgstr = o_blocks_index[msgid]
                                    msgctxt = msgid

Name change from tlrun.py to rttk/run.py.

Name change from test_tlparser.py to rttk/test_tlparser.py.

Name change from tlparser.py to rttk/tlparser.py.

Changes to tl2po.py.

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
# - import your game's translation started in Ren'Py format
# - import default Ren'Py translated strings from "The Question"

from __future__ import print_function
import sys, os, fnmatch, operator
import re
import shutil
import tlparser, tlrun


def tl2po(projectpath, language, outfile=None):
    if not re.match('^[a-z_]+$', language):
        raise Exception("Invalid language", language)
    if not os.path.isdir(os.path.join(projectpath,'game','tl',language)):
        raise Exception("Language not found", os.path.join(projectpath,'game','tl',language))

    if outfile is None:
        outfile = language+'.po'

    # Refresh strings
    print("Calling Ren'Py translate to get latest strings")
    try:
        # Ensure Ren'Py keeps the strings order (rather than append new strings)
        shutil.rmtree(os.path.join(projectpath,'game','tl','pot'))
    except OSError:
        pass
    # using --compile otherwise Ren'Py sometimes skips half of the files
    tlrun.renpy([projectpath, 'translate', 'pot', '--compile'])
    
    originals = []
    for curdir, subdirs, filenames in sorted(os.walk(os.path.join(projectpath,'game','tl','pot')), key=operator.itemgetter(0)):
        for filename in sorted(fnmatch.filter(filenames, '*.rpy')):
            print("Parsing  " + os.path.join(curdir,filename))
            f = open(os.path.join(curdir,filename), 'r')
            lines = f.readlines()
            if lines[0].startswith('\xef\xbb\xbf'):
                lines[0] = lines[0][3:]  # BOM

            lines.reverse()
            while len(lines) > 0:
                originals.extend(tlparser.parse_next_block(lines))

    translated = []
    for curdir, subdirs, filenames in os.walk(os.path.join(projectpath,'game','tl',language)):
        for filename in fnmatch.filter(filenames, '*.rpy'):
            print("Parsing  " + os.path.join(curdir,filename))
            f = open(os.path.join(curdir,filename), 'r')
            lines = f.readlines()
            if lines[0].startswith('\xef\xbb\xbf'):
                lines[0] = lines[0][3:]  # BOM

            lines.reverse()
            while len(lines) > 0:
                translated.extend(tlparser.parse_next_block(lines))

    t_blocks_index = {}
    t_basestr_index = {}
    for s in translated:
        if s['id']:
            t_blocks_index[s['id']] = s['text']
        else:







|



















|












|












|







27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
# - import your game's translation started in Ren'Py format
# - import default Ren'Py translated strings from "The Question"

from __future__ import print_function
import sys, os, fnmatch, operator
import re
import shutil
import rttk.run, rttk.tlparser


def tl2po(projectpath, language, outfile=None):
    if not re.match('^[a-z_]+$', language):
        raise Exception("Invalid language", language)
    if not os.path.isdir(os.path.join(projectpath,'game','tl',language)):
        raise Exception("Language not found", os.path.join(projectpath,'game','tl',language))

    if outfile is None:
        outfile = language+'.po'

    # Refresh strings
    print("Calling Ren'Py translate to get latest strings")
    try:
        # Ensure Ren'Py keeps the strings order (rather than append new strings)
        shutil.rmtree(os.path.join(projectpath,'game','tl','pot'))
    except OSError:
        pass
    # using --compile otherwise Ren'Py sometimes skips half of the files
    rttk.run.renpy([projectpath, 'translate', 'pot', '--compile'])
    
    originals = []
    for curdir, subdirs, filenames in sorted(os.walk(os.path.join(projectpath,'game','tl','pot')), key=operator.itemgetter(0)):
        for filename in sorted(fnmatch.filter(filenames, '*.rpy')):
            print("Parsing  " + os.path.join(curdir,filename))
            f = open(os.path.join(curdir,filename), 'r')
            lines = f.readlines()
            if lines[0].startswith('\xef\xbb\xbf'):
                lines[0] = lines[0][3:]  # BOM

            lines.reverse()
            while len(lines) > 0:
                originals.extend(rttk.tlparser.parse_next_block(lines))

    translated = []
    for curdir, subdirs, filenames in os.walk(os.path.join(projectpath,'game','tl',language)):
        for filename in fnmatch.filter(filenames, '*.rpy'):
            print("Parsing  " + os.path.join(curdir,filename))
            f = open(os.path.join(curdir,filename), 'r')
            lines = f.readlines()
            if lines[0].startswith('\xef\xbb\xbf'):
                lines[0] = lines[0][3:]  # BOM

            lines.reverse()
            while len(lines) > 0:
                translated.extend(rttk.tlparser.parse_next_block(lines))

    t_blocks_index = {}
    t_basestr_index = {}
    for s in translated:
        if s['id']:
            t_blocks_index[s['id']] = s['text']
        else:

Changes to tl2pot.py.

23
24
25
26
27
28
29

30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import print_function
import sys, os, fnmatch, operator
import re
import shutil

import tlparser, tlrun

def tl2pot(projectpath, outfile='game.pot'):
    # Refresh strings
    try:
        # Ensure Ren'Py keeps the strings order (rather than append new strings)
        shutil.rmtree(os.path.join(projectpath,'game','tl','pot'))
    except OSError:
        pass

    print("Calling Ren'Py translate to get the latest strings")
    # using --compile otherwise Ren'Py sometimes skips half of the files
    tlrun.renpy([projectpath, 'translate', 'pot', '--compile'])

    strings = []
    for curdir, subdirs, filenames in sorted(os.walk(os.path.join(projectpath,'game','tl','pot')), key=operator.itemgetter(0)):
        for filename in sorted(fnmatch.filter(filenames, '*.rpy')):
            print("Parsing  " + os.path.join(curdir,filename))
            f = open(os.path.join(curdir,filename), 'r')
            lines = f.readlines()
            if lines[0].startswith('\xef\xbb\xbf'):
                lines[0] = lines[0][3:]  # BOM

            lines.reverse()
            cur_strings = []
            while len(lines) > 0:
                cur_strings.extend(tlparser.parse_next_block(lines))
            cur_strings.sort(key=lambda s: (s['source'].split(':')[0], int(s['source'].split(':')[1])))
            strings.extend(cur_strings)
    
    occurrences = {}
    for s in strings:
        occurrences[s['text']] = occurrences.get(s['text'], 0) + 1








>
|











|













|







23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import print_function
import sys, os, fnmatch, operator
import re
import shutil
import rttk.run, rttk.tlparser


def tl2pot(projectpath, outfile='game.pot'):
    # Refresh strings
    try:
        # Ensure Ren'Py keeps the strings order (rather than append new strings)
        shutil.rmtree(os.path.join(projectpath,'game','tl','pot'))
    except OSError:
        pass

    print("Calling Ren'Py translate to get the latest strings")
    # using --compile otherwise Ren'Py sometimes skips half of the files
    rttk.run.renpy([projectpath, 'translate', 'pot', '--compile'])

    strings = []
    for curdir, subdirs, filenames in sorted(os.walk(os.path.join(projectpath,'game','tl','pot')), key=operator.itemgetter(0)):
        for filename in sorted(fnmatch.filter(filenames, '*.rpy')):
            print("Parsing  " + os.path.join(curdir,filename))
            f = open(os.path.join(curdir,filename), 'r')
            lines = f.readlines()
            if lines[0].startswith('\xef\xbb\xbf'):
                lines[0] = lines[0][3:]  # BOM

            lines.reverse()
            cur_strings = []
            while len(lines) > 0:
                cur_strings.extend(rttk.tlparser.parse_next_block(lines))
            cur_strings.sort(key=lambda s: (s['source'].split(':')[0], int(s['source'].split(':')[1])))
            strings.extend(cur_strings)
    
    occurrences = {}
    for s in strings:
        occurrences[s['text']] = occurrences.get(s['text'], 0) + 1