Align TED English and Korean xml corpuses (requires preprocessing/normalizing/tokenizing code)

1. #!/usr/bin/python3
2. import random
3. from normalizer import Normalizer
4. from tokenizer import Tokenizer
6. '''
7. Align TED English and Korean xml corpuses:
8. https://wit3.fbk.eu/mono.php?release=XML_releases&tinfo=cleanedhtml_ted
9. '''
11. # character threshold
12. # if number of characters is less than this, don't output it
13. ko_len_threshold = 10
14. en_len_threshold = 20
16. class TEDVideo(object):
17. def __init__(self):
18. self.url = None
19. self.transcriptions = []
21. '''
22. Parse an array of TEDVideo objects from an xml file
23. '''
24. def parse_ted_file(fn):
25. all_videos = []
26. contents = None
27. with open(fn, 'r', encoding='utf-8') as fd:
28. contents = fd.read()
29. for tmp in contents.split('<file')[1:]:
30. ted_video = TEDVideo()
31. ted_video.url = tmp.split('<url>')[1].split('</url>')[0].lower().strip()
32. transcription_part = tmp.split('<transcription>')[1].split('</transcription')[0]
33. invalid_file = False
34. for ln in transcription_part.split('\n'):
35. ln = ln.strip()
36. if not ln:
37. continue
38. #print(ln)
39. if '" />' in ln:
40. invalid_file = True
41. continue # erroneous line <seekvideo id="551600" />
42. seekvideo_time = int(ln.split(' id="')[1].split('"')[0])
43. seekvideo_text = ln.split('">')[1].split('</seekvideo>')[0].strip()
45. seekvideo_text_normalized = Normalizer.normalize(seekvideo_text)
46. tk = Tokenizer.tokenize(seekvideo_text_normalized,
47. keep_punctuation=True,
48. keep_symbols=True)
50. #print((seekvideo_time, seekvideo_text))
51. ted_video.transcriptions.append((seekvideo_time, ' '.join(tk)))
52. if not invalid_file:
53. all_videos.append(ted_video)
54. return all_videos
56. parsed_en_videos = parse_ted_file('ted_en-20160408.xml')
57. parsed_ko_videos = parse_ted_file('ted_ko-20160408.xml')
59. url_set_en = set([v.url for v in parsed_en_videos])
60. url_set_ko = set([v.url for v in parsed_ko_videos])
61. monoling_set = url_set_en | url_set_ko
62. biling_set = url_set_en & url_set_ko
64. url_to_tedvideo_en = {}
65. for v in parsed_en_videos:
66. url_to_tedvideo_en[v.url] = v
68. url_to_tedvideo_ko = {}
69. for v in parsed_ko_videos:
70. url_to_tedvideo_ko[v.url] = v
72. print('Detected %d/%d bilingual videos total' % (len(biling_set), len(monoling_set)))
74. #with open('ted_ko-20160408.xml', 'r', encoding='utf-8') as fd:
75. # contents = fd.read()
77. sent_ko = []
78. sent_en = []
80. vocab_ko = set()
81. vocab_en = set()
83. # re-order videos in random order each time
84. biling_set = list(biling_set)
85. random.shuffle(biling_set)
87. for url in biling_set:
88. en_video = url_to_tedvideo_en[url]
89. ko_video = url_to_tedvideo_ko[url]
90. transcript_en = en_video.transcriptions
91. transcript_ko = ko_video.transcriptions
93. biling_timestamps = set(list(zip(*transcript_en))[0]) & set(list(zip(*transcript_ko))[0])
94. transcript_en_dict = dict(transcript_en)
95. transcript_ko_dict = dict(transcript_ko)
97. # unzip and check timestamp consistency
98. #if list(zip(*transcript_en))[0] != list(zip(*transcript_ko))[0]:
99. #print('en', list(zip(*transcript_en))[0])
100. #print('ko', list(zip(*transcript_ko))[0])
101. #print('Timestamps inconsistent for: %s. Skipping.' % url)
102. # continue
103. # or should we just print only the matching timestamps?
105. for ts in biling_timestamps:
106. if len(transcript_ko_dict[ts]) >= ko_len_threshold and len(transcript_en_dict[ts]) >= en_len_threshold:
107. sent_ko.append(transcript_ko_dict[ts])
108. sent_en.append(transcript_en_dict[ts])
110. for wd in transcript_ko_dict[ts].split():
111. vocab_ko.add(wd)
113. for wd in transcript_en_dict[ts].split():
114. vocab_en.add(wd)
116. #fd_ko.write(transcript_ko_dict[ts] + '\n')
117. #fd_en.write(transcript_en_dict[ts] + '\n')
119. assert(len(sent_ko) == len(sent_en))
121. # train/test/dev sets
122. # 80/10/10
124. num_train = int(len(sent_ko) * 0.80)
125. num_test = int(len(sent_ko) * 0.10)
126. num_dev = len(sent_ko)-num_train-num_test
128. train_ko = sent_ko[:num_train]
129. test_ko = sent_ko[num_train:num_train+num_test]
130. dev_ko = sent_ko[num_train+num_test:]
132. train_en = sent_en[:num_train]
133. test_en = sent_en[num_train:num_train+num_test]
134. dev_en = sent_en[num_train+num_test:]
136. assert len(train_ko)==num_train
137. assert len(test_ko)==num_test
138. assert len(dev_ko)==num_dev
140. assert len(train_en)==num_train
141. assert len(test_en)==num_test
142. assert len(dev_en)==num_dev
144. for datasets in [('train', train_ko), ('test', test_ko), ('dev', dev_ko)]:
145. with open('ted_biling_%s.ko' % datasets[0], 'w', encoding='utf-8') as fd:
146. for d in datasets[1]:
147. fd.write(d + '\n')
149. for datasets in [('train', train_en), ('test', test_en), ('dev', dev_en)]:
150. with open('ted_biling_%s.en' % datasets[0], 'w', encoding='utf-8') as fd:
151. for d in datasets[1]:
152. fd.write(d + '\n')
154. # write vocab files for NMT
155. std_vocab = list(['<unk>', '<s>', '</s>'])
157. with open('ted_biling_vocab.ko', 'w', encoding='utf-8') as fd:
158. for v in std_vocab + list(vocab_ko):
159. fd.write('%s\n' % v)
161. with open('ted_biling_vocab.en', 'w', encoding='utf-8') as fd:
162. for v in std_vocab + list(vocab_en):
163. fd.write('%s\n' % v)

#!/usr/bin/python3
import random
from normalizer import Normalizer
from tokenizer import Tokenizer

'''
Align TED English and Korean xml corpuses:
https://wit3.fbk.eu/mono.php?release=XML_releases&tinfo=cleanedhtml_ted
'''

# character threshold
# if number of characters is less than this, don't output it
ko_len_threshold = 10
en_len_threshold = 20

class TEDVideo(object):
    def __init__(self):
        self.url = None
        self.transcriptions = []

'''
Parse an array of TEDVideo objects from an xml file
'''
def parse_ted_file(fn):
    all_videos = []
    contents = None
    with open(fn, 'r', encoding='utf-8') as fd:
        contents = fd.read()
    for tmp in contents.split('<file')[1:]:
        ted_video = TEDVideo()
        ted_video.url = tmp.split('<url>')[1].split('</url>')[0].lower().strip()
        transcription_part = tmp.split('<transcription>')[1].split('</transcription')[0]
        invalid_file = False
        for ln in transcription_part.split('\n'):
            ln = ln.strip()
            if not ln:
                continue
            #print(ln)
            if '" />' in ln:
                invalid_file = True
                continue # erroneous line <seekvideo id="551600" />
            seekvideo_time = int(ln.split(' id="')[1].split('"')[0])
            seekvideo_text = ln.split('">')[1].split('</seekvideo>')[0].strip()

            seekvideo_text_normalized = Normalizer.normalize(seekvideo_text)
            tk = Tokenizer.tokenize(seekvideo_text_normalized,
                                    keep_punctuation=True,
                                    keep_symbols=True)

            #print((seekvideo_time, seekvideo_text))
            ted_video.transcriptions.append((seekvideo_time, ' '.join(tk)))
        if not invalid_file:
            all_videos.append(ted_video)
    return all_videos

parsed_en_videos = parse_ted_file('ted_en-20160408.xml')
parsed_ko_videos = parse_ted_file('ted_ko-20160408.xml')

url_set_en = set([v.url for v in parsed_en_videos])
url_set_ko = set([v.url for v in parsed_ko_videos])
monoling_set = url_set_en | url_set_ko
biling_set = url_set_en & url_set_ko

url_to_tedvideo_en = {}
for v in parsed_en_videos:
    url_to_tedvideo_en[v.url] = v

url_to_tedvideo_ko = {}
for v in parsed_ko_videos:
    url_to_tedvideo_ko[v.url] = v

print('Detected %d/%d bilingual videos total' % (len(biling_set), len(monoling_set)))

#with open('ted_ko-20160408.xml', 'r', encoding='utf-8') as fd:
#    contents = fd.read()

sent_ko = []
sent_en = []

vocab_ko = set()
vocab_en = set()

# re-order videos in random order each time
biling_set = list(biling_set)
random.shuffle(biling_set)

for url in biling_set:
    en_video = url_to_tedvideo_en[url]
    ko_video = url_to_tedvideo_ko[url]
    transcript_en = en_video.transcriptions
    transcript_ko = ko_video.transcriptions

    biling_timestamps = set(list(zip(*transcript_en))[0]) & set(list(zip(*transcript_ko))[0])
    transcript_en_dict = dict(transcript_en)
    transcript_ko_dict = dict(transcript_ko)

    # unzip and check timestamp consistency
    #if list(zip(*transcript_en))[0] != list(zip(*transcript_ko))[0]:
        #print('en', list(zip(*transcript_en))[0])
        #print('ko', list(zip(*transcript_ko))[0])
        #print('Timestamps inconsistent for: %s. Skipping.' % url)
    #    continue
        # or should we just print only the matching timestamps?

    for ts in biling_timestamps:
        if len(transcript_ko_dict[ts]) >= ko_len_threshold and len(transcript_en_dict[ts]) >= en_len_threshold:
            sent_ko.append(transcript_ko_dict[ts])
            sent_en.append(transcript_en_dict[ts])

            for wd in transcript_ko_dict[ts].split():
                vocab_ko.add(wd)

            for wd in transcript_en_dict[ts].split():
                vocab_en.add(wd)

            #fd_ko.write(transcript_ko_dict[ts] + '\n')
            #fd_en.write(transcript_en_dict[ts] + '\n')

assert(len(sent_ko) == len(sent_en))

# train/test/dev sets
# 80/10/10

num_train = int(len(sent_ko) * 0.80)
num_test = int(len(sent_ko) * 0.10)
num_dev = len(sent_ko)-num_train-num_test

train_ko = sent_ko[:num_train]
test_ko = sent_ko[num_train:num_train+num_test]
dev_ko = sent_ko[num_train+num_test:]

train_en = sent_en[:num_train]
test_en = sent_en[num_train:num_train+num_test]
dev_en = sent_en[num_train+num_test:]

assert len(train_ko)==num_train
assert len(test_ko)==num_test
assert len(dev_ko)==num_dev

assert len(train_en)==num_train
assert len(test_en)==num_test
assert len(dev_en)==num_dev

for datasets in [('train', train_ko), ('test', test_ko), ('dev', dev_ko)]:
    with open('ted_biling_%s.ko' % datasets[0], 'w', encoding='utf-8') as fd:
        for d in datasets[1]:
            fd.write(d + '\n')

for datasets in [('train', train_en), ('test', test_en), ('dev', dev_en)]:
    with open('ted_biling_%s.en' % datasets[0], 'w', encoding='utf-8') as fd:
        for d in datasets[1]:
            fd.write(d + '\n')

# write vocab files for NMT
std_vocab = list(['<unk>', '<s>', '</s>'])

with open('ted_biling_vocab.ko', 'w', encoding='utf-8') as fd:
    for v in std_vocab + list(vocab_ko):
        fd.write('%s\n' % v)

with open('ted_biling_vocab.en', 'w', encoding='utf-8') as fd:
    for v in std_vocab + list(vocab_en):
        fd.write('%s\n' % v)