Convert Sejong POS-tagged corpus format to CoNLL-U format

xtknightUncategorized0 Comments

Convert Sejong POS-tagged corpus format to CoNLL-U format (useful for training Google SyntaxNet)

  1. #!/usr/bin/python3
  2. # -*- coding: utf-8 -*-
  3. '''
  4. Convert Sejong POS-tagged corpus format to CoNLL-U format for use with Google
  5. SyntaxNet
  7. http://universaldependencies.org/docs/format.html
  9. Outputs training, testing, and tuning sets (60-20-20 ratio, randomly chosen)
  11. Arguments: ./corpus_to_conll.py <sejong_corpus_dir> <output_dir>
  13. sejong_corpus_dir should contain UTF-16 text files
  14. (BTAA0001.txt, etc...)
  16. output_dir will contain:
  17. - tagged-training-corpus.conllu
  18. - tagged-tuning-corpus.conllu
  19. - tagged-dev-corpus.conllu (evaluation set)
  20. '''
  22. import os, sys
  24. if(len(sys.argv) < 3):
  25. print('Arguments: ./corpus_to_conll.py <sejong_corpus_dir> <output_dir>')
  26. sys.exit(1)
  28. SEJONG_PATH = sys.argv[1]
  29. output_dir = sys.argv[2]
  31. from os import listdir
  32. from os.path import isfile, join
  33. SEJONG_FILES = [join(SEJONG_PATH, f) for f in listdir(SEJONG_PATH) if isfile(join(SEJONG_PATH, f))]
  34. # Process in sensible order
  35. SEJONG_FILES.sort()
  37. if output_dir.endswith('/'):
  38. output_dir = output_dir[:-1]
  40. try:
  41. os.mkdir(output_dir)
  42. except FileExistsError:
  43. pass
  45. print('Processing corpus and outputting to %s...\n' % output_dir)
  47. class CoNLLSentence(object):
  48. def __init__(self):
  49. # 장식품/NNG + 으로/JKB + …/SE
  51. # [장식품, 으로, …]
  52. self.FORM = []
  53. # ['NNG', 'JKB', 'SE']
  54. self.XPOSTAG = []
  56. def toString(self):
  57. assert(len(self.FORM) == len(self.XPOSTAG))
  59. self.ID = [(i+1) for i in range(0, len(self.FORM))]
  60. self.LEMMA = self.FORM
  61. self.UPOSTAG = self.XPOSTAG
  62. self.FEATS = ['_' for i in range(0, len(self.FORM))]
  63. self.HEAD = ['_' for i in range(0, len(self.FORM))]
  64. self.DEPREL = ['_' for i in range(0, len(self.FORM))]
  66. s = []
  68. for i in range(0, len(self.FORM)):
  69. s.append('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s' % (self.ID[i],
  70. self.FORM[i],
  71. self.LEMMA[i],
  72. self.UPOSTAG[i],
  73. self.XPOSTAG[i],
  74. self.FEATS[i],
  75. self.HEAD[i],
  76. self.DEPREL[i]))
  78. return '\n'.join(s)
  81. '''
  82. Process a file and return a list of sentences
  83. '''
  84. def processFile(path):
  85. # BTAE0201.txt has encoding error. be lenient.
  86. f = open(path, 'r', encoding='utf16', errors='ignore')
  87. contents = f.read()
  88. f.close()
  90. # Some of the files don't have <group>!
  91. # strings -f -el *.txt |grep "<group>"
  92. # For example, BTAZ0223.txt
  94. contents = contents.replace('<head>', '<p>')
  95. contents = contents.replace('</head>', '</p>')
  97. # Detected Sejong tagset
  98. tagset = set()
  100. # CoNLL sentences
  101. sentences = []
  103. # There's even <p> before </teiHeader>
  104. paras = contents.split('</teiHeader>')[1].split('<p>')[1:]
  105. for p in paras:
  106. p = p.split('</p>')[0].strip()
  108. sentence = CoNLLSentence()
  110. try:
  111. lines = p.split('\n')
  112. #skipPara = False
  113. for ln in lines:
  114. if not ln:
  115. continue
  117. #BTAA0001-00000013 세계적인 세계/NNG + 적/XSN + 이/VCP + ᆫ/ETM
  118. cols = ln.split('\t')
  120. if ('<' in cols[0]):
  121. # Do we want to process dates? Not sure yet...
  122. print('Ignoring tag: ' + cols[0])
  123. #skipPara = True
  124. break
  126. if len(cols) != 3:
  127. #print('Parsing error in %s! Column may contain tab or unexpected tag' % (path))
  128. raise ValueError('Parsing error in %s! Column may contain tab or unexpected tag' % (path))
  130. #print(cols)
  131. lineid = cols[0].strip()
  132. # Don't recommend stripping the other columns
  133. # Sometimes spaces are tagged, and apparently they could be anywhere
  134. # BTHO0432-00032162 주장자(　杖子)를 주장자/NNG + (/SS + 　/SW + 杖子/SH + )/SS + 를/JKO
  135. word = cols[1]
  136. # BTBA0233-00016517 2(남북한)+4(미 2/SN + (/SS + 남/NNP + 북한/NNP + )/SS + +/SW + 4/SN + (/SS + 미/NNP
  137. # Some data even contains +. What we really want is to split by ' + '
  138. tags = cols[2].split(' + ')
  140. for tagelem in tags:
  141. slashes = tagelem.split('/')
  142. # sometimes component contains slash
  143. # only consider tag to be after last /
  144. # >>> slashes = '////////SP'.split('/')
  145. wordcomponent = '/'.join(slashes[0:len(slashes)-1])
  146. tag = slashes[len(slashes)-1]
  148. if (not wordcomponent) or (not tag):
  149. #print('Parsing error in %s! Word or tag empty: %s' % (path, str(tags)))
  150. raise ValueError('Parsing error in %s! Word or tag empty: %s' % (path, str(tags)))
  152. #print(wordcomponent, tag)
  153. assert(not('\t' in wordcomponent))
  154. assert(not('\n' in wordcomponent))
  155. assert(not('\t' in tag))
  156. assert(not('\n' in tag))
  157. sentence.FORM.append(wordcomponent)
  158. sentence.XPOSTAG.append(tag)
  159. tagset.add(tag)
  161. #print(sentence.toString() + '\n')
  162. sentences.append(sentence)
  163. except Exception as e:
  164. print(e)
  165. print('Processing paragraph failed.')
  167. return (tagset, sentences)
  169. tagset = set()
  170. sentences = list()
  172. print('Found %d files to process.' % len(SEJONG_FILES))
  174. m = 0
  175. #SEJONG_FILES = ['../data/sejong_entire/BTHO0136.txt']
  176. for f in SEJONG_FILES:
  177. m += 1
  178. print('Processing %s (%0.2f%%)' % (f, 100.0 * float(m) / float(len(SEJONG_FILES))))
  179. thisFileTagset, thisFileSentences = processFile(f)
  180. tagset = tagset.union(thisFileTagset)
  181. sentences += thisFileSentences
  183. # index of sentences within sentences list for each set
  184. training_set_indices = []
  185. tuning_set_indices = []
  186. testing_set_indices = []
  188. # split to 60-20-20 (training, tuning, testing sets)
  189. for i in range(len(sentences)):
  190. if i % 10 == 0 or i % 10 == 1: # ~20%
  191. tuning_set_indices.append(i)
  192. elif i % 10 == 2 or i % 10 == 3: # ~20%
  193. testing_set_indices.append(i)
  194. else: # ~60%
  195. training_set_indices.append(i)
  197. print()
  198. print('Found %d sentences (%d-%d-%d).' % (len(sentences),
  199. len(training_set_indices), len(tuning_set_indices), len(testing_set_indices)))
  200. print('Complete tagset found: ' + str(tagset))
  201. print()
  203. print('Writing %s/tagged-training-corpus.conllu (%d sentences)...' % (output_dir, len(training_set_indices)))
  205. fd = open('%s/tagged-training-corpus.conllu' % output_dir, 'w', encoding='utf-8')
  206. for i in range(len(training_set_indices)):
  207. idx = training_set_indices[i]
  208. fd.write(sentences[idx].toString() + '\n')
  209. # no extra newline after last element
  210. if(i < len(training_set_indices)-1):
  211. fd.write('\n')
  212. fd.close()
  214. print('Writing %s/tagged-tuning-corpus.conllu (%d sentences)...' % (output_dir, len(tuning_set_indices)))
  216. fd = open('%s/tagged-tuning-corpus.conllu' % output_dir, 'w', encoding='utf-8')
  217. for i in range(len(tuning_set_indices)):
  218. idx = tuning_set_indices[i]
  219. fd.write(sentences[idx].toString() + '\n')
  220. if(i < len(tuning_set_indices)-1):
  221. fd.write('\n')
  222. fd.close()
  224. print('Writing %s/tagged-dev-corpus.conllu (%d sentences)...' % (output_dir, len(testing_set_indices)))
  226. fd = open('%s/tagged-dev-corpus.conllu' % output_dir, 'w', encoding='utf-8')
  227. for i in range(len(testing_set_indices)):
  228. idx = testing_set_indices[i]
  229. fd.write(sentences[idx].toString() + '\n')
  230. if(i < len(testing_set_indices)-1):
  231. fd.write('\n')
  232. fd.close()
#!/usr/bin/python3
# -*- coding: utf-8 -*-
'''
Convert Sejong POS-tagged corpus format to CoNLL-U format for use with Google
SyntaxNet

http://universaldependencies.org/docs/format.html

Outputs training, testing, and tuning sets (60-20-20 ratio, randomly chosen)

Arguments: ./corpus_to_conll.py <sejong_corpus_dir> <output_dir>

sejong_corpus_dir should contain UTF-16 text files
(BTAA0001.txt, etc...)

output_dir will contain:
- tagged-training-corpus.conllu
- tagged-tuning-corpus.conllu
- tagged-dev-corpus.conllu (evaluation set)
'''

import os, sys

if(len(sys.argv) < 3):
    print('Arguments: ./corpus_to_conll.py <sejong_corpus_dir> <output_dir>')
    sys.exit(1)

SEJONG_PATH = sys.argv[1]
output_dir = sys.argv[2]

from os import listdir
from os.path import isfile, join
SEJONG_FILES = [join(SEJONG_PATH, f) for f in listdir(SEJONG_PATH) if isfile(join(SEJONG_PATH, f))]
# Process in sensible order
SEJONG_FILES.sort()

if output_dir.endswith('/'):
    output_dir = output_dir[:-1]

try:
    os.mkdir(output_dir)
except FileExistsError:
    pass

print('Processing corpus and outputting to %s...\n' % output_dir)

class CoNLLSentence(object):
    def __init__(self):
                # 장식품/NNG + 으로/JKB + …/SE

        # [장식품, 으로, …]
        self.FORM = []
        # ['NNG', 'JKB', 'SE']
        self.XPOSTAG = []

    def toString(self):
        assert(len(self.FORM) == len(self.XPOSTAG))

        self.ID = [(i+1) for i in range(0, len(self.FORM))]
        self.LEMMA = self.FORM
        self.UPOSTAG = self.XPOSTAG
        self.FEATS = ['_' for i in range(0, len(self.FORM))]
        self.HEAD = ['_' for i in range(0, len(self.FORM))]
        self.DEPREL = ['_' for i in range(0, len(self.FORM))]

        s = []

        for i in range(0, len(self.FORM)):
            s.append('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s' % (self.ID[i],
                self.FORM[i],
                self.LEMMA[i],
                self.UPOSTAG[i],
                self.XPOSTAG[i],
                self.FEATS[i],
                self.HEAD[i],
                self.DEPREL[i]))

        return '\n'.join(s)
        

'''
Process a file and return a list of sentences
'''
def processFile(path):
    # BTAE0201.txt has encoding error. be lenient.
    f = open(path, 'r', encoding='utf16', errors='ignore')
    contents = f.read()
    f.close()
    
    # Some of the files don't have <group>!
    # strings -f -el *.txt |grep "<group>"
    # For example, BTAZ0223.txt
    
    contents = contents.replace('<head>', '<p>')
    contents = contents.replace('</head>', '</p>')
    
    # Detected Sejong tagset
    tagset = set()

    # CoNLL sentences
    sentences = []
    
    # There's even <p> before </teiHeader>
    paras = contents.split('</teiHeader>')[1].split('<p>')[1:]
    for p in paras:
        p = p.split('</p>')[0].strip()
        
        sentence = CoNLLSentence()

        try:
            lines = p.split('\n')
            #skipPara = False
            for ln in lines:
                if not ln:
                    continue
                
                #BTAA0001-00000013	세계적인	세계/NNG + 적/XSN + 이/VCP + ᆫ/ETM
                cols = ln.split('\t')
                
                if ('<' in cols[0]):
                    # Do we want to process dates? Not sure yet...
                    print('Ignoring tag: ' + cols[0])
                    #skipPara = True
                    break
                
                if len(cols) != 3:
                    #print('Parsing error in %s! Column may contain tab or unexpected tag' % (path))
                    raise ValueError('Parsing error in %s! Column may contain tab or unexpected tag' % (path))
                
                #print(cols)
                lineid = cols[0].strip()
                # Don't recommend stripping the other columns
                # Sometimes spaces are tagged, and apparently they could be anywhere
                # BTHO0432-00032162	주장자(　杖子)를	주장자/NNG + (/SS + 　/SW + 杖子/SH + )/SS + 를/JKO
                word = cols[1]
                # BTBA0233-00016517	2(남북한)+4(미	2/SN + (/SS + 남/NNP + 북한/NNP + )/SS + +/SW + 4/SN + (/SS + 미/NNP
                # Some data even contains +. What we really want is to split by ' + '
                tags = cols[2].split(' + ')
                
                for tagelem in tags:
                    slashes = tagelem.split('/')
                    # sometimes component contains slash
                    # only consider tag to be after last /
                    # >>> slashes = '////////SP'.split('/')
                    wordcomponent = '/'.join(slashes[0:len(slashes)-1])
                    tag = slashes[len(slashes)-1]
                    
                    if (not wordcomponent) or (not tag):
                        #print('Parsing error in %s! Word or tag empty: %s' % (path, str(tags)))
                        raise ValueError('Parsing error in %s! Word or tag empty: %s' % (path, str(tags)))
                    
                    #print(wordcomponent, tag)
                    assert(not('\t' in wordcomponent))
                    assert(not('\n' in wordcomponent))
                    assert(not('\t' in tag))
                    assert(not('\n' in tag))
                    sentence.FORM.append(wordcomponent)
                    sentence.XPOSTAG.append(tag)
                    tagset.add(tag)

            #print(sentence.toString() + '\n')
            sentences.append(sentence)
        except Exception as e:
            print(e)
            print('Processing paragraph failed.')

    return (tagset, sentences)

tagset = set()
sentences = list()

print('Found %d files to process.' % len(SEJONG_FILES))

m = 0
#SEJONG_FILES = ['../data/sejong_entire/BTHO0136.txt']
for f in SEJONG_FILES:
    m += 1
    print('Processing %s (%0.2f%%)' % (f, 100.0 * float(m) / float(len(SEJONG_FILES))))
    thisFileTagset, thisFileSentences = processFile(f)
    tagset = tagset.union(thisFileTagset)
    sentences += thisFileSentences

# index of sentences within sentences list for each set
training_set_indices = []
tuning_set_indices = []
testing_set_indices = []

# split to 60-20-20 (training, tuning, testing sets)
for i in range(len(sentences)):
    if i % 10 == 0 or i % 10 == 1:     # ~20%
        tuning_set_indices.append(i)
    elif i % 10 == 2 or i % 10 == 3:   # ~20%
        testing_set_indices.append(i)
    else:                               # ~60%
        training_set_indices.append(i)

print()
print('Found %d sentences (%d-%d-%d).' % (len(sentences),
    len(training_set_indices), len(tuning_set_indices), len(testing_set_indices)))
print('Complete tagset found: ' + str(tagset))
print()

print('Writing %s/tagged-training-corpus.conllu (%d sentences)...' % (output_dir, len(training_set_indices)))

fd = open('%s/tagged-training-corpus.conllu' % output_dir, 'w', encoding='utf-8')
for i in range(len(training_set_indices)):
    idx = training_set_indices[i]
    fd.write(sentences[idx].toString() + '\n')
    # no extra newline after last element
    if(i < len(training_set_indices)-1):
        fd.write('\n')
fd.close()

print('Writing %s/tagged-tuning-corpus.conllu (%d sentences)...' % (output_dir, len(tuning_set_indices)))

fd = open('%s/tagged-tuning-corpus.conllu' % output_dir, 'w', encoding='utf-8')
for i in range(len(tuning_set_indices)):
    idx = tuning_set_indices[i]
    fd.write(sentences[idx].toString() + '\n')
    if(i < len(tuning_set_indices)-1):
        fd.write('\n')
fd.close()

print('Writing %s/tagged-dev-corpus.conllu (%d sentences)...' % (output_dir, len(testing_set_indices)))

fd = open('%s/tagged-dev-corpus.conllu' % output_dir, 'w', encoding='utf-8')
for i in range(len(testing_set_indices)):
    idx = testing_set_indices[i]
    fd.write(sentences[idx].toString() + '\n')
    if(i < len(testing_set_indices)-1):
        fd.write('\n')
fd.close()

 

Leave a Reply

Your email address will not be published. Required fields are marked *