CoNLL File Parsing Module

1. '''
2. A set of classes to handle input and output of CoNLL-U files
4. http://universaldependencies.org/docs/format.html
6. The Parsed* classes are useful to store extra properties needed during
7. the parsing process that are external to the Conll instances themselves
8. '''
10. import logging
11. import well_formed_filter
13. def isNonCompatibilityJamo(c):
14. assert len(c) == 1
15. # HANGUL JAMO: (U+1100-U+11FF)
16. if ord(c) >= 0x1100 and ord(c) <= 0x11FF:
17. return True
18. else:
19. return False
21. def normalizeToCompatJamo(s):
22. from jamo import h2j, j2hcj
23. out = ''
24. for c in s:
25. if isNonCompatibilityJamo(c):
26. out += j2hcj(c)
27. else:
28. out += c
29. assert len(s) == len(out)
30. return out
32. def encodeNoneAsUnderscore(s):
33. if s == None:
34. return '_'
35. else:
36. return s
38. def encodeNoneAsUnderscore_Int(i):
39. if i == None:
40. return '_'
41. else:
42. return str(i)
44. '''
45. Represents a CoNLL token and all its properties (except index)
46. '''
47. class ConllToken(object):
48. def __init__(self):
49. self.FORM = None
50. self.LEMMA = None
51. self.UPOSTAG = None
52. self.XPOSTAG = None
53. self.FEATS = []
55. '''
56. Make sure to subtract one from the HEAD value in the file
57. Root becomes -1
59. HEAD then becomes n, which refers to the n'th 0-based index entry
60. in the parent ConllSentence
62. Our parser also requires this to start at -1
63. '''
64. self.HEAD = None
66. self.DEPREL = None
67. self.DEPS = None
68. self.MISC = None
70. # morpheme extraction extension
71. self.morphemes = []
73. def __str__(self):
74. return self.toFileOutput('_')
76. def __repr__(self):
77. return self.__str__()
79. def parseMorphemesFromLemma(self):
80. self.morphemes = []
81. if self.LEMMA:
82. for elem in self.LEMMA.split(' + '):
83. word, pos = elem.rsplit('/', 1)
84. #pos = 'SEG' # HACK TEST
85. self.morphemes.append((word, pos))
87. def toFileOutput(self, ID):
88. def checkTab(s):
89. assert '\t' not in s, 'field must not contain a tab: ' + s
90. return s
92. def checkPipe(s):
93. assert '|' not in s, 'field must not contain a pipe: ' + s
94. return s
96. assert self.FORM != None
97. assert type(self.FEATS) is list
99. cols = [str(ID),
100. checkTab(self.FORM),
101. checkTab(encodeNoneAsUnderscore(self.LEMMA)),
102. checkTab(encodeNoneAsUnderscore(self.UPOSTAG)),
103. checkTab(encodeNoneAsUnderscore(self.XPOSTAG)),
104. '|'.join(checkPipe(checkTab(f)) for f in self.FEATS),
105. encodeNoneAsUnderscore_Int(self.HEAD+1), # +1 when writing as file
106. checkTab(encodeNoneAsUnderscore(self.DEPREL)),
107. checkTab(encodeNoneAsUnderscore(self.DEPS)), # TODO
108. checkTab(encodeNoneAsUnderscore(self.MISC))]
110. return '\t'.join(cols)
112. '''
113. Represents a ConllToken, as parsed
114. '''
115. class ParsedConllToken(ConllToken):
116. def __init__(self):
117. super().__init__()
118. self.parsedLabel = None
119. self.parsedHead = None
120. self.HEAD = -1 # match default value in sentence.proto
122. def setParsedLabel(self, label):
123. self.parsedLabel = label
125. def setParsedHead(self, head):
126. self.parsedHead = head
128. def clearParsedHead(self):
129. self.parsedHead = -1 # match ParserState: always use -1 as <ROOT>
131. '''
132. Stores an ordered list of CoNLL tokens
133. '''
134. class ConllSentence(object):
135. def __init__(self):
136. self.tokens = []
138. '''
139. Convert to file output representation
140. '''
141. def toFileOutput(self):
142. return '\n'.join(self.tokens[ID-1].toFileOutput(ID) \
143. for ID in range(1, len(self.tokens)+1))
145. def genSyntaxNetJson(self, token, break_level=None, start_index=0):
146. break_contents = ''
147. if break_level:
148. break_contents = \
149. '''
150. break_level : %s''' % break_level
152. return \
153. '''token: {
154. word : "%s"
155. start : %d
156. end : %d
157. head : %d
158. tag : "%s"
159. category: "%s"
160. label : "%s"%s
161. }''' % (token.FORM, start_index, start_index+len(token.FORM)-1, token.HEAD,
162. token.XPOSTAG, token.UPOSTAG, token.DEPREL, break_contents)
164. def genSyntaxNetTextHeader(self):
165. return 'text : "%s"' % (' '.join(t.FORM for t in self.tokens))
167. '''
168. Convert to SyntaxNet JSON format
169. '''
170. def toSyntaxNetJson(self):
171. out = []
172. start_index = 0
173. out.append(self.genSyntaxNetTextHeader())
174. for i in range(len(self.tokens)):
175. if i == 0:
176. out.append(self.genSyntaxNetJson(self.tokens[i],
177. break_level='SENTENCE_BREAK',
178. start_index=start_index))
179. else:
180. out.append(self.genSyntaxNetJson(self.tokens[i],
181. start_index=start_index))
182. start_index += len(self.tokens[i].FORM) + 1 # assume space
183. return '\n'.join(out)
185. '''
186. Output the token separated by spaces
187. '''
188. def toSimpleRepresentation(self):
189. return ' '.join(t.FORM for t in self.tokens)
191. class ParsedConllSentence(ConllSentence):
192. def __init__(self, docid):
193. super().__init__()
194. self.docid_ = docid
196. def docid(self):
197. return self.docid_
199. ## checked accessor
200. def mutableToken(self, i):
201. assert i >= 0
202. assert i < len(self.tokens)
203. return self.tokens[i]
205. def tokenSize(self):
206. return len(self.tokens)
208. '''
209. Stores an ordered list of sentences within a CoNLL file
211. keepMalformed:
212. Whether to retain non-projective and invalid examples
214. projectivize:
215. Whether to retain non-projective examples by projectivizing them
217. logStats:
218. Log statistics about the corpus
220. enableLemmaMorphemes:
221. Enable parsing of morphemes in the LEMMA column, format: 웅가로/NNP + 가/JKS
223. 6 웅가로가 웅가로/NNP + 가/JKS _ _ _ 11 NP_SBJ _ _
224. '''
225. class ConllFile(object):
226. def __init__(self, parsed=False,
227. checkParserConformity=False,
228. keepMalformed=False, projectivize=False,
229. logStats=False, enableLemmaMorphemes=False,
230. compatibleJamo=False):
231. #self.sentenceIndex = None
232. self.sentences = []
233. # use parsed variant of structures
234. self.parsed = parsed
235. self.logger = logging.getLogger('ConllUtils')
236. self.keepMalformed = keepMalformed
237. self.projectivize = projectivize
238. self.logStats = logStats
239. self.enableLemmaMorphemes = enableLemmaMorphemes
240. self.compatibleJamo = compatibleJamo
241. self.checkParserConformity = checkParserConformity
244. '''
245. Read CoNLL-U from the given string
247. excludeCols: CoNLL column indices to exclude from reading
248. sometimes we just want to get rid of certain
249. attributes of a token
250. 1-based index
251. '''
252. def read(self, s, excludeCols=[]):
253. assert 1 not in excludeCols, 'cannot exclude reading of ID'
254. assert 2 not in excludeCols, 'cannot exclude reading of FORM'
256. if self.checkParserConformity:
257. well_formed_inst = well_formed_filter.WellFormedFilter()
258. else:
259. assert self.keepMalformed, 'in order to discard malformed ' \
260. 'sentences, you must enable parser ' \
261. 'conformity checking'
263. # arbitrary ID that can be used with parser
264. if self.parsed:
265. docid = 0
267. ln_num = 0
269. current_sentence = None
271. # if we encounter an error during processing a sentence
272. invalid_sentence = False
274. # set up iterator
275. # if there is no iterator, set one up
276. # if there was an iterator, leave it at its current position
277. #if self.sentenceIndex == None:
278. # self.sentenceIndex = len(self.sentences)
280. def commit(s):
281. # if we're even getting rid of malformed sentences in the first
282. # place...
283. if self.checkParserConformity:
284. if not self.keepMalformed:
285. if not well_formed_inst.isWellFormed(s,
286. projectivize=self.projectivize):
287. # if the sentence is non-projective and projectivize
288. # is enabled, the sentence will be fixed and not discarded
289. self.logger.debug('line %d: discarding malformed or non' \
290. '-projective sentence: "%s"' % \
291. (ln_num, s.toSimpleRepresentation()))
292. # as long as we discard the sentence here,
293. # discarded sentences' words, tags, and labels
294. # won't be added to the lexicon, which is exactly the
295. # behavior we want.
296. return
298. self.sentences.append(s)
300. def processUnderscore(s):
301. if s == '_':
302. return None
303. else:
304. return s
306. # token index (to check that it's in order)
307. current_ID = 0
309. lines = s.split('\n')
310. for ln in lines:
311. ln_num += 1
312. ln = ln.strip()
313. if ln.startswith(';'):
314. # ignore comments
315. continue
316. if not ln:
317. # a completely blank line indicates we need to commit the
318. # current sentence
319. if current_sentence != None:
320. if not invalid_sentence:
321. commit(current_sentence)
323. current_sentence = None
324. current_ID = 0
325. invalid_sentence = False
326. continue
327. if ln[0] == '#': # ignore comments completely
328. continue
329. if invalid_sentence: # don't process invalid sentences
330. continue
331. cols = [x.strip() for x in ln.split('\t')]
332. assert len(cols) >= 2, \
333. 'line %d: must have at least ID and FORM: ' % ln_num + str(cols)
335. if '-' in cols[0] or '.' in cols[0]:
336. self.logger.warning('line %d: not implemented: ID=%s, ' \
337. 'invalidating sentence' % (ln_num, cols[0]))
338. invalid_sentence = True
339. continue
340. else:
341. ID = int(cols[0])
342. assert ID==current_ID+1, 'line %d: token IDs must be in order' \
343. ' and increment by one' % ln_num
345. current_ID = ID
347. if current_ID == 1:
348. if self.parsed:
349. current_sentence = ParsedConllSentence(docid)
350. docid += 1
351. else:
352. current_sentence = ConllSentence()
354. if self.parsed:
355. current_token = ParsedConllToken()
356. else:
357. current_token = ConllToken()
359. #if self.parsed:
360. # current_token.FORM = normalizeDigits(cols[1])
361. #else:
362. # current_token.FORM = cols[1]
364. # for SyntaxNet,
365. # normalization ONLY happens in lexicon builder
366. # yet numbers and up as <UNKNOWN> during training
367. # interesting...
369. # let this be underscore if needed (don't call processUnderscore())
370. if self.compatibleJamo:
371. cols[1] = normalizeToCompatJamo(cols[1])
372. current_token.FORM = cols[1]
374. if len(cols) > 2 and (3 not in excludeCols):
375. # let this be underscore if needed
376. # (don't call processUnderscore())
377. if self.compatibleJamo:
378. cols[2] = normalizeToCompatJamo(cols[2])
379. current_token.LEMMA = cols[2]
381. if self.enableLemmaMorphemes:
382. try:
383. current_token.parseMorphemesFromLemma()
384. except:
385. self.logger.warning('line %d: invalid morpheme '
386. 'sequence: %s,'
387. ' invalidating sentence' % (ln_num, \
388. current_token.LEMMA))
390. invalid_sentence = True
391. continue
392. if len(cols) > 3 and (4 not in excludeCols):
393. current_token.UPOSTAG = processUnderscore(cols[3])
394. if len(cols) > 4 and (5 not in excludeCols):
395. current_token.XPOSTAG = processUnderscore(cols[4])
396. if len(cols) > 5 and (6 not in excludeCols):
397. if processUnderscore(cols[5]):
398. current_token.FEATS = \
399. [x.strip() for x in cols[5].split('|')]
400. else:
401. current_token.FEATS = []
402. if len(cols) > 6 and (7 not in excludeCols):
403. current_token.HEAD = processUnderscore(cols[6])
404. if current_token.HEAD != None:
405. if '-' in current_token.HEAD or '.' in current_token.HEAD:
406. self.logger.warning('line %d: not implemented: HEAD=%s,'
407. ' invalidating sentence' % (ln_num, \
408. current_token.HEAD))
410. invalid_sentence = True
411. continue
412. else:
413. # it's important for parsing that HEAD start at -1
414. current_token.HEAD = int(current_token.HEAD)-1
415. if len(cols) > 7 and (8 not in excludeCols):
416. current_token.DEPREL = processUnderscore(cols[7])
417. if len(cols) > 8 and (9 not in excludeCols):
418. # TODO
419. current_token.DEPS = processUnderscore(cols[8])
420. if len(cols) > 9 and (10 not in excludeCols):
421. current_token.MISC = processUnderscore(cols[9])
423. current_sentence.tokens.append(current_token)
425. # an EOF indicates we need to commit the current sentence
426. if current_sentence != None:
427. if not invalid_sentence:
428. commit(current_sentence)
430. current_sentence = None
431. current_ID = 0
432. invalid_sentence = False
434. if self.logStats:
435. if self.checkParserConformity:
436. self.logger.info('Projectivized %d/%d non-projective sentences' \
437. ' (%.2f%% of set)' % \
438. (well_formed_inst.projectivizedCount, \
439. well_formed_inst.nonProjectiveCount,
440. 100.0 * float(well_formed_inst.projectivizedCount) \
441. / float(len(self.sentences))
442. ))
444. # if we're even getting rid of malformed sentences in the first
445. # place...
446. if self.checkParserConformity:
447. if not self.keepMalformed:
448. if self.projectivize:
449. # the definition of this variable changes when projectivize
450. # is on
451. self.logger.info('Discarded %d non-well-formed sentences'
452. % (well_formed_inst.nonWellFormedCount))
453. else:
454. self.logger.info('Discarded %d non-well-formed and ' \
455. ' non-projective sentences' % \
456. (well_formed_inst.nonWellFormedCount))
458. self.logger.info('%d valid sentences processed in total' % \
459. len(self.sentences))
461. '''
462. Write the current CoNLL-U data to the specified file descriptor
463. '''
464. def write(self, fd):
465. data = [s.toFileOutput() for s in self.sentences]
466. fd.write('\n\n'.join(data))
467. fd.flush()
469. def __iter__(self):
470. index = 0
471. while index < len(self.sentences):
472. yield self.sentences[index]
473. index += 1
475. class ParsedConllFile(ConllFile):
476. def __init__(self, checkParserConformity=False, keepMalformed=False,
477. projectivize=False, logStats=False,
478. enableLemmaMorphemes=False, compatibleJamo=False):
479. super().__init__(parsed=True,
480. checkParserConformity=checkParserConformity,
481. keepMalformed=keepMalformed,
482. projectivize=projectivize, logStats=logStats,
483. enableLemmaMorphemes=enableLemmaMorphemes,
484. compatibleJamo=compatibleJamo)

'''
A set of classes to handle input and output of CoNLL-U files

http://universaldependencies.org/docs/format.html

The Parsed* classes are useful to store extra properties needed during
the parsing process that are external to the Conll instances themselves
'''

import logging
import well_formed_filter

def isNonCompatibilityJamo(c):
    assert len(c) == 1
    # HANGUL JAMO: (U+1100-U+11FF)
    if ord(c) >= 0x1100 and ord(c) <= 0x11FF:
        return True
    else:
        return False

def normalizeToCompatJamo(s):
    from jamo import h2j, j2hcj
    out = ''
    for c in s:
        if isNonCompatibilityJamo(c):
            out += j2hcj(c)
        else:
            out += c
    assert len(s) == len(out)
    return out

def encodeNoneAsUnderscore(s):
    if s == None:
        return '_'
    else:
        return s

def encodeNoneAsUnderscore_Int(i):
    if i == None:
        return '_'
    else:
        return str(i)

'''
Represents a CoNLL token and all its properties (except index)
'''
class ConllToken(object):
    def __init__(self):
        self.FORM = None
        self.LEMMA = None
        self.UPOSTAG = None
        self.XPOSTAG = None
        self.FEATS = []

        '''
        Make sure to subtract one from the HEAD value in the file
        Root becomes -1

        HEAD then becomes n, which refers to the n'th 0-based index entry
        in the parent ConllSentence

        Our parser also requires this to start at -1
        '''
        self.HEAD = None

        self.DEPREL = None
        self.DEPS = None
        self.MISC = None

        # morpheme extraction extension
        self.morphemes = []

    def __str__(self):
        return self.toFileOutput('_')

    def __repr__(self):
        return self.__str__()

    def parseMorphemesFromLemma(self):
        self.morphemes = []
        if self.LEMMA:
            for elem in self.LEMMA.split(' + '):
                word, pos = elem.rsplit('/', 1)
                #pos = 'SEG' # HACK TEST
                self.morphemes.append((word, pos))

    def toFileOutput(self, ID):
        def checkTab(s):
            assert '\t' not in s, 'field must not contain a tab: ' + s
            return s

        def checkPipe(s):
            assert '|' not in s, 'field must not contain a pipe: ' + s
            return s

        assert self.FORM != None
        assert type(self.FEATS) is list

        cols = [str(ID),
            checkTab(self.FORM),
            checkTab(encodeNoneAsUnderscore(self.LEMMA)),
            checkTab(encodeNoneAsUnderscore(self.UPOSTAG)),
            checkTab(encodeNoneAsUnderscore(self.XPOSTAG)),
            '|'.join(checkPipe(checkTab(f)) for f in self.FEATS),
            encodeNoneAsUnderscore_Int(self.HEAD+1), # +1 when writing as file
            checkTab(encodeNoneAsUnderscore(self.DEPREL)),
            checkTab(encodeNoneAsUnderscore(self.DEPS)),   # TODO
            checkTab(encodeNoneAsUnderscore(self.MISC))]

        return '\t'.join(cols)

'''
Represents a ConllToken, as parsed
'''
class ParsedConllToken(ConllToken):
    def __init__(self):
        super().__init__()
        self.parsedLabel = None
        self.parsedHead = None
        self.HEAD = -1 # match default value in sentence.proto

    def setParsedLabel(self, label):
        self.parsedLabel = label

    def setParsedHead(self, head):
        self.parsedHead = head

    def clearParsedHead(self):
        self.parsedHead = -1 # match ParserState: always use -1 as <ROOT>

'''
Stores an ordered list of CoNLL tokens
'''
class ConllSentence(object):
    def __init__(self):
        self.tokens = []

    '''
    Convert to file output representation
    '''
    def toFileOutput(self):
        return '\n'.join(self.tokens[ID-1].toFileOutput(ID) \
            for ID in range(1, len(self.tokens)+1))

    def genSyntaxNetJson(self, token, break_level=None, start_index=0):
        break_contents = ''
        if break_level:
            break_contents = \
'''
  break_level       : %s''' % break_level

        return \
'''token: {
  word    : "%s"
  start   : %d
  end     : %d
  head    : %d
  tag     : "%s"
  category: "%s"
  label   : "%s"%s
}''' % (token.FORM, start_index, start_index+len(token.FORM)-1, token.HEAD,
        token.XPOSTAG, token.UPOSTAG, token.DEPREL, break_contents)

    def genSyntaxNetTextHeader(self):
        return 'text       : "%s"' % (' '.join(t.FORM for t in self.tokens))

    '''
    Convert to SyntaxNet JSON format
    '''
    def toSyntaxNetJson(self):
        out = []
        start_index = 0
        out.append(self.genSyntaxNetTextHeader())
        for i in range(len(self.tokens)):
            if i == 0:
                out.append(self.genSyntaxNetJson(self.tokens[i],
                                                 break_level='SENTENCE_BREAK',
                                                 start_index=start_index))
            else:
                out.append(self.genSyntaxNetJson(self.tokens[i],
                                                 start_index=start_index))
            start_index += len(self.tokens[i].FORM) + 1 # assume space
        return '\n'.join(out)

    '''
    Output the token separated by spaces
    '''
    def toSimpleRepresentation(self):
        return ' '.join(t.FORM for t in self.tokens)

class ParsedConllSentence(ConllSentence):
    def __init__(self, docid):
        super().__init__()
        self.docid_ = docid

    def docid(self):
        return self.docid_

    ## checked accessor
    def mutableToken(self, i):
        assert i >= 0
        assert i < len(self.tokens)
        return self.tokens[i]

    def tokenSize(self):
        return len(self.tokens)
    
'''
Stores an ordered list of sentences within a CoNLL file

keepMalformed:
Whether to retain non-projective and invalid examples

projectivize:
Whether to retain non-projective examples by projectivizing them

logStats:
Log statistics about the corpus

enableLemmaMorphemes:
Enable parsing of morphemes in the LEMMA column, format: 웅가로/NNP + 가/JKS

6	웅가로가	웅가로/NNP + 가/JKS	_	_	_	11	NP_SBJ	_	_
'''
class ConllFile(object):
    def __init__(self, parsed=False,
                 checkParserConformity=False,
                 keepMalformed=False, projectivize=False,
                 logStats=False, enableLemmaMorphemes=False,
                 compatibleJamo=False):
        #self.sentenceIndex = None
        self.sentences = []
        # use parsed variant of structures
        self.parsed = parsed
        self.logger = logging.getLogger('ConllUtils')
        self.keepMalformed = keepMalformed
        self.projectivize = projectivize
        self.logStats = logStats
        self.enableLemmaMorphemes = enableLemmaMorphemes
        self.compatibleJamo = compatibleJamo
        self.checkParserConformity = checkParserConformity


    '''
    Read CoNLL-U from the given string

    excludeCols: CoNLL column indices to exclude from reading
                 sometimes we just want to get rid of certain
                 attributes of a token
                 1-based index
    '''
    def read(self, s, excludeCols=[]):
        assert 1 not in excludeCols, 'cannot exclude reading of ID'
        assert 2 not in excludeCols, 'cannot exclude reading of FORM'

        if self.checkParserConformity:
            well_formed_inst = well_formed_filter.WellFormedFilter()
        else:
            assert self.keepMalformed, 'in order to discard malformed ' \
                                       'sentences, you must enable parser ' \
                                       'conformity checking'

        # arbitrary ID that can be used with parser
        if self.parsed:
            docid = 0

        ln_num = 0

        current_sentence = None
        
        # if we encounter an error during processing a sentence
        invalid_sentence = False

        # set up iterator
        # if there is no iterator, set one up
        # if there was an iterator, leave it at its current position
        #if self.sentenceIndex == None:
        #    self.sentenceIndex = len(self.sentences)

        def commit(s):
            # if we're even getting rid of malformed sentences in the first
            # place...
            if self.checkParserConformity:
                if not self.keepMalformed:
                    if not well_formed_inst.isWellFormed(s,
                            projectivize=self.projectivize):
                        # if the sentence is non-projective and projectivize
                        # is enabled, the sentence will be fixed and not discarded
                        self.logger.debug('line %d: discarding malformed or non' \
                            '-projective sentence: "%s"' % \
                            (ln_num, s.toSimpleRepresentation()))
                        # as long as we discard the sentence here,
                        # discarded sentences' words, tags, and labels
                        # won't be added to the lexicon, which is exactly the
                        # behavior we want.
                        return

            self.sentences.append(s)

        def processUnderscore(s):
            if s == '_':
                return None
            else:
                return s

        # token index (to check that it's in order)
        current_ID = 0

        lines = s.split('\n')
        for ln in lines:
            ln_num += 1
            ln = ln.strip()
            if ln.startswith(';'):
                # ignore comments
                continue
            if not ln:
                # a completely blank line indicates we need to commit the
                # current sentence
                if current_sentence != None:
                    if not invalid_sentence:
                        commit(current_sentence)

                    current_sentence = None
                    current_ID = 0
                    invalid_sentence = False
                continue
            if ln[0] == '#': # ignore comments completely
                continue
            if invalid_sentence: # don't process invalid sentences
                continue
            cols = [x.strip() for x in ln.split('\t')]
            assert len(cols) >= 2, \
                'line %d: must have at least ID and FORM: ' % ln_num + str(cols)

            if '-' in cols[0] or '.' in cols[0]:
                self.logger.warning('line %d: not implemented: ID=%s, ' \
                                    'invalidating sentence' % (ln_num, cols[0]))
                invalid_sentence = True
                continue
            else:
                ID = int(cols[0])
                assert ID==current_ID+1, 'line %d: token IDs must be in order' \
                   ' and increment by one' % ln_num

            current_ID = ID

            if current_ID == 1:
                if self.parsed:
                    current_sentence = ParsedConllSentence(docid)
                    docid += 1
                else:
                    current_sentence = ConllSentence()

            if self.parsed:
                current_token = ParsedConllToken()
            else:
                current_token = ConllToken()

            #if self.parsed:
            #    current_token.FORM = normalizeDigits(cols[1])
            #else:
            #    current_token.FORM = cols[1]

            # for SyntaxNet,
            # normalization ONLY happens in lexicon builder
            # yet numbers and up as <UNKNOWN> during training
            # interesting...

            # let this be underscore if needed (don't call processUnderscore())
            if self.compatibleJamo:
                cols[1] = normalizeToCompatJamo(cols[1])
            current_token.FORM = cols[1]

            if len(cols) > 2 and (3 not in excludeCols):
                # let this be underscore if needed
                # (don't call processUnderscore())
                if self.compatibleJamo:
                    cols[2] = normalizeToCompatJamo(cols[2])
                current_token.LEMMA = cols[2]

                if self.enableLemmaMorphemes:
                    try:
                        current_token.parseMorphemesFromLemma()
                    except:
                        self.logger.warning('line %d: invalid morpheme '
                                            'sequence: %s,'
                            ' invalidating sentence' % (ln_num, \
                            current_token.LEMMA))

                        invalid_sentence = True
                        continue
            if len(cols) > 3 and (4 not in excludeCols):
                current_token.UPOSTAG = processUnderscore(cols[3])
            if len(cols) > 4 and (5 not in excludeCols):
                current_token.XPOSTAG = processUnderscore(cols[4])
            if len(cols) > 5 and (6 not in excludeCols):
                if processUnderscore(cols[5]):
                    current_token.FEATS = \
                        [x.strip() for x in cols[5].split('|')]
                else:
                    current_token.FEATS = []
            if len(cols) > 6 and (7 not in excludeCols):
                current_token.HEAD = processUnderscore(cols[6])
                if current_token.HEAD != None:
                    if '-' in current_token.HEAD or '.' in current_token.HEAD:
                        self.logger.warning('line %d: not implemented: HEAD=%s,'
                            ' invalidating sentence' % (ln_num, \
                            current_token.HEAD))

                        invalid_sentence = True
                        continue
                    else:
                        # it's important for parsing that HEAD start at -1
                        current_token.HEAD = int(current_token.HEAD)-1
            if len(cols) > 7 and (8 not in excludeCols):
                current_token.DEPREL = processUnderscore(cols[7])
            if len(cols) > 8 and (9 not in excludeCols):
                # TODO
                current_token.DEPS = processUnderscore(cols[8])
            if len(cols) > 9 and (10 not in excludeCols):
                current_token.MISC = processUnderscore(cols[9])

            current_sentence.tokens.append(current_token)

        # an EOF indicates we need to commit the current sentence
        if current_sentence != None:
            if not invalid_sentence:
                commit(current_sentence)

            current_sentence = None
            current_ID = 0
            invalid_sentence = False

        if self.logStats:
            if self.checkParserConformity:
                self.logger.info('Projectivized %d/%d non-projective sentences' \
                    ' (%.2f%% of set)' % \
                    (well_formed_inst.projectivizedCount, \
                    well_formed_inst.nonProjectiveCount,
                    100.0 * float(well_formed_inst.projectivizedCount) \
                        / float(len(self.sentences))
                    ))

            # if we're even getting rid of malformed sentences in the first
            # place...
            if self.checkParserConformity:
                if not self.keepMalformed:
                    if self.projectivize:
                        # the definition of this variable changes when projectivize
                        # is on
                        self.logger.info('Discarded %d non-well-formed sentences'
                                         % (well_formed_inst.nonWellFormedCount))
                    else:
                        self.logger.info('Discarded %d non-well-formed and ' \
                            ' non-projective sentences' % \
                            (well_formed_inst.nonWellFormedCount))

            self.logger.info('%d valid sentences processed in total' % \
                len(self.sentences))

    '''
    Write the current CoNLL-U data to the specified file descriptor
    '''
    def write(self, fd):
        data = [s.toFileOutput() for s in self.sentences]
        fd.write('\n\n'.join(data))
        fd.flush()

    def __iter__(self):
        index = 0
        while index < len(self.sentences):
            yield self.sentences[index]
            index += 1

class ParsedConllFile(ConllFile):
    def __init__(self, checkParserConformity=False, keepMalformed=False,
                 projectivize=False, logStats=False,
                 enableLemmaMorphemes=False, compatibleJamo=False):
        super().__init__(parsed=True,
                         checkParserConformity=checkParserConformity,
                         keepMalformed=keepMalformed,
                         projectivize=projectivize, logStats=logStats,
                         enableLemmaMorphemes=enableLemmaMorphemes,
                         compatibleJamo=compatibleJamo)