Source Code for Module nltk.corpus.reader.indian

 1  # Natural Language Toolkit: Indian Language POS-Tagged Corpus Reader 
 2  # 
 3  # Copyright (C) 2001-2011 NLTK Project 
 4  # Author: Steven Bird <sb@ldc.upenn.edu> 
 5  #         Edward Loper <edloper@gradient.cis.upenn.edu> 
 6  # URL: <http://www.nltk.org/> 
 7  # For license information, see LICENSE.TXT 
 8   
 9  """ 
10  Indian Language POS-Tagged Corpus 
11  Collected by A Kumaran, Microsoft Research, India 
12  Distributed with permission 
13   
14  Contents: 
15    - Bangla: IIT Kharagpur 
16    - Hindi: Microsoft Research India 
17    - Marathi: IIT Bombay 
18    - Telugu: IIIT Hyderabad 
19  """        
20   
21  import codecs 
22   
23  from nltk.tag.util import str2tuple 
24   
25  from util import * 
26  from api import * 
27   
28 -class IndianCorpusReader(CorpusReader): 
29      """ 
30      List of words, one per line.  Blank lines are ignored. 
31      """ 
32 -    def words(self, fileids=None): 
33          return concat([IndianCorpusView(fileid, enc, 
34                                          False, False) 
35                         for (fileid, enc) in self.abspaths(fileids, True)]) 
36   
37 -    def tagged_words(self, fileids=None, simplify_tags=False): 
38          if simplify_tags: 
39              tag_mapping_function = self._tag_mapping_function 
40          else: 
41              tag_mapping_function = None 
42          return concat([IndianCorpusView(fileid, enc, 
43                                          True, False, tag_mapping_function) 
44                         for (fileid, enc) in self.abspaths(fileids, True)]) 
45   
46 -    def sents(self, fileids=None): 
47          return concat([IndianCorpusView(fileid, enc, 
48                                          False, True) 
49                         for (fileid, enc) in self.abspaths(fileids, True)]) 
50   
51 -    def tagged_sents(self, fileids=None, simplify_tags=False): 
52          if simplify_tags: 
53              tag_mapping_function = self._tag_mapping_function 
54          else: 
55              tag_mapping_function = None 
56          return concat([IndianCorpusView(fileid, enc, 
57                                          True, True, tag_mapping_function) 
58                         for (fileid, enc) in self.abspaths(fileids, True)]) 
59   
60 -    def raw(self, fileids=None): 
61          if fileids is None: fileids = self._fileids 
62          elif isinstance(fileids, basestring): fileids = [fileids] 
63          return concat([self.open(f).read() for f in fileids]) 
64   
65   
66 -class IndianCorpusView(StreamBackedCorpusView): 
67 -    def __init__(self, corpus_file, encoding, tagged, 
68                   group_by_sent, tag_mapping_function=None): 
69          self._tagged = tagged 
70          self._group_by_sent = group_by_sent 
71          self._tag_mapping_function = tag_mapping_function 
72          StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding) 
73   
74 -    def read_block(self, stream): 
75          line = stream.readline() 
76          if line.startswith('<'): 
77              return [] 
78          sent = [str2tuple(word, sep='_') for word in line.split()] 
79          if self._tag_mapping_function: 
80              sent = [(w, self._tag_mapping_function(t)) for (w,t) in sent] 
81          if not self._tagged: sent = [w for (w,t) in sent] 
82          if self._group_by_sent: 
83              return [sent] 
84          else: 
85              return sent 
86