1
2
3
4
5
6
7
8
9 """
10 API for corpus readers.
11 """
12
13 import os
14 import re
15
16 from nltk.compat import defaultdict
17 from nltk.data import PathPointer, FileSystemPathPointer, ZipFilePathPointer
18 from nltk.sourcedstring import SourcedStringStream
19
20 from util import *
21
23 """
24 A base class for X{corpus reader} classes, each of which can be
25 used to read a specific corpus format. Each individual corpus
26 reader instance is used to read a specific corpus, consisting of
27 one or more files under a common root directory. Each file is
28 identified by its C{file identifier}, which is the relative path
29 to the file from the root directory.
30
31 A separate subclass is be defined for each corpus format. These
32 subclasses define one or more methods that provide 'views' on the
33 corpus contents, such as C{words()} (for a list of words) and
34 C{parsed_sents()} (for a list of parsed sentences). Called with
35 no arguments, these methods will return the contents of the entire
36 corpus. For most corpora, these methods define one or more
37 selection arguments, such as C{fileids} or C{categories}, which can
38 be used to select which portion of the corpus should be returned.
39 """
40 - def __init__(self, root, fileids, encoding=None, tag_mapping_function=None):
41 """
42 @type root: L{PathPointer} or C{str}
43 @param root: A path pointer identifying the root directory for
44 this corpus. If a string is specified, then it will be
45 converted to a L{PathPointer} automatically.
46 @param fileids: A list of the files that make up this corpus.
47 This list can either be specified explicitly, as a list of
48 strings; or implicitly, as a regular expression over file
49 paths. The absolute path for each file will be constructed
50 by joining the reader's root to each file name.
51 @param encoding: The default unicode encoding for the files
52 that make up the corpus. C{encoding}'s value can be any
53 of the following:
54
55 - B{A string}: C{encoding} is the encoding name for all
56 files.
57 - B{A dictionary}: C{encoding[file_id]} is the encoding
58 name for the file whose identifier is C{file_id}. If
59 C{file_id} is not in C{encoding}, then the file
60 contents will be processed using non-unicode byte
61 strings.
62 - B{A list}: C{encoding} should be a list of C{(regexp,
63 encoding)} tuples. The encoding for a file whose
64 identifier is C{file_id} will be the C{encoding} value
65 for the first tuple whose C{regexp} matches the
66 C{file_id}. If no tuple's C{regexp} matches the
67 C{file_id}, the file contents will be processed using
68 non-unicode byte strings.
69 - C{None}: the file contents of all files will be
70 processed using non-unicode byte strings.
71 @param tag_mapping_function: A function for normalizing or
72 simplifying the POS tags returned by the tagged_words()
73 or tagged_sents() methods.
74 """
75
76 if isinstance(root, basestring):
77 m = re.match('(.*\.zip)/?(.*)$|', root)
78 zipfile, zipentry = m.groups()
79 if zipfile:
80 root = ZipFilePathPointer(zipfile, zipentry)
81 else:
82 root = FileSystemPathPointer(root)
83 elif not isinstance(root, PathPointer):
84 raise TypeError('CorpusReader: expected a string or a PathPointer')
85
86
87 if isinstance(fileids, basestring):
88 fileids = find_corpus_fileids(root, fileids)
89
90 self._fileids = fileids
91 """A list of the relative paths for the fileids that make up
92 this corpus."""
93
94 self._root = root
95 """The root directory for this corpus."""
96
97
98
99 if isinstance(encoding, list):
100 encoding_dict = {}
101 for fileid in self._fileids:
102 for x in encoding:
103 (regexp, enc) = x
104 if re.match(regexp, fileid):
105 encoding_dict[fileid] = enc
106 break
107 encoding = encoding_dict
108
109 self._encoding = encoding
110 """The default unicode encoding for the fileids that make up
111 this corpus. If C{encoding} is C{None}, then the file
112 contents are processed using byte strings (C{str})."""
113 self._tag_mapping_function = tag_mapping_function
114
121
123 """
124 Return the contents of the corpus README file, if it exists.
125 """
126
127 return self.open("README").read()
128
130 """
131 Return a list of file identifiers for the fileids that make up
132 this corpus.
133 """
134 return self._fileids
135
137 """
138 Return the absolute path for the given file.
139
140 @type file: C{str}
141 @param file: The file identifier for the file whose path
142 should be returned.
143
144 @rtype: L{PathPointer}
145 """
146 return self._root.join(fileid)
147
148 - def abspaths(self, fileids=None, include_encoding=False,
149 include_fileid=False):
150 """
151 Return a list of the absolute paths for all fileids in this corpus;
152 or for the given list of fileids, if specified.
153
154 @type fileids: C{None} or C{str} or C{list}
155 @param fileids: Specifies the set of fileids for which paths should
156 be returned. Can be C{None}, for all fileids; a list of
157 file identifiers, for a specified set of fileids; or a single
158 file identifier, for a single file. Note that the return
159 value is always a list of paths, even if C{fileids} is a
160 single file identifier.
161
162 @param include_encoding: If true, then return a list of
163 C{(path_pointer, encoding)} tuples.
164
165 @rtype: C{list} of L{PathPointer}
166 """
167 if fileids is None:
168 fileids = self._fileids
169 elif isinstance(fileids, basestring):
170 fileids = [fileids]
171
172 paths = [self._root.join(f) for f in fileids]
173
174 if include_encoding and include_fileid:
175 return zip(paths, [self.encoding(f) for f in fileids], fileids)
176 elif include_fileid:
177 return zip(paths, fileid)
178 elif include_encoding:
179 return zip(paths, [self.encoding(f) for f in fileids])
180 else:
181 return paths
182
183 - def open(self, file, sourced=False):
184 """
185 Return an open stream that can be used to read the given file.
186 If the file's encoding is not C{None}, then the stream will
187 automatically decode the file's contents into unicode.
188
189 @param file: The file identifier of the file to read.
190 """
191 encoding = self.encoding(file)
192 stream = self._root.join(file).open(encoding)
193 if sourced:
194 stream = SourcedStringStream(stream, file)
195 return stream
196
198 """
199 Return the unicode encoding for the given corpus file, if known.
200 If the encoding is unknown, or if the given file should be
201 processed using byte strings (C{str}), then return C{None}.
202 """
203 if isinstance(self._encoding, dict):
204 return self._encoding.get(file)
205 else:
206 return self._encoding
207
209 root = property(_get_root, doc="""
210 The directory where this corpus is stored.
211
212 @type: L{PathPointer}""")
213
214
215
216
217
218
220 """
221 A mixin class used to aid in the implementation of corpus readers
222 for categorized corpora. This class defines the method
223 L{categories()}, which returns a list of the categories for the
224 corpus or for a specified set of fileids; and overrides L{fileids()}
225 to take a C{categories} argument, restricting the set of fileids to
226 be returned.
227
228 Subclasses are expected to:
229
230 - Call L{__init__()} to set up the mapping.
231
232 - Override all view methods to accept a C{categories} parameter,
233 which can be used *instead* of the C{fileids} parameter, to
234 select which fileids should be included in the returned view.
235 """
236
238 """
239 Initialize this mapping based on keyword arguments, as
240 follows:
241
242 - cat_pattern: A regular expression pattern used to find the
243 category for each file identifier. The pattern will be
244 applied to each file identifier, and the first matching
245 group will be used as the category label for that file.
246
247 - cat_map: A dictionary, mapping from file identifiers to
248 category labels.
249
250 - cat_file: The name of a file that contains the mapping
251 from file identifiers to categories. The argument
252 C{cat_delimiter} can be used to specify a delimiter.
253
254 The corresponding argument will be deleted from C{kwargs}. If
255 more than one argument is specified, an exception will be
256 raised.
257 """
258 self._f2c = None
259 self._c2f = None
260
261 self._pattern = None
262 self._map = None
263 self._file = None
264 self._delimiter = None
265
266 if 'cat_pattern' in kwargs:
267 self._pattern = kwargs['cat_pattern']
268 del kwargs['cat_pattern']
269 elif 'cat_map' in kwargs:
270 self._map = kwargs['cat_map']
271 del kwargs['cat_map']
272 elif 'cat_file' in kwargs:
273 self._file = kwargs['cat_file']
274 del kwargs['cat_file']
275 if 'cat_delimiter' in kwargs:
276 self._delimiter = kwargs['cat_delimiter']
277 del kwargs['cat_delimiter']
278 else:
279 raise ValueError('Expected keyword argument cat_pattern or '
280 'cat_map or cat_file.')
281
282
283 if ('cat_pattern' in kwargs or 'cat_map' in kwargs or
284 'cat_file' in kwargs):
285 raise ValueError('Specify exactly one of: cat_pattern, '
286 'cat_map, cat_file.')
287
289 self._f2c = defaultdict(set)
290 self._c2f = defaultdict(set)
291
292 if self._pattern is not None:
293 for file_id in self._fileids:
294 category = re.match(self._pattern, file_id).group(1)
295 self._add(file_id, category)
296
297 elif self._map is not None:
298 for (file_id, categories) in self._map.items():
299 for category in categories:
300 self._add(file_id, category)
301
302 elif self._file is not None:
303 for line in self.open(self._file).readlines():
304 line = line.strip()
305 file_id, categories = line.split(self._delimiter, 1)
306 if file_id not in self.fileids():
307 raise ValueError('In category mapping file %s: %s '
308 'not found' % (self._file, file_id))
309 for category in categories.split(self._delimiter):
310 self._add(file_id, category)
311
312 - def _add(self, file_id, category):
313 self._f2c[file_id].add(category)
314 self._c2f[category].add(file_id)
315
317 """
318 Return a list of the categories that are defined for this corpus,
319 or for the file(s) if it is given.
320 """
321 if self._f2c is None:
322 self._init()
323 if fileids is None:
324 return sorted(self._c2f)
325 if isinstance(fileids, basestring):
326 fileids = [fileids]
327 return sorted(set.union(*[self._f2c[d] for d in fileids]))
328
329 - def fileids(self, categories=None):
347
348
349
350
351
352
354 """
355 An abstract base class for reading corpora consisting of
356 syntactically parsed text. Subclasses should define:
357
358 - L{__init__}, which specifies the location of the corpus
359 and a method for detecting the sentence blocks in corpus files.
360 - L{_read_block}, which reads a block from the input stream.
361 - L{_word}, which takes a block and returns a list of list of words.
362 - L{_tag}, which takes a block and returns a list of list of tagged
363 words.
364 - L{_parse}, which takes a block and returns a list of parsed
365 sentences.
366 """
368 raise AssertionError('Abstract method')
370 raise AssertionError('Abstract method')
372 raise AssertionError('Abstract method')
374 raise AssertionError('Abstract method')
375
376 - def raw(self, fileids=None):
380
385
389 return concat([StreamBackedCorpusView(fileid, reader, encoding=enc)
390 for fileid, enc in self.abspaths(fileids, True)])
391
392 - def sents(self, fileids=None):
396
400 return concat([StreamBackedCorpusView(fileid, reader, encoding=enc)
401 for fileid, enc in self.abspaths(fileids, True)])
402
403 - def words(self, fileids=None):
408
409
410
411
414
417
420
422 return filter(None, [self._tag(t, simplify_tags)
423 for t in self._read_block(stream)])
424
427
428
429
430