1
2
3
4
5
6
7
8
9
10 """
11 Corpus reader for the York-Toronto-Helsinki Parsed Corpus of Old
12 English Prose (YCOE), a 1.5 million word syntactically-annotated
13 corpus of Old English prose texts. The corpus is distributed by the
14 Oxford Text Archive: http://www.ota.ahds.ac.uk/ It is not included
15 with NLTK.
16
17 The YCOE corpus is divided into 100 files, each representing
18 an Old English prose text. Tags used within each text complies
19 to the YCOE standard: http://www-users.york.ac.uk/~lang22/YCOE/YcoeHome.htm
20 """
21
22 import os
23 import re
24
25 from nltk.tokenize import RegexpTokenizer
26 from nltk.corpus.reader.bracket_parse import BracketParseCorpusReader
27 from nltk.corpus.reader.tagged import TaggedCorpusReader
28 from string import split
29
30 from util import *
31 from api import *
32
34 """
35 Corpus reader for the York-Toronto-Helsinki Parsed Corpus of Old
36 English Prose (YCOE), a 1.5 million word syntactically-annotated
37 corpus of Old English prose texts.
38 """
39 - def __init__(self, root, encoding=None):
40 CorpusReader.__init__(self, root, [], encoding)
41
42 self._psd_reader = YCOEParseCorpusReader(
43 self.root.join('psd'), '.*', '.psd', encoding=encoding)
44 self._pos_reader = YCOETaggedCorpusReader(
45 self.root.join('pos'), '.*', '.pos')
46
47
48 documents = set(f[:-4] for f in self._psd_reader.fileids())
49 if set(f[:-4] for f in self._pos_reader.fileids()) != documents:
50 raise ValueError('Items in "psd" and "pos" '
51 'subdirectories do not match.')
52
53 fileids = sorted(['%s.psd' % doc for doc in documents] +
54 ['%s.pos' % doc for doc in documents])
55 CorpusReader.__init__(self, root, fileids, encoding)
56 self._documents = sorted(documents)
57
59 """
60 Return a list of document identifiers for all documents in
61 this corpus, or for the documents with the given file(s) if
62 specified.
63 """
64 if fileids is None:
65 return self._documents
66 if isinstance(fileids, basestring):
67 fileids = [fileids]
68 for f in fileids:
69 if f not in self._fileids:
70 raise KeyError('File id %s not found' % fileids)
71
72 return sorted(set(f[:-4] for f in fileids))
73
75 """
76 Return a list of file identifiers for the files that make up
77 this corpus, or that store the given document(s) if specified.
78 """
79 if documents is None:
80 return self._fileids
81 elif isinstance(documents, basestring):
82 documents = [documents]
83 return sorted(set(['%s.pos' % doc for doc in documents] +
84 ['%s.psd' % doc for doc in documents]))
85
87 """
88 Helper that selects the appropriate fileids for a given set of
89 documents from a given subcorpus (pos or psd).
90 """
91 if documents is None:
92 documents = self._documents
93 else:
94 if isinstance(documents, basestring):
95 documents = [documents]
96 for document in documents:
97 if document not in self._documents:
98 if document[-4:] in ('.pos', '.psd'):
99 raise ValueError(
100 'Expected a document identifier, not a file '
101 'identifier. (Use corpus.documents() to get '
102 'a list of document identifiers.')
103 else:
104 raise ValueError('Document identifier %s not found'
105 % document)
106 return ['%s.%s' % (d, subcorpus) for d in documents]
107
108
109 - def words(self, documents=None):
111 - def sents(self, documents=None):
113 - def paras(self, documents=None):
123
124
126 """Specialized version of the standard bracket parse corpus reader
127 that strips out (CODE ...) and (ID ...) nodes."""
132
134 - def __init__(self, root, items, encoding=None):