Source Code for Module nltk.tag.api

  1  # Natural Language Toolkit: Tagger Interface 
  2  # 
  3  # Copyright (C) 2001-2011 NLTK Project 
  4  # Author: Edward Loper <edloper@gradient.cis.upenn.edu> 
  5  #         Steven Bird <sb@csse.unimelb.edu.au> (minor additions) 
  6  # URL: <http://www.nltk.org/> 
  7  # For license information, see LICENSE.TXT 
  8   
  9  """ 
 10  Interface for tagging each token in a sentence with supplementary 
 11  information, such as its part of speech. 
 12  """ 
 13   
 14  from nltk.internals import overridden 
 15  from nltk.metrics import accuracy as _accuracy 
 16  from util import untag 
 17   
 18 -class TaggerI(object): 
 19      """ 
 20      A processing interface for assigning a tag to each token in a list. 
 21      Tags are case sensitive strings that identify some property of each 
 22      token, such as its part of speech or its sense. 
 23   
 24      Some taggers require specific types for their tokens.  This is 
 25      generally indicated by the use of a sub-interface to C{TaggerI}. 
 26      For example, I{featureset taggers}, which are subclassed from 
 27      L{FeaturesetTaggerI}, require that each token be a I{featureset}. 
 28   
 29      Subclasses must define: 
 30        - either L{tag()} or L{batch_tag()} (or both) 
 31      """ 
 32 -    def tag(self, tokens): 
 33          """ 
 34          Determine the most appropriate tag sequence for the given 
 35          token sequence, and return a corresponding list of tagged 
 36          tokens.  A tagged token is encoded as a tuple C{(token, tag)}. 
 37   
 38          @rtype: C{list} of C{(token, tag)} 
 39          """ 
 40          if overridden(self.batch_tag): 
 41              return self.batch_tag([tokens])[0] 
 42          else: 
 43              raise NotImplementedError() 
 44   
 45 -    def batch_tag(self, sentences): 
 46          """ 
 47          Apply L{self.tag()} to each element of C{sentences}.  I.e.: 
 48   
 49              >>> return [self.tag(sent) for sent in sentences] 
 50          """ 
 51          return [self.tag(sent) for sent in sentences] 
 52   
 53 -    def evaluate(self, gold): 
 54          """ 
 55          Score the accuracy of the tagger against the gold standard. 
 56          Strip the tags from the gold standard text, retag it using 
 57          the tagger, then compute the accuracy score. 
 58   
 59          @type gold: C{list} of C{list} of C{(token, tag)} 
 60          @param gold: The list of tagged sentences to score the tagger on. 
 61          @rtype: C{float} 
 62          """ 
 63   
 64          tagged_sents = self.batch_tag([untag(sent) for sent in gold]) 
 65          gold_tokens = sum(gold, []) 
 66          test_tokens = sum(tagged_sents, []) 
 67          return _accuracy(gold_tokens, test_tokens) 
 68   
 69 -    def _check_params(self, train, model): 
 70          if (train and model) or (not train and not model): 
 71              raise ValueError('Must specify either training data or trained model.') 
 72           
 73 -class FeaturesetTaggerI(TaggerI): 
 74      """ 
 75      A tagger that requires tokens to be I{featuresets}.  A featureset 
 76      is a dictionary that maps from I{feature names} to I{feature 
 77      values}.  See L{nltk.classify} for more information about features 
 78      and featuresets. 
 79      """ 
 80   
 81       
 82 -class HiddenMarkovModelTaggerTransformI(object): 
 83      """ 
 84      An interface for a transformation to be used as the transform parameter 
 85      of C{HiddenMarkovModelTagger}. 
 86      """ 
 87 -    def __init__(self): 
 88          if self.__class__ == HiddenMarkovModelTaggerTransformI: 
 89              raise AssertionError, "Interfaces can't be instantiated"       
 90   
 91 -    def transform(self, labeled_symbols): 
 92          """ 
 93          @return: a C{list} of transformed symbols 
 94          @rtype: C{list} 
 95          @param labeled_symbols: a C{list} of labeled untransformed symbols,  
 96              i.e. symbols that are not (token, tag) or (word, tag) 
 97          @type labeled_symbols: C{list} 
 98          """       
 99          raise NotImplementedError() 
100