Source Code for Module jazzparser.evaluation.taggers

  1  """Tagger evaluation routines. 
  2   
  3  @note: This was originally C{jazzparser.utils.stats}, but only ended up having  
  4  tagger evaluation stuff in it. 
  5   
  6  """ 
  7  """ 
  8  ============================== License ======================================== 
  9   Copyright (C) 2008, 2010-12 University of Edinburgh, Mark Granroth-Wilding 
 10    
 11   This file is part of The Jazz Parser. 
 12    
 13   The Jazz Parser is free software: you can redistribute it and/or modify 
 14   it under the terms of the GNU General Public License as published by 
 15   the Free Software Foundation, either version 3 of the License, or 
 16   (at your option) any later version. 
 17    
 18   The Jazz Parser is distributed in the hope that it will be useful, 
 19   but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 21   GNU General Public License for more details. 
 22    
 23   You should have received a copy of the GNU General Public License 
 24   along with The Jazz Parser.  If not, see <http://www.gnu.org/licenses/>. 
 25   
 26  ============================ End license ====================================== 
 27   
 28  """ 
 29  __author__ = "Mark Granroth-Wilding <mark.granroth-wilding@ed.ac.uk>"  
 30   
 31  from math import log 
 32  import logging 
 33   
 34  # Get the logger from the logging system 
 35  logger = logging.getLogger("main_logger") 
 36   
 37 -def tagger_entropy(input, grammar, tagger_class, correct_tags, options={}): 
 38      """ 
 39      Given some input, a tagger and a list of correct tags,  
 40      computes the entropy of the tagger over the input, returning a tuple  
 41      of the total entropy and the number of chords that were involved  
 42      in the computation. 
 43       
 44      The option dictionary options will be passed to the tagger. 
 45      """ 
 46      # Do the tagging of the sequence 
 47      tagger = tagger_class(grammar, input, options=options) 
 48      def _get_word(i): 
 49          if i >= len(correct_tags): 
 50              return None 
 51          else: 
 52              return tagger.get_word(i) 
 53      def _get_context(i): 
 54          if i == 0: 
 55              return "START *%s* %s" % (_get_word(0), _get_word(1)) 
 56          elif i == 1: 
 57              return "START %s *%s* %s" % (_get_word(0), _get_word(1), _get_word(2)) 
 58          else: 
 59              return "%s %s *%s* %s" % (_get_word(i-2), _get_word(i-1), _get_word(i), _get_word(i+1)) 
 60      # Compute the entropy for every chord 
 61      total_entropy = 0.0 
 62      total_chords = 0 
 63      for i in range(tagger.input_length): 
 64          # Don't include chords that aren't annotated - we could never get them right 
 65          if correct_tags[i] not in ["", "?"]: 
 66              # The tag is actually the root and the schema, but the list of  
 67              #  correct tags only includes the schema 
 68              correct = (_get_word(i).root, correct_tags[i]) 
 69              probability = tagger.get_tag_probability(i, correct) 
 70              if probability == 0.0: 
 71                  # Should be -inf 
 72                  # What to do? 
 73                  # Just give it a very small probability to fudge the calculations 
 74                  # This is effectively giving the model a very naive smoothing 
 75                  logger.error("The correct tag (%s) was given 0 probability by the tagger. Context: %s. Assigning a very small probability instead." % (correct_tags[i], _get_context(i))) 
 76                  probability = float('1e-50') 
 77              entropy = log(probability, 2) 
 78              total_entropy += entropy 
 79              total_chords += 1 
 80      total_entropy = -1 * total_entropy 
 81      return (total_entropy, total_chords) 
 82   
 83 -def tagger_agreement(input, grammar, tagger_class, correct_tags, options={}, confusion_matrix=None, topn=1): 
 84      """ 
 85      Like tagger_entropy, but instead of computing the entropy computes  
 86      the proportion of top tags assigned that agree with the gold  
 87      standard. 
 88       
 89      The option dictionary options will be passed to the tagger. 
 90       
 91      Optionally puts a confusion matrix in the dictionary given in  
 92      confusion_matrix. This is a dictionary keyed by correct tags whose  
 93      values are dictionaries keyed by the incorrect tags that were  
 94      confused for them, whose values are a count of the confusions. 
 95       
 96      Returns a tuple (number_agreeing, number_compared). 
 97       
 98      """ 
 99      # Do the tagging of the sequence 
100      tagger = tagger_class(grammar, input, options=options) 
101      # Compute the entropy for every chord 
102      total_agreeing = 0 
103      total_chords = 0 
104       
105      # Get all the spans the tagger can give us 
106      spans = tagger.get_signs(offset=0) 
107      added = len(spans) 
108      offset = 1 
109      while added > 0: 
110          new_spans = tagger.get_signs(offset=offset) 
111          added = len(new_spans) 
112          spans.extend(new_spans) 
113          offset += 1 
114      # Look for all the length 1 spans: single-word categories 
115      word_signs = {} 
116      for (start,end,sign) in spans: 
117          if end == start + 1: 
118              word_signs.setdefault(start, []).append(sign) 
119       
120      for i in range(tagger.input_length): 
121          # Don't include chords that aren't annotated - we could never get them right 
122          if correct_tags[i] not in ["", "?"]: 
123              tags = word_signs.get(i, []) 
124              if len(tags) == 0: 
125                  # Nothing returned by the tagger - must be wrong 
126                  assigned_tag = None 
127              else: 
128                  tags = [schema for __,(root,schema),__ in tags] 
129                  assigned_tag = tags[0] 
130              # Count as correct if the correct tag is in the top n returned 
131              tags = tags[:topn] 
132              if correct_tags[i] in tags: 
133                  # Got the correct one! 
134                  total_agreeing += 1 
135                   
136              # Keep a record of what tags were confused if the top tag wasn't right 
137              if confusion_matrix is not None and correct_tags[i] != assigned_tag: 
138                  conf = confusion_matrix.setdefault(correct_tags[i], {}) 
139                  conf.setdefault(assigned_tag, 0) 
140                  # Increment the count of this particular confusion by one 
141                  conf[assigned_tag] += 1 
142              total_chords += 1 
143      return (total_agreeing, total_chords) 
144