jazzparser.taggers.tagger

41 """ 42 The superclass of all taggers. Subclass this to create tagger components. 43 44 Probabilities are returned by the tagger along with signs. These 45 are posterior probabilities for the C&C supertagging approach: 46 that is, Pr(tag | observations). For the PCFG parser approach, 47 the taggers must yield likelihoods: Pr(observation | tag). A tagger 48 of this sort should have POSTERIOR set to False. 49 50 """ 51 COMPATIBLE_FORMALISMS = [] 52 TAGGER_OPTIONS = [] 53 """ Tagger-specific options. List of ModuleOptions. """ 54 INPUT_TYPES = [] 55 """ List of allowed input datatypes. See L{jazzparser.data.input.INPUT_TYPES}. """ 56 shell_tools = [] 57 """ Interactive shell tools available when this tagger is used. """ 58 LEXICAL_PROBABILITY = False 59 """ 60 Some models provide lexical probabilities that the parsing models 61 can use. They should set this to true. They should also provide a method 62 C{lexical_probability(start_time, end_time, span_label)}. 63 """ 64

65 - def __init__(self, grammar, input, options={}, original_input=None, logger=None):

66 """ 67 The tagger must have reference to the grammar being used to parse 68 the input. It must also be given the full input when instantiated. 69 The format of this input will depend on the tagger: for example, 70 it might be a string or a MIDI file. 71 72 @param original_input: the input in its original, unprocessed form. This 73 will usually be a string. This is optional, but in some 74 circumstances things might fall apart if it hasn't been given. 75 E.g. using a backoff model as backoff from a tagging model requires 76 the original input to be passed to the backoff model. 77 @param logger: optional progress logger. Logging will be sent to this 78 during initialization of the tagger and tagging. If not given, the 79 logging will be lost. Subclasses may access the logger (or a dummy 80 logger if none was given) in C{self.logger}. 81 82 """ 83 self.grammar = grammar 84 # Check the formalism is one that's allowed by this tagger 85 formalism = self.grammar.formalism.get_name() 86 if formalism not in self.COMPATIBLE_FORMALISMS: 87 raise TaggerLoadError, "Formalism '%s' cannot be used with "\ 88 "tagger '%s'" % (formalism,self.name) 89 90 # Check what input type we've received and preprocess it 91 datatype, input = detect_input_type(input, allowed=self.INPUT_TYPES, \ 92 errmess=" for use with tagger '%s'" % self.name) 93 # Store this for the subclass to use as appropriate 94 self.input = input 95 if original_input is None: 96 self.original_input = input 97 else: 98 self.original_input = original_input 99 # Subclasses may redefine self.input to taste 100 # We keep the original wrapped input somewhere where it's sure to remain 101 self.wrapped_input = input 102 # Initialize using tagger-specific options 103 self.options = type(self).check_options(options) 104 105 if logger is not None: 106 self.logger = logger 107 else: 108 self.logger = create_dummy_logger()

109

110 - def _get_name(self):

111 return type(self).__module__.split(".")[2]

112 name = property(_get_name) 113

114 - def _get_input_length(self):

115 """ 116 Should return the number of words (chords) in the input, or 117 some other measure of input length appropriate to the type of 118 tagger. 119 120 """ 121 return len(self.input)

122 input_length = property(_get_input_length) 123

124 - def get_signs(self, offset=0):

125 """ 126 Returns a list of tuples C{(start, end, signtup)}. These represent spans 127 to be added to the chart, C{start} and C{end} being the start and 128 end nodes. 129 130 Each signtup is a (sign,tag,probability) tuple representing a sign 131 that the tagger wishes to add to the chart in this position. 132 How many are returned 133 is up to the tagger (it may wish to return more in cases where there 134 are no clear winners, for example). 135 If the tag is not found in the grammar, sign will be None. 136 137 Returned list is sorted by probability, highest first. 138 139 offset may be set >0 in order to retrieve further signs once some have 140 already been returned. If offset=k, the tagger should disregard 141 all the signs that would have been returned for offset<k and 142 return the next bunch - as many as it sees fit. offset is incremented 143 each time the parse fails. 144 145 The simplest approach, and that employed by most taggers, has some 146 signs for each word and none spanning more than one word. That is, 147 the tuples in the list would be of the form 148 C{(wordnum, wordnum+1, signtup)}. This is by no means required, though: 149 some taggers will want to add multi-node spans to the chart. 150 151 @note: This functionality used to be provided by C{get_signs_for_word}. 152 For convenience, if a tagger provides C{get_signs_for_word} and not 153 get_signs(), the results of the former will be used to produce the 154 latter. New taggers should not do this, but override this method 155 directly. 156 157 """ 158 if hasattr(self, "get_signs_for_word"): 159 # The tagger doesn't provide get_signs(), but is an old class 160 # that provides get_signs_for_word() instead 161 # We can use the output of get_signs_for_word() to produce 162 # the required output of get_signs() 163 signs = [] 164 for start_node in range(self.input_length): 165 word_signs = self.get_signs_for_word(start_node, offset=offset) 166 for sign in word_signs: 167 signs.append((start_node, start_node+1, sign)) 168 return signs 169 else: 170 # Must be overridden 171 raise NotImplementedError, "Cannot use the Tagger abstract superclass."

172

173 - def get_word(self, index):

174 """ 175 Returns the input word at this index. This does not need to be 176 a string, but must have a sensible __str__, so that it can 177 be converted to a readable string. 178 The purpose of this is to provide a readable form of the input 179 for the parser to store in derivation traces. 180 """ 181 # Must be overridden 182 raise NotImplementedError, "Tagger.get_word() must be overridden by tagger classes."

183

184 - def get_word_duration(self, index):

185 """ 186 Returns the duration of the word at this index if durations 187 are available. Otherwise raises an AttributeError. 188 189 """ 190 if not hasattr(self, 'durations'): 191 raise AttributeError, "tagger has not stored durations. "\ 192 "Tried to get the duration of an input word" 193 return self.durations[index]

194

195 - def get_string_input(self):

196 """ 197 Returns a list of string representations of the inputs. 198 This is just a convenience function, which uses whatever 199 representation gets returned by get_word() to produce a 200 representation of the whole input. 201 202 """ 203 return [str(self.get_word(i)) for i in range(self.input_length)]

204

205 - def get_tag_probability(self, index, tag, end_index=None):

206 """ 207 Returns as a float the probability with which the tagger judges 208 the given span will be assigned the given sign. 209 210 If C{end_index} is not given, it defaults to C{index}+1. 211 212 """ 213 if end_index is None: 214 end_index = index+1 215 # Check all the signs for a match 216 all_signs = self.get_all_signs() 217 218 if (index,end_index) not in all_signs: 219 # The sign was not assigned at all 220 return 0.0 221 else: 222 for __,assigned_tag,prob in all_signs[(index,end_index)]: 223 if assigned_tag == tag: 224 return prob 225 return 0.0

226

227 - def get_all_signs(self):

228 """ 229 Gets all signs that the tagger will return, regardless of offset. 230 This just uses L{get_signs} to get the signs for every offset. 231 232 @rtype: dict 233 @return: all the signs, keyed by (start,end) tuple 234 235 """ 236 if not hasattr(self, "_all_signs"): 237 # Compute and cache the signs list 238 signs = {} 239 offset = 0 240 new_signs = None 241 # Get all the signs by incrementing the offset until there are no more 242 while new_signs is None or len(new_signs) > 0: 243 new_signs = self.get_signs(offset=offset) 244 # Index the signs by their span tuple 245 for (start,end,sign) in new_signs: 246 signs.setdefault((start,end), []).append(sign) 247 offset += 1 248 self._all_signs = signs 249 return self._all_signs

250 251 @classmethod

252 - def check_options(cls, options):

253 """ 254 Normally, options are validated when the tagger is instantiated. 255 This allows you to check them before that. 256 257 """ 258 return ModuleOption.process_option_dict(options, cls.TAGGER_OPTIONS)

Source Code for Module jazzparser.taggers.tagger