jazzparser.formalisms.base.semantics.distance

1 """Base distance metric representation. 2 3 """ 4 """ 5 ============================== License ======================================== 6 Copyright (C) 2008, 2010-12 University of Edinburgh, Mark Granroth-Wilding 7 8 This file is part of The Jazz Parser. 9 10 The Jazz Parser is free software: you can redistribute it and/or modify 11 it under the terms of the GNU General Public License as published by 12 the Free Software Foundation, either version 3 of the License, or 13 (at your option) any later version. 14 15 The Jazz Parser is distributed in the hope that it will be useful, 16 but WITHOUT ANY WARRANTY; without even the implied warranty of 17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 GNU General Public License for more details. 19 20 You should have received a copy of the GNU General Public License 21 along with The Jazz Parser. If not, see <http://www.gnu.org/licenses/>. 22 23 ============================ End license ====================================== 24 25 """ 26 __author__ = "Mark Granroth-Wilding <mark.granroth-wilding@ed.ac.uk>" 27 28 from jazzparser.utils.options import ModuleOption, choose_from_list 29

30 -class DistanceMetric(object):

31 """ 32 Base class for semantic distance metrics. Formalism-specific distance 33 metrics should be provided as subclasses of this. 34 35 """ 36 OPTIONS = [] 37 name = "base" 38

39 - def __init__(self, options={}):

40 self.options = ModuleOption.process_option_dict(options, self.OPTIONS)

41

42 - def _get_identifier(self):

43 """ 44 A human-readable identifier for the metric represented by this instance. 45 """ 46 return self.name

47 identifier = property(_get_identifier) 48

49 - def distance(self, sem1, sem2):

50 """ 51 Compares the two semantics instances and returns a float distance 52 between them. 53 54 """ 55 raise NotImplementedError, "called distance() on base DistanceMetric"

56

57 - def print_computation(self, sem1, sem2):

58 """ 59 Produces a string showing a derivation of the distance metric that 60 would be returned for the given inputs. This is useful for debugging 61 the metric. 62 63 Subclasses are not required to provide this, and there may not always 64 be something sensible to show. 65 66 """ 67 return "Metric '%s' does not provide any computation trace information"\ 68 % self.name

69

70 - def total_distance(self, input_pairs):

71 """ 72 Returns the distances summed over the given pairs of inputs. By 73 default, this will just be the sum of the individual distances, 74 but in some cases it may be necessary to do something else, as with 75 f-score. 76 77 The input pairs may continue C{None} values. For example, if evaluating 78 the distance of parse results from the gold standard, there may be 79 inputs for which no parse result is obtained. It doesn't make sense, 80 however, for both of the pair to be C{None}. 81 82 """ 83 return sum([self.distance(*pair) for pair in input_pairs], 0.0)

84

85 - def format_distance(self, dist):

86 """ 87 Format a distance value (as returned from L{distance} or 88 L{total_distance}) as a string suitable for human-readable output. 89 90 """ 91 return "%s" % dist

92

93 -class FScoreMetric(DistanceMetric):

94 """ 95 Metrics that compute their distance by f-score share a lot of processing. 96 There's no need to put in every one the code for computing f-score from 97 matching stats. Instead, subclasses of this only need to provide the 98 C{fscore_match} method. 99 100 """ 101 OPTIONS = [ 102 # Subclasses may add more values to the output option, but should at 103 # least include these ones 104 ModuleOption('output', filter=choose_from_list( 105 ['f','precision','recall','inversef']), 106 usage="output=O, where O is one of 'f', 'precision', "\ 107 "'recall', 'inversef'", 108 default='inversef', 109 help_text="Select what metric to output. Choose recall "\ 110 "or precision for asymmetric metrics. F-score ('f') "\ 111 "combines these two. This is inverted ('inversef') "\ 112 "to get a distance, rather than similarity"), 113 ] 114

115 - def fscore_match(self, sem1, sem2):

116 """ 117 Subclasses must provide this. It should return a tuple. The first three 118 values must be floats: score given to the matching between the two 119 inputs; max score that could be given to the first; max score for the 120 second. There may be more values in the tuple. 121 122 """ 123 raise NotImplementedError, "f-score metric %s does not provide the "\ 124 "fscore_match method" % self.name

125

126 - def distance(self, sem1, sem2):

127 scores = self.fscore_match(sem1, sem2) 128 alignment = scores[0] 129 max_score1 = scores[1] 130 max_score2 = scores[2] 131 132 # Compute recall and precision 133 if alignment == 0: 134 recall = 0.0 135 else: 136 recall = alignment / max_score2 137 if self.options['output'] == 'recall': 138 return recall 139 140 if alignment == 0: 141 precision = 0.0 142 else: 143 precision = alignment / max_score1 144 if self.options['output'] == 'precision': 145 return precision 146 147 # Harmonic mean: f-score 148 if alignment == 0: 149 f_score = 0.0 150 else: 151 f_score = 2 * recall * precision / (recall+precision) 152 153 if self.options['output'] == 'f': 154 return f_score 155 else: 156 # Assume it must be 'inversef' output 157 return 1.0-f_score

158

159 - def total_distance(self, input_pairs):

160 """ 161 We don't just sum up f-scores to get another f-score. 162 163 """ 164 max_score1 = 0.0 165 max_score2 = 0.0 166 alignment = 0.0 167 168 for (input1,input2) in input_pairs: 169 scores = self.fscore_match(input1, input2) 170 alignment += scores[0] 171 max_score1 += scores[1] 172 max_score2 += scores[2] 173 174 # Now compute the appropriate metric over the whole set 175 if alignment == 0: 176 recall = 0 177 else: 178 recall = alignment / max_score2 179 if self.options['output'] == 'recall': 180 return recall 181 182 if alignment == 0: 183 precision = 0 184 else: 185 precision = alignment / max_score1 186 if self.options['output'] == 'precision': 187 return precision 188 189 # Harmonic mean: f-score 190 if alignment == 0: 191 f_score = 0 192 else: 193 f_score = 2 * recall * precision / (recall+precision) 194 195 if self.options['output'] == 'f': 196 return f_score 197 else: 198 # Assume it must be 'inversef' output 199 return 1.0-f_score

200

201 - def format_distance(self, dist):

202 return "%f%%" % (dist * 100.0)

203 204

205 -def command_line_metric(formalism, metric_name=None, options=""):

206 """ 207 Utility function to make it easy to load a metric, with user-specified 208 options, from the command line. Takes care of printing help output. 209 210 Typical options:: 211 parser.add_option("-m", "--metric", dest="metric", action="store", 212 help="semantics distance metric to use. Use '-m help' for a list of available metrics") 213 parser.add_option("--mopt", "--metric-options", dest="mopts", action="append", 214 help="options to pass to the semantics metric. Use with '--mopt help' with -m to see available options") 215 216 You could then call this as:: 217 metric = command_line_metric(formalism, options.metric, options.mopts) 218 219 @return: the metric instantiated with given options 220 221 """ 222 import sys 223 from jazzparser.utils.options import ModuleOption, options_help_text 224 225 # Get a distance metric 226 # Just check this, as it'll cause problems 227 if len(formalism.semantics_distance_metrics) == 0: 228 print "ERROR: the formalism defines no distance metrics, so this "\ 229 "script won't work" 230 sys.exit(1) 231 232 # First get the metric 233 if metric_name == "help": 234 # Print out a list of metrics available 235 print "Available distance metrics:" 236 print ", ".join([metric.name for metric in \ 237 formalism.semantics_distance_metrics]) 238 sys.exit(0) 239 240 if metric_name is None: 241 # Use the first in the list as default 242 metric_cls = formalism.semantics_distance_metrics[0] 243 else: 244 # Look for the named metric 245 for m in formalism.semantics_distance_metrics: 246 if m.name == metric_name: 247 metric_cls = m 248 break 249 else: 250 # No metric found matching this name 251 print "No metric '%s'" % metric_name 252 sys.exit(1) 253 254 # Options might be given as a list, if the option action was "append" 255 if isinstance(options, str): 256 options = [options] 257 # Now process the metric options 258 if options is not None: 259 moptstr = options 260 if "help" in [s.strip().lower() for s in options]: 261 # Output this parser's option help 262 print options_help_text(metric_cls.OPTIONS, 263 intro="Available options for metric '%s'" % metric_cls.name) 264 sys.exit(0) 265 moptstr = ":".join(moptstr) 266 else: 267 moptstr = "" 268 mopts = ModuleOption.process_option_string(moptstr) 269 # Instantiate the metric with these options 270 metric = metric_cls(options=mopts) 271 272 return metric

273

Source Code for Module jazzparser.formalisms.base.semantics.distance