jazzparser.grammar

1 """Grammar processing for the Jazz Parser. 2 3 This module is used to read the XML grammar files for the parser. 4 They are stored in the OpenCCG grammar format (roughly). This 5 provides the interface to the grammar and the formalism for the parser 6 and tagger. 7 8 """ 9 """ 10 ============================== License ======================================== 11 Copyright (C) 2008, 2010-12 University of Edinburgh, Mark Granroth-Wilding 12 13 This file is part of The Jazz Parser. 14 15 The Jazz Parser is free software: you can redistribute it and/or modify 16 it under the terms of the GNU General Public License as published by 17 the Free Software Foundation, either version 3 of the License, or 18 (at your option) any later version. 19 20 The Jazz Parser is distributed in the hope that it will be useful, 21 but WITHOUT ANY WARRANTY; without even the implied warranty of 22 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 23 GNU General Public License for more details. 24 25 You should have received a copy of the GNU General Public License 26 along with The Jazz Parser. If not, see <http://www.gnu.org/licenses/>. 27 28 ============================ End license ====================================== 29 30 """ 31 __author__ = "Mark Granroth-Wilding <mark.granroth-wilding@ed.ac.uk>" 32 33 import xml.dom.minidom 34 from jazzparser.formalisms.base.modalities import ModalityTree, \ 35 ModalityTreeNode 36 from jazzparser.utils.domxml import attrs_to_dict, remove_unwanted_elements, \ 37 get_single_element_by_tag_name 38 from jazzparser.utils.chords import generalise_chord_name 39 from jazzparser.data import Chord 40 from jazzparser.data.db_mirrors import Chord as DbChord 41 from jazzparser.formalisms import FORMALISMS 42 from jazzparser.formalisms.loader import get_default_formalism, get_formalism, \ 43 FormalismLoadError 44 45 import logging, os, copy 46 import settings 47 48 # Get the logger from the logging system 49 logger = logging.getLogger("main_logger")

50 51 -class Grammar(object):

52 """ 53 Represents a grammar read in from an XML grammar file. 54 Initialised with the location of the XML file. 55 Can be consulted to retrieve lexical entries in the grammar. 56 57 """ 58 ######### All public attributes ######### 59 # These are all create set when the class is instantiated, but are 60 # here so they can be fully documented. 61 formalism = None 62 """ 63 Formalism definition (L{jazzparser.formalisms.FormalismBase}) for the 64 formalism used by the grammar instance. 65 66 Always set. 67 68 """ 69 grammar_file = None 70 """ 71 Path to the grammar.xml file from which the grammar definition was loaded. 72 73 Always set. 74 75 """ 76 literal_functions = None 77 """ 78 Dictionary of builtin semantic literal functions. 79 80 May be None. 81 82 @deprecated: Only used in very early versions of the grammar. Left here 83 for backwards compatibility. May be removed altogether soon. 84 85 """ 86 families = None 87 """ 88 Dictionary of lexical entry families (each a list of lexical signs), 89 keyed by POS. 90 91 """ 92 inactive_families = None 93 """ 94 Like families. Dictionary of families defined in the XML but marked as 95 inactive in their definition. 96 97 Always a list. May be empty. 98 99 """ 100 morphs = None 101 """ 102 List of morph entries (L{MorphItem}). Mapping from word classes to 103 families. You probably don't want to access this directly usually, but 104 to use L{Grammar}'s methods like L{get_signs_for_word}. 105 106 """ 107 morph_items = None 108 """ 109 Same as L{morphs}, but a dictionary indexed by the word attribute 110 (identifier of word or word class) for easy lookup. 111 112 """ 113 modality_tree = None 114 """ 115 Instance of L{jazzparser.formalisms.base.modalities.ModalityTree} 116 representing the hierarchical structure of modalities used in this grammar. 117 118 """ 119 rules = None 120 """ 121 List of all the rules used by the grammar. These are instances of the 122 rule classes, instantiated with the correct parameters and ready to apply 123 to signs. Does not include lexical rules: see L{lexical_rules}. 124 125 """ 126 unary_rules = None 127 """ 128 List of unary rules. Subset of L{rules}. 129 130 May be empty, but should always be a list. 131 132 """ 133 binary_rules = None 134 """ 135 List of binary rules. Subset of L{rules}. 136 137 May be empty, but should always be a list. 138 139 """ 140 rules_by_name = None 141 """ 142 Dictionary of rules, containing same entries as L{rules}, keyed by 143 the internal_name attribute. There should not be more than one rule 144 instance with the same internal name. If multiple are created when the 145 grammar is instantiated, a L{GrammarReadError} will be raised. 146 147 """ 148 lexical_rules = None 149 """ 150 List of special unary rules that are applied to lexical families when the 151 grammar is instantiated to expand the lexicon. The lexicon will then 152 include all the original lexical entries, plus any results of applying 153 these rules to them. 154 155 Empty list if no lexical rules are given. 156 157 """ 158 chord_classes = None 159 """ 160 Dict of L{ChordClass} objects, indexed by name. 161 162 """ 163 ######################################### 164

165 - def __init__(self, grammar_name=None):

166 """ 167 Creates a new grammar by reading from an XML grammar file. 168 169 Words (morph items) are stored in morph_items. 170 Families (lexical families) are stored in families. 171 172 Instantiate this directly only if you want, for some reason, to be sure 173 of getting a new instance of Grammar. Most of the time, you can 174 load a named grammar using L{get_grammar}, which will cache already 175 loaded grammars and return the same instance again if you ask for the 176 same name. 177 178 @type grammar_name: string 179 @param grammar_name: name of the grammar definition to be loaded. 180 Call L{get_grammar_names} for a list of available grammars. If 181 None, loads the default grammar. 182 183 """ 184 if grammar_name is None: 185 grammar_name = settings.DEFAULT_GRAMMAR 186 self.name = grammar_name 187 188 filename_base = os.path.join(settings.GRAMMAR_DATA_DIR, grammar_name) 189 self.grammar_file = os.path.join(filename_base, "grammar.xml") 190 # Read in the grammar 191 logger.debug("Grammar: %s" % self.grammar_file) 192 193 # Read in the XML from the file 194 self.grammar_dom = xml.dom.minidom.parse(self.grammar_file) 195 196 grammar_tag = get_single_element_by_tag_name(self.grammar_dom, "grammar") 197 # Get a named formalism, or the default one 198 formalism_attr = grammar_tag.attributes.getNamedItem("formalism") 199 if formalism_attr is None: 200 formalism = get_default_formalism() 201 else: 202 formalism_name = str(formalism_attr.value) 203 try: 204 formalism = get_formalism(formalism_name) 205 except FormalismLoadError: 206 logger.error("The formalism '%s' does not exist. Possible "\ 207 "formalisms are: %s" % (formalism_name, ", ".join(FORMALISMS))) 208 raise 209 self.formalism = formalism 210 211 ############################### 212 ### Reading in the lexicon 213 lex_tag = get_single_element_by_tag_name(self.grammar_dom, "lexicon") 214 lexicon_file = os.path.join(filename_base, lex_tag.attributes.getNamedItem("file").value) 215 logger.debug("Lexicon: %s" % lexicon_file) 216 # Read in the lexicon 217 self.lexicon_dom = xml.dom.minidom.parse(lexicon_file) 218 219 ############################### 220 ### Reading in the words 221 morph_tag = get_single_element_by_tag_name(self.grammar_dom, "morphology") 222 morph_file = os.path.join(filename_base, morph_tag.attributes.getNamedItem("file").value) 223 logger.debug( "Morphology: %s" % morph_file) 224 # Read in the lexicon 225 self.morph_dom = xml.dom.minidom.parse(morph_file) 226 227 ############################### 228 ### Reading in the rules 229 rules_tag = get_single_element_by_tag_name(self.grammar_dom, "rules") 230 rules_file = os.path.join(filename_base, rules_tag.attributes.getNamedItem("file").value) 231 logger.debug( "Rules: %s" % rules_file) 232 # Read in the lexicon 233 self.rules_dom = xml.dom.minidom.parse(rules_file) 234 235 ############################### 236 ### Reading in the functions list (only used for certain formalisms) 237 functions_tag = get_single_element_by_tag_name(self.grammar_dom, "functions", optional=True) 238 self.literal_functions = {} 239 available_funs = formalism.literal_functions 240 if functions_tag is not None: 241 functions_file = os.path.join(filename_base, functions_tag.attributes.getNamedItem("file").value) 242 logger.debug( "Functions: %s" % functions_file) 243 # Read in the functions from the XML 244 functions_dom = xml.dom.minidom.parse(functions_file) 245 functions_xml = get_single_element_by_tag_name(functions_dom, "functions") 246 functions = remove_unwanted_elements(functions_xml.getElementsByTagName("function")) 247 # Try adding each of the functions, using the formalism's definitions 248 for func_el in functions: 249 func_name = func_el.attributes.getNamedItem("name").value 250 if func_name in available_funs: 251 lit_fun = available_funs[func_name] 252 self.literal_functions[lit_fun.name] = lit_fun 253 else: 254 raise GrammarReadError, "The literal function \"%s\" is not defined in the code for the %s formalism." % formalism.get_name() 255 256 ############################### 257 ### Reading in the modality hierarchy 258 modalities_tag = get_single_element_by_tag_name(self.grammar_dom, "modalities", optional=True) 259 if modalities_tag is not None: 260 modalities_file = os.path.join(filename_base, modalities_tag.attributes.getNamedItem("file").value) 261 logger.debug( "Modalities: %s" % modalities_file) 262 # Read in the modalities 263 self.modalities_dom = get_single_element_by_tag_name(xml.dom.minidom.parse(modalities_file), "modalities") 264 else: 265 self.modalities_dom = None 266 267 ############################### 268 ### Read in grammar-level meta data 269 attrs = self.grammar_dom.getElementsByTagName("attr") 270 # Initialize values that might not get set 271 self.max_categories = None 272 # Read in the values from the XML 273 for el in attrs: 274 name = el.getAttribute("name") 275 value = el.getAttribute("value") 276 # Check for all the attributes we recognize 277 if name == "max_categories": 278 self.max_categories = int(value) 279 280 ############################### 281 ### Prepare the morph word classes 282 self.chord_classes = {} 283 for entry in self.morph_dom.getElementsByTagName("class"): 284 chord_class = ChordClass.from_dom(entry) 285 self.chord_classes[chord_class.name] = chord_class 286 287 # Maybe handle macros here. Not currently using them. 288 289 ############################### 290 ### Prepare lexical entries 291 # Use a hash table for this too, indexed by pos 292 self.families = {} 293 self.inactive_families = [] 294 for family in self.lexicon_dom.getElementsByTagName("family"): 295 fam = Family.from_dom(formalism, family) 296 # Check whether the family has any entries and don't use it if not 297 if len(fam.entries) > 0: 298 # Put a new Family in the table for every family entry 299 if fam.pos in self.families: 300 # Already an entry for this POS: add to the list 301 self.families[fam.pos].append(fam) 302 else: 303 # No occurence of this POS yet: add a new list 304 self.families[fam.pos] = [fam] 305 else: 306 self.inactive_families.append(fam.pos) 307 308 ############################### 309 ### Prepare the morph items 310 self.morphs = [] 311 for entry in self.morph_dom.getElementsByTagName("entry"): 312 morph = MorphItem.from_dom(formalism,entry,self.chord_classes) 313 self.morphs.append(morph) 314 315 # Check that all the morphs correspond to a defined POS 316 for morph in self.morphs: 317 if morph.pos not in self.families: 318 raise GrammarReadError, "morph item refers to undefined "\ 319 "part-of-speech '%s': %s" % (morph.pos, morph.element.toxml()) 320 321 ############################### 322 ### Prepare modalities hierarchy 323 if self.modalities_dom: 324 self.modality_tree = ModalityTree.from_dom(self.modalities_dom) 325 else: 326 # The modalities that existed before they were added to the 327 # XML spec were just "c" and "." 328 self.modality_tree = ModalityTree([ 329 ModalityTreeNode("", 330 [ModalityTreeNode("c")]) ]) 331 332 ############################### 333 ### Prepare rules 334 self.rules = [] 335 # Go through each different type of rule and add appropriate Rule subclasses 336 rule_block = get_single_element_by_tag_name(self.rules_dom, "rules") 337 338 for rule_tag in remove_unwanted_elements(rule_block.childNodes): 339 rulename = rule_tag.tagName 340 if rulename == "lexrules": 341 # We'll deal with these later 342 continue 343 if rulename not in self.formalism.rules: 344 raise GrammarReadError, "unknown rule '%s' (formalism "\ 345 "defines: %s)" % (rulename, ", ".join(formalism.rules.keys())) 346 ruleclass = self.formalism.rules[rulename] 347 # Instantiate the rule, using options from the XML 348 self.rules.append(ruleclass(modalities=self.modality_tree, grammar=self, **attrs_to_dict(rule_tag.attributes))) 349 350 # Keep rules sorted by arity for ease of access 351 self.unary_rules = [] 352 self.binary_rules = [] 353 for rule in self.rules: 354 if rule.arity == 1: 355 self.unary_rules.append(rule) 356 elif rule.arity == 2: 357 self.binary_rules.append(rule) 358 359 # Index rules by internal name for ease of access 360 self.rules_by_name = {} 361 for rule in self.rules: 362 if rule.internal_name in self.rules_by_name: 363 # This shouldn't happen: each rule name should only be used once 364 raise GrammarReadError, "instantiated two rules with the same "\ 365 "internal name: %s. Either the XML has mistakenly "\ 366 "instantiated the same thing twice, or the rule class has "\ 367 "failed to give different varieties of the rule different "\ 368 "names" % rule.internal_name 369 self.rules_by_name[rule.internal_name] = rule 370 371 # Optionally read in a lexrules element and expand the lexicon 372 # using its entries 373 self.lexical_rules = [] 374 lexrules_tag = get_single_element_by_tag_name(self.rules_dom, "lexrules", optional=True) 375 if lexrules_tag is not None: 376 for rule_tag in remove_unwanted_elements(lexrules_tag.childNodes): 377 rulename = rule_tag.tagName 378 if rulename not in self.formalism.rules: 379 raise GrammarReadError, "unknown lexical expansion "\ 380 "rule '%s' (formalism defines: %s)" % \ 381 (rulename, ", ".join(formalism.rules.keys())) 382 ruleclass = self.formalism.rules[rulename] 383 attrs = attrs_to_dict(rule_tag.attributes) 384 # Make sure expanded category has a suffix to put on 385 # POSs. If one isn't given, set a default. 386 if "pos_suffix" in attrs: 387 pos_suffix = attrs["pos_suffix"] 388 del attrs["pos_suffix"] 389 else: 390 pos_suffix = "_Rep" 391 # Instantiate the rule, using any options given 392 rule = ruleclass(modalities=self.modality_tree, 393 grammar=self, 394 **attrs) 395 rule.pos_suffix = pos_suffix 396 # Can only use unary rules - check this one is 397 if rule.arity != 1: 398 raise "can only use unary rules as lexical "\ 399 "expansions. Tried to use %s, which has arity "\ 400 "%d." % (rulename, rule.arity) 401 self.lexical_rules.append(rule) 402 # Use each lexical rule to expand the lexicon 403 for rule in self.lexical_rules: 404 for fam in sum(self.families.values(), []): 405 for entry in fam.entries: 406 # Try apply the expansion rule to this entry 407 new_signs = rule.apply_rule([entry.sign]) 408 if new_signs is not None and len(new_signs) > 0: 409 # Make a new POS for this expanded category 410 new_pos = "%s%s" % (fam.pos, rule.pos_suffix) 411 new_entries = [EntriesItem(self.formalism, "Expanded", new_sign) \ 412 for new_sign in new_signs] 413 new_family = Family(self.formalism, 414 new_pos, 415 new_pos, 416 new_entries, 417 chordfn=fam.chordfn, 418 expanded=rule.internal_name) 419 self.families.setdefault(new_pos, []).append(new_family) 420 # Also create morph items for each of those 421 # that referenced the old unexpanded rules 422 for morph in [m for m in self.morphs if m.pos == fam.pos]: 423 self.morphs.append( 424 MorphItem( 425 self.formalism, 426 copy.deepcopy(morph.words), 427 new_pos, 428 optional_minor=morph.optional_minor, 429 chord_class=morph.chord_class)) 430 431 ############### 432 # Index the morph items by word to make lookup easier 433 self.morph_items = {} 434 for morph in self.morphs: 435 # If the pos is completely inactive in the lexicon, ignore this morph 436 if not morph.pos in self.inactive_families: 437 # Go through each of this morph's words 438 for word in morph.words: 439 # Put a new MorphItem in the table for every entry 440 if word in self.morph_items: 441 # Already a list for this word: add to it 442 self.morph_items[word].append(morph) 443 else: 444 # First occurence of this word: add a new list 445 self.morph_items[word] = [morph] 446 447 ############### 448 # Read in an equivalence map if one is given for morph entries 449 equiv_map_el = get_single_element_by_tag_name(self.morph_dom, "equivmap", optional=True) 450 if equiv_map_el is not None: 451 self.equiv_map = EquivalenceMap.from_dom(formalism, 452 equiv_map_el, 453 self.chord_classes, 454 self.morphs) 455 else: 456 self.equiv_map = EquivalenceMap() 457 458 ########### 459 # Prepare a version of the family list for MIDI input 460 self.midi_families = {} 461 for pos,fams in self.families.items(): 462 new_fams = [] 463 for fam in fams: 464 # Exclude any generated by lexical expansions, unless they're 465 # tonic function 466 if fam.expanded is not None and fam.chordfn != "T": 467 continue 468 new_fams.append(fam) 469 if new_fams: 470 # Exclude any that are mapped onto another entry by an equivalence 471 # mapping that changes the root 472 if pos in self.equiv_map: 473 continue 474 self.midi_families[pos] = new_fams 475 476 ####### Debugging output 477 logger.debug( "Read the following information from the grammar:") 478 logger.debug( "Morphology:") 479 logger.debug("\n".join(["%s: %s" % (word, ", ".join(["%s" % item.pos for item in items])) \ 480 for word,items in self.morph_items.items()])) 481 logger.debug("Lexicon:") 482 logger.debug("\n".join([", ".join(["%s" % initem for initem in item]) \ 483 for item in self.families.values()])) 484 logger.debug("Rules:") 485 logger.debug("\n".join([" %s" % item for item in self.rules])) 486 logger.debug("Lexical expansion rules:") 487 logger.debug("\n".join([" %s" % item for item in self.lexical_rules])) 488 logger.debug("Modalities:") 489 logger.debug("%s" % self.modality_tree) 490 if len(self.literal_functions): 491 logger.debug("Literal functions:") 492 logger.debug("\n".join([" %s: %s" % (name,val) for (name,val) in self.literal_functions.items()]))

493 494

495 - def get_signs_for_word(self, word, tags=None, extra_features=None):

496 """ 497 Given a word string, returns a list of the possible signs 498 (as CCGSigns) that the grammar can assign to it. 499 word may also be a Chord object. 500 For now, this assumes that the input is a single chord in 501 roman numeral notation and that spelling issues have already 502 been resolved (e.g. that 6s have been removed). 503 504 If tags is given it should be a list of strings. Signs will be 505 restricted to those whose entry's tag name/POS is in the list. 506 507 If you need to get an instantiated category from a lexical entry, 508 use the methods on L{EntriesItem} directly, or L{get_signs_for_tag}. 509 510 """ 511 if isinstance(word, Chord): 512 chord = word.to_db_mirror() 513 elif isinstance(word, basestring): 514 chord = Chord.from_name(word, permissive=True).to_db_mirror() 515 elif isinstance(word, DbChord): 516 chord = word 517 else: 518 raise GrammarLookupError, "Tried to get signs for a word of type "\ 519 "'%s': %s" % (type(word), word) 520 # Get a chord type string to look up in the grammar 521 chord_lookup = "X%s" % chord.type 522 523 # Check whether we know this word 524 if not chord_lookup in self.morph_items: 525 # Word not recognised 526 raise GrammarLookupError, "The word \"%s\" was not found in the "\ 527 "lexicon. (Looked up %s in %s)" \ 528 % (word, chord_lookup, 529 ",".join(["%s" % item for item in self.morph_items.keys()])) 530 # Get the list of interpretations of this word 531 morphs = self.morph_items[chord_lookup] 532 533 # Limit to tag list if one was given 534 if tags is not None: 535 morphs = [m for m in morphs if m.pos in tags] 536 537 # Build a sign for each morph-family pair 538 category_list = [] 539 for morph in morphs: 540 # Look for families corresponding to the POS 541 if not morph.pos in self.families: 542 raise GrammarLookupError, "There is no family in the lexicon "\ 543 "for the POS %s." % morph.pos 544 for family in self.families[morph.pos]: 545 # Build a CCGCategory for each entry in each family found 546 for entry in family.entries: 547 sign = entry.sign.copy() 548 sign.tag = entry.tag_name 549 features = { 550 'root' : chord.root, 551 'morph' : morph 552 } 553 if extra_features is not None: 554 features.update(extra_features) 555 sign.apply_lexical_features(features) 556 category_list.append(sign) 557 558 return category_list

559

560 - def get_sign_for_word_by_tag(self, word, tag, extra_features=None):

561 """ 562 Returns a sign that can be assigned to the given word and that 563 has the given tag. If the word cannot have that tag, returns 564 None. 565 """ 566 possibles = self.get_signs_for_word(word, tags=[tag], extra_features=extra_features) 567 if len(possibles) == 0: 568 return None 569 else: 570 return possibles[0]

571

572 - def get_signs_for_tag(self, tag, features):

573 """ 574 Instantiates all the signs associated with a pos tag, using the given 575 dictionary of lexical features. 576 577 """ 578 # Get all the entries for this tag 579 entries = sum([fam.entries for fam in self.families[tag]], []) 580 # Instantiate each lexically 581 return [entry.get_lexical_sign(features, self) for entry in entries]

582

583 - def _get_entries_by_tag(self):

584 """ 585 Gets all the tags recognised by the grammar. The tags are 586 returned as indices in a dict to the entry they represent. 587 Note that these tags are unique identifiers of lexical items, 588 not pos tags. 589 If you want entries indexed by POS tags, just use 590 grammar.families. 591 """ 592 families = self.families 593 entries = {} 594 # Get all the lexical entries 595 for pos in sorted(families.keys()): 596 for fam in families[pos]: 597 for entry in fam.entries: 598 entries[entry.tag_name] = entry 599 return entries

600 entries_by_tag = property(_get_entries_by_tag) 601

602 - def _get_pos_tags(self):

603 """ 604 Convenience function to get the full list of POS tags allowed 605 by the grammar. 606 607 """ 608 return self.families.keys()

609 pos_tags = property(_get_pos_tags) 610

611 - def tag_to_function(self, tag):

612 """ 613 Given a POS tag (roughly denoting a category), returns the 614 function of the chord that is implicit in this interpretation. 615 If the tag is not in the grammar or a function is not given 616 for this tag in the lexicon, returns None. 617 618 """ 619 if tag in self.families: 620 return self.families[tag][0].chordfn 621 else: 622 return None

623

624 - def _get_tags_by_function(self):

625 """ 626 Returns a dictionary mapping chord functions to POS tags (i.e. 627 category family names). 628 629 """ 630 return dict([(fam.chordfn,tag) for (tag,fam) in self.families.items()])

631 tags_by_function = property(_get_tags_by_function)

632

633 ########################## 634 ## Class structure ## 635 ## for grammar from xml ## 636 ########################## 637 # Based on OpenCCG # 638 -class Family(object):

639 """ 640 A lexical family. 641 642 """

643 - def __init__(self, formalism, name, pos, entries, chordfn=None, expanded=None):

644 """ 645 @type expanded: string or None 646 @param expanded: if the family is generated by a lexical expansion 647 rule, should contain the name of the rule that generated it. 648 Otherwise None 649 650 """ 651 self.formalism = formalism 652 self.name = name 653 self.pos = pos 654 self.chordfn = chordfn 655 self.entries = entries 656 for entry in self.entries: 657 entry.family = self 658 self.expanded = expanded

659

660 - def __str__(self):

661 return "<Family: \"%s\", \"%s\">%s</Family>" % (self.name, self.pos, \ 662 "".join(["%s" % entry for entry in self.entries]))

663 664 @staticmethod

665 - def from_dom(formalism, element):

666 """ 667 Builds a Family instance from a DOM element read in from the 668 XML grammar definition. 669 670 """ 671 name = element.attributes.getNamedItem("name").value 672 pos = element.attributes.getNamedItem("pos").value 673 # The chordfn attr is optional, but we hope it's given for everything 674 chordfn = element.attributes.getNamedItem("chordfn") 675 if chordfn is None: 676 # Not given, just use None so it's clear where it came from 677 chordfn = None 678 else: 679 chordfn = chordfn.value 680 # No need to read "closed" attr, since all families are closed 681 # Same applies to "member" elements. 682 683 entries = [] 684 # Create an Entry instance for every entry child 685 for entry in element.getElementsByTagName("entry"): 686 entry = EntriesItem.from_dom(formalism,entry) 687 # Only use the entry if it's active 688 if entry.active: 689 entries.append(entry) 690 691 return Family(formalism, 692 name, 693 pos, 694 entries, 695 chordfn=chordfn)

696

697 -class MorphItem(object):

698 """ 699 A morphological item - a word. Stores the word and the POS. 700 Words are stored as strings, unlike in OpenCCG. We don't need all the 701 extra information about words that OpenCCG holds. 702 703 The word field stores a list of strings, defining all the words 704 represented by this morph item. In the case of a 705 C{<entry pos="POS" word="WORD"/>} item, this is a single word, 706 but in the case of a C{<entry pos="POS" class="CLASS"/>} 707 item, this is all the words contained in the class CLASS. 708 709 The argument classes should be a dictionary mapping class names to 710 L{ChordClass} objects, for all the classes that have been 711 read in for the grammar. 712 713 """

714 - def __init__(self, formalism, words, pos, optional_minor=None, 715 chord_class=None, mark_class=True):

716 self.formalism = formalism 717 self.words = words 718 self.pos = pos 719 self.optional_minor = optional_minor 720 self.chord_class = chord_class 721 if mark_class: 722 # Mark the chord class to indicate that it was used by a morph item 723 if chord_class is not None: 724 chord_class.used = True

725

726 - def __str__(self):

727 return "<MorphItem: %s>" % self.desc

728

729 - def __repr__(self):

730 return str(self)

731

732 - def __get_desc(self):

733 if self.chord_class is not None: 734 return "cls %s := %s" % (self.chord_class, self.pos) 735 else: 736 return "%s := %s" % \ 737 (",".join(["%s" % word for word in self.words]), self.pos)

738 desc = property(__get_desc) 739 740 @staticmethod

741 - def from_dom(formalism, element, classes, mark_class=True):

742 """ 743 Builds a MorphItem instance from a DOM XML specification. 744 745 """ 746 word_attr = element.attributes.getNamedItem("word") 747 if word_attr is not None: 748 words = [word_attr.value] 749 chord_class = None 750 else: 751 # Must be a class 752 class_attr = element.attributes.getNamedItem("class") 753 if class_attr is None: 754 raise GrammarReadError, "Morph item has no word or class attribute" 755 class_name = class_attr.value 756 if not class_name in classes: 757 raise GrammarReadError, "Referenced word class has not been defined" 758 759 # Get word list from the class dictionary 760 chord_class = classes[class_name] 761 words = copy.deepcopy(chord_class.words) 762 763 pos = element.attributes.getNamedItem("pos").value 764 # Optional minors in the category may be forced to be major or minor, 765 # or left ambiguous 766 optmin_element = element.attributes.getNamedItem("optional_minor") 767 if optmin_element is None: 768 # No constraint is imposed: don't change the categories at all 769 optional_minor = None 770 else: 771 optmin_val = optmin_element.value 772 if optmin_val == "minor": 773 # Force the optional minors to be minor 774 optional_minor = True 775 elif optmin_val == "major": 776 # Force the optional minors to be major 777 optional_minor = False 778 else: 779 # Not a valid value: don't impose a constraint 780 logger.warning("An invalid value was found for the optional_"\ 781 "minors attribute: %s. It must be \"major\" or "\ 782 "\"minor\"" % optmin_val) 783 optional_minor = None 784 return MorphItem(formalism, words, pos, optional_minor=optional_minor, \ 785 chord_class=chord_class, mark_class=mark_class)

786

787 788 -class MacroItem(object):

789 """ 790 Not yet implemented. Might be needed 791 """ 792 pass

793

794 795 -class EntriesItem(object):

796 - def __init__(self, formalism, name, sign, active=True, family=None):

797 self.formalism = formalism 798 self.family = family 799 self.name = name 800 self.active = active 801 self.sign = sign

802

803 - def copy(self):

804 return EntriesItem(self.formalism, 805 copy.copy(self.name), 806 self.sign.copy(), 807 active=self.active, 808 family=self.family)

809

810 - def __str__(self):

811 return "<Entry: %s>" % self.category

812

813 - def __repr__(self):

814 return str(self)

815

816 - def _get_tag_name(self):

817 """ The unique tag name used for this lexical entry. """ 818 if self.family is None: 819 return 820 elif len(self.family.entries) == 1: 821 return "%s" % self.family.name 822 else: 823 return "%s%s" % (self.family.name, self.name)

824 tag_name = property(_get_tag_name) 825 826 @property

827 - def category(self):

828 """ 829 Alias for C{sign} attribute, for backwards compatibility. 830 """ 831 return self.sign

832

833 - def get_lexical_sign(self, features, grammar):

834 """ 835 The sign stored in the EntriesItem is an abstraction of the lexical 836 signs, or a lexical schema, that may be instantiated. This method 837 produces a specialised lexical sign using the lexical information. 838 839 @type features: dict 840 @param features: lexical features to be passed directly to the sign's 841 C{apply_lexical_features} method. 842 843 """ 844 # Take a full copy of the prototypical sign 845 sign = self.sign.copy() 846 sign.tag = self.tag_name 847 # Use the sign's own method to specialise it 848 sign.apply_lexical_features(features) 849 return sign

850 851 @staticmethod

852 - def from_dom(formalism, element):

853 name = element.attributes.getNamedItem("name").value 854 active_el = element.attributes.getNamedItem("active") 855 if active_el is not None: 856 active = (active_el.value == "true") 857 else: 858 active = True 859 sign = formalism.lexicon_builder(element) 860 861 return EntriesItem(formalism, name, sign, active=active)

862

863 -class ChordClass(object):

864 """ 865 Representation of a chord class. Primarily, this links morph entries to 866 words. 867 868 """

869 - def __init__(self, name, words, notes=[]):

870 self.name = name 871 self.words = words 872 self.notes = notes 873 self.used = False

874 875 @staticmethod

876 - def from_dom(element):

877 # Read off the attributes of the chord class 878 name = element.attributes.getNamedItem("name").value 879 word_string = element.attributes.getNamedItem("words").value 880 words = word_string.split() 881 # The notes list is technically optional, so default to no notes 882 notes_el = element.attributes.getNamedItem("notes") 883 if notes_el is None: 884 notes = [] 885 else: 886 notes = [int(note) for note in notes_el.value.split()] 887 return ChordClass(name, words, notes=notes)

888

889 - def __str__(self):

890 return self.name

891

892 - def __repr__(self):

893 return "<ChordClass %s>" % self.name

894

895 -class EquivalenceMap(dict):

896 """ 897 A mapping from some of the morph entries to others via a root change. 898 899 These define entries that can be considered equivalent to another 900 entry given a change of root. These exist to deal with inversions 901 that show up in chord sequences as different chords. 902 903 E.g., a diminished chord may be written as if any of its notes 904 are the root, but this decision is made mainly on the basis of 905 what is easiest to read. Categories must be included to handle 906 each case for the chord grammar, but only one is required for a 907 model that can recognise inversions itself, such as a MIDI model. 908 909 Here we tell such models how to construct a smaller set of 910 categories, so that they can understand annotated chord data. 911 912 """

913 - def __init__(self, map={}):

914 self.update(map)

915 916 @staticmethod

917 - def from_dom(formalism, element, classes, morphs):

918 equiv_map = {} 919 # This should contain only <map> elements 920 for map_entry in element.getElementsByTagName("map"): 921 # There should be a "pos" attribute, giving the pos of the source 922 pos = map_entry.attributes.getNamedItem("pos").value 923 # This should contain exactly 1 equiv 924 equiv = get_single_element_by_tag_name(map_entry, "equiv") 925 926 if pos in equiv_map: 927 raise GrammarReadError, "two equivalence mappings for the same "\ 928 "morph entry: %s" % (source) 929 930 # Read the map target element 931 target = EquivalenceEntry.from_dom(formalism, equiv, classes, morphs) 932 933 # Store this mapping 934 equiv_map[pos] = target 935 return EquivalenceMap(equiv_map)

936

937 -class EquivalenceEntry(object):

938 """ 939 Like a L{MorphItem}. Represents the target in a mapping from one morph 940 entry to another. For now this only works with chord class entries, 941 not words. 942 943 """

944 - def __init__(self, target, root):

945 self.target = target 946 self.root = root

947

948 - def __str__(self):

949 return "<Equiv: %d, %s>" % (self.root, self.target.desc)

950

951 - def __repr__(self):

952 return str(self)

953 954 @staticmethod

955 - def from_dom(formalism, element, classes, morphs):

956 # First build a morph item, since this shares all of its attributes 957 morph = MorphItem.from_dom(formalism, element, classes, mark_class=False) 958 # Get the root specifier 959 root = int(element.attributes.getNamedItem("root").value) 960 961 # Check that a chord class was given 962 if morph.chord_class is None: 963 raise GrammarReadError, "equivalence mappings can only currently "\ 964 "map to morph entries that use chord classes" 965 966 # Look for a morph entry that matches this 967 for old_morph in morphs: 968 if old_morph.chord_class == morph.chord_class and \ 969 old_morph.pos == morph.pos: 970 target = old_morph 971 break 972 else: 973 # No matching morph entry was found 974 raise GrammarReadError, "could not find a morph entry to use as "\ 975 "equivalence target matching pos=%s and chord_class=%s" % \ 976 (morph.pos, morph.chord_class) 977 return EquivalenceEntry(target, root)

978

979 -def get_grammar_names():

980 """ Returns a list of all valid grammar names. """ 981 dirs = [d for d in os.listdir(settings.GRAMMAR_DATA_DIR) if not d.startswith(".")] 982 grammars = [] 983 # Work out which of these directories are valid(ish) grammars 984 for name in dirs: 985 dirname = os.path.abspath(os.path.join(settings.GRAMMAR_DATA_DIR, name)) 986 if not os.path.exists(dirname): 987 continue 988 if not os.path.isdir(dirname): 989 continue 990 if not os.path.exists(os.path.join(dirname, "grammar.xml")): 991 continue 992 grammars.append(name) 993 return grammars

994 995 996 _loaded_grammars = {}

997 998 -def get_grammar(name=None):

999 """ 1000 Returns an instance of L{Grammar} for the named grammar. 1001 1002 This is like instantiating L{Grammar} with the grammar with the name 1003 as an argument, but caches loaded grammars. If the named grammar has 1004 been previously loaded, the same instance will be returned again. 1005 1006 If you want to force a new instance, use C{Grammar(name)}. However, 1007 most of the time there's no need, since Grammar is essentially a read-only 1008 data structure. 1009 1010 """ 1011 if name is None: 1012 name = settings.DEFAULT_GRAMMAR 1013 if name not in _loaded_grammars: 1014 _loaded_grammars[name] = Grammar(name) 1015 return _loaded_grammars[name]

1016

1017 -class GrammarReadError(Exception):

1018 """ 1019 Thrown if there's a problem while reading the grammar description 1020 from the XML file. This will usually be because of missing elements 1021 or some such thing. 1022 """ 1023 pass

1024

1025 -class GrammarLookupError(Exception):

1026 """ 1027 Raised if there are problems consulting the grammar. 1028 """ 1029 pass

1030

Source Code for Module jazzparser.grammar