Package jazzparser :: Module parse
[hide private]
[frames] | no frames]

Source Code for Module jazzparser.parse

  1  """Main command-line interface to the Jazz Parser. 
  2   
  3  Takes input from the command line or a file and parses it. Allows  
  4  specification of formalism, parser and tagger modules. 
  5  See usage info for more details. 
  6   
  7  """ 
  8  """ 
  9  ============================== License ======================================== 
 10   Copyright (C) 2008, 2010-12 University of Edinburgh, Mark Granroth-Wilding 
 11    
 12   This file is part of The Jazz Parser. 
 13    
 14   The Jazz Parser is free software: you can redistribute it and/or modify 
 15   it under the terms of the GNU General Public License as published by 
 16   the Free Software Foundation, either version 3 of the License, or 
 17   (at your option) any later version. 
 18    
 19   The Jazz Parser is distributed in the hope that it will be useful, 
 20   but WITHOUT ANY WARRANTY; without even the implied warranty of 
 21   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 22   GNU General Public License for more details. 
 23    
 24   You should have received a copy of the GNU General Public License 
 25   along with The Jazz Parser.  If not, see <http://www.gnu.org/licenses/>. 
 26   
 27  ============================ End license ====================================== 
 28   
 29  """ 
 30  __author__ = "Mark Granroth-Wilding <mark.granroth-wilding@ed.ac.uk>"  
 31   
 32  from jazzparser.utils.tableprint import pprint_table, print_latex_table, \ 
 33                              format_table 
 34  from jazzparser.grammar import get_grammar, get_grammar_names 
 35  from jazzparser.data import Fraction, Chord 
 36  from jazzparser.data.input import DbInput, ChordInput, get_input_type, \ 
 37                              is_bulk_type, get_input_type_names, \ 
 38                              command_line_input, SegmentedMidiInput, \ 
 39                              DbBulkInput, SegmentedMidiBulkInput 
 40  from jazzparser.taggers.loader import get_tagger, get_default_tagger, \ 
 41                                  TaggerLoadError 
 42  from jazzparser.utils.base import ExecutionTimer, group_pairs, check_directory, \ 
 43                              exception_tuple 
 44  from jazzparser.utils.latex import filter_latex 
 45  from jazzparser.utils.options import ModuleOption, ModuleOptionError 
 46  from jazzparser.utils.loggers import init_logging, create_plain_stderr_logger, \ 
 47                              create_logger 
 48  from jazzparser.utils.tonalspace import coordinates_to_roman_names, add_z_coordinates 
 49  from jazzparser.utils.strings import slugify 
 50  from jazzparser.utils.data import partition 
 51  from jazzparser.utils.interface import input_iterator 
 52  from jazzparser.utils.system import set_proc_title 
 53  from jazzparser.parsers.loader import get_default_parser, get_parser, ParserLoadError 
 54  from jazzparser.formalisms import FORMALISMS 
 55  from jazzparser.utils.config import parse_args_with_config 
 56  from jazzparser.harmonical.tones import render_path_to_file as render_path_to_wave_file 
 57  from jazzparser.harmonical.midi.chords import render_path_to_file as render_path_to_midi_file 
 58  from jazzparser.harmonical.files import save_wave_data 
 59  from jazzparser.backoff.loader import get_backoff_builder, BackoffLoadError 
 60  from jazzparser.data.parsing import ParseResults 
 61   
 62  import sys, os, readline, copy 
 63  import logging, traceback 
 64  from optparse import OptionParser, OptionGroup 
 65  from itertools import count, imap 
 66  from multiprocessing import Pool 
 67   
 68  from jazzparser import settings 
 69   
 70  # Get the logger from the logging system 
 71  logger = logging.getLogger("main_logger") 
 72   
73 -def main():
74 set_proc_title("jazzparser") 75 ######################################################## 76 usage = "jazzparser [<options>]" 77 description = "The main parser interface for the Jazz Parser" 78 ## Process the input options 79 optparser = OptionParser(usage=usage, description=description) 80 ### 81 # File input options 82 group = OptionGroup(optparser, "Input", "Input type and location") 83 optparser.add_option_group(group) 84 group.add_option("--file", "-f", dest="file", action="store", help="use a file to get parser input from. Use --filetype to specify the type of the file.") 85 group.add_option("--filetype", "--ft", dest="filetype", action="store", help="select the file type for the input file (--file). Use '--filetype help' for a list of available types. Default: chords", default='chords') 86 group.add_option("--file-options", "--fopt", dest="file_options", action="store", help="options for the input file (--file). Type '--fopt help', using '--ft <type>' to select file type, for a list of available options.") 87 group.add_option("--index", "--indices", dest="input_index", action="store", help="select individual inputs to process. Specify as a comma-separated list of indices. All inputs are loaded as usual, but only the ith input is processed, for each i in the list") 88 group.add_option("--only-load", dest="only_load", action="store_true", help="don't do anything with the inputs, just load and list them. Handy for checking the inputs load and getting their indices") 89 group.add_option("--partitions", dest="partitions", action="store", type="int", help="divide the input data into this number of partitions and use a different set of models for each. For any parser, tagger and backoff that takes a 'model' argument, the partition number will be appended to the given value") 90 group.add_option("--seq-parts", "--sequence-partitions", dest="sequence_partitions", action="store", help="use a chord sequence index to partition the inputs. Input type (bulk) must support association of the inputs with chord sequences by id. Sequences in the given sequence index file are partitioned n ways (--partitions) and the inputs are processed according to their associated sequence.") 91 group.add_option("--continue", "--skip-done", dest="skip_done", action="store_true", help="skip any inputs for which a readable results file already exists. This is useful for continuing a bulk job that was stopped in the middle") 92 ### 93 group = OptionGroup(optparser, "Parser", "Parser, supertagger and backoff parser") 94 optparser.add_option_group(group) 95 group.add_option("-d", "--derivations", dest="derivations", action="store_true", help="keep derivation logs during parse.") 96 group.add_option("-g", "--grammar", dest="grammar", action="store", help="use the named grammar instead of the default.") 97 # Parser options 98 group.add_option("-p", "--parser", dest="parser", action="store", help="use the named parser algorithm instead of the default. Use '-p help' to see the list of available parsers. Default: %s" % settings.DEFAULT_PARSER, default=settings.DEFAULT_PARSER) 99 group.add_option("--popt", "--parser-options", dest="popts", action="append", help="specify options for the parser. Type '--popt help', using '--parser <name>' to select a parser module, to get a list of options.") 100 # Tagger options 101 group.add_option("-t", "--tagger", "--supertagger", dest="supertagger", action="store", help="run the parser using the named supertagger. Use '-t help' to see the list of available taggers. Default: %s" % settings.DEFAULT_SUPERTAGGER, default=settings.DEFAULT_SUPERTAGGER) 102 group.add_option("--topt", "--tagger-options", dest="topts", action="append", help="specify options for the tagger. Type '--topt help', using '-u <name>' to select a tagger module, to get a list of options.") 103 # Backoff options 104 group.add_option("-b", "--backoff", "--noparse", dest="backoff", action="store", help="use the named backoff model as a backoff if the parser produces no results") 105 group.add_option("--bopt", "--backoff-options", "--backoff-options", "--npo", dest="backoff_opts", action="append", help="specify options for the backoff model. Type '--npo help', using '--backoff <name>' to select a backoff modules, to get a list of options.") 106 ### 107 # Multiprocessing options 108 group = OptionGroup(optparser, "Multiprocessing") 109 optparser.add_option_group(group) 110 group.add_option("--processes", dest="processes", action="store", type="int", help="number of processes to create to perform parses in parallel. Default: 1, i.e. no process pool. Use -1 to create a process for every input", default=1) 111 ### 112 # Output options 113 group = OptionGroup(optparser, "Output") 114 optparser.add_option_group(group) 115 group.add_option("--output", dest="output", action="store", help="directory name to output parse results to. A filename specific to the individual input will be appended to this") 116 group.add_option("--topn", dest="topn", action="store", type="int", help="limit the number of final results to store in the output file to the top n by probability. By default, stores all") 117 group.add_option("--output-opts", "--oopts", dest="output_opts", action="store", help="options that affect the output formatting. Use '--output-opts help' for a list of options.") 118 group.add_option("-a", "--atomic-results", dest="atoms_only", action="store_true", help="only include atomic categories in the results.") 119 group.add_option("-l", "--latex", dest="latex", action="store_true", help="output all results as Latex source. Used to produce a whole Latex document, but doesn't any more") 120 group.add_option("--all-times", dest="all_times", action="store_true", help="display all timing information on semantics in output.") 121 group.add_option("-v", "--debug", dest="debug", action="store_true", help="output verbose debugging information.") 122 group.add_option("--time", dest="time", action="store_true", help="time how long the parse takes and output with the results.") 123 group.add_option("--no-results", dest="no_results", action="store_true", help="don't print out the parse results at the end. Obviously you'll want to make sure they're going to a file (--output). This is useful for bulk parse jobs, where the results produce a lot of unnecessary output") 124 group.add_option("--no-progress", dest="no_progress", action="store_true", help="don't output the summary of completed sequences after each one finishes") 125 ### 126 # Output analysis and harmonical 127 group = OptionGroup(optparser, "Output processing", "Output analysis and harmonical") 128 optparser.add_option_group(group) 129 group.add_option("--harmonical", dest="harmonical", action="store", help="use the harmonical to play the chords justly intoned according to the top result and output to a wave file.") 130 group.add_option("--enharmonical", dest="enharmonical", action="store", help="use the harmonical to play the chords in equal temperament and output to a wave file.") 131 group.add_option("--midi", dest="midi", action="store_true", help="generate MIDI files from the harmonical, instead of wave files.") 132 group.add_option("--tempo", dest="tempo", action="store", type=int, help="tempo to use for the generated music (see --harmonical/--enharmonical). Default: 120", default=120) 133 group.add_option("--lh-analysis", dest="lh_analysis", action="store_true", help="output the Longuet-Higgins space interpretation of the semantics for each result.") 134 group.add_option("--lh-coordinates", dest="lh_coord", action="store_true", help="like lh-analysis, but displays the coordinates of the points instead of their names.") 135 ### 136 # Logging options 137 group = OptionGroup(optparser, "Logging") 138 optparser.add_option_group(group) 139 group.add_option("--long-progress", dest="long_progress", action="store_true", help="print a summary of the chart so far after each chord/word has been processed.") 140 group.add_option("--progress", "--short-progress", dest="short_progress", action="store_true", help="print a small amount of information out during parsing to indicate progress.") 141 group.add_option("--logger", dest="logger", action="store", help="directory to put parser logging in. A filename based on an identifier for each individual input will be appended.") 142 ### 143 # Shell options 144 group = OptionGroup(optparser, "Shell", "Interactive shell for inspecting results and parser state") 145 optparser.add_option_group(group) 146 group.add_option("-i", "--interactive", dest="interactive", action="store_true", help="enter interactive mode after parsing.") 147 group.add_option("--error", dest="error_shell", action="store_true", help="catch any errors, report them and then enter the interactive shell. This also catches keyboard interrupts, so you can use it to halt parsing and enter the shell.") 148 149 # Read in command line options and args 150 options, clinput = parse_args_with_config(optparser) 151 152 ########################### Option processing #################### 153 154 # Get log level option first, so we can start using the logger 155 if options.debug: 156 log_level = logging.DEBUG 157 else: 158 log_level = logging.INFO 159 # Set up a logger 160 init_logging(log_level) 161 162 if options.latex: 163 settings.OPTIONS.OUTPUT_LATEX = True 164 165 if options.logger: 166 # Directory 167 parse_logger_dir = options.logger 168 check_directory(parse_logger_dir) 169 else: 170 parse_logger_dir = None 171 172 ######## Grammar ######## 173 # Check the grammar actually exists 174 grammar_names = get_grammar_names() 175 if options.grammar is not None and options.grammar not in grammar_names: 176 # This is not a valid grammar name 177 logger.error("The grammar '%s' does not exist. Possible "\ 178 "grammars are: %s." % (options.grammar, ", ".join(grammar_names))) 179 return 1 180 grammar = get_grammar(options.grammar) 181 182 ######## Parser ######## 183 # Load the requested parser 184 from jazzparser.parsers import PARSERS 185 if options.parser.lower() == "help": 186 print "Available parsers are: %s" % ", ".join(PARSERS) 187 return 0 188 try: 189 parser_cls = get_parser(options.parser) 190 except ParserLoadError: 191 logger.error("The parser '%s' could not be loaded. Possible "\ 192 "parsers are: %s" % (options.parser, ", ".join(PARSERS))) 193 return 1 194 195 # Get parser options 196 if options.popts is not None: 197 poptstr = options.popts 198 if "help" in [s.strip().lower() for s in poptstr]: 199 # Output this tagger's option help 200 from jazzparser.utils.options import options_help_text 201 print options_help_text(parser_cls.PARSER_OPTIONS, intro="Available options for selected parser") 202 return 0 203 poptstr = ":".join(poptstr) 204 else: 205 poptstr = "" 206 popts = ModuleOption.process_option_string(poptstr) 207 # Check that the options are valid 208 try: 209 parser_cls.check_options(popts) 210 except ModuleOptionError, err: 211 logger.error("Problem with parser options (--popt): %s" % err) 212 return 1 213 214 ######## Supertagger ######## 215 # Now load the supertagger requested 216 from jazzparser.taggers import TAGGERS 217 if options.supertagger.lower() == "help": 218 print "Available taggers are: %s" % ", ".join(TAGGERS) 219 return 0 220 try: 221 tagger_cls = get_tagger(options.supertagger) 222 except TaggerLoadError: 223 logger.error("The tagger '%s' could not be loaded. Possible "\ 224 "taggers are: %s" % (options.supertagger, ", ".join(TAGGERS))) 225 return 1 226 227 # Get supertagger options before initializing the tagger 228 if options.topts is not None: 229 toptstr = options.topts 230 if "help" in [s.strip().lower() for s in toptstr]: 231 # Output this tagger's option help 232 from jazzparser.utils.options import options_help_text 233 print options_help_text(tagger_cls.TAGGER_OPTIONS, intro="Available options for selected tagger") 234 return 0 235 toptstr = ":".join(toptstr) 236 else: 237 toptstr = "" 238 topts = ModuleOption.process_option_string(toptstr) 239 # Check that the options are valid 240 try: 241 tagger_cls.check_options(topts) 242 except ModuleOptionError, err: 243 logger.error("Problem with tagger options (--topt): %s" % err) 244 return 1 245 246 ######## Backoff ######## 247 # Load the requested backoff model, if any 248 if options.backoff is not None: 249 from jazzparser.backoff import BUILDERS 250 if options.backoff.lower() == "help": 251 print "Available backoff model types are: %s" % ", ".join(BUILDERS) 252 return 0 253 try: 254 backoff = get_backoff_builder(options.backoff) 255 except BackoffLoadError: 256 logger.error("The backoff model '%s' could not be loaded. Possible "\ 257 "models are: %s" % (options.backoff, ", ".join(BUILDERS))) 258 return 1 259 else: 260 backoff = None 261 262 # Get backoff options for initializing the backoff model 263 if options.backoff_opts is not None: 264 npoptstr = options.backoff_opts 265 if "help" in [s.strip().lower() for s in npoptstr]: 266 # Output this tagger's option help 267 from jazzparser.utils.options import options_help_text 268 print options_help_text(backoff.BUILDER_OPTIONS, intro="Available options for selected backoff module") 269 return 0 270 npoptstr = ":".join(npoptstr) 271 else: 272 npoptstr = "" 273 npopts = ModuleOption.process_option_string(npoptstr) 274 # Check that the options are valid 275 if backoff is not None: 276 try: 277 backoff.check_options(npopts) 278 except ModuleOptionError, err: 279 logger.error("Problem with backoff options (--backoff-options): %s" % err) 280 return 1 281 282 ######## Other misc options ######## 283 # Time the process and output timing info if requested 284 time_parse = options.time 285 286 # Display all time information on semantics in output 287 settings.OPTIONS.OUTPUT_ALL_TIMES = options.all_times 288 289 # Set output options according to the command line spec 290 grammar.formalism.cl_output_options(options.output_opts) 291 292 # Prepare output directory 293 if options.output is None: 294 output_dir = None 295 else: 296 # Check the output directory exists before starting 297 output_dir = os.path.abspath(options.output) 298 check_directory(output_dir, is_dir=True) 299 300 if options.partitions and options.partitions > 1: 301 partitions = options.partitions 302 else: 303 partitions = 1 304 305 ############################ Input processing ##################### 306 stdinput = False 307 # Try getting a file from the command-line options 308 input_data = command_line_input(filename=options.file, 309 filetype=options.filetype, 310 options=options.file_options) 311 # Record progress in this for helpful output 312 global completed_parses 313 partition_numbers = None 314 if input_data is None: 315 # No input file: process command line input 316 input_string = " ".join(clinput) 317 input_list = [input_string] 318 name_getter = iter(["commandline"]) 319 # Take input from stdin if nothing else is given 320 if len(input_string) == 0: 321 stdinput = True 322 # Use integers to identify each input 323 name_getter = count() 324 num_inputs = None 325 # Let the progress record be filled as we go 326 completed_parses = {} 327 else: 328 num_inputs = 1 329 completed_parses = {"commandline":False} 330 else: 331 # Input file was given 332 if is_bulk_type(type(input_data)): 333 # If this is a bulk filetype, we can just iterate over it 334 input_list = input_data 335 # Get the bulk input to supply names 336 name_getter = iter(input_data.get_identifiers()) 337 num_inputs = len(input_data) 338 # Fill the progress record with names and mark as incomplete 339 completed_parses = dict([(name,False) \ 340 for name in input_data.get_identifiers()]) 341 if partitions > 1: 342 if options.sequence_partitions is not None: 343 # Split the inputs up into partitions on the basis of 344 # an even partitioning of chord sequences 345 # This can only be done with 346 if not isinstance(input_data, SegmentedMidiBulkInput): 347 logger.error("option --sequence-partitions is only "\ 348 "valid with bulk midi input data") 349 return 1 350 chord_seqs = DbBulkInput.from_file(options.sequence_partitions) 351 # Partition the chord sequences: we only need indices 352 seq_indices = enumerate(partition( 353 [i for i in range(len(chord_seqs))], partitions)) 354 seq_partitions = dict( 355 sum([[(index,part_num) for index in part] for 356 (part_num,part) in seq_indices], []) ) 357 # Associate a partition num with each midi input 358 partition_numbers = [ 359 seq_partitions[midi.sequence_index] for midi in input_data] 360 else: 361 # Prepare a list of partition numbers to append to model names 362 partition_numbers = sum([ 363 [partnum for i in part] for (partnum,part) in \ 364 enumerate(partition(range(num_inputs), partitions))], []) 365 else: 366 # Otherwise, there's just one input 367 input_list = [input_data] 368 num_inputs = 1 369 # Try getting a name for this 370 if input_data.name is None: 371 name = "unnamed" 372 else: 373 name = input_data.name 374 name_getter = iter([name]) 375 completed_parses = {name:False} 376 377 select_indices = None 378 if options.input_index is not None: 379 indices = [] 380 try: 381 for selector in options.input_index.split(","): 382 if "-" in selector: 383 # This is a range 384 start,end = selector.split("-") 385 start,end = int(start),int(end) 386 indices.extend(range(start, end+1)) 387 else: 388 indices.append(int(selector)) 389 except ValueError: 390 logger.error("Could not parse index values: %s" % options.input_index) 391 return 1 392 if len(indices): 393 select_indices = indices 394 395 if stdinput: 396 input_getter = input_iterator("Enter chord sequence:\n# ") 397 print "No input string given: accepting input from stdin. Hit Ctrl+d to exit" 398 # Load the shell history if possible 399 try: 400 readline.read_history_file(settings.INPUT_PROMPT_HISTORY_FILE) 401 except IOError: 402 # No history file found. No problem 403 pass 404 else: 405 input_getter = iter(input_list) 406 407 if partitions > 1 and partition_numbers is None: 408 # We can only partition certain types of input 409 logger.error("Got partitions=%d, but can only partition bulk input "\ 410 "data" % (partitions)) 411 return 1 412 413 ############################ Process pool options ############## 414 if options.processes == 0 or options.processes < -1: 415 # Doesn't make sense 416 logger.error("Cannot create %d processes!" % options.processes) 417 sys.exit(1) 418 419 if select_indices is not None and len(select_indices) == 1 or options.only_load: 420 # Selecting a single input: don't use a pool 421 processes = 1 422 elif num_inputs is None: 423 # We don't know how many inputs there will be 424 if options.processes == -1: 425 logger.error("Could not create a process pool the size of the "\ 426 "input because we don't know the size of the input") 427 processes = 1 428 else: 429 # Just create the number we were asked to 430 processes = options.processes 431 else: 432 # We know the number of inputs 433 if select_indices is not None and len(select_indices) < options.processes: 434 # We're only processing a subset of the inputs 435 processes = len(select_indices) 436 elif options.processes > num_inputs or options.processes == -1: 437 # No point creating more processes than inputs 438 processes = num_inputs 439 else: 440 processes = options.processes 441 multiprocessing = (processes > 1) 442 443 ############# Parameter output ################ 444 # Display the important parameter settings 445 # This is useful for the user, because they see what settings they're 446 # using even if they're defaults 447 print >>sys.stderr, "===== The Jazz Parser =====" 448 print >>sys.stderr, "Supertagger: %s" % options.supertagger 449 print >>sys.stderr, "Supertagger options: %s" % toptstr 450 print >>sys.stderr, "Parser: %s" % options.parser 451 print >>sys.stderr, "Parser options: %s" % poptstr 452 print >>sys.stderr, "Backoff: %s" % (options.backoff or "") 453 print >>sys.stderr, "Backoff options: %s" % npoptstr 454 print >>sys.stderr, "Grammar: %s" % grammar.name 455 print >>sys.stderr, "Derivation traces: %s" % ("yes" if options.derivations else "no") 456 print >>sys.stderr, "Input file: %s" % options.file 457 print >>sys.stderr, "Input filetype: %s" % options.filetype 458 print >>sys.stderr, "Input options: %s" % (options.file_options or "") 459 print >>sys.stderr, "Input selection: %s" % \ 460 (",".join(str(i) for i in select_indices) \ 461 if select_indices else "all") 462 if multiprocessing: 463 print >>sys.stderr, "Process pool size: %d" % processes 464 else: 465 print >>sys.stderr, "No process pool" 466 print >>sys.stderr, "===========================\n" 467 if options.skip_done: 468 print >>sys.stderr, "Skipping any completed parses" 469 470 ###################### Process pool init ########### 471 if multiprocessing: 472 print >>sys.stderr, "Spawning %d worker processes" % processes 473 pool = Pool(processes=processes) 474 475 #################### Result callback ############### 476 def get_output_filename(identifier): 477 if output_dir is not None: 478 return os.path.join(output_dir, \ 479 "%s.res" % slugify(str(identifier)))
480 # Callback for getting results back 481 # This will be called when the parsing of a sequence is finished 482 def _result_callback(response): 483 if response is None: 484 # Empty input, or the subprocess doesn't want us to do anything 485 return 486 else: 487 # Mark this input as completed 488 global completed_parses 489 completed_parses[response['identifier']] = True 490 491 if response['results'] is None: 492 # There was some error: check what it was 493 error = response['error'] 494 print >> sys.stderr, "Error parsing %s" % str(response['input']) 495 print >> sys.stderr, "The error was:" 496 print >>sys.stderr, error[2] 497 global parse_exit_status 498 parse_exit_status = 1 499 else: 500 # Keep this together with all the other processes' responses 501 all_results.append(response) 502 print "Parsed: %s" % response['input'] 503 504 # Run any cleanup routines that the formalism defines 505 grammar.formalism.clean_results(response['results']) 506 507 # Remove complex results if atomic-only option has been set 508 if options.atoms_only: 509 response['results'] = remove_complex_categories(response['results'], grammar.formalism) 510 511 if not options.no_results: 512 print "Results:" 513 list_results(response['results']) 514 515 if output_dir is not None: 516 # Try getting a gold standard analysis if one has been 517 # associated with the input 518 gold = response['input'].get_gold_analysis() 519 520 # Get the results with their probabilities 521 top_results = [(getattr(res, 'probability', None), res) \ 522 for res in response['results']] 523 if options.topn is not None: 524 # Limit the results that get stored 525 top_results = list(reversed(sorted( 526 top_results)))[:options.topn] 527 # Output the results to a file 528 presults = ParseResults( 529 top_results, 530 signs=True, 531 gold_parse=gold, 532 timed_out=response['timed_out'], 533 cpu_time=response['time']) 534 filename = get_output_filename(response['identifier']) 535 presults.save(filename) 536 print "Parse results output to %s" % filename 537 538 if time_parse: 539 print "Parse took %f seconds" % response['time'] 540 541 if options.lh_analysis: 542 print >>sys.stderr, "\nLonguet-Higgins tonal space analysis for each result:" 543 # Output the tonal space path for each result 544 for i,result in enumerate(response['results']): 545 path = grammar.formalism.sign_to_coordinates(result) 546 coords,times = zip(*path) 547 print "%d> %s" % (i, ", ".join( 548 ["%s@%s" % (crd,time) for (crd,time) in 549 zip(coordinates_to_roman_names(coords),times)])) 550 551 if options.lh_coord: 552 print >>sys.stderr, "\nLonguet-Higgins tonal space coordinates for each result:" 553 # Output the tonal space path for each result 554 for i,result in enumerate(response['results']): 555 path = grammar.formalism.sign_to_coordinates(result) 556 print "%d> %s" % (i, ", ".join(["(%d,%d)@%s" % (x,y,t) for ((x,y),t) in path])) 557 558 # Print out any messages the parse routine sent to us 559 for message in response['messages']: 560 print message 561 562 # Print as summary of what we've completed 563 num_completed = len(filter(lambda x:x[1], completed_parses.items())) 564 if not stdinput: 565 if not options.no_progress: 566 print format_table([ 567 [str(ident), 568 "Complete" if completed_parses[ident] else ""] 569 for ident in sorted(completed_parses.keys())]) 570 if num_inputs is None: 571 print "\nCompleted %d parses" % num_completed 572 else: 573 print "\nCompleted %d/%d parses" % (num_completed, num_inputs) 574 575 # Enter interactive mode now if requested in options 576 # Don't do this is we're in a process pool 577 if not multiprocessing and options.interactive: 578 print 579 from jazzparser.shell import interactive_shell 580 env = {} 581 env.update(globals()) 582 env.update(locals()) 583 interactive_shell(response['results'], 584 options, 585 response['tagger'], 586 response['parser'], 587 grammar.formalism, 588 env, 589 input_data=response['input']) 590 print 591 # Flush the output to make sure everything gets out before we start the next one 592 sys.stderr.flush() 593 sys.stdout.flush() 594 #### End of _result_callback 595 596 597 ########################### Input loop #################### 598 # Process each input one by one 599 all_results = [] 600 jobs = [] 601 # This will get set to 1 if any errors are encountered during parsing 602 global parser_exit_status 603 parser_exit_status = 0 604 605 for input_index,input in enumerate(input_getter): 606 # Get an identifier for this input 607 input_identifier = name_getter.next() 608 609 if select_indices is not None: 610 # Check whether this is the input we're supposed to process 611 if input_index not in select_indices: 612 continue 613 else: 614 print >>sys.stderr, "Restricting to input %d" % input_index 615 616 if options.only_load: 617 # Just output that we'd process this input, but don't do anything 618 print "Input %d: %s" % (input_index,input_identifier) 619 continue 620 621 if options.skip_done: 622 # Skip any inputs for which a readable output file already exists 623 outfile = get_output_filename(input_identifier) 624 if os.path.exists(outfile): 625 # Try loading the output file 626 try: 627 old_res = ParseResults.from_file(outfile) 628 except ParseResults.LoadError, err: 629 pass 630 else: 631 # File loaded ok: don't process this input 632 # Mark it as complete 633 completed_parses[input_identifier] = True 634 continue 635 636 # Mark this as incomplete 637 completed_parses[input_identifier] = False 638 639 # Get a filename for a logger for this input 640 if parse_logger_dir: 641 parse_logger = os.path.join(parse_logger_dir, "%s.log" % \ 642 slugify(input_identifier)) 643 else: 644 parse_logger = None 645 646 # Create a new copy of the options dicts for this partition 647 input_topts = copy.deepcopy(topts) 648 input_popts = copy.deepcopy(popts) 649 input_npopts = copy.deepcopy(npopts) 650 651 if partitions > 1: 652 partition_num = partition_numbers[input_index] 653 # All ModelTagger, PcfgParser and BackoffBuilder 654 # subclasses take this option, so it's safe 655 # to assume that if we're giving a model name we can also give 656 # a partition number 657 if 'model' in input_topts and input_topts['model'] is not None: 658 input_topts['partition'] = partition_num 659 if 'model' in input_popts and input_popts['model'] is not None: 660 input_popts['partition'] = partition_num 661 if 'model' in input_npopts and input_npopts['model'] is not None: 662 input_npopts['partition'] = partition_num 663 664 if multiprocessing: 665 # Add a job to the process pool 666 jobs.append( 667 pool.apply_async(do_parse, \ 668 (grammar, tagger_cls, parser_cls, input, input_topts, 669 input_popts, backoff, input_npopts, options, 670 input_identifier), 671 { 'multiprocessing' : True, 672 'logfile' : parse_logger }, 673 _result_callback)) 674 else: 675 # Just run do_parse on this input 676 response = do_parse(grammar, tagger_cls, parser_cls, input, 677 input_topts, input_popts, backoff, input_npopts, options, 678 input_identifier, multiprocessing=False, 679 logfile=parse_logger) 680 _result_callback(response) 681 682 if multiprocessing: 683 # Block until all processes are done 684 try: 685 pool.close() 686 print >>sys.stderr, "Waiting for parse jobs to complete" 687 pool.join() 688 except KeyboardInterrupt: 689 # Subprocesses return on keyboard interrupt, so we should receive 690 # it here 691 print >>sys.stderr, "Exiting on keyboard interrupt" 692 sys.exit(1) 693 694 # Check that each process completed and display the errors if not 695 for job in jobs: 696 if not job.successful(): 697 try: 698 # Get the exception 699 job.get() 700 except Exception, err: 701 # Unfortunately, it's impossible to get any 702 # more info on where the error came from 703 print >>sys.stderr, "\nError in worker thread: %s" % err 704 parser_exit_status = 1 705 706 if stdinput: 707 print 708 # Write the history out to a file 709 readline.write_history_file(settings.INPUT_PROMPT_HISTORY_FILE) 710 sys.exit(parser_exit_status) 711 # End of main() function 712 713
714 -def do_parse(grammar, tagger_cls, parser_cls, input, topts, popts, backoff, 715 npopts, options, identifier, multiprocessing=False, 716 logfile=None, partition=None):
717 """ 718 Function called for each input to do tagging and parsing and return the 719 results. It's a separate function so that we can hand it over to worker 720 processes to do multiprocessing. 721 722 @type logfile: str 723 @param logfile: filename to send logging output to. If None, will log 724 to stderr 725 726 """ 727 # If the input's a string, preprocess it 728 if isinstance(input, str): 729 input = input.rstrip("\n") 730 if len(input) == 0: 731 return 732 input = ChordInput.from_string(input) 733 734 print "Processing input: %s (%s)" % (input, identifier) 735 736 if logfile is None: 737 # Sending logging output to stderr 738 logger = create_plain_stderr_logger() 739 else: 740 logger = create_logger(filename=logfile) 741 print "Logging parser progress to %s" % logfile 742 743 # Prepare an initial response 744 # We'll fill in some values of this later 745 response = { 746 'tagger' : None, 747 'parser' : None, 748 'input' : input, 749 'error' : None, 750 'messages' : [], 751 'time' : None, 752 'identifier' : identifier, 753 'results' : None, 754 'timed_out' : False, 755 } 756 tagger = None 757 parser = None 758 messages = [] 759 760 if options.short_progress: 761 # Only output the short form of the progress reports 762 progress = 2 763 elif options.long_progress: 764 progress = 1 765 else: 766 progress = 0 767 768 # Start a timer now to time the parse 769 timer = ExecutionTimer(clock=True) 770 771 # Catch any errors and continue to the next input, instead of giving up 772 try: 773 ######### Do that parsing thang 774 logger.info("Tagging sequence (%d timesteps)" % len(input)) 775 776 # Prepare a suitable tagger component 777 tagger = tagger_cls(grammar, input, options=topts.copy(), logger=logger) 778 if not multiprocessing: 779 response['tagger'] = tagger 780 781 # Create a parser using this tagger 782 parser = parser_cls(grammar, tagger, options=popts.copy(), 783 backoff=backoff, 784 backoff_options=npopts.copy(), 785 logger=logger) 786 if not multiprocessing: 787 response['parser'] = parser 788 try: 789 # Parse to produce a list of results 790 results = parser.parse(derivations=options.derivations, summaries=progress) 791 except (KeyboardInterrupt, Exception), err: 792 if multiprocessing: 793 # Don't go interactive if we're in a subprocess 794 # Instead, just return with an error 795 response.update({ 796 'error' : exception_tuple(str_tb=True), 797 }) 798 return response 799 else: 800 # Drop into the shell 801 if type(err) == KeyboardInterrupt: 802 print "Dropping out on keyboard interrupt" 803 print "Entering shell: use 'chart' command to see current state of parse" 804 elif options.error_shell: 805 print >> sys.stderr, "Error parsing %s" % str(input) 806 print >> sys.stderr, "The error was:" 807 traceback.print_exc(file=sys.stderr) 808 # If we keyboard interrupted, always go into the shell, so 809 # the user can see how far we got 810 if options.error_shell or type(err) == KeyboardInterrupt: 811 # Instead of exiting, enter the interactive shell 812 print 813 from jazzparser.shell import interactive_shell 814 env = {} 815 env.update(globals()) 816 env.update(locals()) 817 interactive_shell(parser.chart.parses,options,tagger,parser, 818 grammar.formalism,env,input_data=input) 819 return 820 else: 821 raise 822 except (KeyboardInterrupt, Exception), err: 823 if multiprocessing: 824 response.update({ 825 'error' : exception_tuple(str_tb=True), 826 }) 827 return response 828 else: 829 if type(err) == KeyboardInterrupt: 830 print "Exiting on keyboard interrupt" 831 sys.exit(1) 832 else: 833 response.update({ 834 'error' : exception_tuple(str_tb=True), 835 'messages' : messages, 836 'time' : timer.get_time(), 837 }) 838 return response 839 else: 840 # Parsed successfully 841 # Do some postprocessing and return to the main function 842 843 # Output audio files from the harmonical 844 if (options.harmonical is not None or \ 845 options.enharmonical is not None) and len(results) > 0: 846 path = grammar.formalism.sign_to_coordinates(results[0]) 847 # Assuming we used a temporal formalism, the times should be 848 # available as a list from the semantics 849 times = results[0].semantics.get_path_times() 850 point_durations = [next-current for current,next in group_pairs(times)] + [0] 851 # Get 3d coordinates as well 852 path3d = zip(add_z_coordinates(path, pitch_range=2), point_durations) 853 path2d = zip(path,point_durations) 854 # Get chord types out of the input 855 chords = tagger.get_string_input() 856 chord_durs = [tagger.get_word_duration(i) for i in range(tagger.input_length)] 857 chord_types = [(Chord.from_name(c).type,dur) for c,dur in zip(chords,chord_durs)] 858 859 if options.midi: 860 # Maybe set this as a CL option or a setting 861 # 73 - flute 862 # 0 - piano 863 # 4 - e-piano 864 instrument = 73 865 # TODO: make these filenames different for multiple inputs 866 if options.harmonical is not None: 867 filename = os.path.abspath(options.harmonical) 868 render_path_to_midi_file(filename, path3d, chord_types=chord_types, tempo=options.tempo, instrument=instrument, bass_root=True, root_octave=-1) 869 messages.append("Output JI MIDI data to %s" % filename) 870 if options.enharmonical is not None: 871 filename = os.path.abspath(options.enharmonical) 872 render_path_to_midi_file(filename, path3d, chord_types=chord_types, equal_temperament=True, tempo=options.tempo, instrument=instrument, bass_root=True, root_octave=-1) 873 messages.append("Output ET MIDI data to %s" % filename) 874 else: 875 if options.harmonical is not None: 876 filename = os.path.abspath(options.harmonical) 877 render_path_to_wave_file(filename, path2d, chord_types=chord_types, double_root=True, tempo=options.tempo) 878 messages.append("Output JI wave data to %s" % filename) 879 if options.enharmonical is not None: 880 filename = os.path.abspath(options.enharmonical) 881 render_path_to_wave_file(filename, path2d, chord_types=chord_types, double_root=True, equal_temperament=True, tempo=options.tempo) 882 messages.append("Output ET wave data to %s" % filename) 883 884 response.update({ 885 'results' : results, 886 'time' : timer.get_time(), 887 'messages' : messages, 888 'timed_out' : parser.timed_out, 889 }) 890 return response
891 892
893 -def list_results(results):
894 """ 895 Prints out a list of the results in the given results list. 896 This is used after parsing and during interactive results viewing. 897 """ 898 # Print out what we got 899 if len(results) == 0: 900 if settings.OPTIONS.OUTPUT_LATEX: 901 print "\\textit{No results}\n" 902 else: 903 print "No results" 904 else: 905 # Print the results with numbers and pretty arrows otherwise 906 if settings.OPTIONS.OUTPUT_LATEX: 907 print "\\begin{enumerate}" 908 for i in range(len(results)): 909 print "\\item %s" % (filter_latex(results[i].format_latex_result())) 910 print "\\end{enumerate}" 911 else: 912 for i in range(len(results)): 913 print "%d> %s" % (i, results[i].format_result())
914
915 -def remove_complex_categories(result_list, formalism):
916 # Filter out results that aren't atomic categories 917 return [sign for sign in result_list if type(sign.category) == formalism.Syntax.AtomicCategory]
918 919 if __name__ == "__main__": 920 sys.exit(main()) 921