1 """Main command-line interface to the Jazz Parser.
2
3 Takes input from the command line or a file and parses it. Allows
4 specification of formalism, parser and tagger modules.
5 See usage info for more details.
6
7 """
8 """
9 ============================== License ========================================
10 Copyright (C) 2008, 2010-12 University of Edinburgh, Mark Granroth-Wilding
11
12 This file is part of The Jazz Parser.
13
14 The Jazz Parser is free software: you can redistribute it and/or modify
15 it under the terms of the GNU General Public License as published by
16 the Free Software Foundation, either version 3 of the License, or
17 (at your option) any later version.
18
19 The Jazz Parser is distributed in the hope that it will be useful,
20 but WITHOUT ANY WARRANTY; without even the implied warranty of
21 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 GNU General Public License for more details.
23
24 You should have received a copy of the GNU General Public License
25 along with The Jazz Parser. If not, see <http://www.gnu.org/licenses/>.
26
27 ============================ End license ======================================
28
29 """
30 __author__ = "Mark Granroth-Wilding <mark.granroth-wilding@ed.ac.uk>"
31
32 from jazzparser.utils.tableprint import pprint_table, print_latex_table, \
33 format_table
34 from jazzparser.grammar import get_grammar, get_grammar_names
35 from jazzparser.data import Fraction, Chord
36 from jazzparser.data.input import DbInput, ChordInput, get_input_type, \
37 is_bulk_type, get_input_type_names, \
38 command_line_input, SegmentedMidiInput, \
39 DbBulkInput, SegmentedMidiBulkInput
40 from jazzparser.taggers.loader import get_tagger, get_default_tagger, \
41 TaggerLoadError
42 from jazzparser.utils.base import ExecutionTimer, group_pairs, check_directory, \
43 exception_tuple
44 from jazzparser.utils.latex import filter_latex
45 from jazzparser.utils.options import ModuleOption, ModuleOptionError
46 from jazzparser.utils.loggers import init_logging, create_plain_stderr_logger, \
47 create_logger
48 from jazzparser.utils.tonalspace import coordinates_to_roman_names, add_z_coordinates
49 from jazzparser.utils.strings import slugify
50 from jazzparser.utils.data import partition
51 from jazzparser.utils.interface import input_iterator
52 from jazzparser.utils.system import set_proc_title
53 from jazzparser.parsers.loader import get_default_parser, get_parser, ParserLoadError
54 from jazzparser.formalisms import FORMALISMS
55 from jazzparser.utils.config import parse_args_with_config
56 from jazzparser.harmonical.tones import render_path_to_file as render_path_to_wave_file
57 from jazzparser.harmonical.midi.chords import render_path_to_file as render_path_to_midi_file
58 from jazzparser.harmonical.files import save_wave_data
59 from jazzparser.backoff.loader import get_backoff_builder, BackoffLoadError
60 from jazzparser.data.parsing import ParseResults
61
62 import sys, os, readline, copy
63 import logging, traceback
64 from optparse import OptionParser, OptionGroup
65 from itertools import count, imap
66 from multiprocessing import Pool
67
68 from jazzparser import settings
69
70
71 logger = logging.getLogger("main_logger")
72
74 set_proc_title("jazzparser")
75
76 usage = "jazzparser [<options>]"
77 description = "The main parser interface for the Jazz Parser"
78
79 optparser = OptionParser(usage=usage, description=description)
80
81
82 group = OptionGroup(optparser, "Input", "Input type and location")
83 optparser.add_option_group(group)
84 group.add_option("--file", "-f", dest="file", action="store", help="use a file to get parser input from. Use --filetype to specify the type of the file.")
85 group.add_option("--filetype", "--ft", dest="filetype", action="store", help="select the file type for the input file (--file). Use '--filetype help' for a list of available types. Default: chords", default='chords')
86 group.add_option("--file-options", "--fopt", dest="file_options", action="store", help="options for the input file (--file). Type '--fopt help', using '--ft <type>' to select file type, for a list of available options.")
87 group.add_option("--index", "--indices", dest="input_index", action="store", help="select individual inputs to process. Specify as a comma-separated list of indices. All inputs are loaded as usual, but only the ith input is processed, for each i in the list")
88 group.add_option("--only-load", dest="only_load", action="store_true", help="don't do anything with the inputs, just load and list them. Handy for checking the inputs load and getting their indices")
89 group.add_option("--partitions", dest="partitions", action="store", type="int", help="divide the input data into this number of partitions and use a different set of models for each. For any parser, tagger and backoff that takes a 'model' argument, the partition number will be appended to the given value")
90 group.add_option("--seq-parts", "--sequence-partitions", dest="sequence_partitions", action="store", help="use a chord sequence index to partition the inputs. Input type (bulk) must support association of the inputs with chord sequences by id. Sequences in the given sequence index file are partitioned n ways (--partitions) and the inputs are processed according to their associated sequence.")
91 group.add_option("--continue", "--skip-done", dest="skip_done", action="store_true", help="skip any inputs for which a readable results file already exists. This is useful for continuing a bulk job that was stopped in the middle")
92
93 group = OptionGroup(optparser, "Parser", "Parser, supertagger and backoff parser")
94 optparser.add_option_group(group)
95 group.add_option("-d", "--derivations", dest="derivations", action="store_true", help="keep derivation logs during parse.")
96 group.add_option("-g", "--grammar", dest="grammar", action="store", help="use the named grammar instead of the default.")
97
98 group.add_option("-p", "--parser", dest="parser", action="store", help="use the named parser algorithm instead of the default. Use '-p help' to see the list of available parsers. Default: %s" % settings.DEFAULT_PARSER, default=settings.DEFAULT_PARSER)
99 group.add_option("--popt", "--parser-options", dest="popts", action="append", help="specify options for the parser. Type '--popt help', using '--parser <name>' to select a parser module, to get a list of options.")
100
101 group.add_option("-t", "--tagger", "--supertagger", dest="supertagger", action="store", help="run the parser using the named supertagger. Use '-t help' to see the list of available taggers. Default: %s" % settings.DEFAULT_SUPERTAGGER, default=settings.DEFAULT_SUPERTAGGER)
102 group.add_option("--topt", "--tagger-options", dest="topts", action="append", help="specify options for the tagger. Type '--topt help', using '-u <name>' to select a tagger module, to get a list of options.")
103
104 group.add_option("-b", "--backoff", "--noparse", dest="backoff", action="store", help="use the named backoff model as a backoff if the parser produces no results")
105 group.add_option("--bopt", "--backoff-options", "--backoff-options", "--npo", dest="backoff_opts", action="append", help="specify options for the backoff model. Type '--npo help', using '--backoff <name>' to select a backoff modules, to get a list of options.")
106
107
108 group = OptionGroup(optparser, "Multiprocessing")
109 optparser.add_option_group(group)
110 group.add_option("--processes", dest="processes", action="store", type="int", help="number of processes to create to perform parses in parallel. Default: 1, i.e. no process pool. Use -1 to create a process for every input", default=1)
111
112
113 group = OptionGroup(optparser, "Output")
114 optparser.add_option_group(group)
115 group.add_option("--output", dest="output", action="store", help="directory name to output parse results to. A filename specific to the individual input will be appended to this")
116 group.add_option("--topn", dest="topn", action="store", type="int", help="limit the number of final results to store in the output file to the top n by probability. By default, stores all")
117 group.add_option("--output-opts", "--oopts", dest="output_opts", action="store", help="options that affect the output formatting. Use '--output-opts help' for a list of options.")
118 group.add_option("-a", "--atomic-results", dest="atoms_only", action="store_true", help="only include atomic categories in the results.")
119 group.add_option("-l", "--latex", dest="latex", action="store_true", help="output all results as Latex source. Used to produce a whole Latex document, but doesn't any more")
120 group.add_option("--all-times", dest="all_times", action="store_true", help="display all timing information on semantics in output.")
121 group.add_option("-v", "--debug", dest="debug", action="store_true", help="output verbose debugging information.")
122 group.add_option("--time", dest="time", action="store_true", help="time how long the parse takes and output with the results.")
123 group.add_option("--no-results", dest="no_results", action="store_true", help="don't print out the parse results at the end. Obviously you'll want to make sure they're going to a file (--output). This is useful for bulk parse jobs, where the results produce a lot of unnecessary output")
124 group.add_option("--no-progress", dest="no_progress", action="store_true", help="don't output the summary of completed sequences after each one finishes")
125
126
127 group = OptionGroup(optparser, "Output processing", "Output analysis and harmonical")
128 optparser.add_option_group(group)
129 group.add_option("--harmonical", dest="harmonical", action="store", help="use the harmonical to play the chords justly intoned according to the top result and output to a wave file.")
130 group.add_option("--enharmonical", dest="enharmonical", action="store", help="use the harmonical to play the chords in equal temperament and output to a wave file.")
131 group.add_option("--midi", dest="midi", action="store_true", help="generate MIDI files from the harmonical, instead of wave files.")
132 group.add_option("--tempo", dest="tempo", action="store", type=int, help="tempo to use for the generated music (see --harmonical/--enharmonical). Default: 120", default=120)
133 group.add_option("--lh-analysis", dest="lh_analysis", action="store_true", help="output the Longuet-Higgins space interpretation of the semantics for each result.")
134 group.add_option("--lh-coordinates", dest="lh_coord", action="store_true", help="like lh-analysis, but displays the coordinates of the points instead of their names.")
135
136
137 group = OptionGroup(optparser, "Logging")
138 optparser.add_option_group(group)
139 group.add_option("--long-progress", dest="long_progress", action="store_true", help="print a summary of the chart so far after each chord/word has been processed.")
140 group.add_option("--progress", "--short-progress", dest="short_progress", action="store_true", help="print a small amount of information out during parsing to indicate progress.")
141 group.add_option("--logger", dest="logger", action="store", help="directory to put parser logging in. A filename based on an identifier for each individual input will be appended.")
142
143
144 group = OptionGroup(optparser, "Shell", "Interactive shell for inspecting results and parser state")
145 optparser.add_option_group(group)
146 group.add_option("-i", "--interactive", dest="interactive", action="store_true", help="enter interactive mode after parsing.")
147 group.add_option("--error", dest="error_shell", action="store_true", help="catch any errors, report them and then enter the interactive shell. This also catches keyboard interrupts, so you can use it to halt parsing and enter the shell.")
148
149
150 options, clinput = parse_args_with_config(optparser)
151
152
153
154
155 if options.debug:
156 log_level = logging.DEBUG
157 else:
158 log_level = logging.INFO
159
160 init_logging(log_level)
161
162 if options.latex:
163 settings.OPTIONS.OUTPUT_LATEX = True
164
165 if options.logger:
166
167 parse_logger_dir = options.logger
168 check_directory(parse_logger_dir)
169 else:
170 parse_logger_dir = None
171
172
173
174 grammar_names = get_grammar_names()
175 if options.grammar is not None and options.grammar not in grammar_names:
176
177 logger.error("The grammar '%s' does not exist. Possible "\
178 "grammars are: %s." % (options.grammar, ", ".join(grammar_names)))
179 return 1
180 grammar = get_grammar(options.grammar)
181
182
183
184 from jazzparser.parsers import PARSERS
185 if options.parser.lower() == "help":
186 print "Available parsers are: %s" % ", ".join(PARSERS)
187 return 0
188 try:
189 parser_cls = get_parser(options.parser)
190 except ParserLoadError:
191 logger.error("The parser '%s' could not be loaded. Possible "\
192 "parsers are: %s" % (options.parser, ", ".join(PARSERS)))
193 return 1
194
195
196 if options.popts is not None:
197 poptstr = options.popts
198 if "help" in [s.strip().lower() for s in poptstr]:
199
200 from jazzparser.utils.options import options_help_text
201 print options_help_text(parser_cls.PARSER_OPTIONS, intro="Available options for selected parser")
202 return 0
203 poptstr = ":".join(poptstr)
204 else:
205 poptstr = ""
206 popts = ModuleOption.process_option_string(poptstr)
207
208 try:
209 parser_cls.check_options(popts)
210 except ModuleOptionError, err:
211 logger.error("Problem with parser options (--popt): %s" % err)
212 return 1
213
214
215
216 from jazzparser.taggers import TAGGERS
217 if options.supertagger.lower() == "help":
218 print "Available taggers are: %s" % ", ".join(TAGGERS)
219 return 0
220 try:
221 tagger_cls = get_tagger(options.supertagger)
222 except TaggerLoadError:
223 logger.error("The tagger '%s' could not be loaded. Possible "\
224 "taggers are: %s" % (options.supertagger, ", ".join(TAGGERS)))
225 return 1
226
227
228 if options.topts is not None:
229 toptstr = options.topts
230 if "help" in [s.strip().lower() for s in toptstr]:
231
232 from jazzparser.utils.options import options_help_text
233 print options_help_text(tagger_cls.TAGGER_OPTIONS, intro="Available options for selected tagger")
234 return 0
235 toptstr = ":".join(toptstr)
236 else:
237 toptstr = ""
238 topts = ModuleOption.process_option_string(toptstr)
239
240 try:
241 tagger_cls.check_options(topts)
242 except ModuleOptionError, err:
243 logger.error("Problem with tagger options (--topt): %s" % err)
244 return 1
245
246
247
248 if options.backoff is not None:
249 from jazzparser.backoff import BUILDERS
250 if options.backoff.lower() == "help":
251 print "Available backoff model types are: %s" % ", ".join(BUILDERS)
252 return 0
253 try:
254 backoff = get_backoff_builder(options.backoff)
255 except BackoffLoadError:
256 logger.error("The backoff model '%s' could not be loaded. Possible "\
257 "models are: %s" % (options.backoff, ", ".join(BUILDERS)))
258 return 1
259 else:
260 backoff = None
261
262
263 if options.backoff_opts is not None:
264 npoptstr = options.backoff_opts
265 if "help" in [s.strip().lower() for s in npoptstr]:
266
267 from jazzparser.utils.options import options_help_text
268 print options_help_text(backoff.BUILDER_OPTIONS, intro="Available options for selected backoff module")
269 return 0
270 npoptstr = ":".join(npoptstr)
271 else:
272 npoptstr = ""
273 npopts = ModuleOption.process_option_string(npoptstr)
274
275 if backoff is not None:
276 try:
277 backoff.check_options(npopts)
278 except ModuleOptionError, err:
279 logger.error("Problem with backoff options (--backoff-options): %s" % err)
280 return 1
281
282
283
284 time_parse = options.time
285
286
287 settings.OPTIONS.OUTPUT_ALL_TIMES = options.all_times
288
289
290 grammar.formalism.cl_output_options(options.output_opts)
291
292
293 if options.output is None:
294 output_dir = None
295 else:
296
297 output_dir = os.path.abspath(options.output)
298 check_directory(output_dir, is_dir=True)
299
300 if options.partitions and options.partitions > 1:
301 partitions = options.partitions
302 else:
303 partitions = 1
304
305
306 stdinput = False
307
308 input_data = command_line_input(filename=options.file,
309 filetype=options.filetype,
310 options=options.file_options)
311
312 global completed_parses
313 partition_numbers = None
314 if input_data is None:
315
316 input_string = " ".join(clinput)
317 input_list = [input_string]
318 name_getter = iter(["commandline"])
319
320 if len(input_string) == 0:
321 stdinput = True
322
323 name_getter = count()
324 num_inputs = None
325
326 completed_parses = {}
327 else:
328 num_inputs = 1
329 completed_parses = {"commandline":False}
330 else:
331
332 if is_bulk_type(type(input_data)):
333
334 input_list = input_data
335
336 name_getter = iter(input_data.get_identifiers())
337 num_inputs = len(input_data)
338
339 completed_parses = dict([(name,False) \
340 for name in input_data.get_identifiers()])
341 if partitions > 1:
342 if options.sequence_partitions is not None:
343
344
345
346 if not isinstance(input_data, SegmentedMidiBulkInput):
347 logger.error("option --sequence-partitions is only "\
348 "valid with bulk midi input data")
349 return 1
350 chord_seqs = DbBulkInput.from_file(options.sequence_partitions)
351
352 seq_indices = enumerate(partition(
353 [i for i in range(len(chord_seqs))], partitions))
354 seq_partitions = dict(
355 sum([[(index,part_num) for index in part] for
356 (part_num,part) in seq_indices], []) )
357
358 partition_numbers = [
359 seq_partitions[midi.sequence_index] for midi in input_data]
360 else:
361
362 partition_numbers = sum([
363 [partnum for i in part] for (partnum,part) in \
364 enumerate(partition(range(num_inputs), partitions))], [])
365 else:
366
367 input_list = [input_data]
368 num_inputs = 1
369
370 if input_data.name is None:
371 name = "unnamed"
372 else:
373 name = input_data.name
374 name_getter = iter([name])
375 completed_parses = {name:False}
376
377 select_indices = None
378 if options.input_index is not None:
379 indices = []
380 try:
381 for selector in options.input_index.split(","):
382 if "-" in selector:
383
384 start,end = selector.split("-")
385 start,end = int(start),int(end)
386 indices.extend(range(start, end+1))
387 else:
388 indices.append(int(selector))
389 except ValueError:
390 logger.error("Could not parse index values: %s" % options.input_index)
391 return 1
392 if len(indices):
393 select_indices = indices
394
395 if stdinput:
396 input_getter = input_iterator("Enter chord sequence:\n# ")
397 print "No input string given: accepting input from stdin. Hit Ctrl+d to exit"
398
399 try:
400 readline.read_history_file(settings.INPUT_PROMPT_HISTORY_FILE)
401 except IOError:
402
403 pass
404 else:
405 input_getter = iter(input_list)
406
407 if partitions > 1 and partition_numbers is None:
408
409 logger.error("Got partitions=%d, but can only partition bulk input "\
410 "data" % (partitions))
411 return 1
412
413
414 if options.processes == 0 or options.processes < -1:
415
416 logger.error("Cannot create %d processes!" % options.processes)
417 sys.exit(1)
418
419 if select_indices is not None and len(select_indices) == 1 or options.only_load:
420
421 processes = 1
422 elif num_inputs is None:
423
424 if options.processes == -1:
425 logger.error("Could not create a process pool the size of the "\
426 "input because we don't know the size of the input")
427 processes = 1
428 else:
429
430 processes = options.processes
431 else:
432
433 if select_indices is not None and len(select_indices) < options.processes:
434
435 processes = len(select_indices)
436 elif options.processes > num_inputs or options.processes == -1:
437
438 processes = num_inputs
439 else:
440 processes = options.processes
441 multiprocessing = (processes > 1)
442
443
444
445
446
447 print >>sys.stderr, "===== The Jazz Parser ====="
448 print >>sys.stderr, "Supertagger: %s" % options.supertagger
449 print >>sys.stderr, "Supertagger options: %s" % toptstr
450 print >>sys.stderr, "Parser: %s" % options.parser
451 print >>sys.stderr, "Parser options: %s" % poptstr
452 print >>sys.stderr, "Backoff: %s" % (options.backoff or "")
453 print >>sys.stderr, "Backoff options: %s" % npoptstr
454 print >>sys.stderr, "Grammar: %s" % grammar.name
455 print >>sys.stderr, "Derivation traces: %s" % ("yes" if options.derivations else "no")
456 print >>sys.stderr, "Input file: %s" % options.file
457 print >>sys.stderr, "Input filetype: %s" % options.filetype
458 print >>sys.stderr, "Input options: %s" % (options.file_options or "")
459 print >>sys.stderr, "Input selection: %s" % \
460 (",".join(str(i) for i in select_indices) \
461 if select_indices else "all")
462 if multiprocessing:
463 print >>sys.stderr, "Process pool size: %d" % processes
464 else:
465 print >>sys.stderr, "No process pool"
466 print >>sys.stderr, "===========================\n"
467 if options.skip_done:
468 print >>sys.stderr, "Skipping any completed parses"
469
470
471 if multiprocessing:
472 print >>sys.stderr, "Spawning %d worker processes" % processes
473 pool = Pool(processes=processes)
474
475
476 def get_output_filename(identifier):
477 if output_dir is not None:
478 return os.path.join(output_dir, \
479 "%s.res" % slugify(str(identifier)))
480
481
482 def _result_callback(response):
483 if response is None:
484
485 return
486 else:
487
488 global completed_parses
489 completed_parses[response['identifier']] = True
490
491 if response['results'] is None:
492
493 error = response['error']
494 print >> sys.stderr, "Error parsing %s" % str(response['input'])
495 print >> sys.stderr, "The error was:"
496 print >>sys.stderr, error[2]
497 global parse_exit_status
498 parse_exit_status = 1
499 else:
500
501 all_results.append(response)
502 print "Parsed: %s" % response['input']
503
504
505 grammar.formalism.clean_results(response['results'])
506
507
508 if options.atoms_only:
509 response['results'] = remove_complex_categories(response['results'], grammar.formalism)
510
511 if not options.no_results:
512 print "Results:"
513 list_results(response['results'])
514
515 if output_dir is not None:
516
517
518 gold = response['input'].get_gold_analysis()
519
520
521 top_results = [(getattr(res, 'probability', None), res) \
522 for res in response['results']]
523 if options.topn is not None:
524
525 top_results = list(reversed(sorted(
526 top_results)))[:options.topn]
527
528 presults = ParseResults(
529 top_results,
530 signs=True,
531 gold_parse=gold,
532 timed_out=response['timed_out'],
533 cpu_time=response['time'])
534 filename = get_output_filename(response['identifier'])
535 presults.save(filename)
536 print "Parse results output to %s" % filename
537
538 if time_parse:
539 print "Parse took %f seconds" % response['time']
540
541 if options.lh_analysis:
542 print >>sys.stderr, "\nLonguet-Higgins tonal space analysis for each result:"
543
544 for i,result in enumerate(response['results']):
545 path = grammar.formalism.sign_to_coordinates(result)
546 coords,times = zip(*path)
547 print "%d> %s" % (i, ", ".join(
548 ["%s@%s" % (crd,time) for (crd,time) in
549 zip(coordinates_to_roman_names(coords),times)]))
550
551 if options.lh_coord:
552 print >>sys.stderr, "\nLonguet-Higgins tonal space coordinates for each result:"
553
554 for i,result in enumerate(response['results']):
555 path = grammar.formalism.sign_to_coordinates(result)
556 print "%d> %s" % (i, ", ".join(["(%d,%d)@%s" % (x,y,t) for ((x,y),t) in path]))
557
558
559 for message in response['messages']:
560 print message
561
562
563 num_completed = len(filter(lambda x:x[1], completed_parses.items()))
564 if not stdinput:
565 if not options.no_progress:
566 print format_table([
567 [str(ident),
568 "Complete" if completed_parses[ident] else ""]
569 for ident in sorted(completed_parses.keys())])
570 if num_inputs is None:
571 print "\nCompleted %d parses" % num_completed
572 else:
573 print "\nCompleted %d/%d parses" % (num_completed, num_inputs)
574
575
576
577 if not multiprocessing and options.interactive:
578 print
579 from jazzparser.shell import interactive_shell
580 env = {}
581 env.update(globals())
582 env.update(locals())
583 interactive_shell(response['results'],
584 options,
585 response['tagger'],
586 response['parser'],
587 grammar.formalism,
588 env,
589 input_data=response['input'])
590 print
591
592 sys.stderr.flush()
593 sys.stdout.flush()
594
595
596
597
598
599 all_results = []
600 jobs = []
601
602 global parser_exit_status
603 parser_exit_status = 0
604
605 for input_index,input in enumerate(input_getter):
606
607 input_identifier = name_getter.next()
608
609 if select_indices is not None:
610
611 if input_index not in select_indices:
612 continue
613 else:
614 print >>sys.stderr, "Restricting to input %d" % input_index
615
616 if options.only_load:
617
618 print "Input %d: %s" % (input_index,input_identifier)
619 continue
620
621 if options.skip_done:
622
623 outfile = get_output_filename(input_identifier)
624 if os.path.exists(outfile):
625
626 try:
627 old_res = ParseResults.from_file(outfile)
628 except ParseResults.LoadError, err:
629 pass
630 else:
631
632
633 completed_parses[input_identifier] = True
634 continue
635
636
637 completed_parses[input_identifier] = False
638
639
640 if parse_logger_dir:
641 parse_logger = os.path.join(parse_logger_dir, "%s.log" % \
642 slugify(input_identifier))
643 else:
644 parse_logger = None
645
646
647 input_topts = copy.deepcopy(topts)
648 input_popts = copy.deepcopy(popts)
649 input_npopts = copy.deepcopy(npopts)
650
651 if partitions > 1:
652 partition_num = partition_numbers[input_index]
653
654
655
656
657 if 'model' in input_topts and input_topts['model'] is not None:
658 input_topts['partition'] = partition_num
659 if 'model' in input_popts and input_popts['model'] is not None:
660 input_popts['partition'] = partition_num
661 if 'model' in input_npopts and input_npopts['model'] is not None:
662 input_npopts['partition'] = partition_num
663
664 if multiprocessing:
665
666 jobs.append(
667 pool.apply_async(do_parse, \
668 (grammar, tagger_cls, parser_cls, input, input_topts,
669 input_popts, backoff, input_npopts, options,
670 input_identifier),
671 { 'multiprocessing' : True,
672 'logfile' : parse_logger },
673 _result_callback))
674 else:
675
676 response = do_parse(grammar, tagger_cls, parser_cls, input,
677 input_topts, input_popts, backoff, input_npopts, options,
678 input_identifier, multiprocessing=False,
679 logfile=parse_logger)
680 _result_callback(response)
681
682 if multiprocessing:
683
684 try:
685 pool.close()
686 print >>sys.stderr, "Waiting for parse jobs to complete"
687 pool.join()
688 except KeyboardInterrupt:
689
690
691 print >>sys.stderr, "Exiting on keyboard interrupt"
692 sys.exit(1)
693
694
695 for job in jobs:
696 if not job.successful():
697 try:
698
699 job.get()
700 except Exception, err:
701
702
703 print >>sys.stderr, "\nError in worker thread: %s" % err
704 parser_exit_status = 1
705
706 if stdinput:
707 print
708
709 readline.write_history_file(settings.INPUT_PROMPT_HISTORY_FILE)
710 sys.exit(parser_exit_status)
711
712
713
714 -def do_parse(grammar, tagger_cls, parser_cls, input, topts, popts, backoff,
715 npopts, options, identifier, multiprocessing=False,
716 logfile=None, partition=None):
717 """
718 Function called for each input to do tagging and parsing and return the
719 results. It's a separate function so that we can hand it over to worker
720 processes to do multiprocessing.
721
722 @type logfile: str
723 @param logfile: filename to send logging output to. If None, will log
724 to stderr
725
726 """
727
728 if isinstance(input, str):
729 input = input.rstrip("\n")
730 if len(input) == 0:
731 return
732 input = ChordInput.from_string(input)
733
734 print "Processing input: %s (%s)" % (input, identifier)
735
736 if logfile is None:
737
738 logger = create_plain_stderr_logger()
739 else:
740 logger = create_logger(filename=logfile)
741 print "Logging parser progress to %s" % logfile
742
743
744
745 response = {
746 'tagger' : None,
747 'parser' : None,
748 'input' : input,
749 'error' : None,
750 'messages' : [],
751 'time' : None,
752 'identifier' : identifier,
753 'results' : None,
754 'timed_out' : False,
755 }
756 tagger = None
757 parser = None
758 messages = []
759
760 if options.short_progress:
761
762 progress = 2
763 elif options.long_progress:
764 progress = 1
765 else:
766 progress = 0
767
768
769 timer = ExecutionTimer(clock=True)
770
771
772 try:
773
774 logger.info("Tagging sequence (%d timesteps)" % len(input))
775
776
777 tagger = tagger_cls(grammar, input, options=topts.copy(), logger=logger)
778 if not multiprocessing:
779 response['tagger'] = tagger
780
781
782 parser = parser_cls(grammar, tagger, options=popts.copy(),
783 backoff=backoff,
784 backoff_options=npopts.copy(),
785 logger=logger)
786 if not multiprocessing:
787 response['parser'] = parser
788 try:
789
790 results = parser.parse(derivations=options.derivations, summaries=progress)
791 except (KeyboardInterrupt, Exception), err:
792 if multiprocessing:
793
794
795 response.update({
796 'error' : exception_tuple(str_tb=True),
797 })
798 return response
799 else:
800
801 if type(err) == KeyboardInterrupt:
802 print "Dropping out on keyboard interrupt"
803 print "Entering shell: use 'chart' command to see current state of parse"
804 elif options.error_shell:
805 print >> sys.stderr, "Error parsing %s" % str(input)
806 print >> sys.stderr, "The error was:"
807 traceback.print_exc(file=sys.stderr)
808
809
810 if options.error_shell or type(err) == KeyboardInterrupt:
811
812 print
813 from jazzparser.shell import interactive_shell
814 env = {}
815 env.update(globals())
816 env.update(locals())
817 interactive_shell(parser.chart.parses,options,tagger,parser,
818 grammar.formalism,env,input_data=input)
819 return
820 else:
821 raise
822 except (KeyboardInterrupt, Exception), err:
823 if multiprocessing:
824 response.update({
825 'error' : exception_tuple(str_tb=True),
826 })
827 return response
828 else:
829 if type(err) == KeyboardInterrupt:
830 print "Exiting on keyboard interrupt"
831 sys.exit(1)
832 else:
833 response.update({
834 'error' : exception_tuple(str_tb=True),
835 'messages' : messages,
836 'time' : timer.get_time(),
837 })
838 return response
839 else:
840
841
842
843
844 if (options.harmonical is not None or \
845 options.enharmonical is not None) and len(results) > 0:
846 path = grammar.formalism.sign_to_coordinates(results[0])
847
848
849 times = results[0].semantics.get_path_times()
850 point_durations = [next-current for current,next in group_pairs(times)] + [0]
851
852 path3d = zip(add_z_coordinates(path, pitch_range=2), point_durations)
853 path2d = zip(path,point_durations)
854
855 chords = tagger.get_string_input()
856 chord_durs = [tagger.get_word_duration(i) for i in range(tagger.input_length)]
857 chord_types = [(Chord.from_name(c).type,dur) for c,dur in zip(chords,chord_durs)]
858
859 if options.midi:
860
861
862
863
864 instrument = 73
865
866 if options.harmonical is not None:
867 filename = os.path.abspath(options.harmonical)
868 render_path_to_midi_file(filename, path3d, chord_types=chord_types, tempo=options.tempo, instrument=instrument, bass_root=True, root_octave=-1)
869 messages.append("Output JI MIDI data to %s" % filename)
870 if options.enharmonical is not None:
871 filename = os.path.abspath(options.enharmonical)
872 render_path_to_midi_file(filename, path3d, chord_types=chord_types, equal_temperament=True, tempo=options.tempo, instrument=instrument, bass_root=True, root_octave=-1)
873 messages.append("Output ET MIDI data to %s" % filename)
874 else:
875 if options.harmonical is not None:
876 filename = os.path.abspath(options.harmonical)
877 render_path_to_wave_file(filename, path2d, chord_types=chord_types, double_root=True, tempo=options.tempo)
878 messages.append("Output JI wave data to %s" % filename)
879 if options.enharmonical is not None:
880 filename = os.path.abspath(options.enharmonical)
881 render_path_to_wave_file(filename, path2d, chord_types=chord_types, double_root=True, equal_temperament=True, tempo=options.tempo)
882 messages.append("Output ET wave data to %s" % filename)
883
884 response.update({
885 'results' : results,
886 'time' : timer.get_time(),
887 'messages' : messages,
888 'timed_out' : parser.timed_out,
889 })
890 return response
891
892
914
918
919 if __name__ == "__main__":
920 sys.exit(main())
921