From 97b83a34745f8f34c8a55b72fd6700203a256bc7 Mon Sep 17 00:00:00 2001 From: Eleftherios Avramidis Date: Tue, 13 Jan 2015 12:06:14 +0100 Subject: [PATCH 1/7] Added perl scripts that do moses pre- and post- processing --- worker/src/util/compound-splitter.perl | 293 +++++++++++ worker/src/util/detokenizer.perl | 366 ++++++++++++++ worker/src/util/detruecase.perl | 88 ++++ worker/src/util/normalize-punctuation.perl | 78 +++ worker/src/util/tokenizer.perl | 539 +++++++++++++++++++++ worker/src/util/truecase.perl | 87 ++++ 6 files changed, 1451 insertions(+) create mode 100755 worker/src/util/compound-splitter.perl create mode 100644 worker/src/util/detokenizer.perl create mode 100755 worker/src/util/detruecase.perl create mode 100644 worker/src/util/normalize-punctuation.perl create mode 100755 worker/src/util/tokenizer.perl create mode 100755 worker/src/util/truecase.perl diff --git a/worker/src/util/compound-splitter.perl b/worker/src/util/compound-splitter.perl new file mode 100755 index 0000000..0628557 --- /dev/null +++ b/worker/src/util/compound-splitter.perl @@ -0,0 +1,293 @@ +#!/usr/bin/perl -w + +use strict; +use Getopt::Long "GetOptions"; +use IO::Handle; +autoflush STDOUT 1; + +my ($CORPUS,$MODEL,$TRAIN,$HELP,$VERBOSE); +my $FILLER = ":s:es"; +my $MIN_SIZE = 3; +my $MIN_COUNT = 5; +my $MAX_COUNT = 5; +my $FACTORED = 0; +my $SYNTAX = 0; +my $MARK_SPLIT = 0; +my $BINARIZE = 0; +$HELP = 1 + unless &GetOptions('corpus=s' => \$CORPUS, + 'model=s' => \$MODEL, + 'filler=s' => \$FILLER, + 'factored' => \$FACTORED, + 'min-size=i' => \$MIN_SIZE, + 'min-count=i' => \$MIN_COUNT, + 'max-count=i' => \$MAX_COUNT, + 'help' => \$HELP, + 'verbose' => \$VERBOSE, + 'syntax' => \$SYNTAX, + 'binarize' => \$BINARIZE, + 'mark-split' => \$MARK_SPLIT, + 'train' => \$TRAIN); + +if ($HELP || + ( $TRAIN && !$CORPUS) || + (!$TRAIN && !$MODEL)) { + print "Compound splitter\n"; + print "-----------------\n\n"; + print "train: compound-splitter -train -corpus txt-file -model new-model\n"; + print "apply: compound-splitter -model trained-model < in > out\n"; + print "options: -min-size: minimum word size (default $MIN_SIZE)\n"; + print " -min-count: minimum word count (default $MIN_COUNT)\n"; + print " -filler: filler letters between words (default $FILLER)\n"; + print " -factor: factored data, assuming factor 0 as surface (default $FACTORED)\n"; + print " -syntax: syntactically parsed data (default $SYNTAX)\n"; + print " -mark-split: mark non-terminal label of split words (default $MARK_SPLIT)\n"; + print " -binarize: binarize subtree for split word (default $BINARIZE)\n"; + exit; +} + +if ($TRAIN) { + if ($SYNTAX) { &train_syntax(); } + elsif ($FACTORED) { &train_factored(); } + else { &train(); } +} +else { + &apply(); +} + +sub train { + my %COUNT; + open(CORPUS,$CORPUS) || die("ERROR: could not open corpus '$CORPUS'"); + while() { + chop; s/\s+/ /g; s/^ //; s/ $//; + foreach (split) { + $COUNT{$_}++; + } + } + close(CORPUS); + &save_trained_model(\%COUNT); +} + +sub save_trained_model { + my ($COUNT) = @_; + my $id = 0; + open(MODEL,">".$MODEL); + foreach my $word (keys %$COUNT) { + print MODEL "".(++$id)."\t".$word."\t".$$COUNT{$word}."\n"; + } + close(MODEL); + print STDERR "written model file with ".(scalar keys %$COUNT)." words.\n"; +} + +sub train_factored { + my (%COUNT,%FACTORED_COUNT); + # collect counts for interpretations for each surface word + open(CORPUS,$CORPUS) || die("ERROR: could not open corpus '$CORPUS'"); + while() { + chop; s/\s+/ /g; s/^ //; s/ $//; + foreach my $factored_word (split) { + my $word = $factored_word; + $word =~ s/\|.+//g; # just first factor + $FACTORED_COUNT{$word}{$factored_word}++; + } + } + close(CORPUS); + # only preserve most frequent interpretation, assign sum of counts + foreach my $word (keys %FACTORED_COUNT) { + my ($max,$best,$total) = (0,"",0); + foreach my $factored_word (keys %{$FACTORED_COUNT{$word}}) { + my $count = $FACTORED_COUNT{$word}{$factored_word}; + $total += $count; + if ($count > $max) { + $max = $count; + $best = $factored_word; + } + } + $COUNT{$best} = $total; + } + &save_trained_model(\%COUNT); +} + +sub train_syntax { + my (%COUNT,%LABELED_COUNT); + # collect counts for interpretations for each surface word + open(CORPUS,$CORPUS) || die("ERROR: could not open corpus '$CORPUS'"); + while() { + chop; s/\s+/ /g; s/^ //; s/ $//; + my $label; + foreach (split) { + if (/^label="([^\"]+)"/) { + $label = $1; + } + elsif (! /^ $max) { + $max = $count; + $best = "$word $label"; + } + } + $COUNT{$best} = $total; + } + &save_trained_model(\%COUNT); +} + +sub apply { + my (%COUNT,%TRUECASE,%LABEL); + open(MODEL,$MODEL) || die("ERROR: could not open model '$MODEL'"); + while() { + chomp; + my ($id,$factored_word,$count) = split(/\t/); + my $label; + ($factored_word,$label) = split(/ /,$factored_word); + my $word = $factored_word; + $word =~ s/\|.+//g; # just first factor + my $lc = lc($word); + # if word exists with multipe casings, only record most frequent + next if defined($COUNT{$lc}) && $COUNT{$lc} > $count; + $COUNT{$lc} = $count; + $TRUECASE{$lc} = $factored_word; + $LABEL{$lc} = $label if $SYNTAX; + } + close(MODEL); + + while() { + my $first = 1; + chop; s/\s+/ /g; s/^ //; s/ $//; + my @BUFFER; # for xml tags + foreach my $factored_word (split) { + print " " unless $first; + $first = 0; + + # syntax: don't split xml + if ($SYNTAX && ($factored_word =~ /^$/)) { + push @BUFFER,$factored_word; + $first = 1; + next; + } + + # get case class + my $word = $factored_word; + $word =~ s/\|.+//g; # just first factor + my $lc = lc($word); + + print STDERR "considering $word ($lc)...\n" if $VERBOSE; + # don't split frequent words + if ((defined($COUNT{$lc}) && $COUNT{$lc}>=$MAX_COUNT) || + $lc !~ /[a-zA-Z]/) {; # has to have at least one letter + print join(" ",@BUFFER)." " if scalar(@BUFFER); @BUFFER = (); # clear buffer + print $factored_word; + print STDERR "\tfrequent word ($COUNT{$lc}>=$MAX_COUNT), skipping\n" if $VERBOSE; + next; + } + + # consider possible splits + my $final = length($word)-1; + my %REACHABLE; + for(my $i=0;$i<=$final;$i++) { $REACHABLE{$i} = (); } + + print STDERR "splitting $word:\n" if $VERBOSE; + for(my $end=$MIN_SIZE;$end= $MIN_COUNT; + print STDERR "\tmatching word $start .. $end ($filler)$subword $COUNT{$subword}\n" if $VERBOSE; + push @{$REACHABLE{$end}},"$start $TRUECASE{$subword} $COUNT{$subword}"; + } + } + } + + # no matches at all? + if (!defined($REACHABLE{$final})) { + print join(" ",@BUFFER)." " if scalar(@BUFFER); @BUFFER = (); # clear buffer + print $factored_word; + next; + } + + my ($best_split,$best_score) = ("",0); + + my %ITERATOR; + for(my $i=0;$i<=$final;$i++) { $ITERATOR{$i}=0; } + my $done = 0; + while(1) { + # read off word + my ($pos,$decomp,$score,$num,@INDEX) = ($final,"",1,0); + while($pos>0) { + last unless scalar @{$REACHABLE{$pos}} > $ITERATOR{$pos}; # dead end? + my ($nextpos,$subword,$count) + = split(/ /,$REACHABLE{$pos}[ $ITERATOR{$pos} ]); + $decomp = $subword." ".$decomp; + $score *= $count; + $num++; + push @INDEX,$pos; +# print STDERR "($nextpos-$pos,$decomp,$score,$num)\n"; + $pos = $nextpos-1; + } + + chop($decomp); + print STDERR "\tsplit: $decomp ($score ** 1/$num) = ".($score ** (1/$num))."\n" if $VERBOSE; + $score **= 1/$num; + if ($score>$best_score) { + $best_score = $score; + $best_split = $decomp; + } + + # increase iterator + my $increase = -1; + while($increase<$final) { + $increase = pop @INDEX; + $ITERATOR{$increase}++; + last if scalar @{$REACHABLE{$increase}} > $ITERATOR{$increase}; + } + last unless scalar @{$REACHABLE{$final}} > $ITERATOR{$final}; + for(my $i=0;$i<$increase;$i++) { $ITERATOR{$i}=0; } + } + if ($best_split !~ / /) { + print join(" ",@BUFFER)." " if scalar(@BUFFER); @BUFFER = (); # clear buffer + print $factored_word; # do not change case for unsplit words + next; + } + if (!$SYNTAX) { + print $best_split; + } + else { + $BUFFER[$#BUFFER] =~ s/label=\"/label=\"SPLIT-/ if $MARK_SPLIT; + $BUFFER[$#BUFFER] =~ /label=\"([^\"]+)\"/ || die("ERROR: $BUFFER[$#BUFFER]\n"); + my $pos = $1; + print join(" ",@BUFFER)." " if scalar(@BUFFER); @BUFFER = (); # clear buffer + + my @SPLIT = split(/ /,$best_split); + my @OUT = (); + if ($BINARIZE) { + for(my $w=0;$w"; + } + } + for(my $w=0;$w=2) { push @OUT, ""; } + push @OUT," $SPLIT[$w] "; + } + print join(" ",@OUT); + } + } + print " ".join(" ",@BUFFER) if scalar(@BUFFER); @BUFFER = (); # clear buffer + print "\n"; + } +} diff --git a/worker/src/util/detokenizer.perl b/worker/src/util/detokenizer.perl new file mode 100644 index 0000000..b9641b1 --- /dev/null +++ b/worker/src/util/detokenizer.perl @@ -0,0 +1,366 @@ +#!/usr/bin/perl -w + +# $Id: detokenizer.perl 4134 2011-08-08 15:30:54Z bgottesman $ +# Sample De-Tokenizer +# written by Josh Schroeder, based on code by Philipp Koehn +# further modifications by Ondrej Bojar + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +use strict; +use utf8; # tell perl this script file is in UTF-8 (see all funny punct below) + +my $language = "en"; +my $QUIET = 0; +my $HELP = 0; +my $UPPERCASE_SENT = 0; +my $PENN = 0; + +use IO::Handle; +autoflush STDOUT 1; + +while (@ARGV) { + $_ = shift; + /^-b$/ && ($| = 1, next); + /^-l$/ && ($language = shift, next); + /^-q$/ && ($QUIET = 1, next); + /^-h$/ && ($HELP = 1, next); + /^-u$/ && ($UPPERCASE_SENT = 1, next); + /^-penn$/ && ($PENN = 1, next); +} + +if ($HELP) { + print "Usage ./detokenizer.perl (-l [en|fr|it|cs|...]) < tokenizedfile > detokenizedfile\n"; + print "Options:\n"; + print " -u ... uppercase the first char in the final sentence.\n"; + print " -q ... don't report detokenizer revision.\n"; + print " -b ... disable Perl buffering.\n"; + print " -penn ... assume input is tokenized as per tokenizer.perl's -penn option.\n"; + exit; +} + +if ($language !~ /^(cs|en|fr|it)$/) { + print STDERR "Warning: No built-in rules for language $language.\n" +} + +if ($PENN && $language ne "en") { + print STDERR "Error: -penn option only supported for English text.\n"; + exit; +} + +if (!$QUIET) { + print STDERR "Detokenizer Version ".'$Revision: 4134 $'."\n"; + print STDERR "Language: $language\n"; +} + +while() { + if (/^<.+>$/ || /^\s*$/) { + #don't try to detokenize XML/HTML tag lines + print $_; + } elsif ($PENN) { + print &detokenize_penn($_); + } else { + print &detokenize($_); + } +} + + +sub ucsecondarg { + # uppercase the second argument + my $arg1 = shift; + my $arg2 = shift; + return $arg1.uc($arg2); +} + +sub deescape { + # de-escape special chars + my ($text) = @_; + $text =~ s/\&bar;/\|/g; # factor separator (legacy) + $text =~ s/\|/\|/g; # factor separator + $text =~ s/\</\/g; # xml + $text =~ s/\&bra;/\[/g; # syntax non-terminal (legacy) + $text =~ s/\&ket;/\]/g; # syntax non-terminal (legacy) + $text =~ s/\"/\"/g; # xml + $text =~ s/\'/\'/g; # xml + $text =~ s/\[/\[/g; # syntax non-terminal + $text =~ s/\]/\]/g; # syntax non-terminal + $text =~ s/\&/\&/g; # escape escape + return $text; +} + +sub detokenize { + my($text) = @_; + chomp($text); + $text = " $text "; + $text =~ s/ \@\-\@ /-/g; + $text = &deescape($text); + + my $word; + my $i; + my @words = split(/ /,$text); + $text = ""; + my %quoteCount = ("\'"=>0,"\""=>0); + my $prependSpace = " "; + for ($i=0;$i<(scalar(@words));$i++) { + if (&startsWithCJKChar($words[$i])) { + if ($i > 0 && &endsWithCJKChar($words[$i-1])) { + # perform left shift if this is a second consecutive CJK (Chinese/Japanese/Korean) word + $text=$text.$words[$i]; + } else { + # ... but do nothing special if this is a CJK word that doesn't follow a CJK word + $text=$text.$prependSpace.$words[$i]; + } + $prependSpace = " "; + } elsif ($words[$i] =~ /^[\p{IsSc}\(\[\{\¿\¡]+$/) { + #perform right shift on currency and other random punctuation items + $text = $text.$prependSpace.$words[$i]; + $prependSpace = ""; + } elsif ($words[$i] =~ /^[\,\.\?\!\:\;\\\%\}\]\)]+$/){ + if (($language eq "fr") && ($words[$i] =~ /^[\?\!\:\;\\\%]$/)) { + #these punctuations are prefixed with a non-breakable space in french + $text .= " "; } + #perform left shift on punctuation items + $text=$text.$words[$i]; + $prependSpace = " "; + } elsif (($language eq "en") && ($i>0) && ($words[$i] =~ /^[\'][\p{IsAlpha}]/) && ($words[$i-1] =~ /[\p{IsAlnum}]$/)) { + #left-shift the contraction for English + $text=$text.$words[$i]; + $prependSpace = " "; + } elsif (($language eq "cs") && ($i>1) && ($words[$i-2] =~ /^[0-9]+$/) && ($words[$i-1] =~ /^[.,]$/) && ($words[$i] =~ /^[0-9]+$/)) { + #left-shift floats in Czech + $text=$text.$words[$i]; + $prependSpace = " "; + } elsif ((($language eq "fr") ||($language eq "it")) && ($i<=(scalar(@words)-2)) && ($words[$i] =~ /[\p{IsAlpha}][\']$/) && ($words[$i+1] =~ /^[\p{IsAlpha}]/)) { + #right-shift the contraction for French and Italian + $text = $text.$prependSpace.$words[$i]; + $prependSpace = ""; + } elsif (($language eq "cs") && ($i<(scalar(@words)-3)) + && ($words[$i] =~ /[\p{IsAlpha}]$/) + && ($words[$i+1] =~ /^[-–]$/) + && ($words[$i+2] =~ /^li$|^mail.*/i) + ) { + #right-shift "-li" in Czech and a few Czech dashed words (e-mail) + $text = $text.$prependSpace.$words[$i].$words[$i+1]; + $i++; # advance over the dash + $prependSpace = ""; + } elsif ($words[$i] =~ /^[\'\"„“`]+$/) { + #combine punctuation smartly + my $normalized_quo = $words[$i]; + $normalized_quo = '"' if $words[$i] =~ /^[„“”]+$/; + $quoteCount{$normalized_quo} = 0 + if !defined $quoteCount{$normalized_quo}; + if ($language eq "cs" && $words[$i] eq "„") { + # this is always the starting quote in Czech + $quoteCount{$normalized_quo} = 0; + } + if ($language eq "cs" && $words[$i] eq "“") { + # this is usually the ending quote in Czech + $quoteCount{$normalized_quo} = 1; + } + if (($quoteCount{$normalized_quo} % 2) eq 0) { + if(($language eq "en") && ($words[$i] eq "'") && ($i > 0) && ($words[$i-1] =~ /[s]$/)) { + #single quote for posesssives ending in s... "The Jones' house" + #left shift + $text=$text.$words[$i]; + $prependSpace = " "; + } else { + #right shift + $text = $text.$prependSpace.$words[$i]; + $prependSpace = ""; + $quoteCount{$normalized_quo} ++; + + } + } else { + #left shift + $text=$text.$words[$i]; + $prependSpace = " "; + $quoteCount{$normalized_quo} ++; + + } + + } else { + $text=$text.$prependSpace.$words[$i]; + $prependSpace = " "; + } + } + + # clean up spaces at head and tail of each line as well as any double-spacing + $text =~ s/ +/ /g; + $text =~ s/\n /\n/g; + $text =~ s/ \n/\n/g; + $text =~ s/^ //g; + $text =~ s/ $//g; + + #add trailing break + $text .= "\n" unless $text =~ /\n$/; + + $text =~ s/^([[:punct:]\s]*)([[:alpha:]])/ucsecondarg($1, $2)/e if $UPPERCASE_SENT; + + return $text; +} + +sub detokenize_penn { + my($text) = @_; + + chomp($text); + $text = " $text "; + $text =~ s/ \@\-\@ /-/g; + $text =~ s/ \@\/\@ /\//g; + $text = &deescape($text); + + # merge de-contracted forms except where the second word begins with an + # apostrophe (those are handled later) + $text =~ s/ n't /n't /g; + $text =~ s/ N'T /N'T /g; + $text =~ s/ ([Cc])an not / $1annot /g; + $text =~ s/ ([Dd])' ye / $1'ye /g; + $text =~ s/ ([Gg])im me / $1imme /g; + $text =~ s/ ([Gg])on na / $1onna /g; + $text =~ s/ ([Gg])ot ta / $1otta /g; + $text =~ s/ ([Ll])em me / $1emme /g; + $text =~ s/ '([Tt]) is / '$1is /g; + $text =~ s/ '([Tt]) was / '$1was /g; + $text =~ s/ ([Ww])an na / $1anna /g; + + # restore brackets + $text =~ s/-LRB-/\(/g; + $text =~ s/-RRB-/\)/g; + $text =~ s/-LSB-/\[/g; + $text =~ s/-RSB-/\]/g; + $text =~ s/-LCB-/{/g; + $text =~ s/-RCB-/}/g; + + my $i; + my @words = split(/ /,$text); + $text = ""; + my $prependSpace = " "; + for ($i=0;$i<(scalar(@words));$i++) { + if ($words[$i] =~ /^[\p{IsSc}\(\[\{\¿\¡]+$/) { + # perform right shift on currency and other random punctuation items + $text = $text.$prependSpace.$words[$i]; + $prependSpace = ""; + } elsif ($words[$i] =~ /^[\,\.\?\!\:\;\\\%\}\]\)]+$/){ + # perform left shift on punctuation items + $text=$text.$words[$i]; + $prependSpace = " "; + } elsif (($i>0) && ($words[$i] =~ /^[\'][\p{IsAlpha}]/) && ($words[$i-1] =~ /[\p{IsAlnum}]$/)) { + # left-shift the contraction + $text=$text.$words[$i]; + $prependSpace = " "; + } elsif ($words[$i] eq "`") { # Assume that punctuation has been normalized and is one of `, ``, ', '' only + # opening single quote: convert to straight quote and right-shift + $text = $text.$prependSpace."\'"; + $prependSpace = ""; + } elsif ($words[$i] eq "``") { + # opening double quote: convert to straight quote and right-shift + $text = $text.$prependSpace."\""; + $prependSpace = ""; + } elsif ($words[$i] eq "\'") { + # closing single quote: convert to straight quote and left shift + $text = $text."\'"; + $prependSpace = " "; + } elsif ($words[$i] eq "\'\'") { + # closing double quote: convert to straight quote and left shift + $text = $text."\""; + $prependSpace = " "; + } else { + $text = $text.$prependSpace.$words[$i]; + $prependSpace = " "; + } + } + + # clean up spaces at head and tail of each line as well as any double-spacing + $text =~ s/ +/ /g; + $text =~ s/\n /\n/g; + $text =~ s/ \n/\n/g; + $text =~ s/^ //g; + $text =~ s/ $//g; + + # add trailing break + $text .= "\n" unless $text =~ /\n$/; + + $text =~ s/^([[:punct:]\s]*)([[:alpha:]])/ucsecondarg($1, $2)/e if $UPPERCASE_SENT; + + return $text; +} + +sub startsWithCJKChar { + my ($str) = @_; + return 0 if length($str) == 0; + my $firstChar = substr($str, 0, 1); + return &charIsCJK($firstChar); +} + +sub endsWithCJKChar { + my ($str) = @_; + return 0 if length($str) == 0; + my $lastChar = substr($str, length($str)-1, 1); + return &charIsCJK($lastChar); +} + +# Given a string consisting of one character, returns true iff the character +# is a CJK (Chinese/Japanese/Korean) character +sub charIsCJK { + my ($char) = @_; + # $char should be a string of length 1 + my $codepoint = &codepoint_dec($char); + + # The following is based on http://en.wikipedia.org/wiki/Basic_Multilingual_Plane#Basic_Multilingual_Plane + + # Hangul Jamo (1100–11FF) + return 1 if (&between_hexes($codepoint, '1100', '11FF')); + + # CJK Radicals Supplement (2E80–2EFF) + # Kangxi Radicals (2F00–2FDF) + # Ideographic Description Characters (2FF0–2FFF) + # CJK Symbols and Punctuation (3000–303F) + # Hiragana (3040–309F) + # Katakana (30A0–30FF) + # Bopomofo (3100–312F) + # Hangul Compatibility Jamo (3130–318F) + # Kanbun (3190–319F) + # Bopomofo Extended (31A0–31BF) + # CJK Strokes (31C0–31EF) + # Katakana Phonetic Extensions (31F0–31FF) + # Enclosed CJK Letters and Months (3200–32FF) + # CJK Compatibility (3300–33FF) + # CJK Unified Ideographs Extension A (3400–4DBF) + # Yijing Hexagram Symbols (4DC0–4DFF) + # CJK Unified Ideographs (4E00–9FFF) + # Yi Syllables (A000–A48F) + # Yi Radicals (A490–A4CF) + return 1 if (&between_hexes($codepoint, '2E80', 'A4CF')); + + # Phags-pa (A840–A87F) + return 1 if (&between_hexes($codepoint, 'A840', 'A87F')); + + # Hangul Syllables (AC00–D7AF) + return 1 if (&between_hexes($codepoint, 'AC00', 'D7AF')); + + # CJK Compatibility Ideographs (F900–FAFF) + return 1 if (&between_hexes($codepoint, 'F900', 'FAFF')); + + # CJK Compatibility Forms (FE30–FE4F) + return 1 if (&between_hexes($codepoint, 'FE30', 'FE4F')); + + # Range U+FF65–FFDC encodes halfwidth forms, of Katakana and Hangul characters + return 1 if (&between_hexes($codepoint, 'FF65', 'FFDC')); + + # Supplementary Ideographic Plane 20000–2FFFF + return 1 if (&between_hexes($codepoint, '20000', '2FFFF')); + + return 0; +} + +# Returns the code point of a Unicode char, represented as a decimal number +sub codepoint_dec { + if (my $char = shift) { + return unpack('U0U*', $char); + } +} + +sub between_hexes { + my ($num, $left, $right) = @_; + return $num >= hex($left) && $num <= hex($right); +} diff --git a/worker/src/util/detruecase.perl b/worker/src/util/detruecase.perl new file mode 100755 index 0000000..012c143 --- /dev/null +++ b/worker/src/util/detruecase.perl @@ -0,0 +1,88 @@ +#!/usr/bin/perl -w + +use strict; +use Getopt::Long "GetOptions"; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); + +my ($SRC,$INFILE,$UNBUFFERED); +die("detruecase.perl < in > out") + unless &GetOptions('headline=s' => \$SRC, + 'in=s' => \$INFILE, + 'b|unbuffered' => \$UNBUFFERED); +if (defined($UNBUFFERED) && $UNBUFFERED) { $|=1; } + +my %SENTENCE_END = ("."=>1,":"=>1,"?"=>1,"!"=>1); +my %DELAYED_SENTENCE_START = ("("=>1,"["=>1,"\""=>1,"'"=>1,"""=>1,"'"=>1,"["=>1,"]"=>1); + +# lowercase even in headline +my %ALWAYS_LOWER; +foreach ("a","after","against","al-.+","and","any","as","at","be","because","between","by","during","el-.+","for","from","his","in","is","its","last","not","of","off","on","than","the","their","this","to","was","were","which","will","with") { $ALWAYS_LOWER{$_} = 1; } + +# find out about the headlines +my @HEADLINE; +if (defined($SRC)) { + open(SRC,$SRC); + my $headline_flag = 0; + while() { + $headline_flag = 1 if //; + $headline_flag = 0 if /<.hl>/; + next unless /^) { + &process($_,$sentence++); + } + close(IN); +} +else { + while() { + &process($_,$sentence++); + } +} + +sub process { + my $line = $_[0]; + chomp($line); + $line =~ s/^\s+//; + $line =~ s/\s+$//; + my @WORD = split(/\s+/,$line); + + # uppercase at sentence start + my $sentence_start = 1; + for(my $i=0;$i) { + s/\r//g; + # remove extra spaces + s/\(/ \(/g; + s/\)/\) /g; s/ +/ /g; + s/\) ([\.\!\:\?\;\,])/\)$1/g; + s/\( /\(/g; + s/ \)/\)/g; + s/(\d) \%/$1\%/g; + s/ :/:/g; + s/ ;/;/g; + # normalize unicode punctuation + s/„/\"/g; + s/“/\"/g; + s/”/\"/g; + s/–/-/g; + s/—/ - /g; s/ +/ /g; + s/´/\'/g; + s/([a-z])‘([a-z])/$1\'$2/gi; + s/([a-z])’([a-z])/$1\'$2/gi; + s/‘/\"/g; + s/‚/\"/g; + s/’/\"/g; + s/''/\"/g; + s/´´/\"/g; + s/…/.../g; + # French quotes + s/ « / \"/g; + s/« /\"/g; + s/«/\"/g; + s/ » /\" /g; + s/ »/\"/g; + s/»/\"/g; + # handle pseudo-spaces + s/ \%/\%/g; + s/nº /nº /g; + s/ :/:/g; + s/ ºC/ ºC/g; + s/ cm/ cm/g; + s/ \?/\?/g; + s/ \!/\!/g; + s/ ;/;/g; + s/, /, /g; s/ +/ /g; + + # English "quotation," followed by comma, style + if ($language eq "en") { + s/\"([,\.]+)/$1\"/g; + } + # Czech is confused + elsif ($language eq "cs" || $language eq "cz") { + } + # German/Spanish/French "quotation", followed by comma, style + else { + s/,\"/\",/g; + s/(\.+)\"(\s*[^<])/\"$1$2/g; # don't fix period at end of sentence + } + + print STDERR $_ if //; + + if ($language eq "de" || $language eq "es" || $language eq "cz" || $language eq "cs" || $language eq "fr") { + s/(\d) (\d)/$1,$2/g; + } + else { + s/(\d) (\d)/$1.$2/g; + } + print $_; +} diff --git a/worker/src/util/tokenizer.perl b/worker/src/util/tokenizer.perl new file mode 100755 index 0000000..186dbed --- /dev/null +++ b/worker/src/util/tokenizer.perl @@ -0,0 +1,539 @@ +#!/usr/bin/perl -w + +# Sample Tokenizer +### Version 1.1 +# written by Pidong Wang, based on the code written by Josh Schroeder and Philipp Koehn +# Version 1.1 updates: +# (1) add multithreading option "-threads NUM_THREADS" (default is 1); +# (2) add a timing option "-time" to calculate the average speed of this tokenizer; +# (3) add an option "-lines NUM_SENTENCES_PER_THREAD" to set the number of lines for each thread (default is 2000), and this option controls the memory amount needed: the larger this number is, the larger memory is required (the higher tokenization speed); +### Version 1.0 +# $Id: tokenizer.perl 915 2009-08-10 08:15:49Z philipp $ +# written by Josh Schroeder, based on code by Philipp Koehn + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); + +use FindBin qw($RealBin); +use strict; +use Time::HiRes; +use Thread; + +my $mydir = "$RealBin/../share/nonbreaking_prefixes"; +print $mydir; + +my %NONBREAKING_PREFIX = (); +my @protected_patterns = (); +my $protected_patterns_file = ""; +my $language = "en"; +my $QUIET = 0; +my $HELP = 0; +my $AGGRESSIVE = 0; +my $SKIP_XML = 0; +my $TIMING = 0; +my $NUM_THREADS = 1; +my $NUM_SENTENCES_PER_THREAD = 2000; +my $PENN = 0; + +while (@ARGV) +{ + $_ = shift; + /^-b$/ && ($| = 1, next); + /^-l$/ && ($language = shift, next); + /^-q$/ && ($QUIET = 1, next); + /^-h$/ && ($HELP = 1, next); + /^-x$/ && ($SKIP_XML = 1, next); + /^-a$/ && ($AGGRESSIVE = 1, next); + /^-time$/ && ($TIMING = 1, next); + # Option to add list of regexps to be protected + /^-protected/ && ($protected_patterns_file = shift, next); + /^-threads$/ && ($NUM_THREADS = int(shift), next); + /^-lines$/ && ($NUM_SENTENCES_PER_THREAD = int(shift), next); + /^-penn$/ && ($PENN = 1, next); +} + +# for time calculation +my $start_time; +if ($TIMING) +{ + $start_time = [ Time::HiRes::gettimeofday( ) ]; +} + +# print help message +if ($HELP) +{ + print "Usage ./tokenizer.perl (-l [en|de|...]) (-threads 4) < textfile > tokenizedfile\n"; + print "Options:\n"; + print " -q ... quiet.\n"; + print " -a ... aggressive hyphen splitting.\n"; + print " -b ... disable Perl buffering.\n"; + print " -time ... enable processing time calculation.\n"; + print " -penn ... use Penn treebank-like tokenization.\n"; + print " -protected FILE ... specify file with patters to be protected in tokenisation.\n"; + exit; +} + +if (!$QUIET) +{ + print STDERR "Tokenizer Version 1.1\n"; + print STDERR "Language: $language\n"; + print STDERR "Number of threads: $NUM_THREADS\n"; +} + +# load the language-specific non-breaking prefix info from files in the directory nonbreaking_prefixes +load_prefixes($language,\%NONBREAKING_PREFIX); + +if (scalar(%NONBREAKING_PREFIX) eq 0) +{ + print STDERR "Warning: No known abbreviations for language '$language'\n"; +} + +# Load protected patterns +if ($protected_patterns_file) +{ + open(PP,$protected_patterns_file) || die "Unable to open $protected_patterns_file"; + while() { + chomp; + push @protected_patterns, $_; + } +} + +my @batch_sentences = (); +my @thread_list = (); +my $count_sentences = 0; + +if ($NUM_THREADS > 1) +{# multi-threading tokenization + while() + { + $count_sentences = $count_sentences + 1; + push(@batch_sentences, $_); + if (scalar(@batch_sentences)>=($NUM_SENTENCES_PER_THREAD*$NUM_THREADS)) + { + # assign each thread work + for (my $i=0; $i<$NUM_THREADS; $i++) + { + my $start_index = $i*$NUM_SENTENCES_PER_THREAD; + my $end_index = $start_index+$NUM_SENTENCES_PER_THREAD-1; + my @subbatch_sentences = @batch_sentences[$start_index..$end_index]; + my $new_thread = new Thread \&tokenize_batch, @subbatch_sentences; + push(@thread_list, $new_thread); + } + foreach (@thread_list) + { + my $tokenized_list = $_->join; + foreach (@$tokenized_list) + { + print $_; + } + } + # reset for the new run + @thread_list = (); + @batch_sentences = (); + } + } + # the last batch + if (scalar(@batch_sentences)>0) + { + # assign each thread work + for (my $i=0; $i<$NUM_THREADS; $i++) + { + my $start_index = $i*$NUM_SENTENCES_PER_THREAD; + if ($start_index >= scalar(@batch_sentences)) + { + last; + } + my $end_index = $start_index+$NUM_SENTENCES_PER_THREAD-1; + if ($end_index >= scalar(@batch_sentences)) + { + $end_index = scalar(@batch_sentences)-1; + } + my @subbatch_sentences = @batch_sentences[$start_index..$end_index]; + my $new_thread = new Thread \&tokenize_batch, @subbatch_sentences; + push(@thread_list, $new_thread); + } + foreach (@thread_list) + { + my $tokenized_list = $_->join; + foreach (@$tokenized_list) + { + print $_; + } + } + } +} +else +{# single thread only + while() + { + if (($SKIP_XML && /^<.+>$/) || /^\s*$/) + { + #don't try to tokenize XML/HTML tag lines + print $_; + } + else + { + print &tokenize($_); + } + } +} + +if ($TIMING) +{ + my $duration = Time::HiRes::tv_interval( $start_time ); + print STDERR ("TOTAL EXECUTION TIME: ".$duration."\n"); + print STDERR ("TOKENIZATION SPEED: ".($duration/$count_sentences*1000)." milliseconds/line\n"); +} + +##################################################################################### +# subroutines afterward + +# tokenize a batch of texts saved in an array +# input: an array containing a batch of texts +# return: another array containing a batch of tokenized texts for the input array +sub tokenize_batch +{ + my(@text_list) = @_; + my(@tokenized_list) = (); + foreach (@text_list) + { + if (($SKIP_XML && /^<.+>$/) || /^\s*$/) + { + #don't try to tokenize XML/HTML tag lines + push(@tokenized_list, $_); + } + else + { + push(@tokenized_list, &tokenize($_)); + } + } + return \@tokenized_list; +} + +# the actual tokenize function which tokenizes one input string +# input: one string +# return: the tokenized string for the input string +sub tokenize +{ + my($text) = @_; + + if ($PENN) { + return tokenize_penn($text); + } + + chomp($text); + $text = " $text "; + + # remove ASCII junk + $text =~ s/\s+/ /g; + $text =~ s/[\000-\037]//g; + + # Find protected patterns + my @protected = (); + foreach my $protected_pattern (@protected_patterns) { + foreach ($text =~ /($protected_pattern)/) { + push @protected, $_; + } + } + + for (my $i = 0; $i < scalar(@protected); ++$i) { + my $subst = sprintf("THISISPROTECTED%.3d", $i); + $text =~ s,\Q$protected[$i],$subst,g; + } + + # seperate out all "other" special characters + $text =~ s/([^\p{IsAlnum}\s\.\'\`\,\-])/ $1 /g; + + # aggressive hyphen splitting + if ($AGGRESSIVE) + { + $text =~ s/([\p{IsAlnum}])\-([\p{IsAlnum}])/$1 \@-\@ $2/g; + } + + #multi-dots stay together + $text =~ s/\.([\.]+)/ DOTMULTI$1/g; + while($text =~ /DOTMULTI\./) + { + $text =~ s/DOTMULTI\.([^\.])/DOTDOTMULTI $1/g; + $text =~ s/DOTMULTI\./DOTDOTMULTI/g; + } + + # seperate out "," except if within numbers (5,300) + #$text =~ s/([^\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g; + + # separate out "," except if within numbers (5,300) + # previous "global" application skips some: A,B,C,D,E > A , B,C , D,E + # first application uses up B so rule can't see B,C + # two-step version here may create extra spaces but these are removed later + # will also space digit,letter or letter,digit forms (redundant with next section) + $text =~ s/([^\p{IsN}])[,]/$1 , /g; + $text =~ s/[,]([^\p{IsN}])/ , $1/g; + + # separate , pre and post number + #$text =~ s/([\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g; + #$text =~ s/([^\p{IsN}])[,]([\p{IsN}])/$1 , $2/g; + + # turn `into ' + $text =~ s/\`/\'/g; + + #turn '' into " + $text =~ s/\'\'/ \" /g; + + if ($language eq "en") + { + #split contractions right + $text =~ s/([^\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([^\p{IsAlpha}\p{IsN}])[']([\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([\p{IsAlpha}])[']([\p{IsAlpha}])/$1 '$2/g; + #special case for "1990's" + $text =~ s/([\p{IsN}])[']([s])/$1 '$2/g; + } + elsif (($language eq "fr") or ($language eq "it")) + { + #split contractions left + $text =~ s/([^\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([^\p{IsAlpha}])[']([\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([\p{IsAlpha}])[']([\p{IsAlpha}])/$1' $2/g; + } + else + { + $text =~ s/\'/ \' /g; + } + + #word token method + my @words = split(/\s/,$text); + $text = ""; + for (my $i=0;$i<(scalar(@words));$i++) + { + my $word = $words[$i]; + if ( $word =~ /^(\S+)\.$/) + { + my $pre = $1; + if (($pre =~ /\./ && $pre =~ /\p{IsAlpha}/) || ($NONBREAKING_PREFIX{$pre} && $NONBREAKING_PREFIX{$pre}==1) || ($i/\>/g; # xml + $text =~ s/\'/\'/g; # xml + $text =~ s/\"/\"/g; # xml + $text =~ s/\[/\[/g; # syntax non-terminal + $text =~ s/\]/\]/g; # syntax non-terminal + + #ensure final line break + $text .= "\n" unless $text =~ /\n$/; + + return $text; +} + +sub tokenize_penn +{ + # Improved compatibility with Penn Treebank tokenization. Useful if + # the text is to later be parsed with a PTB-trained parser. + # + # Adapted from Robert MacIntyre's sed script: + # http://www.cis.upenn.edu/~treebank/tokenizer.sed + + my($text) = @_; + chomp($text); + + # remove ASCII junk + $text =~ s/\s+/ /g; + $text =~ s/[\000-\037]//g; + + # attempt to get correct directional quotes + $text =~ s/^``/`` /g; + $text =~ s/^"/`` /g; + $text =~ s/^`([^`])/` $1/g; + $text =~ s/^'/` /g; + $text =~ s/([ ([{<])"/$1 `` /g; + $text =~ s/([ ([{<])``/$1 `` /g; + $text =~ s/([ ([{<])`([^`])/$1 ` $2/g; + $text =~ s/([ ([{<])'/$1 ` /g; + # close quotes handled at end + + $text =~ s=\.\.\.= _ELLIPSIS_ =g; + + # separate out "," except if within numbers (5,300) + $text =~ s/([^\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g; + # separate , pre and post number + $text =~ s/([\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g; + $text =~ s/([^\p{IsN}])[,]([\p{IsN}])/$1 , $2/g; + + #$text =~ s=([;:@#\$%&\p{IsSc}])= $1 =g; +$text =~ s=([;:@#\$%&\p{IsSc}\p{IsSo}])= $1 =g; + + # Separate out intra-token slashes. PTB tokenization doesn't do this, so + # the tokens should be merged prior to parsing with a PTB-trained parser + # (see syntax-hyphen-splitting.perl). + $text =~ s/([\p{IsAlnum}])\/([\p{IsAlnum}])/$1 \@\/\@ $2/g; + + # Assume sentence tokenization has been done first, so split FINAL periods + # only. + $text =~ s=([^.])([.])([\]\)}>"']*) ?$=$1 $2$3 =g; + # however, we may as well split ALL question marks and exclamation points, + # since they shouldn't have the abbrev.-marker ambiguity problem + $text =~ s=([?!])= $1 =g; + + # parentheses, brackets, etc. + $text =~ s=([\]\[\(\){}<>])= $1 =g; + $text =~ s/\(/-LRB-/g; + $text =~ s/\)/-RRB-/g; + $text =~ s/\[/-LSB-/g; + $text =~ s/\]/-RSB-/g; + $text =~ s/{/-LCB-/g; + $text =~ s/}/-RCB-/g; + + $text =~ s=--= -- =g; + + # First off, add a space to the beginning and end of each line, to reduce + # necessary number of regexps. + $text =~ s=$= =; + $text =~ s=^= =; + + $text =~ s="= '' =g; + # possessive or close-single-quote + $text =~ s=([^'])' =$1 ' =g; + # as in it's, I'm, we'd + $text =~ s='([sSmMdD]) = '$1 =g; + $text =~ s='ll = 'll =g; + $text =~ s='re = 're =g; + $text =~ s='ve = 've =g; + $text =~ s=n't = n't =g; + $text =~ s='LL = 'LL =g; + $text =~ s='RE = 'RE =g; + $text =~ s='VE = 'VE =g; + $text =~ s=N'T = N'T =g; + + $text =~ s= ([Cc])annot = $1an not =g; + $text =~ s= ([Dd])'ye = $1' ye =g; + $text =~ s= ([Gg])imme = $1im me =g; + $text =~ s= ([Gg])onna = $1on na =g; + $text =~ s= ([Gg])otta = $1ot ta =g; + $text =~ s= ([Ll])emme = $1em me =g; + $text =~ s= ([Mm])ore'n = $1ore 'n =g; + $text =~ s= '([Tt])is = '$1 is =g; + $text =~ s= '([Tt])was = '$1 was =g; + $text =~ s= ([Ww])anna = $1an na =g; + + #word token method + my @words = split(/\s/,$text); + $text = ""; + for (my $i=0;$i<(scalar(@words));$i++) + { + my $word = $words[$i]; + if ( $word =~ /^(\S+)\.$/) + { + my $pre = $1; + if (($pre =~ /\./ && $pre =~ /\p{IsAlpha}/) || ($NONBREAKING_PREFIX{$pre} && $NONBREAKING_PREFIX{$pre}==1) || ($i/\>/g; # xml + $text =~ s/\'/\'/g; # xml + $text =~ s/\"/\"/g; # xml + $text =~ s/\[/\[/g; # syntax non-terminal + $text =~ s/\]/\]/g; # syntax non-terminal + + #ensure final line break + $text .= "\n" unless $text =~ /\n$/; + + return $text; +} + +sub load_prefixes +{ + my ($language, $PREFIX_REF) = @_; + + my $prefixfile = "$mydir/nonbreaking_prefix.$language"; + + #default back to English if we don't have a language-specific prefix file + if (!(-e $prefixfile)) + { + $prefixfile = "$mydir/nonbreaking_prefix.en"; + print STDERR "WARNING: No known abbreviations for language '$language', attempting fall-back to English version...\n"; + die ("ERROR: No abbreviations files found in $mydir\n") unless (-e $prefixfile); + } + + if (-e "$prefixfile") + { + open(PREFIX, "<:utf8", "$prefixfile"); + while () + { + my $item = $_; + chomp($item); + if (($item) && (substr($item,0,1) ne "#")) + { + if ($item =~ /(.*)[\s]+(\#NUMERIC_ONLY\#)/) + { + $PREFIX_REF->{$1} = 2; + } + else + { + $PREFIX_REF->{$item} = 1; + } + } + } + close(PREFIX); + } +} + diff --git a/worker/src/util/truecase.perl b/worker/src/util/truecase.perl new file mode 100755 index 0000000..811654f --- /dev/null +++ b/worker/src/util/truecase.perl @@ -0,0 +1,87 @@ +#!/usr/bin/perl -w + +# $Id: train-recaser.perl 1326 2007-03-26 05:44:27Z bojar $ +use strict; +use Getopt::Long "GetOptions"; +$|=1; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); + +# apply switches +my $MODEL; +die("truecase.perl --model truecaser < in > out") + unless &GetOptions('model=s' => \$MODEL); + +my (%BEST,%KNOWN); +open(MODEL,$MODEL) || die("ERROR: could not open '$MODEL'"); +binmode(MODEL, ":utf8"); +while() { + my ($word,@OPTIONS) = split; + $BEST{ lc($word) } = $word; + $KNOWN{ $word } = 1; + for(my $i=1;$i<$#OPTIONS;$i+=2) { + $KNOWN{ $OPTIONS[$i] } = 1; + } +} +close(MODEL); + +my %SENTENCE_END = ("."=>1,":"=>1,"?"=>1,"!"=>1); +my %DELAYED_SENTENCE_START = ("("=>1,"["=>1,"\""=>1,"'"=>1); + +while() { + chop; + my ($WORD,$MARKUP) = split_xml($_); + my $sentence_start = 1; + for(my $i=0;$i<=$#$WORD;$i++) { + print " " if $i; + print $$MARKUP[$i]; + + $$WORD[$i] =~ /^([^\|]+)(.*)/; + my $word = $1; + my $otherfactors = $2; + + if ($sentence_start && defined($BEST{lc($word)})) { + print $BEST{lc($word)}; # truecase sentence start + } + elsif (defined($KNOWN{$word})) { + print $word; # don't change known words + } + elsif (defined($BEST{lc($word)})) { + print $BEST{lc($word)}; # truecase otherwise unknown words + } + else { + print $word; # unknown, nothing to do + } + print $otherfactors; + + if ( defined($SENTENCE_END{ $word })) { $sentence_start = 1; } + elsif (!defined($DELAYED_SENTENCE_START{ $word })) { $sentence_start = 0; } + } + print " ".$$MARKUP[$#$MARKUP]; + print "\n"; +} + +# store away xml markup +sub split_xml { + my ($line) = @_; + my (@WORD,@MARKUP); + my $i = 0; + $MARKUP[0] = ""; + while($line =~ /\S/) { + if ($line =~ /^\s*(<\S[^>]*>)(.*)$/) { + $MARKUP[$i] .= $1." "; + $line = $2; + } + elsif ($line =~ /^\s*(\S+)(.*)$/) { + $WORD[$i++] = $1; + $MARKUP[$i] = ""; + $line = $2; + } + else { + die("ERROR: huh? $line\n"); + } + } + chop($MARKUP[$#MARKUP]); + return (\@WORD,\@MARKUP); +} From 639dd9f2144ae42de55d6fcc6b216a654834b797 Mon Sep 17 00:00:00 2001 From: Eleftherios Avramidis Date: Tue, 13 Jan 2015 14:59:48 +0100 Subject: [PATCH 2/7] Modifications to allow perl piped pre- and post-processors --- worker/src/tasks/translate.py | 84 ++++++++++++++++++++++++++-------- worker/src/util/detokenize.py | 4 ++ worker/src/util/tokenize.py | 5 ++ worker/src/util/tokenizer.perl | 4 +- worker/src/worker.py | 6 ++- 5 files changed, 81 insertions(+), 22 deletions(-) diff --git a/worker/src/tasks/translate.py b/worker/src/tasks/translate.py index 532522b..33224ae 100644 --- a/worker/src/tasks/translate.py +++ b/worker/src/tasks/translate.py @@ -5,11 +5,19 @@ import xmlrpclib import operator import os +import logging as logger from util.parallel import parallel_map from util.tokenize import Tokenizer from util.detokenize import Detokenizer from util.split_sentences import SentenceSplitter +from util.preprocessor import Tokenizer as PerlTokenizer +from util.preprocessor import Detokenizer as PerlDetokenizer +from util.preprocessor import Truecaser +from util.preprocessor import Detruecaser +from util.preprocessor import Normalizer +from util.preprocessor import CompoundSplitter + class Translator(object): """Base class for all classes that handle the 'translate' task for MTMonkeyWorkers""" @@ -62,10 +70,20 @@ def process_task(self, task): class MosesTranslator(Translator): """Handles the 'translate' task for MTMonkeyWorkers using Moses XML-RPC servers - and built-in segmentation, tokenization, and detokenization. + and built-in segmentation, tokenization, and detokenization. + @ivar translate_proxy_addr: proxy address for the translation server + @ivar recase_proxy_addr: proxy address for the recaser-decoder server + @ivar splitter: the sentence splitter class to be used along all threads + @type splitter: SentenceSplitter + @ivar preprocessors: a list of pre-processing classes supporting the function + "process_string" to run before text is sent to the decoder + @ivar postprocessors: a list of post-processing classes to run after text is + sent to the decoder + @ivar threads: desired number of threads + @type threads: int """ - def __init__(self, translate_port, recase_port, source_lang, target_lang, threads): + def __init__(self, translate_port, recase_port, source_lang, target_lang, threads, truecaser_model=None, splitter_model=None, perl_tokenizer=None, normalizer=None): """Initialize a MosesTranslator object according to the given configuration settings. @@ -77,16 +95,42 @@ def __init__(self, translate_port, recase_port, source_lang, target_lang, thread # precompile XML-RPC Moses server addresses self.translate_proxy_addr = "http://localhost:" + translate_port + "/RPC2" self.recase_proxy_addr = None - if recase_port is not None: + if recase_port is not None and recase_port.strip() != "": self.recase_proxy_addr = "http://localhost:" + recase_port + "/RPC2" # initialize text processing tools (can be shared among threads) self.splitter = SentenceSplitter({'language': source_lang}) - self.tokenizer = Tokenizer({'lowercase': True, + # put sentence-level pre- and post-processors in two lists + # depending on whether they are enabled from the settings + self.preprocessors = [] + self.postprocessors = [] + if normalizer: + self.normalizer = Normalizer(source_lang) + if not perl_tokenizer: + tokenizer = Tokenizer({'lowercase': True, 'moses_escape': True}) - self.detokenizer = Detokenizer({'moses_deescape': True, + self.preprocessors.append(tokenizer) + detokenizer = Detokenizer({'moses_deescape': True, 'capitalize_sents': True, - 'language': target_lang}) + 'language': target_lang}) + self.postprocessors.append(tokenizer) + else: + tokenizer = PerlTokenizer(source_lang) + self.preprocessors.append(tokenizer) + detokenizer = PerlDetokenizer(target_lang) + self.postprocessors.append(detokenizer) + if truecaser_model: + truecaser = Truecaser(source_lang, truecaser_model) + self.preprocessors.append(truecaser) + detruecaser = Detruecaser(target_lang) + self.postprocessors.append(detruecaser) + if splitter_model: + compound_splitter = CompoundSplitter(source_lang, splitter_model) + self.preprocessors.append(compound_splitter) + + #post-processors to run in the opposite order as pre-processors + self.postprocessors.reverse() + self.threads = threads @@ -132,12 +176,14 @@ def _translate(self, src, doalign, dodetok, nbestsize, ret_src_tok, dotok, doseg if self.recase_proxy_addr is not None: # recasing only if there is a recaser set up recase_proxy = xmlrpclib.ServerProxy(self.recase_proxy_addr) - # tokenize - src_tokenized = self.tokenizer.tokenize(src) if dotok else src - + # preprocess + if dotok: + for preprocessor in self.preprocessors: + src = preprocessor.process_string(src) + logger.warning("Preprocessed source after {}: {}".format(preprocessor.__class__.__name__, src)) # translate translation = translate_proxy.translate({ - "text": src_tokenized, + "text": src, "align": doalign, "nbest": nbestsize, "nbest-distinct": True, @@ -149,22 +195,24 @@ def _translate(self, src, doalign, dodetok, nbestsize, ret_src_tok, dotok, doseg for hypo in translation['nbest']: # recase (if there is a recaser set up) if recase_proxy is not None: - recased = recase_proxy.translate({"text": hypo['hyp']})['text'].strip() + postprocessed = recase_proxy.translate({"text": hypo['hyp']})['text'].strip() else: - recased = hypo['hyp'] + postprocessed = hypo['hyp'] # construct the output parsed_hypo = { - 'text': recased, + 'text': postprocessed, + 'text-unprocessed': hypo['hyp'], 'score': hypo['totalScore'], 'rank': rank, } - if dodetok: # detokenize if needed - parsed_hypo['text'] = self.detokenizer.detokenize(recased) - + if dodetok: # postprocess if needed + for postprocessor in self.postprocessors: + postprocessed = postprocessor.process_string(postprocessed) + parsed_hypo['text'] = postprocessed if doalign: # provide alignment information if needed - parsed_hypo['tokenized'] = recased - parsed_hypo['alignment-raw'] = _add_tgt_end(hypo['align'], recased) + parsed_hypo['tokenized'] = postprocessed + parsed_hypo['alignment-raw'] = _add_tgt_end(hypo['align'], postprocessed) rank += 1 hypos.append(parsed_hypo) diff --git a/worker/src/util/detokenize.py b/worker/src/util/detokenize.py index 256b460..e6e934f 100755 --- a/worker/src/util/detokenize.py +++ b/worker/src/util/detokenize.py @@ -150,6 +150,10 @@ def detokenize(self, text): text = text[0].upper() + text[1:] return text + def process_string(self, text): + return self.detokenize(text) + + def display_usage(): """\ diff --git a/worker/src/util/tokenize.py b/worker/src/util/tokenize.py index 3ea4f41..b62b6a7 100755 --- a/worker/src/util/tokenize.py +++ b/worker/src/util/tokenize.py @@ -136,6 +136,11 @@ def tokenize(self, text): text = text.lower() return text + def process_string(self, text): + """ + Redirect function to make it compatible with other pre-/post-processors + """ + return self.tokenize(text) def display_usage(): """\ diff --git a/worker/src/util/tokenizer.perl b/worker/src/util/tokenizer.perl index 186dbed..4e61880 100755 --- a/worker/src/util/tokenizer.perl +++ b/worker/src/util/tokenizer.perl @@ -19,9 +19,7 @@ use Time::HiRes; use Thread; -my $mydir = "$RealBin/../share/nonbreaking_prefixes"; -print $mydir; - +my $mydir = "$RealBin/nonbreaking_prefixes"; my %NONBREAKING_PREFIX = (); my @protected_patterns = (); my $protected_patterns_file = ""; diff --git a/worker/src/worker.py b/worker/src/worker.py index 2cde64b..8f502a3 100755 --- a/worker/src/worker.py +++ b/worker/src/worker.py @@ -34,7 +34,11 @@ def __init__(self, config, logger): config.get('RECASE_PORT'), config.get('SOURCE_LANG', 'en'), config.get('TARGET_LANG', 'en'), - int(config.get('THREADS', '4'))) + int(config.get('THREADS', '4')), + config.get("TRUECASER_MODEL", None), + config.get("SPLITTER_MODEL", None), + config.get("PERL_TOKENIZER", None), + config.get("NORMALIZER", None)) self._logger = logger def process_task(self, task): From 63c40b229dc84e9d202434349459e1f1f201f1f8 Mon Sep 17 00:00:00 2001 From: Eleftherios Avramidis Date: Tue, 13 Jan 2015 19:13:10 +0100 Subject: [PATCH 3/7] Sample configuration file for worker with pipeline pre-processors --- config-example/worker.processors.cfg | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 config-example/worker.processors.cfg diff --git a/config-example/worker.processors.cfg b/config-example/worker.processors.cfg new file mode 100644 index 0000000..c0a45a7 --- /dev/null +++ b/config-example/worker.processors.cfg @@ -0,0 +1,10 @@ +PORT = 7001 +TRANSLATE_PORT = 8081 +#RECASE_PORT = # comment out if you use a truecaser +PERL_TOKENIZER = TRUE #comment out if you prefer the python tokenizer and de-tokenizer +#specify absolute path of pre-processor models or comment out if not needed +SPLITTER_MODEL = /project/qtleap/subprojects/metamue/software/worker_de-en/mt-2.1.1/models/deen/split-model.7.de +TRUECASER_MODEL = /project/qtleap/subprojects/metamue/software/worker_de-en/mt-2.1.1/models/deen/truecase-model.7.de +SOURCE_LANG = de +TARGET_LANG = en +THREADS = 4 From 805de823514c04afbfce60050f4e22bd5dbe96e0 Mon Sep 17 00:00:00 2001 From: Eleftherios Avramidis Date: Tue, 13 Jan 2015 19:23:05 +0100 Subject: [PATCH 4/7] Better worker configuration samples --- config-example/worker.cfg | 7 +++++++ config-example/worker.processors.cfg | 19 +++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/config-example/worker.cfg b/config-example/worker.cfg index da70146..348e391 100644 --- a/config-example/worker.cfg +++ b/config-example/worker.cfg @@ -4,3 +4,10 @@ RECASE_PORT = 9000 # comment out if you do not need a recaser SOURCE_LANG = en TARGET_LANG = de THREADS = 4 +#comment in and specify absolute path of pre-processor models or comment out if not needed +#SPLITTER_MODEL = +#TRUECASER_MODEL = +#comment in for the punctuation normalizer +#NORMALIZER = TRUE +#comment in for the original moses/perl tokenizer and detokenizer +#PERL_TOKENIZER = TRUE diff --git a/config-example/worker.processors.cfg b/config-example/worker.processors.cfg index c0a45a7..971341f 100644 --- a/config-example/worker.processors.cfg +++ b/config-example/worker.processors.cfg @@ -1,10 +1,13 @@ PORT = 7001 -TRANSLATE_PORT = 8081 -#RECASE_PORT = # comment out if you use a truecaser -PERL_TOKENIZER = TRUE #comment out if you prefer the python tokenizer and de-tokenizer -#specify absolute path of pre-processor models or comment out if not needed -SPLITTER_MODEL = /project/qtleap/subprojects/metamue/software/worker_de-en/mt-2.1.1/models/deen/split-model.7.de -TRUECASER_MODEL = /project/qtleap/subprojects/metamue/software/worker_de-en/mt-2.1.1/models/deen/truecase-model.7.de -SOURCE_LANG = de -TARGET_LANG = en +TRANSLATE_PORT = 8080 +#RECASE_PORT = 9000 # comment out if you use a truecaser +SOURCE_LANG = en +TARGET_LANG = de THREADS = 4 +#comment out if no pre- or post-processors needed +SPLITTER_MODEL = +TRUECASER_MODEL = +#comment out to disable normalizer +NORMALIZER = TRUE +#comment out to use the embedded python tokenizer +PERL_TOKENIZER = TRUE From 8fba2ae18efe6c4566be5e61215333e2de01358c Mon Sep 17 00:00:00 2001 From: Eleftherios Avramidis Date: Wed, 4 Feb 2015 13:00:04 +0100 Subject: [PATCH 5/7] Added preprocessor.py and basic-protected-patterns in the git repository --- worker/src/util/basic-protected-patterns | 4 + worker/src/util/preprocessor.py | 165 +++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 worker/src/util/basic-protected-patterns create mode 100644 worker/src/util/preprocessor.py diff --git a/worker/src/util/basic-protected-patterns b/worker/src/util/basic-protected-patterns new file mode 100644 index 0000000..276bb84 --- /dev/null +++ b/worker/src/util/basic-protected-patterns @@ -0,0 +1,4 @@ +<\/?\S+\/?> +<\S+( [a-zA-Z0-9]+\=\"?[^\"]\")+ ?\/?> +<\S+( [a-zA-Z0-9]+\=\'?[^\']\')+ ?\/?> +(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? diff --git a/worker/src/util/preprocessor.py b/worker/src/util/preprocessor.py new file mode 100644 index 0000000..cf90d83 --- /dev/null +++ b/worker/src/util/preprocessor.py @@ -0,0 +1,165 @@ +''' +Created on 24 Mar 2012 +@author: Lefteris Avramidis +''' +import subprocess +import codecs +import os +import Queue +import threading +import logging as logger + +class Preprocessor(object): + """ + """ + def __init__(self, lang): + self.lang = lang + + def add_features_src(self, simplesentence, parallelsentence = None): + src_lang = parallelsentence.get_attribute("langsrc") #TODO: make this format independent by adding it as an attribute of the sentence objects + if src_lang == self.lang: + simplesentence.string = self.process_string(simplesentence.string) + return simplesentence + + def add_features_tgt(self, simplesentence, parallelsentence = None): + tgt_lang = parallelsentence.get_attribute("langtgt") + if tgt_lang == self.lang: + simplesentence.string = self.process_string(simplesentence.string) + return simplesentence + + + def process_string(self, string): + raise NotImplementedError + + +class CommandlinePreprocessor(Preprocessor): + + def _enqueue_output(self, stdout, queue): + out = 0 + for line in iter(stdout.readline, ''): + print "thread received response: ", line + queue.put(line) + + def __init__(self, path, lang, params = {}, command_template = ""): + self.lang = lang + params["lang"] = lang + params["path"] = path + self.command = command_template.format(**params) + command_items = self.command.split(' ') + self.output = [] + self.running = True + + self.process = subprocess.Popen(command_items, + shell=False, + bufsize=1, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + + def process_string(self, string): + + string = string.encode('utf-8') + #write to the stdin pipe followed by some space bytes in order to flush + self.process.stdin.write('{0}{1}\n'.format(string, ' '*10240)) + self.process.stdin.flush() + self.process.stdout.flush() + + output = self.process.stdout.readline().strip() + + #some preprocessors occasionally return an empty string. In that case read once more + if output == "" and len(string) > 1: + output = self.process.stdout.readline().strip() + + + output = output.decode('utf-8') + return output + + def close(self): + self.running = False + try: + self.process.stdin.close() + self.process.terminate() + except: + pass + + def __del__(self): + self.close() + + def _get_temporary_file(self, strings): + import tempfile + + f, filename = tempfile.mkstemp(text=True) + os.close(f) + print filename + f = open(filename, 'w') + for string in strings: + f.write(string) + f.write('\n') + f.close() + return filename + + def _get_tool_output(self, strings): + tmpfilename = self._get_temporary_file(strings) + tmpfile = open(tmpfilename, 'r') + commanditems = self.command.split(' ') + output = subprocess.check_output(commanditems, stdin=tmpfile).split('\n') + tmpfile.close() + #os.remove(tmpfile) + return output + +class Normalizer(CommandlinePreprocessor): + def __init__(self, lang): + directory = os.path.dirname(__file__) + path = os.path.join(directory, 'normalize-punctuation.perl') + command_template = "perl {path} -b -l {lang}" + super(Normalizer, self).__init__(path, lang, {}, command_template) + +class Tokenizer(CommandlinePreprocessor): + def __init__(self, lang): + directory = os.path.dirname(__file__) + path = os.path.join(directory, 'tokenizer.perl') + #TODO: protected parameters causes errors + #protected = os.path.join(directory, "basic-protected-patterns") + #logger.warning("Protected patterns loaded from {}".format(protected)) + #command_template = "".join(["perl {path} -p -b -l {lang}", " -protected {}".format(protected)]) + command_template = "perl {path} -p -b -l {lang}" + super(Tokenizer, self).__init__(path, lang, {}, command_template) + +class Detokenizer(CommandlinePreprocessor): + def __init__(self, lang): + directory = os.path.dirname(__file__) + path = os.path.join(directory, 'detokenizer.perl') + command_template = "perl {path} -l {lang}" + super(Detokenizer, self).__init__(path, lang, {}, command_template) + +class Truecaser(CommandlinePreprocessor): + def __init__(self, lang, model): + directory = os.path.dirname(__file__) + path = os.path.join(directory, 'truecase.perl') + command_template = "perl {path} -model {model}" + super(Truecaser, self).__init__(path, lang, {"model": model}, command_template) + +class Detruecaser(CommandlinePreprocessor): + def __init__(self, lang): + directory = os.path.dirname(__file__) + path = os.path.join(directory, 'detruecase.perl') + command_template = "perl {path} -b" + super(Detruecaser, self).__init__(path, lang, {}, command_template) + +class CompoundSplitter(CommandlinePreprocessor): + def __init__(self, lang, model): + directory = os.path.dirname(__file__) + path = os.path.join(directory, 'compound-splitter.perl') + command_template = "perl {path} -model {model}" + super(CompoundSplitter, self).__init__(path, lang, {"model": model}, command_template) + + +if __name__ == '__main__': + import sys + tokenizer = Tokenizer("en") + string = sys.argv[1] + print tokenizer.process_string(string) + + + From 24a2a83450646913ccde5c39eb795f80539c611c Mon Sep 17 00:00:00 2001 From: Eleftherios Avramidis Date: Wed, 4 Feb 2015 16:12:27 +0100 Subject: [PATCH 6/7] Commented out warning message that failed when utf-8 characters are included --- worker/src/tasks/translate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/worker/src/tasks/translate.py b/worker/src/tasks/translate.py index 33224ae..69553d6 100644 --- a/worker/src/tasks/translate.py +++ b/worker/src/tasks/translate.py @@ -180,7 +180,8 @@ def _translate(self, src, doalign, dodetok, nbestsize, ret_src_tok, dotok, doseg if dotok: for preprocessor in self.preprocessors: src = preprocessor.process_string(src) - logger.warning("Preprocessed source after {}: {}".format(preprocessor.__class__.__name__, src)) + #commented out as causing utf-8 errors + #logger.warning("Preprocessed source after {}: {}".format(preprocessor.__class__.__name__, src)) # translate translation = translate_proxy.translate({ "text": src, From b3dd211bab708c2d9db55c41b2fe399be5ea4dea Mon Sep 17 00:00:00 2001 From: Eleftherios Avramidis Date: Thu, 12 Mar 2015 16:24:57 +0100 Subject: [PATCH 7/7] Fixed subprocess piping issues by bypassing multithreaded map on multiple sentences --- worker/src/tasks/translate.py | 23 ++++++++++++++++------- worker/src/util/preprocessor.py | 1 + 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/worker/src/tasks/translate.py b/worker/src/tasks/translate.py index 69553d6..34fcdaa 100644 --- a/worker/src/tasks/translate.py +++ b/worker/src/tasks/translate.py @@ -105,7 +105,8 @@ def __init__(self, translate_port, recase_port, source_lang, target_lang, thread self.preprocessors = [] self.postprocessors = [] if normalizer: - self.normalizer = Normalizer(source_lang) + normalizer = Normalizer(source_lang) + self.preprocessors.append(normalizer) if not perl_tokenizer: tokenizer = Tokenizer({'lowercase': True, 'moses_escape': True}) @@ -113,7 +114,7 @@ def __init__(self, translate_port, recase_port, source_lang, target_lang, thread detokenizer = Detokenizer({'moses_deescape': True, 'capitalize_sents': True, 'language': target_lang}) - self.postprocessors.append(tokenizer) + self.postprocessors.append(detokenizer) else: tokenizer = PerlTokenizer(source_lang) self.preprocessors.append(tokenizer) @@ -150,10 +151,14 @@ def process_task(self, task): src_lines = self.splitter.split_sentences(task['text']) if dosegment else [ task['text'] ] ret_src_tok = doalign or len(src_lines) > 1 - def _translator(line): - return self._translate(line, doalign, dodetok, nbestsize, ret_src_tok, dotok, dosegment) + #def _translator(line): + # return self._translate(line, doalign, dodetok, nbestsize, ret_src_tok, dotok, dosegment) + # + #translated = parallel_map(_translator, src_lines) + translated = [] + for line in src_lines: + translated.append(self._translate(line, doalign, dodetok, nbestsize, ret_src_tok, dotok, dosegment)) - translated = parallel_map(_translator, src_lines) return { 'translationId': uuid.uuid4().hex, @@ -177,8 +182,10 @@ def _translate(self, src, doalign, dodetok, nbestsize, ret_src_tok, dotok, doseg recase_proxy = xmlrpclib.ServerProxy(self.recase_proxy_addr) # preprocess + src_original = src if dotok: for preprocessor in self.preprocessors: + #logger.warning("Preprocessed source before {}: {}".format(preprocessor.__class__.__name__, src)) src = preprocessor.process_string(src) #commented out as causing utf-8 errors #logger.warning("Preprocessed source after {}: {}".format(preprocessor.__class__.__name__, src)) @@ -209,7 +216,9 @@ def _translate(self, src, doalign, dodetok, nbestsize, ret_src_tok, dotok, doseg } if dodetok: # postprocess if needed for postprocessor in self.postprocessors: + #logger.warning("Postprocessed output before {}: {}".format(postprocessor.__class__.__name__, postprocessed)) postprocessed = postprocessor.process_string(postprocessed) + #logger.warning("Postprocessed output after {}: {}".format(postprocessor.__class__.__name__, postprocessed)) parsed_hypo['text'] = postprocessed if doalign: # provide alignment information if needed parsed_hypo['tokenized'] = postprocessed @@ -219,12 +228,12 @@ def _translate(self, src, doalign, dodetok, nbestsize, ret_src_tok, dotok, doseg hypos.append(parsed_hypo) result = { - 'src': src, + 'src': src_original, 'translated': hypos, } if ret_src_tok: - result['src-tokenized'] = src_tokenized + result['src-tokenized'] = src return result diff --git a/worker/src/util/preprocessor.py b/worker/src/util/preprocessor.py index cf90d83..30795d1 100644 --- a/worker/src/util/preprocessor.py +++ b/worker/src/util/preprocessor.py @@ -62,6 +62,7 @@ def process_string(self, string): string = string.encode('utf-8') #write to the stdin pipe followed by some space bytes in order to flush self.process.stdin.write('{0}{1}\n'.format(string, ' '*10240)) + #self.process.stdin.write(string.strip()) self.process.stdin.flush() self.process.stdout.flush()