summary | shortlog | log | commit | commitdiff | tree
raw | patch | inline | side by side (parent: 517123f)
raw | patch | inline | side by side (parent: 517123f)
| author | Dan Povey <dpovey@gmail.com> | |
| Tue, 23 Apr 2013 22:24:52 +0000 (22:24 +0000) | ||
| committer | Dan Povey <dpovey@gmail.com> | |
| Tue, 23 Apr 2013 22:24:52 +0000 (22:24 +0000) |
22 files changed:
index 81b517ec4e2c0f44f45f3728b4e5faefaba600ae..ca638bdf058ed000468cd8894d6a026380a59184 100755 (executable)
+#/bin/bash
+
#Simple utility script to convert the gzipped ARPA lm into a G.fst file
lmfile=$1
index db415f31c82c1580be88875ae002524a78646c1c..6fe0fcbed70472ea45c8192e6d2d3dfedc57b68b 100755 (executable)
use_icu=true
icu_transform="'Any-Lower()'"
silence_word= # Optional silence word to insert (once) between words of the transcript.
-
# End configuration section.
echo $0 "$@"
diff --git a/egs/babel/s5/local/kws_data_prep_syllables.sh b/egs/babel/s5/local/kws_data_prep_syllables.sh
index d13ecd77c61a3c583d7a0b9b4d0b070b16997c80..c6245e52c9e5ff417940b39e3da8a7a59add1bd4 100755 (executable)
# Apache 2.0.
# Begin configuration section.
-case_insensitive=true
+silence_word= # Optional silence word to insert (once) between words of the transcript.
# End configuration section.
echo $0 "$@"
. parse_options.sh || exit 1;
-if [ $# -ne 5 ]; then
- echo "Usage: local/kws_data_prep.sh <lang-dir> <data-dir> <syllable-lexicon> <silphone> <kws-data-dir>"
- echo " e.g.: local/kws_data_prep.sh data/lang/ data/eval/ SIL data/kws/"
+if [ $# -ne 4 ]; then
+ echo "Usage: local/kws_data_prep_syllables.sh [options] <lang-dir> <data-dir> <syllable-lexicon> <kws-data-dir>"
+ echo " e.g.: local/kws_data_prep_syllables.sh data/lang/ data/dev10h/ SIL data/kws/"
echo "Input is in <kws-data-dir>: kwlist.xml, ecf.xml (rttm file not needed)."
echo "The lang directory is expected to be syllable-level. The syllable-lexicon "
echo "is a text file with lines of the form:"
echo " kwlist_outvocab.xml, keywords.fsts; note that the only syllable-level"
echo " output (and the only one that really matters) is keywords.fsts"
echo "Note: most important output is keywords.fsts"
+ echo " Options:"
+ echo " --silence-word <silence-word> # Note, this is required. It is a word, e.g. SIL,"
+ echo " # in the syllable lexicon, that's optional."
exit 1;
fi
langdir=$1;
datadir=$2;
syllable_lexicon=$3
-silphone=$4
-kwsdatadir=$5
+kwsdatadir=$4
keywords=$kwsdatadir/kwlist.xml
+[ -z $silence_word ] && echo "--silence-word option is required" && exit 1;
mkdir -p $kwsdatadir;
}
' > $kwsdatadir/keywords.txt
-[ ! -s "$syllable_lexicon" ] && echo "No such file '$f' (syllable lexicon), or empty file." && exit 1;
+[ ! -s "$syllable_lexicon" ] && echo "No such file '$syllable_lexicon' (syllable lexicon), or empty file." && exit 1;
# The word symbols on the first entry of $syllable_lexicon will be given a symbol-table
# file. We just use this symbol table in this script; the values will never appear
egrep " 0 | 0$" | cut -f 1 -d ' ' | \
local/subset_kwslist.pl $keywords > $kwsdatadir/kwlist_outvocab.xml
-local/make_lexicon_fst_special.pl $kwsdatadir/temp/syllable_lexicon.txt $silphone | \
+local/make_lexicon_fst_special.pl $kwsdatadir/temp/syllable_lexicon.txt $silence_word | \
sym2int.pl -f 4 $kwsdatadir/temp/words.txt | \
sym2int.pl -f 3 $langdir/words.txt | \
fstcompile | \
diff --git a/egs/babel/s5/local/lattice_to_ctm_syllable.sh b/egs/babel/s5/local/lattice_to_ctm_syllable.sh
--- /dev/null
@@ -0,0 +1,115 @@
+#!/bin/bash
+# Copyright Johns Hopkins University (Author: Daniel Povey) 2012. Apache 2.0.
+
+# begin configuration section.
+cmd=run.pl
+stage=0
+decode_mbr=true
+beam=4 # Use a fairly narrow beam because lattice-align-words is slow-ish.
+word_ins_penalty=0.5
+min_lmwt=7
+max_lmwt=17
+cleanup=true
+model=
+
+#end configuration section.
+
+#debugging stuff
+echo $0 $@
+
+[ -f ./path.sh ] && . ./path.sh
+[ -f ./cmd.sh ] && . ./cmd.sh
+. parse_options.sh || exit 1;
+
+if [ $# -ne 4 ]; then
+ echo "Usage: $0 [options] <dataDir> <langDir|graphDir> <w2s-dir> <decodeDir>" && exit;
+ echo "This is as lattice_to_ctm.sh, but for syllable-based systems where we want to"
+ echo "obtain word-level ctms. Here, <w2s-dir> is a directory like data/local/w2s,"
+ echo "as created by run-6-syllables.sh. It contains:"
+ echo " G.fst, Ldet.fst, words.txt, word_align_lexicon.int"
+ echo " Options:"
+ echo " --cmd (run.pl|queue.pl...) # specify how to run the sub-processes."
+ echo " --stage (0|1) # (createCTM | filterCTM )."
+ exit 1;
+fi
+
+data=$1
+lang=$2 # Note: may be graph directory not lang directory, but has the necessary stuff copied.
+w2sdir=$3
+dir=$4
+
+if [ -z "$model" ] ; then
+ model=`dirname $dir`/final.mdl # Relative path does not work in some cases
+ #model=$dir/../final.mdl # assume model one level up from decoding dir.
+ #[ ! -f $model ] && model=`(set +P; cd $dir/../; pwd)`/final.mdl
+fi
+
+for f in $lang/words.txt $lang/phones/word_boundary.int \
+ $model $data/segments $data/reco2file_and_channel $dir/lat.1.gz \
+ $w2sdir/{G.fst,Ldet.fst,words.txt,word_align_lexicon.int}; do
+ [ ! -f $f ] && echo "$0: expecting file $f to exist" && exit 1;
+done
+
+name=`basename $data`; # e.g. eval2000
+
+mkdir -p $dir/scoring/log
+
+# we are counting the LM twice since we have both the original, syllable-level LM
+# and the new, word-level one, so we scale by 0.5 to get a reasonably scaled
+# LM cost.
+
+if [ $stage -le 0 ]; then
+ nj=`cat $dir/num_jobs` || exit 1;
+ $cmd JOB=1:$nj $dir/scoring/log/get_word_lats.JOB.log \
+ lattice-compose "ark:gunzip -c $dir/lat.JOB.gz|" $w2sdir/Ldet.fst ark:- \| \
+ lattice-determinize ark:- ark:- \| \
+ lattice-compose ark:- $w2sdir/G.fst ark:- \| \
+ lattice-scale --lm-scale=0.5 ark:- "ark:|gzip -c >$dir/wlat.JOB.gz" || exit 1;
+fi
+
+if [ $stage -le 1 ]; then
+ $cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/get_ctm.LMWT.log \
+ mkdir -p $dir/score_LMWT/ '&&' \
+ lattice-scale --inv-acoustic-scale=LMWT "ark:gunzip -c $dir/wlat.*.gz|" ark:- \| \
+ lattice-add-penalty --word-ins-penalty=$word_ins_penalty ark:- ark:- \| \
+ lattice-prune --beam=$beam ark:- ark:- \| \
+ lattice-push ark:- ark:- \| \
+ lattice-align-words-lexicon --max-expand=10 --output-if-empty=true $w2sdir/word_align_lexicon.int $model ark:- ark:- \| \
+ lattice-to-ctm-conf --decode-mbr=$decode_mbr ark:- - \| \
+ utils/int2sym.pl -f 5 $w2sdir/words.txt \| \
+ utils/convert_ctm.pl $data/segments $data/reco2file_and_channel \
+ '>' $dir/score_LMWT/$name.ctm || exit 1;
+fi
+
+if [ $stage -le 2 ]; then
+ # Remove some stuff we don't want to score, from the ctm.
+ for x in $dir/score_*/$name.ctm; do
+ cp $x $x.bkup1;
+ cat $x.bkup1 | grep -v -E '\[NOISE|LAUGHTER|VOCALIZED-NOISE\]' | \
+ grep -v -E '<UNK>|%HESITATION|\(\(\)\)' | \
+ grep -v -E '<eps>' | \
+ grep -v -E '<noise>' | \
+ grep -v -E '<silence>' | \
+ grep -v -E '<unk>' | \
+ grep -v -E '<v-noise>' | \
+ perl -e '@list = (); %list = ();
+ while(<>) {
+ chomp;
+ @col = split(" ", $_);
+ push(@list, $_);
+ $key = "$col[0]" . " $col[1]";
+ $list{$key} = 1;
+ }
+ foreach(sort keys %list) {
+ $key = $_;
+ foreach(grep(/$key/, @list)) {
+ print "$_\n";
+ }
+ }' > $x;
+ done
+fi
+
+$cleanup && rm $dir/wlat.*.gz
+
+echo "Lattice2CTM finished on " `date`
+exit 0
diff --git a/egs/babel/s5/local/make_syllable_lexicon.sh b/egs/babel/s5/local/make_syllable_lexicon.sh
index bcd52e7c7a8fe508d5f2b49d95c445a33a44d3b3..118845982b999067896dc0f91472708041cd670d 100755 (executable)
w/o w o"
# config vars:
-
+pron_probs=false # If you set --pron-probs true, will expect pron-prob on input lexicon and produce
+ # pron-probs on word2syllable lexicon.
# end configs.
. utils/parse_options.sh
mkdir -p `dirname $s2p_lex_out`
cat $lex_in | perl -e '
- $w2s = shift @ARGV;
+ ($w2s, $pron_probs) = @ARGV;
open(W2S, ">$w2s") || die "opening word to syllable lexicon";
$saw_tabs = 0;
- while(<>) {
+ while(<STDIN>) {
chop;
- @A = split("\t", $_);
+ if ($pron_probs eq "true") {
+ m:(\S+)\s+(\S+)\s+(.+): || die "Bad line $_ (note: have pron probs).";
+ $word = $1;
+ $prob = $2;
+ $pron = $3;
+ ($prob > 0.0 && $prob <= 1.0) || die "Bad pron-prob $prob in line $_";
+ print W2S "$word $prob";
+ } else {
+ m:(\S+)\s+(.+): || die "Bad line $_ (note: do not have pron probs).";
+ $word = $1;
+ $pron = $2;
+ print W2S "$word";
+ }
+ @A = split("\t", $pron);
@A >= 1 || die "Bad lexicon line $_\n";
- $word = shift @A;
- split(" ", $word) > 1 && die "Bad lexicon line $_ (expecting word to be followed by tab)";
- print W2S $word;
if (@A > 1) { $saw_tabs = 1; }
foreach $s (@A) {
$s =~ s/^\s+//; # Remove leading space.
die "You seem to be using as input to this script, a lexicon that does not have " .
"syllables separated by tabs.";
}
- ' $w2s_lex_out | sort | uniq > $s2p_lex_out || exit 1;
+ ' $w2s_lex_out $pron_probs | sort | uniq > $s2p_lex_out || exit 1;
exit 0;
index b39fb6436e7fd54c91ab86b307a79332ca24846f..cef470421a0e49d1b97edf57cd175a1a7e340d90 100755 (executable)
name=`basename $data`; # e.g. eval2000
if [ $stage -le 1 ] ; then
- local/lattice_to_ctm.sh --decode-mbr $decode_mbr --min-lmwt $min_lmwt \
- --max-lmwt $max_lmwt --cmd "$cmd" --beam $beam --stage $stage \
- --word-ins-penalty $word_ins_penalty $data $lang $dir
+ if [ -d data/local/w2s_extended ]; then # this is for the syllable-based system.
+ local/lattice_to_ctm_syllable.sh --decode-mbr $decode_mbr --min-lmwt $min_lmwt \
+ --max-lmwt $max_lmwt --cmd "$cmd" --beam $beam --stage $stage \
+ --word-ins-penalty $word_ins_penalty $data $lang data/local/w2s_extended $dir
+ else
+ local/lattice_to_ctm.sh --decode-mbr $decode_mbr --min-lmwt $min_lmwt \
+ --max-lmwt $max_lmwt --cmd "$cmd" --beam $beam --stage $stage \
+ --word-ins-penalty $word_ins_penalty $data $lang $dir
+ fi
fi
if [ $stage -le 2 ] ; then
diff --git a/egs/babel/s5/path.sh b/egs/babel/s5/path.sh
index c1bb2bf47f661d7d7e34b925eece4d22c11395b5..720a5c0d6088eac3698343c50823011ff639137e 100755 (executable)
--- a/egs/babel/s5/path.sh
+++ b/egs/babel/s5/path.sh
export KALDI_ROOT=`pwd`/../../..
. /export/babel/data/software/env.sh
-export PATH=$PWD/utils/:$KALDI_ROOT/tools/sph2pipe_v2.5/:$KALDI_ROOT/src/bin:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/src/fstbin/:$KALDI_ROOT/src/gmmbin/:$KALDI_ROOT/src/featbin/:$KALDI_ROOT/src/lm/:$KALDI_ROOT/src/sgmmbin/:$KALDI_ROOT/src/sgmm2bin/:$KALDI_ROOT/src/fgmmbin/:$KALDI_ROOT/src/latbin/:$KALDI_ROOT/src/nnetbin:$KALDI_ROOT/src/nnet-cpubin/:$KALDI_ROOT/src/kwsbin:$SRILM:$PWD:$PATH
+export PATH=$PWD/utils/:$KALDI_ROOT/tools/sph2pipe_v2.5/:$KALDI_ROOT/src/bin:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/src/fstbin/:$KALDI_ROOT/src/gmmbin/:$KALDI_ROOT/src/featbin/:$KALDI_ROOT/src/lm/:$KALDI_ROOT/src/sgmmbin/:$KALDI_ROOT/src/sgmm2bin/:$KALDI_ROOT/src/fgmmbin/:$KALDI_ROOT/src/latbin/:$KALDI_ROOT/src/nnetbin:$KALDI_ROOT/src/nnet-cpubin/:$KALDI_ROOT/src/kwsbin:$PWD:$PATH
export LC_ALL=C
diff --git a/egs/babel/s5/run-6-syllables.sh b/egs/babel/s5/run-6-syllables.sh
--- /dev/null
@@ -0,0 +1,591 @@
+#!/bin/bash
+
+# This script creates the data directories for us to run a syllable-based system
+# in the directory with the same name as this, but with the -syllables suffix.
+
+# dependency and build a LM on the phone level. We get the syllable sequences from one
+# of the existing systems' alignments.
+
+set -e #Exit on non-zero return code from any command
+set -o pipefail #Exit if any of the commands in the pipeline will
+ #return non-zero return code
+set -u #Fail on an undefined variable
+
+. conf/common_vars.sh || exit 1;
+. ./lang.conf || exit 1;
+
+# extra config:
+syllable_ins_prob=0.01 # relative to prob of <unk>
+
+# We will create a directory with the same name as this one, but
+# with "-syllables" at the end of its name.
+target=`pwd`-syllables
+
+
+if [ ! -d $target ]; then
+ echo ---------------------------------------------------------------------
+ echo "Creating directory $target for the syllable system"
+ echo ---------------------------------------------------------------------
+ mkdir -p $target
+ cp *.sh $target
+ for x in steps utils local conf lang.conf; do
+ [ ! -s $x ] && echo "No such file or directory $x" && exit 1;
+ if [ -L $x ]; then # if these are links
+ cp -d $x $target # copy the link over.
+ else # create a link to here.
+ ln -s ../`basename $PWD`/$x $target
+ fi
+ done
+fi
+
+mkdir -p $target/data
+
+for dir in raw_train_data raw_dev2h_data raw_dev10h_data; do
+ ln -s `pwd`/data/$dir $target/data/ || true;
+done
+
+if [ ! -f data/local_withprob/.done ]; then
+ echo -------------------------------------------------------------------------------
+ echo "Creating lexicon with probabilities, data/local_withprob/lexiconp.txt on `date`"
+ echo -------------------------------------------------------------------------------
+
+ cp -rT data/local data/local_withprob
+ steps/get_lexicon_probs.sh data/train data/lang exp/tri5 data/local/lexicon.txt \
+ exp/tri5_lexprobs data/local_withprob/lexiconp.txt || exit 1;
+ touch data/local_withprob/.done
+fi
+
+
+mkdir -p $target/data/local # we'll put some stuff here...
+
+if [ ! -f $target/data/local/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Creating syllable-level lexicon in directory $target/data/local/w2s"
+ echo ---------------------------------------------------------------------
+
+ mkdir -p $target/data/local/w2s
+
+ ! local/make_syllable_lexicon.sh --pron-probs true data/local_withprob/lexiconp.txt \
+ $target/data/local/w2s/lexiconp.txt $target/data/local/lexicon.txt && \
+ echo "Error creating syllable lexicon" && exit 1;
+
+ for f in extra_questions.txt silence_phones.txt nonsilence_phones.txt optional_silence.txt; do
+ cp data/local/$f $target/data/local/$f
+ done
+
+ touch $target/data/local/.done
+fi
+
+if [ ! -f $target/data/lang/.done ]; then
+ echo ------------------------------------------------------------------------
+ echo "Creating lang directories $target/data/lang and $target/data/lang_nopos"
+ echo ------------------------------------------------------------------------
+
+ # create the "lang" directory for the per-syllable setup.
+ # note: the position dependency will now be on the syllable level, not the phone level.
+ # Now we specify zero silence prob, because we'll be putting the silence in
+ # the language model in order to reduce the chances of it being inserted between syllables
+ # where it shouldn't belong.
+ # The oov symbols '<oov>' doesn't really matter for this setup as there will be no OOVs;
+ # the scripts just require it.
+ utils/prepare_lang.sh --sil-prob 0.0 \
+ $target/data/local/ '<oov>' $target/data/local/tmp $target/data/lang
+
+ utils/prepare_lang.sh --sil-prob 0.0 --position-dependent-phones false \
+ $target/data/local/ '<oov>' $target/data/local/tmp $target/data/lang_nopos
+
+ # later stages require word-symbol tables of lang and lang_nopos be the same.
+ cmp $target/data/lang/words.txt $target/data/lang_nopos/words.txt || exit 1;
+
+ touch $target/data/lang/.done
+fi
+
+if [ ! -f $target/data/local/w2s/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Creating word-to-syllable lexicon FSTs"
+ echo --------------------------------------------------------------------
+
+ cp data/lang/words.txt $target/data/local/w2s/words.txt
+ cp $target/data/lang/words.txt $target/data/local/w2s/phones.txt
+
+ # This FST will be used for alignment of the current system to get syllable-level
+ # alignments.
+ utils/make_lexicon_fst.pl --pron-probs $target/data/local/w2s/lexiconp.txt \
+ 0.5 `cat data/lang/phones/optional_silence.txt` | \
+ fstcompile --osymbols=data/lang/words.txt --isymbols=$target/data/lang/words.txt | \
+ fstarcsort --sort_type=olabel > $target/data/local/w2s/L.fst
+
+ touch $target/data/local/w2s/.done
+fi
+
+if [ ! -f $target/data/local/w2s_extended/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Creating lexicon in $target/data/local/w2s_extended, for word decoding"
+ echo "(allows penalized insertion of syllables we can't form into words)."
+ echo --------------------------------------------------------------------
+ dir=$target/data/local/w2s_extended
+ mkdir -p $dir
+
+ cat $target/data/local/w2s/lexiconp.txt | \
+ perl -e ' $syll_prob = shift @ARGV;
+ ($syll_prob > 0.0 && $syll_prob <= 1.0) || die "Bad prob $prob";
+ while(<STDIN>) {
+ @A = split;
+ $word = shift @A; $prob = shift @A;
+ if (!($prob > 0.0 && $prob <= 1.0)) { die "Bad pron-prob $prob in line $_"; }
+ if (@A == 1) { $seen{$A[0]} = 1; } # saw this syllable as singleton.
+ foreach $a ( @A ) { $is_syllable{$a} = 1; }
+ print; # print out the line; all lines of the lexicon get printed, plus some more.
+ }
+ foreach $syllable (keys %is_syllable) {
+ if (! defined $seen{$syllable}) { # did not see as singleton pron.
+ print "<unk> $syll_prob $syllable\n"; # print new pron, <unk>.
+ } } ' $syllable_ins_prob > $dir/lexiconp.txt
+ ndisambig=`utils/add_lex_disambig.pl --pron-probs $dir/lexiconp.txt $dir/lexiconp_disambig.txt`
+ ndisambig=$[$ndisambig+1]; # add one disambig symbol for silence in lexicon FST. [may not be needed here]
+
+ ( for n in `seq 0 $ndisambig`; do echo '#'$n; done ) >$dir/disambig.txt
+ cat $target/data/lang/words.txt | awk '{print $1}' | cat - <(for n in `seq $ndisambig`; do echo '#'$n; done) | \
+ awk '{n=NR-1; print $1, n;}' > $dir/syllables.txt # syllable-level symbol table.
+ utils/sym2int.pl $dir/syllables.txt $dir/disambig.txt >$dir/disambig.int
+ cp data/lang/words.txt $dir/words.txt # word-level symbol table.
+
+ # Make the lexicon with disambig symbols; determinize; remove disambig symbols.
+ utils/make_lexicon_fst.pl --pron-probs $dir/lexiconp_disambig.txt 0.5 SIL "#$ndisambig" | \
+ fstcompile --osymbols=$dir/words.txt --isymbols=$dir/syllables.txt | \
+ fstdeterminizestar | fstrmsymbols $dir/disambig.int > $dir/Ldet.fst
+
+ # Remove the #0 symbol from the input of word-level G.fst, and remove
+ # <silence> from the output, and copy it over. We remove <silence> from the
+ # output because it won't matter for scoring and because it was causing some
+ # problems for phone-alignment which is part of word alignment. (a kind of blowup
+ # from symbols getting out of sync; problems only happens if have 1 symbol per phone).
+ disambig_word=`grep -w "#0" data/lang/words.txt | awk '{print $2}'`
+ silence_word=`grep -w "<silence>" data/lang/words.txt | awk '{print $2}'`
+ fstrmsymbols --remove-from-output=false "echo $disambig_word|" data/lang/G.fst | \
+ fstrmsymbols --remove-from-output=true "echo $silence_word|" | fstarcsort > $dir/G.fst
+
+
+ mkdir -p $dir/tmp
+ # remove pron-probs from syllable-level lexicon
+
+ # Get syllable-level lexicon with word-position dependent phones.
+ cat $target/data/lang/phones/align_lexicon.txt | cut -d ' ' -f 2- > $dir/tmp/syllable_lexicon.txt
+
+ # remove pron-probs from word-level lexicon.
+ cat $dir/lexiconp.txt | perl -ape 's/(\S+\s+)\S+\s+(.+)/$1$2/;' >$dir/lexicon.txt
+
+ utils/apply_map.pl -f 2- $dir/tmp/syllable_lexicon.txt < $dir/lexicon.txt \
+ >$dir/tmp/word2phone_lexicon.txt
+
+ echo "<eps> SIL_S" >> $dir/tmp/word2phone_lexicon.txt # add optional silence. It only appears
+ # in the form SIL_S, because silence is always a word in the syllable setup, never optional.
+ awk '{print $1, $0;}' < $dir/tmp/word2phone_lexicon.txt > $dir/word_align_lexicon.txt # duplicate 1st field
+
+ utils/sym2int.pl -f 1-2 $dir/words.txt < $dir/word_align_lexicon.txt | \
+ utils/sym2int.pl -f 3- $target/data/lang/phones.txt > $dir/word_align_lexicon.int
+fi
+
+if [ ! -f $target/data/train/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Using the data alignments to get syllable-level pronunciations"
+ echo --------------------------------------------------------------------
+ local/get_syllable_text.sh data/train data/lang $target/data/lang_nopos \
+ $target/data/local/w2s/L.fst \
+ exp/tri5_ali exp/tri5_align_syllables $target/data/train
+ touch $target/data/train/.done
+fi
+
+if [ ! -f exp/tri5_dev2h_ali/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Aligning dev data using transcripts (needed for LM dev set)"
+ echo --------------------------------------------------------------------
+
+ steps/align_fmllr.sh --retry-beam 80 \
+ --boost-silence $boost_sil --nj $decode_nj --cmd "$train_cmd" \
+ data/dev2h data/lang exp/tri5 exp/tri5_dev2h_ali
+
+ touch exp/tri5_dev2h_ali/.done
+fi
+
+if [ ! -f $target/data/dev2h/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Using the dev-data alignments to get syllable-level pronunciations"
+ echo --------------------------------------------------------------------
+ local/get_syllable_text.sh data/dev2h data/lang $target/data/lang_nopos \
+ $target/data/local/w2s/L.fst \
+ exp/tri5_dev2h_ali exp/tri5_align_dev2h_syllables $target/data/dev2h
+ touch $target/data/dev2h/.done
+fi
+
+
+if [ ! -d $target/data/dev10h ]; then
+ echo ---------------------------------------------------------------------
+ echo "Copying data directory $target/data/dev10h/"
+ echo --------------------------------------------------------------------
+ ## Note: the "text" in this directory is not really correct, it is word-based
+ ## not syllable based. This doesn't really matter as we won't ever use that.
+ cp -rT data/dev10h $target/data/dev10h
+ rm -rf $target/data/dev10h/split*
+ rm -rf $target/data/dev10h/kws
+fi
+
+for n in 2 10; do
+ if [ ! -d $target/data/dev${n}h/kws/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Preparing kws directory in $target/data/dev${n}h/kws"
+ echo --------------------------------------------------------------------
+ cp -rT data/dev${n}h/kws $target/data/dev${n}h/kws # first copy the old dir, then
+ # run a script for syllable-specific setup.
+
+ perl -ape 's/(\S+\s+)\S+\s+(.+)/$1$2/;' \
+ <$target/data/local/w2s/lexiconp.txt >$target/data/local/w2s/lexicon.txt
+
+ local/kws_data_prep_syllables.sh --silence-word SIL \
+ $target/data/lang $target/data/dev${n}h $target/data/local/w2s/lexicon.txt \
+ $target/data/dev${n}h/kws
+ fi
+done
+
+if [ ! -f $target/data/srilm/lm.gz ]; then
+ echo ---------------------------------------------------------------------
+ echo "Training LM in $target/data/srilm"
+ echo --------------------------------------------------------------------
+
+ local/train_lms_srilm.sh --dev-text $target/data/dev2h/text --train-text $target/data/train/text \
+ --words-file $target/data/lang/words.txt $target/data $target/data/srilm
+fi
+
+
+if [ ! -f $target/data/lang/G.fst ]; then
+ echo ---------------------------------------------------------------------
+ echo "Creating $target/data/lang/G.fst"
+ echo --------------------------------------------------------------------
+ local/arpa2G_syllables.sh $target/data/srilm/lm.gz $target/data/lang $target/data/lang
+fi
+
+
+
+exit 0;
+
+
+
+
+
+
+
+# Get phone sequences with word-boundary info removed from training and dev
+# data, we'll use these to get the "text" at the syllable level.
+
+
+utils/fix_data_dir.sh data/train_syllables/
+
+for n in 2 10; do
+ steps/align_fmllr.sh --retry-beam 80 \
+ --boost-silence $boost_sil --nj $decode_nj --cmd "$train_cmd" \
+ data/dev${n}h data/lang exp/tri5 exp/tri5_dev${n}h_ali
+
+ # repeat the steps above, for the dev data.
+ mkdir -p exp/tri5_dev${n}h_syllables
+ gunzip -c exp/tri5_dev${n}h_ali/ali.{?,??}.gz | ali-to-phones exp/tri5_ali/final.mdl ark:- ark,t:- | \
+ utils/int2sym.pl -f 2- data/lang/phones.txt - | \
+ sed -E 's/_I( |$)/ /g' | sed -E 's/_E( |$)/ /g' | sed -E 's/_B( |$)/ /g' | sed -E 's/_S( |$)/ /g' | \
+ utils/sym2int.pl -f 2- data/lang_syllables_nopos/phones.txt | \
+ gzip -c > exp/tri5_dev${n}h_syllables/phones.ark.gz
+
+ # Note: this replaces partial words with OOVs (3 warnings).
+ cat data/dev${n}h/text | utils/sym2int.pl --map-oov `cat data/lang/oov.int` -f 2- data/lang/words.txt | \
+ transcripts-to-fsts ark:- ark:- | fsttablecompose data/local/word2syllable_lexicon.fst ark:- ark,t:- | \
+ awk '{if (NF < 4) { print; } else { print $1, $2, $3, $3, $5; }}' | gzip -c > exp/tri5_dev${n}h_syllables/syllables.ark.gz
+
+
+ cp -rT data/dev${n}h data/dev${n}h_syllables
+ rm -r data/dev${n}h_syllables/split*
+
+ fsttablecompose data/lang_syllables_nopos/L.fst "ark:gunzip -c exp/tri5_dev${n}h_syllables/syllables.ark.gz|" ark:- | \
+ fsttablecompose "ark:gunzip -c exp/tri5_dev${n}h_syllables/phones.ark.gz | transcripts-to-fsts ark:- ark:- |" \
+ ark,s,cs:- ark,t:- | fsts-to-transcripts ark:- ark,t:- | int2sym.pl -f 2- data/lang_syllables_nopos/words.txt | \
+ sed 's/SIL SIL/SIL/g' > data/dev${n}h_syllables/text
+
+ # This version of the dev${n}h kws setup does not do anything with the keywords that are OOV w.r.t.
+ # the limitedLP training set; it's not expected to be better than the baseline.
+
+ local/kws_data_prep_syllables.sh --case-insensitive true data/lang_syllables \
+ data/dev${n}h_syllables data/local/word2syllable_lexicon.txt SIL data/dev${n}h_syllables/kws || exit 1
+
+ # Temp: this command makes a word->syllable lexicon from the FullLP lexicon.
+ local/make_syllable_lexicon.sh ../s5-vietnamese-full-phones/data/local/lexicon.txt \
+ data/local/word2syllable_lexicon_fullLP.txt /dev/null || exit 1;
+
+ # We'll create a version of the kws dir that has a fullLP lexicon.
+ cp -rT data/dev${n}h_syllables data/dev${n}h_syllables_fullLP
+ local/kws_data_prep_syllables.sh --case-insensitive true data/lang_syllables \
+ data/dev${n}h_syllables_fullLP data/local/word2syllable_lexicon_fullLP.txt SIL data/dev${n}h_syllables_fullLP/kws
+
+done
+ # Done preparing dev data.
+
+
+local/train_lms_srilm.sh --dev-text data/dev2h_syllables/text --train-text data/train_syllables/text \
+ --words-file data/lang_syllables/words.txt data data/srilm_syllables
+
+
+local/arpa2G_syllables.sh data/srilm_syllables/lm.gz data/lang_syllables data/lang_syllables
+
+
+
+
+if [ ! -f data/train_syllables_sub3/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Subsetting monophone training data in data/train_syllables_sub[123] on" `date`
+ echo ---------------------------------------------------------------------
+ numutt=`cat data/train_syllables/feats.scp | wc -l`;
+ utils/subset_data_dir.sh data/train_syllables 5000 data/train_syllables_sub1 || exit 1
+ if [ $numutt -gt 10000 ] ; then
+ utils/subset_data_dir.sh data/train_syllables 10000 data/train_syllables_sub2 || exit 1
+ else
+ (cd data; ln -s train_syllables train_syllables_sub2 ) || exit 1
+ fi
+ if [ $numutt -gt 20000 ] ; then
+ utils/subset_data_dir.sh data/train_syllables 20000 data/train_syllables_sub3 || exit 1
+ else
+ (cd data; ln -s train_syllables train_syllables_sub3 ) || exit 1
+ fi
+
+ touch data/train_syllables_sub3/.done
+fi
+
+mkdir -p exp_syllables
+
+if [ ! -f exp_syllables/mono/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Starting (small) monophone training in exp_syllables/mono on" `date`
+ echo ---------------------------------------------------------------------
+ steps/train_mono.sh \
+ --boost-silence $boost_sil --nj 8 --cmd "$train_cmd" \
+ data/train_syllables_sub1 data/lang_syllables exp_syllables/mono || exit 1
+ touch exp_syllables/mono/.done
+fi
+
+if [ ! -f exp_syllables/tri1/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Starting (small) triphone training in exp_syllables/tri1 on" `date`
+ echo ---------------------------------------------------------------------
+ steps/align_si.sh \
+ --boost-silence $boost_sil --nj 12 --cmd "$train_cmd" \
+ data/train_syllables_sub2 data/lang_syllables exp_syllables/mono exp_syllables/mono_ali_sub2 || exit 1
+ steps/train_deltas.sh \
+ --boost-silence $boost_sil --cmd "$train_cmd" \
+ $numLeavesTri1 $numGaussTri1 data/train_syllables_sub2 data/lang_syllables exp_syllables/mono_ali_sub2 exp_syllables/tri1 || exit 1
+ touch exp_syllables/tri1/.done
+fi
+
+
+echo ---------------------------------------------------------------------
+echo "Starting (medium) triphone training in exp_syllables/tri2 on" `date`
+echo ---------------------------------------------------------------------
+if [ ! -f exp_syllables/tri2/.done ]; then
+ steps/align_si.sh \
+ --boost-silence $boost_sil --nj 24 --cmd "$train_cmd" \
+ data/train_syllables_sub3 data/lang_syllables exp_syllables/tri1 exp_syllables/tri1_ali_sub3 || exit 1
+ steps/train_deltas.sh \
+ --boost-silence $boost_sil --cmd "$train_cmd" \
+ $numLeavesTri2 $numGaussTri2 data/train_syllables_sub3 data/lang_syllables exp_syllables/tri1_ali_sub3 exp_syllables/tri2 || exit 1
+ touch exp_syllables/tri2/.done
+fi
+
+echo ---------------------------------------------------------------------
+echo "Starting (full) triphone training in exp_syllables/tri3 on" `date`
+echo ---------------------------------------------------------------------
+if [ ! -f exp_syllables/tri3/.done ]; then
+ steps/align_si.sh \
+ --boost-silence $boost_sil --nj $train_nj --cmd "$train_cmd" \
+ data/train_syllables data/lang_syllables exp_syllables/tri2 exp_syllables/tri2_ali || exit 1
+ steps/train_deltas.sh \
+ --boost-silence $boost_sil --cmd "$train_cmd" \
+ $numLeavesTri3 $numGaussTri3 data/train_syllables data/lang_syllables exp_syllables/tri2_ali exp_syllables/tri3 || exit 1
+ touch exp_syllables/tri3/.done
+fi
+
+echo ---------------------------------------------------------------------
+echo "Starting (lda_mllt) triphone training in exp_syllables/tri4 on" `date`
+echo ---------------------------------------------------------------------
+if [ ! -f exp_syllables/tri4/.done ]; then
+ steps/align_si.sh \
+ --boost-silence $boost_sil --nj $train_nj --cmd "$train_cmd" \
+ data/train_syllables data/lang_syllables exp_syllables/tri3 exp_syllables/tri3_ali || exit 1
+ steps/train_lda_mllt.sh \
+ --boost-silence $boost_sil --cmd "$train_cmd" \
+ $numLeavesMLLT $numGaussMLLT data/train_syllables data/lang_syllables exp_syllables/tri3_ali exp_syllables/tri4 || exit 1
+ touch exp_syllables/tri4/.done
+fi
+
+if [ ! -f exp_syllables/tri5/.done ]; then
+ steps/align_si.sh \
+ --boost-silence $boost_sil --nj $train_nj --cmd "$train_cmd" \
+ data/train_syllables data/lang_syllables exp_syllables/tri4 exp_syllables/tri4_ali || exit 1
+ steps/train_sat.sh \
+ --boost-silence $boost_sil --cmd "$train_cmd" \
+ $numLeavesSAT $numGaussSAT data/train_syllables data/lang_syllables exp_syllables/tri4_ali exp_syllables/tri5 || exit 1
+ touch exp_syllables/tri5/.done
+fi
+
+
+utils/mkgraph.sh \
+ data/lang_syllables exp_syllables/tri5 exp_syllables/tri5/graph
+
+steps/decode_fmllr.sh --skip-scoring true --nj $decode_nj --cmd "$decode_cmd" --num-threads 6 \
+ --parallel-opts "-pe smp 6 -l mem_free=4G,ram_free=0.7G" \
+ exp_syllables/tri5/graph data/dev2h_syllables exp_syllables/tri5/decode_dev2h
+
+
+
+local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
+ data/lang_syllables data/dev2h_syllables exp_syllables/tri5/decode_dev2h &
+
+cp -rT exp_syllables/tri5/decode_dev2h exp_syllables/tri5/decode_dev2h_fullLP
+
+local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
+ data/lang_syllables data/dev2h_syllables_fullLP exp_syllables/tri5/decode_dev2h_fullLP &
+
+(
+ cp -rT exp_syllables/tri5/decode_dev2h exp_syllables/tri5/decode_dev2h_ntrue2.0
+ local/kws_search.sh --stage 2 --ntrue-scale 2.0 --cmd "$decode_cmd" --duptime $duptime \
+ data/lang_syllables data/dev2h_syllables exp_syllables/tri5/decode_dev2h_ntrue2.0
+)
+
+steps/decode_fmllr.sh --skip-scoring true --nj $decode_nj --cmd "$decode_cmd" --num-threads 6 \
+ --parallel-opts "-pe smp 6 -l mem_free=4G,ram_free=0.7G" \
+ exp_syllables/tri5/graph data/dev10h_syllables exp_syllables/tri5/decode_dev10h
+
+local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
+ data/lang_syllables data/dev10h_syllables exp_syllables/tri5/decode_dev10h
+
+(
+ utils/mkgraph.sh \
+ data/lang_syllables_norepeatsil exp_syllables/tri5 exp_syllables/tri5/graph_norepeatsil
+
+ steps/decode_fmllr.sh --skip-scoring true --nj $decode_nj --cmd "$decode_cmd" --num-threads 6 \
+ --parallel-opts "-pe smp 6 -l mem_free=4G,ram_free=0.7G" \
+ exp_syllables/tri5/graph_norepeatsil data/dev10h_syllables exp_syllables/tri5/decode_dev10h_norepeatsil
+
+ local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
+ data/lang_syllables_norepeatsil data/dev10h_syllables exp_syllables/tri5/decode_dev10h_norepeatsil
+)
+
+(
+ cp -rT exp_syllables/tri5/decode_dev10h exp_syllables/tri5/decode_dev10h_fullLP
+ local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
+ data/lang_syllables data/dev10h_syllables_fullLP exp_syllables/tri5/decode_dev10h_fullLP
+)
+
+(
+ cp -rT exp_syllables/tri5/decode_dev10h exp_syllables/tri5/decode_dev10h_ntrue2.0
+ local/kws_search.sh --stage 2 --ntrue-scale 2.0 --cmd "$decode_cmd" --duptime $duptime \
+ data/lang_syllables data/dev10h_syllables exp_syllables/tri5/decode_dev10h_ntrue2.0
+)
+
+( # In this block we will be using the syllable-level models together with an LM
+ # from the word-level setup to produce a G.fst for the syllable models that
+ # produces syllable level output that is constrained by the lexicon. This
+ # will let us know whether the improvement was from better acoustic modeling
+ # (unlikely, I think) or from language-model effects such as better pronunciation
+ # modeling.
+
+ cp -rT data/lang_syllables data/lang_syllables_constrained
+
+ disambig_id=`grep '#0' data/lang_syllables/words.txt | awk '{print $2}'` || exit 1;
+
+ syllable_disambig_symbol=`grep \#0 data/lang_syllables/words.txt | awk '{print $2}'`
+ word_disambig_symbol=`grep \#0 data/lang/words.txt | awk '{print $2}'`
+
+ ndisambig=`utils/add_lex_disambig.pl data/local/word2syllable_lexicon.txt data/local/word2syllable_lexicon_disambig.txt`
+ if ! grep '#1' data/lang_syllables/words.txt; then
+ cur_sym=`tail -n 1 data/lang_syllables/words.txt | awk '{print $2}'`
+ for n in `seq $ndisambig`; do
+ echo "#$n $[$n+$cur_sym]" >> data/lang_syllables/words.txt
+ done
+ fi
+ grep '^#' data/lang_syllables/words.txt | awk '{print $1}' > data/lang_syllables/phones/disambig.txt
+ grep '^#' data/lang_syllables/words.txt | awk '{print $2}' > data/lang_syllables/phones/disambig.int
+
+ utils/make_lexicon_fst.pl data/local/word2syllable_lexicon_disambig.txt 0.5 SIL | \
+ fstcompile --isymbols=data/lang_syllables/words.txt --osymbols=data/lang/words.txt \
+ --keep_isymbols=false --keep_osymbols=false | \
+ fstaddselfloops "echo $syllable_disambig_symbol |" "echo $word_disambig_symbol |" | \
+ fstcompose - data/lang/G.fst | \
+ fstproject | [remove disambigs] > data/lang_syllables_constrained/G.fst
+
+ utils/mkgraph.sh \
+ data/lang_syllables_constrained exp_syllables/tri5 exp_syllables/tri5/graph_constrained
+
+ steps/decode_fmllr.sh --skip-scoring true --nj $decode_nj --cmd "$decode_cmd" --num-threads 6 \
+ --parallel-opts "-pe smp 6 -l mem_free=4G,ram_free=0.7G" \
+ exp_syllables/tri5/graph data/dev10h_syllables exp_syllables/tri5/decode_dev10h
+)
+
+
+fi ##TEMP
+
+
+if [ ! -f exp_syllables/tri5_ali/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Starting exp_syllables/tri5_ali on" `date`
+ echo ---------------------------------------------------------------------
+ steps/align_fmllr.sh \
+ --boost-silence $boost_sil --nj $train_nj --cmd "$train_cmd" \
+ data/train_syllables data/lang_syllables exp_syllables/tri5 exp_syllables/tri5_ali || exit 1
+ touch exp_syllables/tri5_ali/.done
+fi
+
+if [ ! -f exp_syllables/ubm5/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Starting exp_syllables/ubm5 on" `date`
+ echo ---------------------------------------------------------------------
+ steps/train_ubm.sh \
+ --cmd "$train_cmd" \
+ $numGaussUBM data/train_syllables data/lang_syllables exp_syllables/tri5_ali exp_syllables/ubm5 || exit 1
+ touch exp_syllables/ubm5/.done
+fi
+
+
+if [ ! -f exp_syllables/sgmm5/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Starting exp_syllables/sgmm5 on" `date`
+ echo ---------------------------------------------------------------------
+ steps/train_sgmm2_group.sh \
+ --cmd "$train_cmd" --group 3 --parallel-opts "-l mem_free=6G,ram_free=2G" \
+ $numLeavesSGMM $numGaussSGMM data/train_syllables data/lang_syllables exp_syllables/tri5_ali exp_syllables/ubm5/final.ubm exp_syllables/sgmm5 || exit 1
+ touch exp_syllables/sgmm5/.done
+fi
+
+(
+ if [ ! -f exp_syllables/sgmm5/decode_dev2h_fmllr/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Spawning exp_syllables/sgmm5/decode_dev2h_fmllr on" `date`
+ echo ---------------------------------------------------------------------
+ echo "exp_syllables/sgmm5/decode_dev2h will wait on tri5 decode if necessary"
+ while [ ! -f exp_syllables/tri5/decode_dev2h/.done ]; do sleep 30; done
+ echo "[done]"
+ mkdir -p exp_syllables/sgmm5/graph
+ utils/mkgraph.sh \
+ data/lang_syllables exp_syllables/sgmm5 exp_syllables/sgmm5/graph &> exp_syllables/sgmm5/mkgraph.log
+
+ steps/decode_sgmm2.sh --use-fmllr true --nj $decode_nj --cmd "$decode_cmd" \
+ --num-threads 6 --parallel-opts "-pe smp 6 -l mem_free=5G,ram_free=0.8G" \
+ --transform-dir exp_syllables/tri5/decode_dev2h \
+ exp_syllables/sgmm5/graph data/dev2h/ exp_syllables/sgmm5/decode_dev2h_fmllr &> exp_syllables/sgmm5/decode_dev2h_fmllr.log
+ touch exp_syllables/sgmm5/decode_dev2h_fmllr/.done
+ fi
+
+ if [ ! -f exp_syllables/sgmm5/decode_dev2h_fmllr/kws/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Starting exp_syllables/sgmm5/decode_dev2h_fmllr/kws on" `date`
+ echo ---------------------------------------------------------------------
+ local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
+ data/lang_syllables data/dev2h exp_syllables/sgmm5/decode_dev2h_fmllr
+ touch exp_syllables/sgmm5/decode_dev2h_fmllr/kws/.done
+ fi
+) &
+
+wait
diff --git a/egs/babel/s5/run-7-pronprobs.sh b/egs/babel/s5/run-7-pronprobs.sh
--- /dev/null
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+
+# Working out pronunciation probabilities and testing with them.
+
+set -e #Exit on non-zero return code from any command
+set -o pipefail #Exit if any of the commands in the pipeline will
+ #return non-zero return code
+set -u #Fail on an undefined variable
+
+. conf/common_vars.sh || exit 1;
+. ./lang.conf || exit 1;
+
+if [ ! -f data/local_withprob/.done ]; then
+ echo -------------------------------------------------------------------------------
+ echo "Creating lexicon with probabilities, data/local_withprob/lexicon.txt on `date`"
+ echo -------------------------------------------------------------------------------
+
+ cp -rT data/local data/local_withprob
+ steps/get_lexicon_probs.sh data/train data/lang exp/tri5 data/local/lexicon.txt \
+ exp/tri5_lexprobs data/local_withprob/lexiconp.txt || exit 1;
+ touch data/local_withprob/.done
+fi
+
+mkdir -p data/lang_withprob
+if [[ ! -f data/lang_withprob/.done || data/lang_withprob/L.fst -ot data/local/lexiconp.txt ]]; then
+ echo ---------------------------------------------------------------------
+ echo "Creating L.fst etc in data/lang_withprob" `date`
+ echo ---------------------------------------------------------------------
+ utils/prepare_lang.sh --share-silence-phones true \
+ data/local_withprob $oovSymbol data/local_withprob/tmp.lang data/lang_withprob
+
+ cp data/lang/G.fst data/lang_withprob/G.fst
+ cmp data/lang/words.txt data/lang_withprob/words.txt || exit 1;
+ touch data/lang_withprob/.done
+fi
+
+
+type=dev10h
+
+eval my_data_dir=\$${type}_data_dir
+eval my_data_list=\$${type}_data_list
+if [ "$type" == dev2h ] ; then
+ my_nj=$decode_nj
+
+ eval my_ecf_file=$ecf_file
+ eval my_subset_ecf=$subset_ecf
+ eval my_kwlist_file=$kwlist_file
+ eval my_rttm_file=$rttm_file
+elif [ "$type" == dev10h ] ; then
+ eval my_nj=\$${type}_nj
+
+ eval my_ecf_file=$ecf_file
+ eval my_subset_ecf=false
+ eval my_kwlist_file=$kwlist_file
+ eval my_rttm_file=$rttm_file
+else
+ eval my_nj=\$${type}_nj
+
+ eval my_ecf_file=\$${type}_ecf_file
+ eval my_subset_ecf=\$${type}_subset_ecf
+ eval my_kwlist_file=\$${type}_kwlist_file
+ eval my_rttm_file=\$${type}_rttm_file
+fi
+
+
+if [ ! -f exp/tri5/decode_withprob_${type}/.done ]; then
+ echo ---------------------------------------------------------------------
+ echo "Spawning decoding with SAT models on" `date`
+ echo ---------------------------------------------------------------------
+ mkdir -p exp/tri5/graph
+ utils/mkgraph.sh \
+ data/lang_withprob exp/tri5 exp/tri5/graph_withprob |tee exp/tri5/mkgraph_withprob.log
+ mkdir -p exp/tri5/decode_withprob_${type}
+
+ steps/decode_fmllr.sh --nj $my_nj \
+ --cmd "$decode_cmd" "${decode_extra_opts[@]}" \
+ exp/tri5/graph_withprob data/${type} exp/tri5/decode_withprob_${type} |tee exp/tri5/decode_withprob_${type}.log
+ touch exp/tri5/decode_withprob_${type}/.done
+fi
+
+if [ ! -f exp/tri5/decode_withprob_${type}/kws/.done ]; then
+ local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
+ data/lang_withprob data/${type} exp/tri5/decode_withprob_${type}
+ local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
+ data/lang_withprob data/${type} exp/tri5/decode_withprob_${type}.si
+ touch exp/tri5/decode_withprob_${type}/kws/.done
+fi
+
+
+exit 0;
+
diff --git a/egs/wsj/s5/run.sh b/egs/wsj/s5/run.sh
index e5bc1e09ea11241efac23abe2f1e6aef9bc56d3e..be70cc511d6d77045b4aef4fad8742506dbe0f5f 100755 (executable)
--- a/egs/wsj/s5/run.sh
+++ b/egs/wsj/s5/run.sh
steps/word_align_lattices.sh --cmd "$train_cmd" --silence-label $sil_label \
data/lang_test_tgpr exp/tri1/decode_tgpr_dev93 exp/tri1/decode_tgpr_dev93_aligned || exit 1;
-
-# Align tri1 system with si84 data.
steps/align_si.sh --nj 10 --cmd "$train_cmd" \
data/train_si84 data/lang exp/tri1 exp/tri1_ali_si84 || exit 1;
-
# Train tri2a, which is deltas + delta-deltas, on si84 data.
steps/train_deltas.sh --cmd "$train_cmd" \
2500 15000 data/train_si84 data/lang exp/tri1_ali_si84 exp/tri2a || exit 1;
index 40fa47a96c671302923d6bfe91836181ed914ec6..4d171a2a47d4ed5d5050e14adb49597f5fb51820 100755 (executable)
@@ -199,9 +199,9 @@ feats="$sifeats transform-feats --utt2spk=ark:$sdata/JOB/utt2spk ark:$dir/trans.
if [ $stage -le 4 ]; then
echo "$0: doing a final pass of acoustic rescoring."
- $cmd JOB=1:$nj $dir/log/acoustic_rescore.JOB.log \
+ $cmd $parallel_opts JOB=1:$nj $dir/log/acoustic_rescore.JOB.log \
gmm-rescore-lattice $final_model "ark:gunzip -c $dir/lat.tmp.JOB.gz|" "$feats" ark:- \| \
- lattice-determinize-pruned --acoustic-scale=$acwt --beam=$lattice_beam ark:- \
+ lattice-determinize-pruned$thread_string --acoustic-scale=$acwt --beam=$lattice_beam ark:- \
"ark:|gzip -c > $dir/lat.JOB.gz" '&&' rm $dir/lat.tmp.JOB.gz || exit 1;
fi
diff --git a/egs/wsj/s5/steps/get_lexicon_probs.sh b/egs/wsj/s5/steps/get_lexicon_probs.sh
--- /dev/null
@@ -0,0 +1,225 @@
+#!/bin/bash
+# Copyright 2013 Johns Hopkins University (Author: Daniel Povey)
+# Apache 2.0
+
+
+# From a training or alignment directory, and an original lexicon.txt and lang/
+# directory, obtain a new lexicon with pronunciation probabilities.
+
+
+# Begin configuration section.
+stage=0
+smooth_count=1.0 # Amount of count to add corresponding to each original lexicon entry;
+ # this corresponds to add-one smoothing of the pron-probs.
+max_one=true # If true, normalize the pron-probs so the maximum value for each word is 1.0,
+ # rather than summing to one. This is quite standard.
+
+# End configuration options.
+
+echo "$0 $@" # Print the command line for logging
+
+[ -f path.sh ] && . ./path.sh # source the path.
+. parse_options.sh || exit 1;
+
+if [ $# != 6 ]; then
+ echo "Usage: steps/get_lexicon_probs.sh <data-dir> <lang-dir> <src-dir|ali-dir> <old-lexicon> <exp-dir> <new-lexicon>"
+ echo "e.g.: steps/get_lexicon_probs.sh data/train data/lang exp/tri5 data/local/lexicon.txt \\"
+ echo " exp/tri5_lexprobs data/local_withprob/lexicon.txt"
+ echo "Note: we assume you ran using word-position-dependent phones but both the old and new lexicon will not have"
+ echo "these markings. We also assume the new lexicon will have pron-probs but the old one does not; this limitation"
+ echo "of the script can be removed later."
+ echo "Main options (for others, see top of script file)"
+ echo " --config <config-file> # config containing options"
+ echo " --stage <stage> # used to control partial re-running."
+ echo " --max-one <true|false> # If true, normalize so max prob of each"
+ echo " # word is one. Default: true"
+ echo " --smooth <smooth-count> # Amount to smooth each count by (default: 1.0)"
+ echo " --cmd (utils/run.pl|utils/queue.pl <queue opts>) # how to run jobs."
+ exit 1;
+fi
+
+data=$1
+lang=$2
+srcdir=$3
+old_lexicon=$4
+dir=$5
+new_lexicon=$6
+
+oov=`cat $lang/oov.int` || exit 1;
+nj=`cat $srcdir/num_jobs` || exit 1;
+
+for f in $data/text $lang/L.fst $lang/phones/word_boundary.int $srcdir/ali.1.gz $old_lexicon; do
+ [ ! -f $f ] && echo "$0: no such file $f" && exit 1;
+done
+
+mkdir -p $dir/log
+utils/split_data.sh $data $nj # Make sure split data-dir exists.
+sdata=$data/split$nj
+
+
+mkdir -p $dir/log
+
+if [ $stage -le 0 ]; then
+
+ ( ( for n in `seq $nj`; do gunzip -c $srcdir/ali.$n.gz; done ) | \
+ linear-to-nbest ark:- "ark:utils/sym2int.pl --map-oov $oov -f 2- $lang/words.txt $data/text |" '' '' ark:- | \
+ lattice-align-words $lang/phones/word_boundary.int $srcdir/final.mdl ark:- ark:- | \
+ lattice-to-phone-lattice --replace-words=false $srcdir/final.mdl ark:- ark,t:- | \
+ awk '{ if (NF == 4) { word_phones = sprintf("%s %s", $3, $4); count[word_phones]++; } }
+ END { for(key in count) { print count[key], key; } }' | \
+ sed s:0,0,:: | awk '{print $2, $1, $3;}' | sed 's/_/ /g' | \
+ utils/int2sym.pl -f 3- $lang/phones.txt | \
+ sed -E 's/_I( |$)/ /g' | sed -E 's/_E( |$)/ /g' | sed -E 's/_B( |$)/ /g' | sed -E 's/_S( |$)/ /g' | \
+ utils/int2sym.pl -f 1 $lang/words.txt > $dir/lexicon_counts.txt
+ ) 2>&1 | tee $dir/log/get_fsts.log
+
+fi
+
+cat $old_lexicon | awk '{if (!($2 > 0.0 && $2 < 1.0)) { exit(1); }}' && \
+ echo "Error: old lexicon $old_lexicon appears to have pron-probs; we don't expect this." && \
+ exit 1;
+
+mkdir -p `dirname $new_lexicon` || exit 1;
+
+if [ $stage -le 1 ]; then
+ grep -v -w '^<eps>' $dir/lexicon_counts.txt | \
+ perl -e ' ($old_lexicon, $smooth_count, $max_one) = @ARGV;
+ ($smooth_count >= 0) || die "Invalid smooth_count $smooth_count";
+ ($max_one eq "true" || $max_one eq "false") || die "Invalid max_one variable $max_one";
+ open(O, "<$old_lexicon")||die "Opening old-lexicon file $old_lexicon";
+ while(<O>) {
+ $_ =~ m/(\S+)\s+(.+)/ || die "Bad old-lexicon line $_";
+ $word = $1;
+ $orig_pron = $2;
+ # Remember the mapping from canonical prons to original prons: in the case of
+ # syllable based systems we want to remember the locations of tabs in
+ # the original lexicon.
+ $pron = join(" ", split(" ", $orig_pron));
+ $orig_pron{$word,$pron} = $orig_pron;
+ $count{$word,$pron} += $smooth_count;
+ $tot_count{$word} += $smooth_count;
+ }
+ while (<STDIN>) {
+ $_ =~ m/(\S+)\s+(\S+)\s+(.+)/ || die "Bad new-lexicon line $_";
+ $word = $1;
+ $this_count = $2;
+ $pron = join(" ", split(" ", $3));
+ $count{$word,$pron} += $this_count;
+ $tot_count{$word} += $this_count;
+ }
+ if ($max_one eq "true") { # replace $tot_count{$word} with max count
+ # of any pron.
+ %tot_count = {}; # set to empty assoc array.
+ foreach $key (keys %count) {
+ ($word, $pron) = split($; , $key); # $; is separator for strings that index assoc. arrays.
+ $this_count = $count{$key};
+ if (!defined $tot_count{$word} || $this_count > $tot_count{$word}) {
+ $tot_count{$word} = $this_count;
+ }
+ }
+ }
+ foreach $key (keys %count) {
+ ($word, $pron) = split($; , $key); # $; is separator for strings that index assoc. arrays.
+ $this_orig_pron = $orig_pron{$key};
+ if (!defined $this_orig_pron) { die "Word $word and pron $pron did not appear in original lexicon."; }
+ if (!defined $tot_count{$word}) { die "Tot-count not defined for word $word."; }
+ $prob = $count{$key} / $tot_count{$word};
+ print "$word\t$prob\t$this_orig_pron\n"; # Output happens here.
+ } ' $old_lexicon $smooth_count $max_one > $new_lexicon || exit 1;
+fi
+
+exit 0;
+
+echo $nj > $dir/num_jobs
+[[ -d $sdata && $data/feats.scp -ot $sdata ]] || split_data.sh $data $nj || exit 1;
+
+cp $srcdir/{tree,final.mdl} $dir || exit 1;
+cp $srcdir/final.occs $dir;
+splice_opts=`cat $srcdir/splice_opts 2>/dev/null` # frame-splicing options.
+cp $srcdir/splice_opts $dir 2>/dev/null # frame-splicing options.
+
+
+if [ -f $srcdir/final.mat ]; then feat_type=lda; else feat_type=delta; fi
+echo "$0: feature type is $feat_type"
+
+case $feat_type in
+ delta) sifeats="ark,s,cs:apply-cmvn --norm-vars=$norm_vars --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp scp:$sdata/JOB/feats.scp ark:- | add-deltas ark:- ark:- |";;
+ lda) sifeats="ark,s,cs:apply-cmvn --norm-vars=$norm_vars --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp scp:$sdata/JOB/feats.scp ark:- | splice-feats $splice_opts ark:- ark:- | transform-feats $srcdir/final.mat ark:- ark:- |"
+ cp $srcdir/final.mat $dir
+ ;;
+ *) echo "Invalid feature type $feat_type" && exit 1;
+esac
+
+## Set up model and alignment model.
+mdl=$srcdir/final.mdl
+if [ -f $srcdir/final.alimdl ]; then
+ alimdl=$srcdir/final.alimdl
+else
+ alimdl=$srcdir/final.mdl
+fi
+[ ! -f $mdl ] && echo "$0: no such model $mdl" && exit 1;
+alimdl_cmd="gmm-boost-silence --boost=$boost_silence `cat $lang/phones/optional_silence.csl` $alimdl - |"
+mdl_cmd="gmm-boost-silence --boost=$boost_silence `cat $lang/phones/optional_silence.csl` $mdl - |"
+
+
+## Work out where we're getting the graphs from.
+if $use_graphs; then
+ [ "$nj" != "`cat $srcdir/num_jobs`" ] && \
+ echo "$0: you specified --use-graphs true, but #jobs mismatch." && exit 1;
+ [ ! -f $srcdir/fsts.1.gz ] && echo "No graphs in $srcdir" && exit 1;
+ graphdir=$srcdir
+else
+ graphdir=$dir
+ if [ $stage -le 0 ]; then
+ echo "$0: compiling training graphs"
+ tra="ark:utils/sym2int.pl --map-oov $oov -f 2- $lang/words.txt $sdata/JOB/text|";
+ $cmd JOB=1:$nj $dir/log/compile_graphs.JOB.log \
+ compile-train-graphs $dir/tree $dir/final.mdl $lang/L.fst "$tra" \
+ "ark:|gzip -c >$dir/fsts.JOB.gz" || exit 1;
+ fi
+fi
+
+
+if [ $stage -le 1 ]; then
+ echo "$0: aligning data in $data using $alimdl and speaker-independent features."
+ $cmd JOB=1:$nj $dir/log/align_pass1.JOB.log \
+ gmm-align-compiled $scale_opts --beam=$beam --retry-beam=$retry_beam "$alimdl_cmd" \
+ "ark:gunzip -c $graphdir/fsts.JOB.gz|" "$sifeats" "ark:|gzip -c >$dir/pre_ali.JOB.gz" || exit 1;
+fi
+
+if [ $stage -le 2 ]; then
+ echo "$0: computing fMLLR transforms"
+ if [ "$alimdl" != "$mdl" ]; then
+ $cmd JOB=1:$nj $dir/log/fmllr.JOB.log \
+ ali-to-post "ark:gunzip -c $dir/pre_ali.JOB.gz|" ark:- \| \
+ weight-silence-post 0.0 $silphonelist $alimdl ark:- ark:- \| \
+ gmm-post-to-gpost $alimdl "$sifeats" ark:- ark:- \| \
+ gmm-est-fmllr-gpost --fmllr-update-type=$fmllr_update_type \
+ --spk2utt=ark:$sdata/JOB/spk2utt $mdl "$sifeats" \
+ ark,s,cs:- ark:$dir/trans.JOB || exit 1;
+ else
+ $cmd JOB=1:$nj $dir/log/fmllr.JOB.log \
+ ali-to-post "ark:gunzip -c $dir/pre_ali.JOB.gz|" ark:- \| \
+ weight-silence-post 0.0 $silphonelist $alimdl ark:- ark:- \| \
+ gmm-est-fmllr --fmllr-update-type=$fmllr_update_type \
+ --spk2utt=ark:$sdata/JOB/spk2utt $mdl "$sifeats" \
+ ark,s,cs:- ark:$dir/trans.JOB || exit 1;
+ fi
+fi
+
+feats="$sifeats transform-feats --utt2spk=ark:$sdata/JOB/utt2spk ark:$dir/trans.JOB ark:- ark:- |"
+
+if [ $stage -le 3 ]; then
+ echo "$0: doing final alignment."
+ $cmd JOB=1:$nj $dir/log/align_pass2.JOB.log \
+ gmm-align-compiled $scale_opts --beam=$beam --retry-beam=$retry_beam "$mdl_cmd" \
+ "ark:gunzip -c $graphdir/fsts.JOB.gz|" "$feats" "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1;
+fi
+
+rm $dir/pre_ali.*.gz
+
+echo "$0: done aligning data."
+
+utils/summarize_warnings.pl $dir/log
+
+exit 0;
index 55e96afbd7d889b4d8f4aaa1199f5cfa166e9dc9..4eef666ad334e5bc45c8525ff1ad43986591028c 100755 (executable)
lmwt=1.0
max_silence_frames=50
max_states=1000000
+max_expand=20 # limit memory blowup in lattice-align-words
strict=true
word_ins_penalty=0
+silence_word= # Specify this only if you did so in kws_setup
skip_optimization=false # If you only search for few thousands of keywords, you probablly
# can skip the optimization; but if you're going to search for
# millions of keywords, you'd better do set this optimization to
echo "Using model: $model"
+if [ ! -z $silence_word ]; then
+ silence_int=`grep -w $silence_word $langdir/words.txt | awk '{print $2}'`
+ [ -z $silence_int ] && \
+ echo "Error: could not find integer representation of silence word $silence_word" && exit 1;
+ silence_opt="--silence-label=$silence_int"
+fi
+
$cmd JOB=1:$nj $kwsdir/log/index.JOB.log \
lattice-add-penalty --word-ins-penalty=$word_ins_penalty "ark:gzip -cdf $decodedir/lat.JOB.gz|" ark:- \| \
- lattice-align-words $word_boundary $model ark:- ark:- \| \
+ lattice-align-words $silence_opt --max-expand=$max_expand $word_boundary $model ark:- ark:- \| \
lattice-scale --acoustic-scale=$acwt --lm-scale=$lmwt ark:- ark:- \| \
lattice-to-kws-index --max-silence-frames=$max_silence_frames --strict=$strict ark:$utter_id ark:- ark:- \| \
kws-index-union --skip-optimization=$skip_optimization --strict=$strict --max-states=$max_states \
index c013f5e1739a8d167bb93f2c856bbfc7cfb32f83..01184bb4e30f569ea0c1b5d40b88b99ae622e8c0 100755 (executable)
# can cancel them per frame.
if [ $stage -le $x ]; then
$cmd JOB=1:$nj $dir/log/acc.$x.JOB.log \
- test -s $dir/den_acc.$x.JOB.gz '||' \
+ test -s $dir/den_acc.$x.JOB.gz -a -s $dir/num_acc.$x.JOB.gz '||' \
sgmm2-rescore-lattice "$gselect_opt" $spkvecs_opt $dir/$x.mdl "$lats" "$feats" ark:- \| \
lattice-to-post --acoustic-scale=$acwt ark:- ark:- \| \
sum-post --zero-if-disjoint=$zero_if_disjoint --merge=$cancel --scale1=-1 \
index 8503698e8f92e3f601b352ed00e90dcbebb0d628..95daf9b7d63ddfb9ced1cd4320493af2c838dbe4 100755 (executable)
max_iter_inc=25 # Last iter to increase #Gauss on.
power=0.2 # Exponent for number of gaussians according to occurrence counts
cluster_thresh=-1 # for build-tree control final bottom-up clustering of leaves
+phone_map=
train_tree=true
# End configuration section.
ciphonelist=`cat $lang/phones/context_indep.csl` || exit 1;
sdata=$data/split$nj;
splice_opts=`cat $alidir/splice_opts 2>/dev/null` # frame-splicing options.
+phone_map_opt=
+[ ! -z "$phone_map" ] && phone_map_opt="--phone-map='$phone_map'"
mkdir -p $dir/log
cp $alidir/splice_opts $dir 2>/dev/null # frame-splicing options.
else
if [ $stage -le -5 ]; then
echo "$0: obtaining initial fMLLR transforms since not present in $alidir"
+ # The next line is necessary because of $silphonelist otherwise being incorrect; would require
+ # old $lang dir which would require another option. Not needed anyway.
+ [ ! -z "$phone_map" ] && \
+ echo "$0: error: you must provide transforms if you use the --phone-map option." && exit 1;
$cmd JOB=1:$nj $dir/log/fmllr.0.JOB.log \
ali-to-post "ark:gunzip -c $alidir/ali.JOB.gz|" ark:- \| \
weight-silence-post $silence_weight $silphonelist $alidir/final.mdl ark:- ark:- \| \
# Convert the alignments.
echo "$0: Converting alignments from $alidir to use current tree"
$cmd JOB=1:$nj $dir/log/convert.JOB.log \
- convert-ali $alidir/final.mdl $dir/1.mdl $dir/tree \
+ convert-ali $phone_map_opt $alidir/final.mdl $dir/1.mdl $dir/tree \
"ark:gunzip -c $alidir/ali.JOB.gz|" "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1;
fi
index e4041f5f1c2fa36de29770ec409dd0d8170b3e11..803e7f632d4aad45ac10536369417843623edddf 100755 (executable)
#!/usr/bin/perl
-# Copyright 2010-2011 Microsoft Corporation
+# Copyright 2010-2011 Microsoft Corporation
+# 2013 Johns Hopkins University (author: Daniel Povey)
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# Disambig syms are numbered #1, #2, #3, etc. (#0
# reserved for symbol in grammar).
# Outputs the number of disambig syms to the standard output.
+# With the --pron-probs option, expects the second field
+# of each lexicon line to be a pron-prob.
+
+$pron_probs = 0;
+
+if ($ARGV[0] eq "--pron-probs") {
+ $pron_probs = 1;
+ shift @ARGV;
+}
if(@ARGV != 2) {
- die "Usage: add_lex_disambig.pl lexicon.txt lexicon_disambig.txt "
+ die "Usage: add_lex_disambig.pl [--pron-probs] lexicon.txt lexicon_disambig.txt "
}
foreach $l (@L) {
@A = split(" ", $l);
shift @A; # Remove word.
+ if ($pron_probs) {
+ $p = shift @A;
+ if (!($p > 0.0 && $p <= 1.0)) { die "Bad lexicon line $l (expecting pron-prob as second field)"; }
+ }
$count{join(" ",@A)}++;
}
foreach $l (@L) {
@A = split(" ", $l);
shift @A; # Remove word.
+ if ($pron_probs) { shift @A; } # remove pron-prob.
while(@A > 0) {
pop @A; # Remove last phone
$issubseq{join(" ",@A)} = 1;
foreach $l (@L) {
@A = split(" ", $l);
$word = shift @A;
+ if ($pron_probs) { $pron_prob = shift @A; }
$phnseq = join(" ",@A);
if(!defined $issubseq{$phnseq}
&& $count{$phnseq} == 1) {
$phnseq = $phnseq . " #" . $curnumber;
}
}
- print O "$word\t$phnseq\n";
+ if ($pron_probs) { print O "$word\t$pron_prob\t$phnseq\n"; }
+ else { print O "$word\t$phnseq\n"; }
}
print $max_disambig . "\n";
index 4f89d584b36d9ea21f285b66e7f15bb2bec56e5f..adea210d62d3af7c1b437e61e1df5650a310b83f 100755 (executable)
# This program is a bit like ./sym2int.pl in that it applies a map
# to things in a file, but it's a bit more general in that it doesn't
# assume the things being mapped to are single tokens, they could
-# be sequences of tokens.
+# be sequences of tokens.
# This program takes two arguments, which may be files or "-" for the
# standard input. Both files must have lines with one or more fields,
# y Q R
# then the output of this program will be
# A P Q R
-#
+#
# Note that if x or y did not appear as the first field of file b, we would
# print a warning and omit the whole line rather than map it to the empty
# string.
+
+if (@ARGV > 0 && $ARGV[0] eq "-f") {
+ shift @ARGV;
+ $field_spec = shift @ARGV;
+ if ($field_spec =~ m/^\d+$/) {
+ $field_begin = $field_spec - 1; $field_end = $field_spec - 1;
+ }
+ if ($field_spec =~ m/^(\d*)[-:](\d*)/) { # accept e.g. 1:10 as a courtesty (properly, 1-10)
+ if ($1 ne "") {
+ $field_begin = $1 - 1; # Change to zero-based indexing.
+ }
+ if ($2 ne "") {
+ $field_end = $2 - 1; # Change to zero-based indexing.
+ }
+ }
+ if (!defined $field_begin && !defined $field_end) {
+ die "Bad argument to -f option: $field_spec";
+ }
+}
+
+
if(@ARGV != 1) {
- print STDERR "Usage: apply_map.pl map <input >output\n" .
- "e.g.: echo A B | apply_map.pl <a.txt\n" .
+ print STDERR "Usage: apply_map.pl [options] map <input >output\n" .
+ "options: [-f <field-range> ]\n" .
+ "note: <field-range> can look like 4-5, or 4-, or 5-, or 1.\n" .
+ "e.g.: echo A B | apply_map.pl a.txt\n" .
"where a.txt is:\n" .
"A a1 a2\n" .
"B b\n" .
"will produce:\n" .
"a1 a2 b\n";
+ exit(1);
}
($map) = @ARGV;
while(<STDIN>) {
@A = split(" ", $_);
for ($x = 0; $x < @A; $x++) {
- $a = $A[$x];
- if (!defined $map{$a}) { die "compose_maps.pl: undefined key $a\n"; }
- $A[$x] = $map{$a};
+ if ( (!defined $field_begin || $x >= $field_begin)
+ && (!defined $field_end || $x <= $field_end)) {
+ $a = $A[$x];
+ if (!defined $map{$a}) { die "compose_maps.pl: undefined key $a\n"; }
+ $A[$x] = $map{$a};
+ }
}
print join(" ", @A) . "\n";
}
index a5334279c8c864a94f03841842d2b4019104d588..6e711b807efb1c9f39351d3f0a4513c835b66048 100755 (executable)
#!/usr/bin/perl -w
-# Copyright 2010-2011 Microsoft Corporation
+# Copyright 2010-2011 Microsoft Corporation
+# 2013 Johns Hopkins University (author: Daniel Povey)
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# limitations under the License.
-# makes lexicon FST (no pron-probs involved).
+# makes lexicon FST, in text form, from lexicon (pronunciation probabilities optional).
-if(@ARGV != 1 && @ARGV != 3 && @ARGV != 4) {
- die "Usage: make_lexicon_fst.pl lexicon.txt [silprob silphone [sil_disambig_sym]] >lexiconfst.txt"
+$pron_probs = 0;
+
+if ($ARGV[0] eq "--pron-probs") {
+ $pron_probs = 1;
+ shift @ARGV;
+}
+
+if (@ARGV != 1 && @ARGV != 3 && @ARGV != 4) {
+ print STDERR
+ "Usage: make_lexicon_fst.pl [--pron-probs] lexicon.txt [silprob silphone [sil_disambig_sym]] >lexiconfst.txt
+Creates a lexicon FST that transduces phones to words, and may allow optional silence.
+Note: ordinarily, each line of lexicon.txt is: word phone1 phone2 ... phoneN; if the --pron-probs option is
+used, each line is: word pronunciation-probability phone1 phone2 ... phoneN. The probability 'prob' will
+typically be between zero and one, and note that it's generally helpful to normalize so the largest one
+for each word is 1.0, but this is your responsibility. The silence disambiguation symbol, e.g. something
+like #5, is used only when creating a lexicon with disambiguation symbols, e.g. L_disambig.fst, and was
+introduced to fix a particular case of non-determinism of decoding graphs.\n";
+ exit(1);
}
$lexfn = shift @ARGV;
-if(@ARGV == 0) {
- $silprob = 0.0;
-} elsif (@ARGV == 2){
- ($silprob,$silphone) = @ARGV;
+if (@ARGV == 0) {
+ $silprob = 0.0;
+} elsif (@ARGV == 2) {
+ ($silprob,$silphone) = @ARGV;
} else {
- ($silprob,$silphone,$sildisambig) = @ARGV;
+ ($silprob,$silphone,$sildisambig) = @ARGV;
}
-if($silprob != 0.0) {
- $silprob < 1.0 || die "Sil prob cannot be >= 1.0";
- $silcost = -log($silprob);
- $nosilcost = -log(1.0 - $silprob);
+if ($silprob != 0.0) {
+ $silprob < 1.0 || die "Sil prob cannot be >= 1.0";
+ $silcost = -log($silprob);
+ $nosilcost = -log(1.0 - $silprob);
}
open(L, "<$lexfn") || die "Error opening lexicon $lexfn";
-
sub is_sil {
- # Return true (1) if provided with a phone-sequence
- # that means silence.
- # @_ is the parameters of the function
- # This function returns true if @_ equals ( $silphone )
- # or something of the form ( "#0", $silphone, "#1" )
- # where the "#0" and "#1" are disambiguation symbols.
- return ( @_ == 1 && $_[0] eq $silphone ||
- (@_ == 3 && $_[1] eq $silphone &&
- $_[0] =~ m/^\#\d+$/ &&
- $_[0] =~ m/^\#\d+$/));
+ # Return true (1) if provided with a phone-sequence
+ # that means silence.
+ # @_ is the parameters of the function
+ # This function returns true if @_ equals ( $silphone )
+ # or something of the form ( "#0", $silphone, "#1" )
+ # where the "#0" and "#1" are disambiguation symbols.
+ return ( @_ == 1 && $_[0] eq $silphone ||
+ (@_ == 3 && $_[1] eq $silphone &&
+ $_[0] =~ m/^\#\d+$/ &&
+ $_[0] =~ m/^\#\d+$/));
}
-if( $silprob == 0.0 ) { # No optional silences: just have one (loop+final) state which is numbered zero.
- $loopstate = 0;
- $nextstate = 1; # next unallocated state.
- while(<L>) {
- @A = split(" ", $_);
- $w = shift @A;
-
- $s = $loopstate;
- $word_or_eps = $w;
- while (@A > 0) {
- $p = shift @A;
- if(@A > 0) {
- $ns = $nextstate++;
- } else {
- $ns = $loopstate;
- }
- print "$s\t$ns\t$p\t$word_or_eps\n";
- $word_or_eps = "<eps>";
- $s = $ns;
- }
+if ( $silprob == 0.0 ) { # No optional silences: just have one (loop+final) state which is numbered zero.
+ $loopstate = 0;
+ $nextstate = 1; # next unallocated state.
+ while (<L>) {
+ @A = split(" ", $_);
+ @A == 0 && die "Empty lexicon line.";
+ $w = shift @A;
+ if (! $pron_probs) {
+ $pron_cost = 0.0;
+ } else {
+ $pron_prob = shift @A;
+ if (! defined $pron_prob || !($pron_prob > 0.0 && $pron_prob <= 1.0)) {
+ die "Bad pronunciation probability in line $_";
+ }
+ $pron_cost = -log($pron_prob);
+ }
+ if ($pron_cost != 0.0) { $pron_cost_string = "\t$pron_cost"; } else { $pron_cost_string = ""; }
+
+ $s = $loopstate;
+ $word_or_eps = $w;
+ while (@A > 0) {
+ $p = shift @A;
+ if (@A > 0) {
+ $ns = $nextstate++;
+ } else {
+ $ns = $loopstate;
+ }
+ print "$s\t$ns\t$p\t$word_or_eps$pron_cost_string\n";
+ $word_or_eps = "<eps>";
+ $pron_cost_string = ""; # so we only print it on the first arc of the word.
+ $s = $ns;
}
- print "$loopstate\t0\n"; # final-cost.
-} else { # have silence probs.
- $startstate = 0;
- $loopstate = 1;
- $silstate = 2; # state from where we go to loopstate after emitting silence.
- print "$startstate\t$loopstate\t<eps>\t<eps>\t$nosilcost\n"; # no silence.
- if (!defined $sildisambig) {
- print "$startstate\t$loopstate\t$silphone\t<eps>\t$silcost\n"; # silence.
- print "$silstate\t$loopstate\t$silphone\t<eps>\n"; # no cost.
- $nextstate = 3;
+ }
+ print "$loopstate\t0\n"; # final-cost.
+} else { # have silence probs.
+ $startstate = 0;
+ $loopstate = 1;
+ $silstate = 2; # state from where we go to loopstate after emitting silence.
+ print "$startstate\t$loopstate\t<eps>\t<eps>\t$nosilcost\n"; # no silence.
+ if (!defined $sildisambig) {
+ print "$startstate\t$loopstate\t$silphone\t<eps>\t$silcost\n"; # silence.
+ print "$silstate\t$loopstate\t$silphone\t<eps>\n"; # no cost.
+ $nextstate = 3;
+ } else {
+ $disambigstate = 3;
+ $nextstate = 4;
+ print "$startstate\t$disambigstate\t$silphone\t<eps>\t$silcost\n"; # silence.
+ print "$silstate\t$disambigstate\t$silphone\t<eps>\n"; # no cost.
+ print "$disambigstate\t$loopstate\t$sildisambig\t<eps>\n"; # silence disambiguation symbol.
+ }
+ while (<L>) {
+ @A = split(" ", $_);
+ $w = shift @A;
+ if (! $pron_probs) {
+ $pron_cost = 0.0;
} else {
- $disambigstate = 3;
- $nextstate = 4;
- print "$startstate\t$disambigstate\t$silphone\t<eps>\t$silcost\n"; # silence.
- print "$silstate\t$disambigstate\t$silphone\t<eps>\n"; # no cost.
- print "$disambigstate\t$loopstate\t$sildisambig\t<eps>\n"; # silence disambiguation symbol.
+ $pron_prob = shift @A;
+ if (! defined $pron_prob || !($pron_prob > 0.0 && $pron_prob <= 1.0)) {
+ die "Bad pronunciation probability in line $_";
+ }
+ $pron_cost = -log($pron_prob);
}
- while(<L>) {
- @A = split(" ", $_);
- $w = shift @A;
-
- $s = $loopstate;
- $word_or_eps = $w;
- while (@A > 0) {
- $p = shift @A;
- if(@A > 0) {
- $ns = $nextstate++;
- print "$s\t$ns\t$p\t$word_or_eps\n";
- $word_or_eps = "<eps>";
- $s = $ns;
- } else {
- if(!is_sil(@A)){
- # This is non-deterministic but relatively compact,
- # and avoids epsilons.
- print "$s\t$loopstate\t$p\t$word_or_eps\t$nosilcost\n";
- print "$s\t$silstate\t$p\t$word_or_eps\t$silcost\n";
- } else {
- # no point putting opt-sil after silence word.
- print "$s\t$loopstate\t$p\t$word_or_eps\n";
- }
- $word_or_eps = "<eps>";
- }
- }
+ if ($pron_cost != 0.0) { $pron_cost_string = "\t$pron_cost"; } else { $pron_cost_string = ""; }
+ $s = $loopstate;
+ $word_or_eps = $w;
+ while (@A > 0) {
+ $p = shift @A;
+ if (@A > 0) {
+ $ns = $nextstate++;
+ print "$s\t$ns\t$p\t$word_or_eps$pron_cost_string\n";
+ $word_or_eps = "<eps>";
+ $pron_cost_string = ""; $pron_cost = 0.0; # so we only print it the 1st time.
+ $s = $ns;
+ } else {
+ if (!is_sil(@A)) {
+ # This is non-deterministic but relatively compact,
+ # and avoids epsilons.
+ $local_nosilcost = $nosilcost + $pron_cost;
+ $local_silcost = $silcost + $pron_cost;
+ print "$s\t$loopstate\t$p\t$word_or_eps\t$local_nosilcost\n";
+ print "$s\t$silstate\t$p\t$word_or_eps\t$local_silcost\n";
+ } else {
+ # no point putting opt-sil after silence word.
+ print "$s\t$loopstate\t$p\t$word_or_eps$pron_cost_string\n";
+ }
+ }
}
- print "$loopstate\t0\n"; # final-cost.
+ }
+ print "$loopstate\t0\n"; # final-cost.
}
index 7edaaa7f4e0d9de6c8adffd0c368385d3977ea59..4d82d1848e1b628eef67c46df3a247ed09a0be6b 100755 (executable)
cp $lang/phones/word_boundary.* $dir/phones/ 2>/dev/null # might be needed for ctm scoring,
# but ignore the error if it's not there.
-# in case of position-independent phones, we need the L_align and word symbols
-cp $lang/L_align.fst $dir/ 2> /dev/null
cp $lang/phones/disambig.{txt,int} $dir/phones/ 2> /dev/null
cp $lang/phones/silence.csl $dir/phones/ || exit 1;
cp $lang/phones.txt $dir/ 2> /dev/null # ignore the error if it's not there.
index 28a8098fe3c6d99908b44a77c516efd26459090f..de7ce0d49c11fe98ce5257e30933cd316d27fa88 100755 (executable)
# This script prepares a directory such as data/lang/, in the standard format,
# given a source directory containing a dictionary lexicon.txt in a form like:
# word phone1 phone2 ... phoneN
-# per line (alternate prons would be separate lines).
+# per line (alternate prons would be separate lines), or a dictionary with probabilities
+# called lexiconp.txt in a form:
+# word pron-prob phone1 phone2 ... phoneN
+# (with 0.0 < pron-prob <= 1.0); note: if lexiconp.txt exists, we use it even if
+# lexicon.txt exists.
# and also files silence_phones.txt, nonsilence_phones.txt, optional_silence.txt
# and extra_questions.txt
# Here, silence_phones.txt and nonsilence_phones.txt are lists of silence and
num_sil_states=5
num_nonsil_states=3
position_dependent_phones=true
-# false also when position dependent phones and word_boundary.txt
+# position_dependent_phones is false also when position dependent phones and word_boundary.txt
# have been generated by another source
reverse=false
share_silence_phones=false # if true, then share pdfs of different silence
# phones together.
sil_prob=0.5
-generate_l_align=false # generate alignment FST with word start/end symbols
make_individual_sil_models=false # enforce individual models for all silence phones
# end configuration sections
echo " --share-silence-phones (true|false) # default: false; if true, share pdfs of "
echo " # all non-silence phones. "
echo " --sil-prob <probability of silence> # default: 0.5 [must have 0 <= silprob < 1]"
- echo " --generate-l-align (true|false) # default: false; generate alignment FST with word start/end symbols"
echo " --make-individual-sil-models (true|false) # default: false; make non-{shared,split} states for each silphone"
exit 1;
fi
[ -f path.sh ] && . ./path.sh
-utils/validate_dict_dir.pl $srcdir || exit 1;
+! utils/validate_dict_dir.pl $srcdir && echo "*Error validating directory $srcdir*" && exit 1;
+
+if [[ ! -f $srcdir/lexicon.txt ]]; then
+ echo "**Creating $dir/lexicon.txt from $dir/lexiconp.txt"
+ perl -ape 's/(\S+\s+)\S+\s+(.+)/$1$2/;' < $srcdir/lexiconp.txt > $srcdir/lexicon.txt || exit 1;
+fi
+if [[ ! -f $srcdir/lexiconp.txt ]]; then
+ echo "**Creating $srcdir/lexiconp.txt from $srcdir/lexicon.txt"
+ perl -ape 's/(\S+\s+)(.+)/${1}1.0\t$2/;' < $srcdir/lexicon.txt > $srcdir/lexiconp.txt || exit 1;
+fi
+
+! utils/validate_dict_dir.pl $srcdir >&/dev/null && \
+ echo "Validation failed (second time)" && exit 1;
if $position_dependent_phones; then
# Create $tmpdir/lexicon.original from $srcdir/lexicon.txt by
# adding the markers _B, _E, _S, _I depending on word position.
# In this recipe, these markers apply to silence also.
+ # Do this starting from lexiconp.txt only.
+
- perl -ane '@A=split(" ",$_); $w = shift @A; @A>0||die;
- if(@A==1) { print "$w $A[0]_S\n"; } else { print "$w $A[0]_B ";
+ perl -ane '@A=split(" ",$_); $w = shift @A; $p = shift @A; @A>0||die;
+ if(@A==1) { print "$w $p $A[0]_S\n"; } else { print "$w $p $A[0]_B ";
for($n=1;$n<@A-1;$n++) { print "$A[$n]_I "; } print "$A[$n]_E\n"; } ' \
- < $srcdir/lexicon.txt > $tmpdir/lexicon.original || exit 1;
-
+ < $srcdir/lexiconp.txt > $tmpdir/lexiconp.original || exit 1;
+
# create $tmpdir/phone_map.txt
# this has the format (on each line)
# <original phone> <version 1 of original phone> <version 2> ...
<(for x in `cat $srcdir/nonsilence_phones.txt`; do for y in "" "_B" "_E" "_I" "_S"; do echo -n "$x$y "; done; echo; done) \
> $tmpdir/phone_map.txt
else
- cp $srcdir/lexicon.txt $tmpdir/lexicon.original
+ cp $srcdir/lexiconp.txt $tmpdir/lexiconp.original
cat $srcdir/silence_phones.txt $srcdir/nonsilence_phones.txt | \
sed 's/ /\n/g' | awk '(NF>0){print}' > $tmpdir/phones
paste -d' ' $tmpdir/phones $tmpdir/phones > $tmpdir/phone_map.txt
if $reverse; then
echo "reversing lexicon."
- cat $tmpdir/lexicon.original \
- | awk '{printf "%s ",$1;for(i=NF;i>1;i--){printf "%s ",$i;}printf "\n"}' \
- > $tmpdir/lexicon.txt
+ cat $tmpdir/lexiconp.original \
+ | awk '{printf "%s %s ",$1, $2;for(i=NF;i>2;i--){printf "%s ",$i;}printf "\n"}' \
+ > $tmpdir/lexiconp.txt
else
- mv $tmpdir/lexicon.original $tmpdir/lexicon.txt
+ mv $tmpdir/lexiconp.original $tmpdir/lexiconp.txt
fi
-
mkdir -p $dir/phones # various sets of phones...
# Sets of phones for use in clustering, and making monophone systems.
done
fi
-# add disambig symbols to the lexicon in $tmpdir/lexicon.txt
+# add disambig symbols to the lexicon in $tmpdir/lexiconp.txt
# and produce $tmpdir/lexicon_disambig.txt
-ndisambig=`utils/add_lex_disambig.pl $tmpdir/lexicon.txt $tmpdir/lexicon_disambig.txt`
+ndisambig=`utils/add_lex_disambig.pl --pron-probs $tmpdir/lexiconp.txt $tmpdir/lexiconp_disambig.txt`
ndisambig=$[$ndisambig+1]; # add one disambig symbol for silence in lexicon FST.
echo $ndisambig > $tmpdir/lex_ndisambig
-# Format of lexicon_disambig.txt:
-# !SIL SIL_S
-# <SPOKEN_NOISE> SPN_S #1
-# <UNK> SPN_S #2
-# <NOISE> NSN_S
-# !EXCLAMATION-POINT EH2_B K_I S_I K_I L_I AH0_I M_I EY1_I SH_I AH0_I N_I P_I OY2_I N_I T_E
+# Format of lexiconp_disambig.txt:
+# !SIL 1.0 SIL_S
+# <SPOKEN_NOISE> 1.0 SPN_S #1
+# <UNK> 1.0 SPN_S #2
+# <NOISE> 1.0 NSN_S
+# !EXCLAMATION-POINT 1.0 EH2_B K_I S_I K_I L_I AH0_I M_I EY1_I SH_I AH0_I N_I P_I OY2_I N_I T_E
( for n in `seq 0 $ndisambig`; do echo '#'$n; done ) >$dir/phones/disambig.txt
fi
# Create word symbol table.
-cat $tmpdir/lexicon.txt | awk '{print $1}' | sort | uniq | \
+cat $tmpdir/lexiconp.txt | awk '{print $1}' | sort | uniq | \
awk 'BEGIN{print "<eps> 0";} {printf("%s %d\n", $1, NR);} END{printf("#0 %d\n", NR+1);} ' \
> $dir/words.txt || exit 1;
echo "but you may use the option --sil-prob 0.0 to stop it being used." ) && \
exit 1;
-# create $dir/phones/align_lexicon.{txt,int}
-( # Note: here, $silphone will have no suffix e.g. _S because it occurs as optional-silence,
- # and is not part of a word.
- [ ! -z "$silphone" ] && echo "<eps> $silphone";
-) | cat - $tmpdir/lexicon.txt | \
- perl -ane '@A = split; print $A[0], " ", join(" ", @A), "\n";' | \
- sort | uniq > $dir/phones/align_lexicon.txt
+# create $dir/phones/align_lexicon.{txt,int}.
+# This is the new-new style of lexicon aligning.
+
+# First remove pron-probs from the lexicon.
+perl -ape 's/(\S+\s+)\S+\s+(.+)/$1$2/;' <$tmpdir/lexiconp.txt >$tmpdir/align_lexicon.txt
+
+# Note: here, $silphone will have no suffix e.g. _S because it occurs as optional-silence,
+# and is not part of a word.
+[ ! -z "$silphone" ] && echo "<eps> $silphone" >> $tmpdir/align_lexicon.txt
+
+cat $tmpdir/align_lexicon.txt | \
+ perl -ane '@A = split; print $A[0], " ", join(" ", @A), "\n";' | sort | uniq > $dir/phones/align_lexicon.txt
# create phones/align_lexicon.int
cat $dir/phones/align_lexicon.txt | utils/sym2int.pl -f 3- $dir/phones.txt | \
- utils/sym2int.pl -f 1-2 $dir/words.txt > $dir/phones/align_lexicon.int
+ utils/sym2int.pl -f 1-2 $dir/words.txt > $dir/phones/align_lexicon.int
# Create the basic L.fst without disambiguation symbols, for use
# in training.
-utils/make_lexicon_fst.pl $tmpdir/lexicon.txt $sil_prob $silphone | \
+utils/make_lexicon_fst.pl --pron-probs $tmpdir/lexiconp.txt $sil_prob $silphone | \
fstcompile --isymbols=$dir/phones.txt --osymbols=$dir/words.txt \
--keep_isymbols=false --keep_osymbols=false | \
fstarcsort --sort_type=olabel > $dir/L.fst || exit 1;
@@ -266,7 +286,6 @@ cat $dir/oov.txt | utils/sym2int.pl $dir/words.txt >$dir/oov.int # integer versi
# symbol, used in some scripts.
-
# Create these lists of phones in colon-separated integer list form too,
# for purposes of being given to programs as command-line options.
for f in silence nonsilence optional_silence disambig context_indep; do
@@ -299,23 +318,13 @@ utils/gen_topo.pl $num_nonsil_states $num_sil_states $nonsilphonelist $silphonel
phone_disambig_symbol=`grep \#0 $dir/phones.txt | awk '{print $2}'`
word_disambig_symbol=`grep \#0 $dir/words.txt | awk '{print $2}'`
-utils/make_lexicon_fst.pl $tmpdir/lexicon_disambig.txt $sil_prob $silphone '#'$ndisambig | \
+utils/make_lexicon_fst.pl --pron-probs $tmpdir/lexiconp_disambig.txt $sil_prob $silphone '#'$ndisambig | \
fstcompile --isymbols=$dir/phones.txt --osymbols=$dir/words.txt \
--keep_isymbols=false --keep_osymbols=false | \
fstaddselfloops "echo $phone_disambig_symbol |" "echo $word_disambig_symbol |" | \
fstarcsort --sort_type=olabel > $dir/L_disambig.fst || exit 1;
-# If desired, create lexicon FST for alignments (has word-start and end symbols)
-if $generate_l_align; then
- cat $tmpdir/lexicon.txt |
- awk '{printf("%s #1 ", $1); for (n=2; n <= NF; n++) { printf("%s ", $n); } print "#2"; }' | \
- utils/make_lexicon_fst.pl - $sil_prob $silphone | \
- fstcompile --isymbols=$dir/phones.txt --osymbols=$dir/words.txt \
- --keep_isymbols=false --keep_osymbols=false | \
- fstarcsort --sort_type=olabel > $dir/L_align.fst || exit 1;
-fi
-
echo "$(basename $0): validating output directory"
! utils/validate_lang.pl $dir && echo "$(basename $0): error validating output" && exit 1;
index c19e16fc77e97b08d5d42ec79758042db7072ee2..2c9b711d2bf228a8c07ae1214f55d60748bbf94f 100755 (executable)
$ignore_oov = 0;
-$ignore_first_field = 0;
+
for($x = 0; $x < 2; $x++) {
if ($ARGV[0] eq "--map-oov") {
shift @ARGV; $map_oov = shift @ARGV;
print join(" ", @B);
print "\n";
}
+if ($num_warning > 0) {
+ print STDERR "** Replaced $num_warning instances of OOVs with $map_oov\n";
+}
exit(0);
index c84a3fdebc1d10e62fca7a58720a8758814d4149..4cfccb272bb0c73edb840fbb356d989703ac3eb3 100755 (executable)
#
# Validation script for data/local/dict
+
if(@ARGV != 1) {
die "Usage: validate_dict_dir.pl dict_directory\n";
}
print "Checking disjoint: silence_phones.txt, nonsilence_phones.txt\n";
@itset = intersect(\%silence, \%nonsilence);
if(@itset == 0) {print "--> disjoint property is OK.\n";}
-else {$exit = 1; print "--> ERROR: silence_phones.txt and nonsilence_phones.txt has overlop: "; foreach(@itset) {print "$_ ";} print "\n";}
+else {$exit = 1; print "--> ERROR: silence_phones.txt and nonsilence_phones.txt has overlap: "; foreach(@itset) {print "$_ ";} print "\n";}
print "\n";
-# Checking lexicon.txt -------------------------------
-print "Checking $dict/lexicon.txt\n";
-if(-z "$dict/lexicon.txt") {$exit = 1; print "--> ERROR: $dict/lexicon.txt is empty or not exists\n";}
-if(!open(L, "<$dict/lexicon.txt")) {$exit = 1; print "--> ERROR: fail to open $dict/lexicon.txt\n";}
-$idx = 1;
-$success = 1;
-print "--> reading $dict/lexicon.txt\n";
-while(<L>) {
- chomp;
- my @col = split(" ", $_);
- $word = shift @col;
- foreach(0 .. @col-1) {
- if(!$silence{@col[$_]} and !$nonsilence{@col[$_]}) {
- $exit = 1; print "--> ERROR: phone \"@col[$_]\" is not in {, non}silence.txt (line $idx)\n";
+
+sub check_lexicon {
+ my ($lexfn, $pron_probs) = @_;
+ print "Checking $lexfn\n";
+ if(-z "$lexfn") {$exit = 1; print "--> ERROR: $lexfn is empty or not exists\n";}
+ if(!open(L, "<$lexfn")) {$exit = 1; print "--> ERROR: fail to open $lexfn\n";}
+ $idx = 1;
+ $success = 1;
+ print "--> reading $lexfn\n";
+ while (<L>) {
+ chomp;
+ my @col = split(" ", $_);
+ $word = shift @col;
+ if (!defined $word) {
+ $exit = 1; print "--> ERROR: empty lexicon line in $lexfn\n";
$success = 0;
}
+ if ($pron_probs) {
+ $prob = shift @col;
+ if (!($prob > 0.0 && $prob <= 1.0)) {
+ $exit = 1; print "--> ERROR: bad pron-prob in lexicon-line '$_', in $lexfn\n";
+ $success = 0;
+ }
+ }
+ foreach (0 .. @col-1) {
+ if (!$silence{@col[$_]} and !$nonsilence{@col[$_]}) {
+ $exit = 1; print "--> ERROR: phone \"@col[$_]\" is not in {, non}silence.txt (line $idx)\n";
+ $success = 0;
+ }
+ }
+ $idx ++;
+ }
+ close(L);
+ $success == 0 || print "--> $lexfn is OK\n";
+ print "\n";
+}
+
+if (-f "$dict/lexicon.txt") { check_lexicon("$dict/lexicon.txt", 0); }
+if (-f "$dict/lexiconp.txt") { check_lexicon("$dict/lexiconp.txt", 1); }
+if (!(-f "$dict/lexicon.txt" || -f "$dict/lexiconp.txt")) {
+ print "--> ERROR: neither lexicon.txt or lexiconp.txt exist in directory $dir\n";
+ $exit = 1;
+}
+# If both lexicon.txt and lexiconp.txt exist, we check that they correspond to
+# each other. If not, it could be that the user overwrote one and we need to
+# regenerate the other, but we don't know which is which.
+if ( (-f "$dict/lexicon.txt") && (-f "$dict/lexiconp.txt")) {
+ print "Checking that lexicon.txt and lexiconp.txt match\n";
+ if (!open(L, "<$dict/lexicon.txt") || !open(P, "<$dict/lexiconp.txt")) {
+ die "Error opening lexicon.txt and/or lexiconp.txt"; # already checked, so would be code error.
+ }
+ while(<L>) {
+ @A = split;
+ $x = <P>;
+ if (!defined $x) {
+ print "--> ERROR: lexicon.txt and lexiconp.txt have different numbers of lines (mismatch); delete one.\n";
+ $exit = 1;
+ last;
+ }
+ @B = split(" ", $x);
+ $w = shift @B;
+ $p = shift @B;
+ unshift @B, $w;
+ # now @A and @B should be the same.
+ if ($#A != $#B) {
+ print "--> ERROR: lexicon.txt and lexiconp.txt have mismatched lines '$_' versus '$x'; delete one.\n";
+ $exit = 1;
+ last;
+ }
+ for ($n = 0; $n < @A; $n++) {
+ if ($A[$n] ne $B[$n]) {
+ print "--> ERROR: lexicon.txt and lexiconp.txt have mismatched lines '$_' versus '$x'; delete one.\n";
+ $exit = 1;
+ last;
+ }
+ }
+ }
+ $x = <P>;
+ if (defined $x && $exit == 0) {
+ print "--> ERROR: lexicon.txt and lexiconp.txt have different numbers of lines (mismatch); delete one.\n";
+ $exit = 1;
}
- $idx ++;
}
-close(L);
-$success == 0 || print "--> $dict/lexicon.txt is OK\n";
-print "\n";
# Checking extra_questions.txt -------------------------------
print "Checking $dict/extra_questions.txt ...\n";