#!/usr/bin/perl
|
# Copyright 2010-2011 Microsoft Corporation
|
# 2013 Johns Hopkins University (author: Daniel Povey)
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
# you may not use this file except in compliance with the License.
|
# You may obtain a copy of the License at
|
#
|
# http://www.apache.org/licenses/LICENSE-2.0
|
#
|
# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
# MERCHANTABLITY OR NON-INFRINGEMENT.
|
# See the Apache 2 License for the specific language governing permissions and
|
# limitations under the License.
|
|
|
# Adds disambiguation symbols to a lexicon.
|
# Outputs still in the normal lexicon format.
|
# Disambig syms are numbered #1, #2, #3, etc. (#0
|
# reserved for symbol in grammar).
|
# Outputs the number of disambig syms to the standard output.
|
# With the --pron-probs option, expects the second field
|
# of each lexicon line to be a pron-prob.
|
|
$pron_probs = 0;
|
|
if ($ARGV[0] eq "--pron-probs") {
|
$pron_probs = 1;
|
shift @ARGV;
|
}
|
|
if(@ARGV != 2) {
|
die "Usage: add_lex_disambig.pl [--pron-probs] lexicon.txt lexicon_disambig.txt "
|
}
|
|
|
$lexfn = shift @ARGV;
|
$lexoutfn = shift @ARGV;
|
|
open(L, "<$lexfn") || die "Error opening lexicon $lexfn";
|
|
# (1) Read in the lexicon.
|
@L = ( );
|
while(<L>) {
|
@A = split(" ", $_);
|
push @L, join(" ", @A);
|
}
|
|
# (2) Work out the count of each phone-sequence in the
|
# lexicon.
|
|
foreach $l (@L) {
|
@A = split(" ", $l);
|
shift @A; # Remove word.
|
if ($pron_probs) {
|
$p = shift @A;
|
if (!($p > 0.0 && $p <= 1.0)) { die "Bad lexicon line $l (expecting pron-prob as second field)"; }
|
}
|
$count{join(" ",@A)}++;
|
}
|
|
# (3) For each left sub-sequence of each phone-sequence, note down
|
# that exists (for identifying prefixes of longer strings).
|
|
foreach $l (@L) {
|
@A = split(" ", $l);
|
shift @A; # Remove word.
|
if ($pron_probs) { shift @A; } # remove pron-prob.
|
while(@A > 0) {
|
pop @A; # Remove last phone
|
$issubseq{join(" ",@A)} = 1;
|
}
|
}
|
|
# (4) For each entry in the lexicon:
|
# if the phone sequence is unique and is not a
|
# prefix of another word, no diambig symbol.
|
# Else output #1, or #2, #3, ... if the same phone-seq
|
# has already been assigned a disambig symbol.
|
|
|
open(O, ">$lexoutfn") || die "Opening lexicon file $lexoutfn for writing.\n";
|
|
$max_disambig = 0;
|
foreach $l (@L) {
|
@A = split(" ", $l);
|
$word = shift @A;
|
if ($pron_probs) { $pron_prob = shift @A; }
|
$phnseq = join(" ",@A);
|
if(!defined $issubseq{$phnseq}
|
&& $count{$phnseq} == 1) {
|
; # Do nothing.
|
} else {
|
if($phnseq eq "") { # need disambig symbols for the empty string
|
# that are not use anywhere else.
|
$max_disambig++;
|
$reserved{$max_disambig} = 1;
|
$phnseq = "#$max_disambig";
|
} else {
|
$curnumber = $disambig_of{$phnseq};
|
if(!defined{$curnumber}) { $curnumber = 0; }
|
$curnumber++; # now 1 or 2, ...
|
while(defined $reserved{$curnumber} ) { $curnumber++; } # skip over reserved symbols
|
if($curnumber > $max_disambig) {
|
$max_disambig = $curnumber;
|
}
|
$disambig_of{$phnseq} = $curnumber;
|
$phnseq = $phnseq . " #" . $curnumber;
|
}
|
}
|
if ($pron_probs) { print O "$word\t$pron_prob\t$phnseq\n"; }
|
else { print O "$word\t$phnseq\n"; }
|
}
|
|
print $max_disambig . "\n";
|
