TimeKeeper/audio/flite-2.0.0-release/lang/usenglish/us_text.c

/*************************************************************************/
/*                                                                       */
/*                  Language Technologies Institute                      */
/*                     Carnegie Mellon University                        */
/*                         Copyright (c) 2001                            */
/*                        All Rights Reserved.                           */
/*                                                                       */
/*  Permission is hereby granted, free of charge, to use and distribute  */
/*  this software and its documentation without restriction, including   */
/*  without limitation the rights to use, copy, modify, merge, publish,  */
/*  distribute, sublicense, and/or sell copies of this work, and to      */
/*  permit persons to whom this work is furnished to do so, subject to   */
/*  the following conditions:                                            */
/*   1. The code must retain the above copyright notice, this list of    */
/*      conditions and the following disclaimer.                         */
/*   2. Any modifications must be clearly marked as such.                */
/*   3. Original authors' names are not deleted.                         */
/*   4. The authors' names are not used to endorse or promote products   */
/*      derived from this software without specific prior written        */
/*      permission.                                                      */
/*                                                                       */
/*  CARNEGIE MELLON UNIVERSITY AND THE CONTRIBUTORS TO THIS WORK         */
/*  DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING      */
/*  ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT   */
/*  SHALL CARNEGIE MELLON UNIVERSITY NOR THE CONTRIBUTORS BE LIABLE      */
/*  FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES    */
/*  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN   */
/*  AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,          */
/*  ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF       */
/*  THIS SOFTWARE.                                                       */
/*                                                                       */
/*************************************************************************/
/*             Author:  Alan W Black (awb@cs.cmu.edu)                    */
/*               Date:  January 2001                                     */
/*************************************************************************/
/*                                                                       */
/*  US English text analysis functions                                   */
/*                                                                       */
/*************************************************************************/

#include <ctype.h>
#include "flite.h"
#include "usenglish.h"
#include "us_text.h"
#include "cst_regex.h"

static int text_splitable(const char *s,int i);
static cst_val *state_name(const char *name,cst_item *t);

/* compiled us regexes */
#include "us_regexes.h"

/* Note you need to also update the wandm regex in make_us_regeses too */
static const char * const wandm_abbrevs[99][2] =
{
    { "LB", "pounds" },
    { "LBS", "pounds" },
    { "lb", "pounds" },
    { "lbs", "pounds" },
    { "ft", "feet" },
    { "FT", "feet" },
    { "kg", "kilograms" },
    { "km", "kilometers" },
    { "cm", "centimeters" },
    { "mm", "millimeters" },
    { "ml", "milliliters" },
    { "oz", "ounces" },
    { "hz", "hertz" },
    { "Hz", "hertz" },
    { "HZ", "hertz" },
    { "KHz", "kilohertz" },
    { "MHz", "megahertz" },
    { "GHz", "gigahertz" },
    { "KB", "kilobytes" },
    { "GB", "gigabytes" },
    { "MB", "megabytes" },
    { "TB", "terabytes" },
    { NULL, NULL },
};

static const char * const eedwords[] = {
    "to",
    "can",
    "can't",
    "cannot",
    "cant",
    "could",
    "couldn't",
    "couldnt",
    "will",
    "shall",
    NULL};

static int rex_like(const cst_item *t)
{
    /* returns 1 if this is in a king like context */
    char *pn = cst_downcase(ffeature_string(t,"p.name"));
    char *ppn = cst_downcase(ffeature_string(t,"p.p.name"));
    int v = 0;
    
    if (cst_streq(pn,"louis") ||
	cst_streq(pn,"henry") ||
	cst_streq(pn,"charles") ||
	cst_streq(pn,"philip") ||
	cst_streq(pn,"george") ||
	cst_streq(pn,"edward") ||
	cst_streq(pn,"pius") ||
	cst_streq(pn,"william") ||
	cst_streq(pn,"richard") ||
	cst_streq(pn,"ptolemy") ||
	cst_streq(pn,"john") ||
	cst_streq(pn,"paul") ||
	cst_streq(pn,"peter") ||
	cst_streq(pn,"nicholas") ||
	cst_streq(pn,"frederick") ||
	cst_streq(pn,"james") ||
	cst_streq(pn,"alfonso") ||
	cst_streq(pn,"ivan") ||
	cst_streq(pn,"napolean") ||
	cst_streq(pn,"leo") ||
	cst_streq(pn,"gregory") ||
	cst_streq(pn,"catherine") ||
	cst_streq(pn,"alexandria") ||
	cst_streq(pn,"pierre") ||
	cst_streq(pn,"elizabeth") ||
	cst_streq(pn,"mary"))
	v = 1;
    else if (cst_streq(ppn,"king") ||
	     cst_streq(ppn,"queen") ||
	     cst_streq(ppn,"pope") ||
	     cst_streq(ppn,"duke") ||
	     cst_streq(ppn,"tsar") ||
	     cst_streq(ppn,"emperor") ||
	     cst_streq(ppn,"shah") ||
	     cst_streq(ppn,"ceasar") ||
	     cst_streq(ppn,"duchess") ||
	     cst_streq(ppn,"tsarina") ||
	     cst_streq(ppn,"empress") ||
	     cst_streq(ppn,"baron") ||
	     cst_streq(ppn,"baroness") ||
	     cst_streq(ppn,"sultan") ||
	     cst_streq(ppn,"count") ||
	     cst_streq(ppn,"countess"))
	v = 1;

    cst_free(pn);
    cst_free(ppn);
    return v;
}

static int section_like(const cst_item *t)
{
    /* returns 1 if this is in a king like context */
    char *pn = cst_downcase(ffeature_string(t,"p.name"));
    int v = 0;

    if (cst_streq(pn,"section") ||
	cst_streq(pn,"chapter") ||
	cst_streq(pn,"part") ||
	cst_streq(pn,"phrase") ||
	cst_streq(pn,"verse") ||
	cst_streq(pn,"scene") ||
	cst_streq(pn,"act") ||
	cst_streq(pn,"book") ||
	cst_streq(pn,"volume") ||
	cst_streq(pn,"chap") ||
	cst_streq(pn,"war") ||
	cst_streq(pn,"apollo") ||
	cst_streq(pn,"trek") ||
	cst_streq(pn,"fortran"))
	v = 1;
    
    cst_free(pn);
    
    return v;
}

cst_utterance *us_textanalysis(cst_utterance *u)
{
    if (!feat_present(u->features, "tokentowords_func"))
	utt_set_feat(u, "tokentowords_func", itemfunc_val(us_tokentowords));

    return default_textanalysis(u);
}

static cst_val *us_tokentowords_one(cst_item *token, const char *name);
cst_val *us_tokentowords(cst_item *token)
{
    return us_tokentowords_one(token, item_feat_string(token, "name"));
}

static cst_val *add_break(cst_val *l)
{
    /* add feature (break 1) to last item in this list */
    const cst_val *i;
    cst_val *t;
    cst_features *f;

    for (i=l; val_cdr(i); i=val_cdr(i));

    if (i)  /* might be empty list */
    {
	f = new_features();
	feat_set_string(f,"break","1");
	t = cons_val(val_car(i),features_val(f));
	set_car((cst_val *)i,t);
    }

    return l;
}

static int contains_unicode_single_quote(const char *name)
{
    static const char *unicode_single_quote = "’";
    int i;

    for (i=0; name[i]; i++)
    {
        /* No check if name is long enough as it'll have NULL before end */
        if ((name[i] == unicode_single_quote[0]) &&
            (name[i+1] == unicode_single_quote[1]) &&
            (name[i+2] == unicode_single_quote[2]))
            return TRUE;
    }
    return FALSE;
}

static char *map_unicode_single_quote(const char *name)
{
    static const char *unicode_single_quote = "’";
    int i,j;
    char *aaa = cst_strdup(name);  /* it'll always get shorter */

    for (i=0,j=0; name[i]; i++,j++)
    {
        if ((name[i] == unicode_single_quote[0]) &&
            (name[i+1] == unicode_single_quote[1]) &&
            (name[i+2] == unicode_single_quote[2]))
        {
            aaa[j] = '\'';
            i+=2;
        }
        else
            aaa[j] = name[i];
    }
    aaa[j] = '\0';

    return aaa;
}

static cst_val *us_tokentowords_one(cst_item *token, const char *name)
{
    /* Return list of words that expand token/name */
    char *p, *aaa, *bbb, *ccc;
    int i,j,k,l;
    cst_val *r, *s, *ss;
    const cst_val *rr;
    const char *nsw = "";
    const char *ssml_alias = "";
    const char *token_name = "";
    cst_lexicon *lex;
    cst_utterance *utt;
    /* printf("token_name %s name %s\n",item_name(token),name); */
    /* FIXME: For SAPI and friends, any tokens with explicit
       pronunciations need to be passed through as-is.  This should be
       done in the interface code rather than here once the
       tokentowords hook is accessible. AWB: no, they should set the
       nsw feature and this function should deal with it (doesn't yet though)*/
    if (item_feat_present(token,"phones"))
	return cons_val(string_val(name),NULL);
    
    if (item_feat_present(token,"nsw"))
	nsw = item_feat_string(token,"nsw");

    token_name = item_name(token);

    if ((item_feat_present(token,"ssml_alias")) &&
        (!cst_streq(token_name,name)))  /* and we are not recursing */
    {
        /* SSML has given a substitute for this (and more) tokens */
        /* NOTE: the alias is not put through text normalization */
        ssml_alias = item_feat_string(token,"ssml_alias");
        if (cst_streq(ssml_alias,ffeature_string(token,"p.ssml_alias")))
            /* The first token gets the substitution */
            return NULL;
        else
        {
            return cons_val(string_val(ssml_alias),NULL);
        }
    }

    utt = item_utt(token);
    lex = val_lexicon(feat_val(utt->features,"lexicon"));

    if (cst_streq("1",get_param_string(item_feats(token),"ssml_comment","0")))
        r = NULL;
    else if ((cst_streq("a",name) || cst_streq("A",name)) &&
        ((item_next(token) == 0) ||
         (!cst_streq(name,item_name(token))) ||
         (!cst_streq("",ffeature_string(token,"punc")))))
    {   /* if A is a sub part of a token, then its ey not ah */
	r = cons_val(string_val("_a"),0);
    }
    else if (cst_strlen(name) == 0)
        r = NULL;
    else if (cst_regex_match(dottedabbrevs,name))
    {   /* X.X.X */
	aaa = cst_strdup(name);
	for (i=j=0; aaa[i]; i++)
	    if (aaa[i] != '.')
	    {
		aaa[j] = aaa[i];
		j++;
	    }
	aaa[j] = '\0';
	r = en_exp_letters(aaa);
	cst_free(aaa);
    }
    else if (cst_regex_match(cst_rx_commaint,name))
    {   /* 99,999,999 */
	aaa = cst_strdup(name);
	for (j=i=0; i < cst_strlen(name); i++)
	    if (name[i] != ',')
	    {
		aaa[j] = name[i];
		j++;
	    }
	aaa[j] = '\0';
	r = en_exp_real(aaa);
	cst_free(aaa);
    }
    else if (cst_regex_match(sevenphonenumber,name))
    {   /* 234-3434 telephone numbers */
	p=strchr(name,'-');
	aaa = cst_strdup(name);
	aaa[cst_strlen(name)-cst_strlen(p)] = '\0';
	bbb = cst_strdup(p+1);
	r = val_append(add_break(en_exp_digits(aaa)),
		       en_exp_digits(bbb));
	cst_free(aaa);
	cst_free(bbb);
    }
    else if 
     ((cst_regex_match(threedigits,name) &&
      ((!cst_regex_match(cst_rx_digits,ffeature_string(token,"p.name"))
	&& cst_regex_match(threedigits,ffeature_string(token,"n.name"))
	&& cst_regex_match(fourdigits,ffeature_string(token,"n.n.name"))) ||
       (cst_regex_match(sevenphonenumber,ffeature_string(token,"n.name"))) ||
       (!cst_regex_match(cst_rx_digits,ffeature_string(token,"p.p.name"))
	&& cst_regex_match(threedigits,ffeature_string(token,"p.name"))
	&& cst_regex_match(fourdigits,ffeature_string(token,"n.name"))))) ||
      (cst_regex_match(fourdigits,name) &&
       (!cst_regex_match(cst_rx_digits,ffeature_string(token,"n.name"))
	&& cst_regex_match(threedigits,ffeature_string(token,"p.name"))
	&& cst_regex_match(threedigits,ffeature_string(token,"p.p.name")))))
    {
	/* part of a telephone number */
	if (cst_streq("",ffeature_string(token,"punc")))
	    item_set_string(token,"punc",",");
	r = add_break(en_exp_digits(name));
    }
    else if (cst_regex_match(numbertime,name))
    {
	p=strchr(name,':');
	aaa = cst_strdup(name);
	aaa[cst_strlen(name)-cst_strlen(p)] = '\0';
	bbb = cst_strdup(p+1);

	r = en_exp_number(aaa);
	if (!cst_streq("00",bbb))
	    r = val_append(r,en_exp_id(bbb));
	/* r = add_break(r); */

	cst_free(aaa);
	cst_free(bbb);
    }
    else if (cst_regex_match(numbertimexm,name))
    {
	p=strchr(name,':');
        if (!p) p=strchr(name,'.');
	aaa = cst_strdup(name);
	aaa[cst_strlen(name)-cst_strlen(p)] = '\0';
	bbb = cst_strdup(p+1);
	bbb[2] = '\0';
        ccc = cst_strdup(p+3);

	r = en_exp_number(aaa);
	if (!cst_streq("00",bbb))
	    r = val_append(r,en_exp_id(bbb));
	/* r = add_break(r); */

        r = val_append(r,en_exp_letters(ccc));

	cst_free(aaa);
	cst_free(bbb);
	cst_free(ccc);
    }
    else if (cst_regex_match(digits2dash,name))
    {   /* 999-999-999 etc */
	bbb = cst_strdup(name);
	for (ss=0,aaa=p=bbb; *p; p++)
	{
	    if (*p == '-')
	    {
		*p = '\0';
		ss = cons_val(string_val(aaa),ss);
		aaa = p+1;
	    }
	}
        ss = cons_val(string_val(aaa),ss);
        if ((val_length(ss) == 2) &&
            (atoi(val_string(val_car(val_cdr(ss)))) <
             atoi(val_string(val_car(ss)))))  /* its a number range */
        {
            /* Should get 22-23 November, or 1998-1999 right */
            r = 
                val_append(us_tokentowords_one(token,val_string(val_car(val_cdr(ss)))),
                  cons_val(string_val("to"),
                   us_tokentowords_one(token,val_string(val_car(ss)))));
        }
        else
        {  /* Its just a bunch of ids */
            r = 0;
            for (rr=ss; rr; rr=val_cdr(rr))
            {
                r = val_append(
                     add_break(en_exp_digits(val_string(val_car(rr)))),r);

            }
        }
        delete_val(ss);
	cst_free(bbb);
    }
    else if (cst_regex_match(cst_rx_digits,name))
    {   /* string of digits (use cart to disambiguate) */
	if (cst_streq("nide",nsw))
	    r = en_exp_id(name);
	else {
	    const cst_val *tv;
	    const char *ts;
	    char *rname;
	    
	    rname = cst_strdup(item_feat_string(token,"name"));
	    if (cst_streq(name,rname))
		tv = cart_interpret(token,&us_nums_cart);
	    else 
	    {   /* in a recursive call */
		item_set_string(token,"name",name);
		tv = cart_interpret(token,&us_nums_cart);
		item_set_string(token,"name",rname);
	    }
	    cst_free(rname);
	    ts = val_string(tv);
	    if (cst_streq(ts,"ordinal"))
		r = en_exp_ordinal(name);
	    else if (cst_streq(ts,"digits"))
		r = en_exp_digits(name);
	    else if (cst_streq(ts,"year"))
		r = en_exp_id(name);
	    else
		r = en_exp_number(name);
	}
    }
    else if (cst_regex_match(romannums,name))
    {   /* Roman numerals */
	if (cst_streq("",ffeature_string(token,"p.punc")))
	{   /* no preceeding punc */
	    char n[10];
	    cst_sprintf(n,"%d",en_exp_roman(name));
	    if (rex_like(token))
		r = cons_val(string_val("the"),
			     en_exp_ordinal(n));
	    else if (section_like(token))
		r = en_exp_number(n);
	    else
		r = en_exp_letters(name);
	}
	else
	    r = en_exp_letters(name);
    }
    else if (cst_regex_match(drst,name))  
    {   /* St Andrew's St, Dr King Dr */
	const char *street;
	const char *saint;
	if ((name[0] == 's') || (name[0] == 'S'))
	{
	    street = "street";
	    saint = "saint";
	}
	else
	{
	    street = "drive";
	    saint = "doctor";
	}
	if ((item_next(token) == 0) ||
	    strchr(item_feat_string(token,"punc"),','))
	    r = cons_val(string_val(street),NULL);
	else if (strchr(ffeature_string(token,"punc"),','))
	    r = cons_val(string_val(saint),NULL);
	else 
	{
	    const char *pname = ffeature_string(token,"p.name");
	    const char *nname = ffeature_string(token,"n.name");
	    if ((pname[0] >= 'A') && (pname[0] <= 'Z') &&
		(nname[0] >= 'a') && (nname[0] <= 'z'))
		r = cons_val(string_val(street),NULL);
	    else if ((pname[0] >= '0') && (pname[0] <= '9') &&
		     (nname[0] >= 'a') && (nname[0] <= 'z'))
		r = cons_val(string_val(street),NULL);
	    else if ((pname[0] >= 'a') && (pname[0] <= 'z') &&
		     (nname[0] >= 'A') && (nname[0] <= 'Z'))
		r = cons_val(string_val(saint),NULL);
	    else if (cst_streq(ffeature_string(token,"n.whitespace")," "))
		r = cons_val(string_val(saint),NULL);
	    else
		r = cons_val(string_val(street),NULL);
	}
	if (cst_streq(item_feat_string(token,"punc"),"."))
	    item_set_string(token,"punc","");
    }
    else if (cst_streq(name,"Mr"))
    {
	item_set_string(token,"punc","");
	r = cons_val(string_val("mister"),NULL);
    }
    else if (cst_streq(name,"Mrs"))
    {
	item_set_string(token,"punc","");
	r = cons_val(string_val("missus"),NULL);
    }
    else if ((cst_streq(name,"read")) ||
             (cst_streq(name,"lead")))
    {   /* checking WSJ examples, this seems a quick and easy way to */
        /* get many of these correct */
        const char *pname = ffeature_string(token,"p.name");

        for (i=0; eedwords[i]; i++)
            if (cst_streq(pname,eedwords[i]))
                break;
        
        if (eedwords[i])
        {  /* reed or leed */
            if (name[0] == 'r')
                r = cons_val(string_val("reed"),NULL);
            else
                r = cons_val(string_val("leed"),NULL);
        }
        else /* red or led */
        {
            if (name[0] == 'r')
                r = cons_val(string_val("red"),NULL);
            else
                r = cons_val(string_val("led"),NULL);
        }
    }
    else if (cst_streq(name,"am") || cst_streq(name,"AM"))
    {
        if (!cst_streq(name,item_name(token)))
            r = en_exp_letters(name);
        else if (item_prev(token) &&
                 (cst_regex_match(numbertime,ffeature_string(token,"p.name")) ||
                  cst_regex_match(cst_rx_digits,ffeature_string(token,"p.name"))))
            r = en_exp_letters(name);
        else 
            r = cons_val(string_val(name),NULL);
    }
    else if ((cst_strlen(name) == 1) &&
	     (name[0] >= 'A') &&
	     (name[0] <= 'Z') &&
	     (cst_streq(" ",ffeature_string(token,"n.whitespace"))) &&
	     (ffeature_string(token,"n.name")[0] >= 'A') &&
	     (ffeature_string(token,"n.name")[0] <= 'Z'))
    {
	item_set_string(token,"punc","");
	aaa = cst_downcase(name);
	if (cst_streq(aaa,"a"))
	    r = cons_val(string_val("_a"),0);
	else
	    r = cons_val(string_val(aaa),0);
	cst_free(aaa);
    }
    else if (cst_regex_match(cst_rx_double,name))
    {   /* real numbers */
	r = en_exp_real(name);
    }
    else if (cst_regex_match(ordinal_number,name))
    {   /* explicit ordinals */
	aaa = cst_strdup(name);
	aaa[cst_strlen(name)-2] = '\0';
	r = en_exp_ordinal(aaa);
	cst_free(aaa);
    }
    else if ((cst_regex_match(illion,name)) &&
	     (cst_regex_match(usmoney,ffeature_string(token,"p.name"))))
    {
	r = cons_val(string_val(name),
		     cons_val(string_val("dollars"),NULL));
    }
    else if (cst_regex_match(usmoney,name))
    {
	/* US money */
/*	printf("money, money, money %s\n", name); */
	p = strchr(name,'.');

	if (cst_regex_match(illion,ffeature_string(token,"n.name")))
	{   /* carl sagan's billions and billions */
	    r = en_exp_real(&name[1]);
	}
	else if (!p)
	{
	    aaa = cst_strdup(&name[1]);
	    if (cst_streq("1",aaa))
		r = cons_val(string_val("dollar"),NULL);
	    else
		r = cons_val(string_val("dollars"),NULL);
	    r = val_append(us_tokentowords_one(token,aaa),r);
	    cst_free(aaa);
	}
	else if ((cst_strlen(p) == 1) || (cst_strlen(p) > 3))
	{   /* simply read as mumble point mumble */
	    r = val_append(en_exp_real(&name[1]),
			   cons_val(string_val("dollars"),NULL));
	}
	else
	{
	    aaa = cst_strdup(name);
	    aaa[cst_strlen(name)-cst_strlen(p)] = '\0';
	    for (i=j=0; aaa[i] != '\0'; i++)
	    {
		if (aaa[i] != ',')
		{
		    aaa[j] = aaa[i];
		    j++;
		}
	    }
	    aaa[j] = '\0';
	    if (cst_streq("00",p+1))
		r = 0;
	    else if (cst_streq("01",p+1))
		r = val_append(en_exp_number(p+1),
			       cons_val(string_val("cent"),NULL));
	    else
		r = val_append(en_exp_number(p+1),
			       cons_val(string_val("cents"),NULL));

	    if (cst_streq("1",aaa+1))
		r = cons_val(string_val("dollar"),r);
	    else
		r = cons_val(string_val("dollars"),r);

	    r = val_append(en_exp_number(aaa+1),r);
	    cst_free(aaa);
	}
    }
    else if (name[cst_strlen(name)-1] == '%')
    {
	aaa = cst_strdup(name);
	aaa[cst_strlen(aaa)-1] = '\0';
	r = val_append(us_tokentowords_one(token,aaa),
		       cons_val(string_val("per"),
				cons_val(string_val("cent"),NULL)));
	cst_free(aaa);

    }
    else if (cst_regex_match(numess,name)) 
    {   /* 60s and 7s and 9s */
	aaa = cst_strdup(name);
	aaa[cst_strlen(name)-1] = '\0';
	r = val_append(us_tokentowords_one(token,aaa),
		       cons_val(string_val("'s"),0));
	cst_free(aaa);
    }
    else if (contains_unicode_single_quote(name))
    {
        /* A single quote is sometimes rendered as unicode "’" */
        /* so we map it back to an ascii single quote ' */
        aaa = map_unicode_single_quote(name);
        r = us_tokentowords_one(token, aaa);
        cst_free(aaa);
        return r;
    }
    else if ((p=(cst_strrchr(name,'\''))))
    {
	static const char * const pc[] = { "'s", "'ll", "'ve", "'d", NULL };

	bbb = cst_downcase(p);
	if (cst_member_string(bbb, pc))
	{
	    aaa = cst_strdup(name);
	    aaa[cst_strlen(name)-cst_strlen(p)] = '\0';
	    r = val_append(us_tokentowords_one(token,aaa),
			   cons_val(string_val(bbb),0));
	    cst_free(aaa);
	}
	else if (cst_streq(p,"'tve")) /* admittedly rare and weird */
	{
	    aaa = cst_strdup(name);
	    aaa[cst_strlen(name)-cst_strlen(p)+2] = '\0';
	    r = val_append(us_tokentowords_one(token,aaa),
			   cons_val(string_val("'ve"),0));
	    cst_free(aaa);
	}
	else
	{
	    aaa = cst_strdup(name);
	    strcpy(&aaa[cst_strlen(name)-cst_strlen(p)],p+1);
	    r = us_tokentowords_one(token,aaa);
	    cst_free(aaa);
	}
	cst_free(bbb);
    }
    else if ((cst_regex_match(digitsslashdigits,name)) &&
	     (cst_streq(name,item_name(token))))
    {   /* might be fraction, or not */
	p=strchr(name,'/');
	aaa = cst_strdup(name);
	aaa[cst_strlen(name)-cst_strlen(p)] = '\0';
	bbb = cst_strdup(p+1);
	if ((cst_streq("1",aaa)) && (cst_streq("2",bbb)))
	    r = cons_val(string_val("a"),
			 cons_val(string_val("half"),0));
	else if (atoi(aaa) < (atoi(bbb)))
	{
	    r = val_append(en_exp_number(aaa),
			   en_exp_ordinal(bbb));
	    if (atoi(aaa) > 1)
		r = val_append(r,cons_val(string_val("'s"),0));
	}
	else
	    r = val_append(en_exp_number(aaa),
			   cons_val(string_val("slash"),
				    en_exp_number(bbb)));

	if ((cst_regex_match(cst_rx_digits,ffeature_string(token,"p.name")))
	    && (item_prev(token)))  /* don't mistake "0" as a number */
	    r = cons_val(string_val("and"),r);
	cst_free(aaa);
	cst_free(bbb);
    }
    else if ((p=(strchr(name,'-'))))
    {   /* aaa-bbb */
	aaa = cst_strdup(name);
	aaa[cst_strlen(name)-cst_strlen(p)] = '\0';
	bbb = cst_strdup(p+1);
	if (cst_regex_match(cst_rx_digits,aaa) &&
	    cst_regex_match(cst_rx_digits,bbb))
	{
            ccc = cst_strdup(name);
	    item_set_string(token,"name",bbb);
	    r = us_tokentowords_one(token,bbb);
	    item_set_string(token,"name",aaa);
	    r = val_append(us_tokentowords_one(token,aaa),
			   cons_val(string_val("to"),r));
	    item_set_string(token,"name",ccc);
            cst_free(ccc);
	}
	else
	    r = val_append(us_tokentowords_one(token,aaa),
			   us_tokentowords_one(token,bbb));
	cst_free(aaa);
	cst_free(bbb);
    }
    else if (cst_regex_match(wandm,name))
    {   /* weights and measures */
        for (j=cst_strlen(name)-1; j > 0; j--)
            if (cst_strchr("0123456789",name[j]))
                break;
        j += 1;
        for (i=0; wandm_abbrevs[i][0]; i++)
            if (cst_streq(name+j,wandm_abbrevs[i][0]))
                break;
        aaa = cst_strdup(name);
        aaa[j] = '\0';
        /* remove any commas */
        for (k=0,l=0; aaa[l]; k++,l++)
        {
            if (aaa[l] == ',') l++;
            aaa[k] = aaa[l];
        }
        aaa[k] = '\0';
        if (!wandm_abbrevs[i][0]) /* didn't find an expansion */
	    r = val_append(en_exp_number(aaa),
			   us_tokentowords_one(token,name+j));
        else
            r = val_append(en_exp_number(aaa),
                           cons_val(string_val(wandm_abbrevs[i][1]),NULL));

        cst_free(aaa);
    }
    else if ((cst_strlen(name) > 1) && (!cst_regex_match(cst_rx_alpha,name)))
    {   /* its not just alphas */
	for (i=0; name[i] != '\0'; i++)
	    if (text_splitable(name,i))
		break;
	aaa = cst_strdup(name);
	aaa[i+1] = '\0';
	bbb = cst_strdup(&name[i+1]);
	item_set_string(token,"nsw","nide");
	r = val_append(us_tokentowords_one(token,aaa),
		       us_tokentowords_one(token,bbb));
	cst_free(aaa);
	cst_free(bbb);
    }
    else if ((s = state_name(name,token)))
    {
	r = s;
    }
    else if ((cst_strlen(name) > 1) && 
	     (cst_regex_match(cst_rx_alpha,name)) &&
             (!in_lex(lex,name,NULL,NULL)) &&  // AUP: Added 4th argument (voice feats) as NULL, needs to be revisited later.
	     (!us_aswd(name)))
        /* Still not quiet right, if there is a user_lex we need to check */
        /* it too -- but user_lex isn't user setable yet */
	/* Need common exception list */
	/* unpronouncable list of alphas */
	r = en_exp_letters(name);

    /* buckets of other stuff missing */

    else  /* just a word */
    {
	aaa = cst_downcase(name);
	r = cons_val(string_val(aaa),0);
	cst_free(aaa);
    }
    return r;
}

static int text_splitable(const char *s,int i)
{
    /* should token be split after this */

    if (strchr("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",s[i]) &&
	strchr("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",s[i+1]))
	return FALSE;
    else if (strchr("0123456789",s[i]) &&
	     strchr("0123456789",s[i+1]))
	return FALSE;
    else
	return TRUE;
}

static const char * const states[99][5] =
{
  { "AL", "ambiguous", "alabama" , NULL, NULL },
  { "Al", "ambiguous", "alabama" , NULL, NULL },
  { "Ala", "", "alabama" , NULL, NULL },
  { "AK", "", "alaska" , NULL, NULL },
  { "Ak", "", "alaska" , NULL, NULL },
  { "AZ", "", "arizona" , NULL, NULL },
  { "Az", "", "arizona" , NULL, NULL },
  { "CA", "", "california" , NULL, NULL },
  { "Ca", "", "california" , NULL, NULL },
  { "Cal", "ambiguous", "california" , NULL, NULL },
  { "Calif", "", "california" , NULL, NULL },
  { "CO", "ambiguous", "colorado" , NULL, NULL },
  { "Co", "ambiguous", "colorado" , NULL, NULL },
  { "Colo", "", "colorado" , NULL, NULL },
  { "DC", "", "d" , "c", NULL },
  { "DE", "", "delaware" , NULL, NULL },
  { "De", "ambiguous", "delaware" , NULL, NULL },
  { "Del", "ambiguous", "delaware" , NULL, NULL },
  { "FL", "", "florida" , NULL, NULL },
  { "Fl", "ambiguous", "florida" , NULL, NULL },
  { "Fla", "", "florida" , NULL, NULL },
  { "GA", "", "georgia" , NULL, NULL },
  { "Ga", "", "georgia" , NULL, NULL },
  { "HI", "", "hawaii" , NULL, NULL },
  { "Hi", "ambiguous", "hawaii" , NULL, NULL },
  { "IA", "", "iowa" , NULL, NULL },
  { "Ia", "ambiguous", "iowa" , NULL, NULL },
  { "Ind", "ambiguous", "indiana" , NULL, NULL },
  { "ID", "ambiguous", "idaho" , NULL, NULL },
  { "IL", "ambiguous", "illinois" , NULL, NULL },
  { "Il", "ambiguous", "illinois" , NULL, NULL },
  { "ILL", "ambiguous", "illinois" , NULL, NULL },
  { "KS", "", "kansas" , NULL, NULL },
  { "Ks", "", "kansas" , NULL, NULL },
  { "Kans", "", "kansas" , NULL, NULL },
  { "KY", "ambiguous", "kentucky" , NULL, NULL },
  { "Ky", "ambiguous", "kentucky" , NULL, NULL },
  { "LA", "ambiguous", "louisiana" , NULL, NULL },
  { "La", "ambiguous", "louisiana" , NULL, NULL },
  { "Lou", "ambiguous", "louisiana" , NULL, NULL },
  { "Lous", "ambiguous", "louisiana" , NULL, NULL },
  { "MA", "ambiguous", "massachusetts" , NULL, NULL },
  { "Mass", "ambiguous", "massachusetts" , NULL, NULL },
  { "Ma", "ambiguous", "massachusetts" , NULL, NULL },
  { "MD", "ambiguous", "maryland" , NULL, NULL },
  { "Md", "ambiguous", "maryland" , NULL, NULL },
  { "ME", "ambiguous", "maine" , NULL, NULL },
  { "Me", "ambiguous", "maine" , NULL, NULL },
  { "MI", "", "michigan" , NULL, NULL },
  { "Mi", "ambiguous", "michigan" , NULL, NULL },
  { "Mich", "ambiguous", "michigan" , NULL, NULL },
  { "MN", "ambiguous", "minnestota" , NULL, NULL },
  { "Minn", "ambiguous", "minnestota" , NULL, NULL },
  { "MS", "ambiguous", "mississippi" , NULL, NULL },
  { "Miss", "ambiguous", "mississippi" , NULL, NULL },
  { "MT", "ambiguous", "montanna" , NULL, NULL },
  { "Mt", "ambiguous", "montanna" , NULL, NULL },
  { "MO", "ambiguous", "missouri" , NULL, NULL },
  { "Mo", "ambiguous", "missouri" , NULL, NULL },
  { "NC", "ambiguous", "north" , "carolina", NULL },
  { "ND", "ambiguous", "north" , "dakota", NULL },
  { "NE", "ambiguous", "nebraska" , NULL, NULL },
  { "Ne", "ambiguous", "nebraska" , NULL, NULL },
  { "Neb", "ambiguous", "nebraska" , NULL, NULL },
  { "NH", "ambiguous", "new" , "hampshire", NULL },
  { "NV", "", "nevada" , NULL, NULL },
  { "Nev", "", "nevada" , NULL, NULL },
  { "NY", "", "new" , "york", NULL },
  { "OH", "ambiguous", "ohio" , NULL, NULL },
  { "OK", "ambiguous", "oklahoma" , NULL, NULL },
  { "Okla", "", "oklahoma" , NULL, NULL },
  { "OR", "ambiguous", "oregon" , NULL, NULL },
  { "Or", "ambiguous", "oregon" , NULL, NULL },
  { "Ore", "ambiguous", "oregon" , NULL, NULL },
  { "PA", "ambiguous", "pennsylvania" , NULL, NULL },
  { "Pa", "ambiguous", "pennsylvania" , NULL, NULL },
  { "Penn", "ambiguous", "pennsylvania" , NULL, NULL },
  { "RI", "ambiguous", "rhode" , "island", NULL },
  { "SC", "ambiguous", "south" , "carlolina", NULL },
  { "SD", "ambiguous", "south" , "dakota", NULL },
  { "TN", "ambiguous", "tennesee" , NULL, NULL },
  { "Tn", "ambiguous", "tennesee" , NULL, NULL },
  { "Tenn", "ambiguous", "tennesee" , NULL, NULL },
  { "TX", "ambiguous", "texas" , NULL, NULL },
  { "Tx", "ambiguous", "texas" , NULL, NULL },
  { "Tex", "ambiguous", "texas" , NULL, NULL },
  { "UT", "ambiguous", "utah" , NULL, NULL },
  { "VA", "ambiguous", "virginia" , NULL, NULL },
  { "WA", "ambiguous", "washington" , NULL, NULL },
  { "Wa", "ambiguous", "washington" , NULL, NULL },
  { "Wash", "ambiguous", "washington" , NULL, NULL },
  { "WI", "ambiguous", "wisconsin" , NULL, NULL },
  { "Wi", "ambiguous", "wisconsin" , NULL, NULL },
  { "WV", "ambiguous", "west" , "virginia", NULL },
  { "WY", "ambiguous", "wyoming" , NULL, NULL },
  { "Wy", "ambiguous", "wyoming" , NULL, NULL },
  { "Wyo", "", "wyoming" , NULL, NULL },
  { "PR", "ambiguous", "puerto" , "rico", NULL },
  { NULL, NULL, "puerto" , "rico", NULL }
};

static cst_val *state_name(const char *name,cst_item *t)
{
    int s,j;
    int do_it = 0;
    cst_val *r = 0;

    for (s=0; states[s][0]; s++)
    {
	if (cst_streq(states[s][0],name))
	{
	    if (cst_streq(states[s][1],"ambiguous"))
	    {
		const char *pname = ffeature_string(t,"p.name");
		const char *nname = ffeature_string(t,"n.name");
		    /* previous name is capitalized */
		if (((strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ",pname[0])) &&
		     (cst_strlen(pname) > 2) &&
		     (cst_regex_match(cst_rx_alpha,pname))) &&
		    ((strchr("abcdefghijklmnopqrstuvwxyz",nname[0])) ||
		     (item_next(t) == 0) ||
		     (cst_streq(".",item_feat_string(t,"punc"))) ||
		     (((cst_strlen(nname) == 5 || (cst_strlen(nname) == 10)) &&
		       cst_regex_match(cst_rx_digits,nname)))))
		    do_it = 1;
		else
		    do_it = 0;
	    }
	    else
		do_it = 1;

	    if (do_it)
	    {
		for (j=2; states[s][j]; j++)
		    r = cons_val(string_val(states[s][j]),r);
		return val_reverse(r);
	    }
	}
    }
    return r;

}