import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

media/sphinxbase/src/libsphinxbase/fe/fe_interface.c

@@ -0,0 +1,776 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1996-2004 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
+#include <stdlib.h>
+#include <assert.h>
+#ifdef _WIN32_WCE
+#include <windows.h>
+#else
+#include <time.h>
+#endif
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "sphinxbase/prim_type.h"
+#include "sphinxbase/byteorder.h"
+#include "sphinxbase/fixpoint.h"
+#include "sphinxbase/genrand.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/cmd_ln.h"
+#include "sphinxbase/ckd_alloc.h"
+
+#include "fe_internal.h"
+#include "fe_warp.h"
+
+static const arg_t fe_args[] = {
+    waveform_to_cepstral_command_line_macro(),
+    { NULL, 0, NULL, NULL }
+};
+
+int
+fe_parse_general_params(cmd_ln_t *config, fe_t * fe)
+{
+    int j, frate;
+
+    fe->config = config;
+    fe->sampling_rate = cmd_ln_float32_r(config, "-samprate");
+    frate = cmd_ln_int32_r(config, "-frate");
+    if (frate > MAX_INT16 || frate > fe->sampling_rate || frate < 1) {
+        E_ERROR
+            ("Frame rate %d can not be bigger than sample rate %.02f\n",
+             frate, fe->sampling_rate);
+        return -1;
+    }
+
+    fe->frame_rate = (int16)frate;
+    if (cmd_ln_boolean_r(config, "-dither")) {
+        fe->dither = 1;
+        fe->seed = cmd_ln_int32_r(config, "-seed");
+    }
+#ifdef WORDS_BIGENDIAN
+    fe->swap = strcmp("big", cmd_ln_str_r(config, "-input_endian")) == 0 ? 0 : 1;
+#else        
+    fe->swap = strcmp("little", cmd_ln_str_r(config, "-input_endian")) == 0 ? 0 : 1;
+#endif
+    fe->window_length = cmd_ln_float32_r(config, "-wlen");
+    fe->pre_emphasis_alpha = cmd_ln_float32_r(config, "-alpha");
+
+    fe->num_cepstra = (uint8)cmd_ln_int32_r(config, "-ncep");
+    fe->fft_size = (int16)cmd_ln_int32_r(config, "-nfft");
+
+    /* Check FFT size, compute FFT order (log_2(n)) */
+    for (j = fe->fft_size, fe->fft_order = 0; j > 1; j >>= 1, fe->fft_order++) {
+        if (((j % 2) != 0) || (fe->fft_size <= 0)) {
+            E_ERROR("fft: number of points must be a power of 2 (is %d)\n",
+                    fe->fft_size);
+            return -1;
+        }
+    }
+    /* Verify that FFT size is greater or equal to window length. */
+    if (fe->fft_size < (int)(fe->window_length * fe->sampling_rate)) {
+        E_ERROR("FFT: Number of points must be greater or equal to frame size (%d samples)\n",
+                (int)(fe->window_length * fe->sampling_rate));
+        return -1;
+    }
+
+    fe->prespch_len = (int16)cmd_ln_int32_r(config, "-vad_prespeech");
+    fe->postspch_len = (int16)cmd_ln_int32_r(config, "-vad_postspeech");
+    fe->vad_threshold = cmd_ln_float32_r(config, "-vad_threshold");
+
+    fe->remove_dc = cmd_ln_boolean_r(config, "-remove_dc");
+    fe->remove_noise = cmd_ln_boolean_r(config, "-remove_noise");
+    fe->remove_silence = cmd_ln_boolean_r(config, "-remove_silence");
+
+    if (0 == strcmp(cmd_ln_str_r(config, "-transform"), "dct"))
+        fe->transform = DCT_II;
+    else if (0 == strcmp(cmd_ln_str_r(config, "-transform"), "legacy"))
+        fe->transform = LEGACY_DCT;
+    else if (0 == strcmp(cmd_ln_str_r(config, "-transform"), "htk"))
+        fe->transform = DCT_HTK;
+    else {
+        E_ERROR("Invalid transform type (values are 'dct', 'legacy', 'htk')\n");
+        return -1;
+    }
+
+    if (cmd_ln_boolean_r(config, "-logspec"))
+        fe->log_spec = RAW_LOG_SPEC;
+    if (cmd_ln_boolean_r(config, "-smoothspec"))
+        fe->log_spec = SMOOTH_LOG_SPEC;
+
+    return 0;
+}
+
+static int
+fe_parse_melfb_params(cmd_ln_t *config, fe_t *fe, melfb_t * mel)
+{
+    mel->sampling_rate = fe->sampling_rate;
+    mel->fft_size = fe->fft_size;
+    mel->num_cepstra = fe->num_cepstra;
+    mel->num_filters = cmd_ln_int32_r(config, "-nfilt");
+
+    if (fe->log_spec)
+        fe->feature_dimension = mel->num_filters;
+    else
+        fe->feature_dimension = fe->num_cepstra;
+
+    mel->upper_filt_freq = cmd_ln_float32_r(config, "-upperf");
+    mel->lower_filt_freq = cmd_ln_float32_r(config, "-lowerf");
+
+    mel->doublewide = cmd_ln_boolean_r(config, "-doublebw");
+
+    mel->warp_type = cmd_ln_str_r(config, "-warp_type");
+    mel->warp_params = cmd_ln_str_r(config, "-warp_params");
+    mel->lifter_val = cmd_ln_int32_r(config, "-lifter");
+
+    mel->unit_area = cmd_ln_boolean_r(config, "-unit_area");
+    mel->round_filters = cmd_ln_boolean_r(config, "-round_filters");
+
+    if (fe_warp_set(mel, mel->warp_type) != FE_SUCCESS) {
+        E_ERROR("Failed to initialize the warping function.\n");
+        return -1;
+    }
+    fe_warp_set_parameters(mel, mel->warp_params, mel->sampling_rate);
+    return 0;
+}
+
+void
+fe_print_current(fe_t const *fe)
+{
+    E_INFO("Current FE Parameters:\n");
+    E_INFO("\tSampling Rate:             %f\n", fe->sampling_rate);
+    E_INFO("\tFrame Size:                %d\n", fe->frame_size);
+    E_INFO("\tFrame Shift:               %d\n", fe->frame_shift);
+    E_INFO("\tFFT Size:                  %d\n", fe->fft_size);
+    E_INFO("\tLower Frequency:           %g\n",
+           fe->mel_fb->lower_filt_freq);
+    E_INFO("\tUpper Frequency:           %g\n",
+           fe->mel_fb->upper_filt_freq);
+    E_INFO("\tNumber of filters:         %d\n", fe->mel_fb->num_filters);
+    E_INFO("\tNumber of Overflow Samps:  %d\n", fe->num_overflow_samps);
+    E_INFO("\tStart Utt Status:          %d\n", fe->start_flag);
+    E_INFO("Will %sremove DC offset at frame level\n",
+           fe->remove_dc ? "" : "not ");
+    if (fe->dither) {
+        E_INFO("Will add dither to audio\n");
+        E_INFO("Dither seeded with %d\n", fe->seed);
+    }
+    else {
+        E_INFO("Will not add dither to audio\n");
+    }
+    if (fe->mel_fb->lifter_val) {
+        E_INFO("Will apply sine-curve liftering, period %d\n",
+               fe->mel_fb->lifter_val);
+    }
+    E_INFO("Will %snormalize filters to unit area\n",
+           fe->mel_fb->unit_area ? "" : "not ");
+    E_INFO("Will %sround filter frequencies to DFT points\n",
+           fe->mel_fb->round_filters ? "" : "not ");
+    E_INFO("Will %suse double bandwidth in mel filter\n",
+           fe->mel_fb->doublewide ? "" : "not ");
+}
+
+fe_t *
+fe_init_auto()
+{
+    return fe_init_auto_r(cmd_ln_get());
+}
+
+fe_t *
+fe_init_auto_r(cmd_ln_t *config)
+{
+    fe_t *fe;
+    int prespch_frame_len;
+
+    fe = (fe_t*)ckd_calloc(1, sizeof(*fe));
+    fe->refcount = 1;
+
+    /* transfer params to front end */
+    if (fe_parse_general_params(cmd_ln_retain(config), fe) < 0) {
+        fe_free(fe);
+        return NULL;
+    }
+
+    /* compute remaining fe parameters */
+    /* We add 0.5 so approximate the float with the closest
+     * integer. E.g., 2.3 is truncate to 2, whereas 3.7 becomes 4
+     */
+    fe->frame_shift = (int32) (fe->sampling_rate / fe->frame_rate + 0.5);
+    fe->frame_size = (int32) (fe->window_length * fe->sampling_rate + 0.5);
+    fe->prior = 0;
+    
+    fe_start_stream(fe);
+
+    assert (fe->frame_shift > 1);
+
+    if (fe->frame_size > (fe->fft_size)) {
+        E_ERROR
+            ("Number of FFT points has to be a power of 2 higher than %d, it is %d\n",
+             fe->frame_size, fe->fft_size);
+        fe_free(fe);
+        return NULL;
+    }
+
+    if (fe->dither)
+        fe_init_dither(fe->seed);
+
+    /* establish buffers for overflow samps and hamming window */
+    fe->overflow_samps = ckd_calloc(fe->frame_size, sizeof(int16));
+    fe->hamming_window = ckd_calloc(fe->frame_size/2, sizeof(window_t));
+
+    /* create hamming window */
+    fe_create_hamming(fe->hamming_window, fe->frame_size);
+
+    /* init and fill appropriate filter structure */
+    fe->mel_fb = ckd_calloc(1, sizeof(*fe->mel_fb));
+
+    /* transfer params to mel fb */
+    fe_parse_melfb_params(config, fe, fe->mel_fb);
+    
+    if (fe->mel_fb->upper_filt_freq > fe->sampling_rate / 2 + 1.0) {
+	E_ERROR("Upper frequency %.1f is higher than samprate/2 (%.1f)\n", 
+		fe->mel_fb->upper_filt_freq, fe->sampling_rate / 2);
+	fe_free(fe);
+	return NULL;
+    }
+    
+    fe_build_melfilters(fe->mel_fb);
+
+    fe_compute_melcosine(fe->mel_fb);
+    if (fe->remove_noise || fe->remove_silence)
+        fe->noise_stats = fe_init_noisestats(fe->mel_fb->num_filters);
+
+    fe->vad_data = (vad_data_t*)ckd_calloc(1, sizeof(*fe->vad_data));
+    prespch_frame_len = fe->log_spec != RAW_LOG_SPEC ? fe->num_cepstra : fe->mel_fb->num_filters;
+    fe->vad_data->prespch_buf = fe_prespch_init(fe->prespch_len + 1, prespch_frame_len, fe->frame_shift);
+
+    /* Create temporary FFT, spectrum and mel-spectrum buffers. */
+    /* FIXME: Gosh there are a lot of these. */
+    fe->spch = ckd_calloc(fe->frame_size, sizeof(*fe->spch));
+    fe->frame = ckd_calloc(fe->fft_size, sizeof(*fe->frame));
+    fe->spec = ckd_calloc(fe->fft_size, sizeof(*fe->spec));
+    fe->mfspec = ckd_calloc(fe->mel_fb->num_filters, sizeof(*fe->mfspec));
+
+    /* create twiddle factors */
+    fe->ccc = ckd_calloc(fe->fft_size / 4, sizeof(*fe->ccc));
+    fe->sss = ckd_calloc(fe->fft_size / 4, sizeof(*fe->sss));
+    fe_create_twiddle(fe);
+
+    if (cmd_ln_boolean_r(config, "-verbose")) {
+        fe_print_current(fe);
+    }
+
+    /*** Initialize the overflow buffers ***/
+    fe_start_utt(fe);
+    return fe;
+}
+
+arg_t const *
+fe_get_args(void)
+{
+    return fe_args;
+}
+
+const cmd_ln_t *
+fe_get_config(fe_t *fe)
+{
+    return fe->config;
+}
+
+void
+fe_init_dither(int32 seed)
+{
+    if (seed < 0) {
+        E_INFO("You are using the internal mechanism to generate the seed.\n");
+#ifdef _WIN32_WCE
+        s3_rand_seed(GetTickCount());
+#else
+        s3_rand_seed((long) time(0));
+#endif
+    } else {
+        E_INFO("You are using %d as the seed.\n", seed);
+        s3_rand_seed(seed);
+    }
+}
+
+static void
+fe_reset_vad_data(vad_data_t * vad_data)
+{
+    vad_data->global_state = 0;
+    vad_data->state_changed = 0;
+    vad_data->prespch_num = 0;
+    vad_data->postspch_num = 0;
+    fe_prespch_reset_cep(vad_data->prespch_buf);
+}
+
+int32
+fe_start_utt(fe_t * fe)
+{
+    fe->num_overflow_samps = 0;
+    memset(fe->overflow_samps, 0, fe->frame_size * sizeof(int16));
+    fe->start_flag = 1;
+    fe->prior = 0;
+    fe_reset_vad_data(fe->vad_data);
+    return 0;
+}
+
+void 
+fe_start_stream(fe_t *fe)
+{
+    fe->sample_counter = 0;
+    fe_reset_noisestats(fe->noise_stats);
+}
+
+int
+fe_get_output_size(fe_t *fe)
+{
+    return (int)fe->feature_dimension;
+}
+
+void
+fe_get_input_size(fe_t *fe, int *out_frame_shift,
+                  int *out_frame_size)
+{
+    if (out_frame_shift)
+        *out_frame_shift = fe->frame_shift;
+    if (out_frame_size)
+        *out_frame_size = fe->frame_size;
+}
+
+uint8
+fe_get_vad_state(fe_t *fe)
+{
+    return fe->vad_data->global_state;
+}
+
+int
+fe_process_frames(fe_t *fe,
+                  int16 const **inout_spch,
+                  size_t *inout_nsamps,
+                  mfcc_t **buf_cep,
+                  int32 *inout_nframes,
+                  int32 *out_frameidx)
+{
+    int outidx, n_overflow, orig_n_overflow;
+    int16 const *orig_spch;
+    size_t orig_nsamps;
+
+    /* In the special case where there is no output buffer, return the
+     * maximum number of frames which would be generated. */
+    if (buf_cep == NULL) {
+        if (*inout_nsamps + fe->num_overflow_samps < (size_t)fe->frame_size)
+            *inout_nframes = 0;
+        else 
+            *inout_nframes = 1
+                + ((*inout_nsamps + fe->num_overflow_samps - fe->frame_size)
+                   / fe->frame_shift);
+        if (fe->vad_data->global_state)
+            *inout_nframes += fe_prespch_ncep(fe->vad_data->prespch_buf);
+        return *inout_nframes;
+    }
+
+    if (out_frameidx)
+        *out_frameidx = 0;
+
+    /* Are there not enough samples to make at least 1 frame? */
+    if (*inout_nsamps + fe->num_overflow_samps < (size_t)fe->frame_size) {
+        if (*inout_nsamps > 0) {
+            /* Append them to the overflow buffer. */
+            memcpy(fe->overflow_samps + fe->num_overflow_samps,
+                   *inout_spch, *inout_nsamps * (sizeof(int16)));
+            fe->num_overflow_samps += *inout_nsamps;
+            /* Update input-output pointers and counters. */
+            *inout_spch += *inout_nsamps;
+            *inout_nsamps = 0;
+        }
+        /* We produced no frames of output, sorry! */
+        *inout_nframes = 0;
+        return 0;
+    }
+
+    /* Can't write a frame?  Then do nothing! */
+    if (*inout_nframes < 1) {
+        *inout_nframes = 0;
+        return 0;
+    }
+
+    /* Index of output frame. */
+    outidx = 0;
+
+    /* Try to read from prespeech buffer */
+    if (fe->vad_data->global_state) {
+        while ((*inout_nframes) > 0 && fe_prespch_read_cep(fe->vad_data->prespch_buf, buf_cep[outidx]) > 0) {
+            outidx++;
+            (*inout_nframes)--;
+        }
+        if ((*inout_nframes) < 1) {
+            /* mfcc buffer is filled from prespeech buffer */
+            *inout_nframes = outidx;
+            return 0;
+        }
+
+        /* Sets the start frame for the returned data so that caller can update timings */
+        if (out_frameidx && fe->vad_data->state_changed) {
+            *out_frameidx = fe->sample_counter / fe->frame_shift - fe->prespch_len;
+        }
+    }
+
+    /* Keep track of the original start of the buffer. */
+    orig_spch = *inout_spch;
+    orig_nsamps = *inout_nsamps;
+    orig_n_overflow = fe->num_overflow_samps;
+
+    /* Start processing, taking care of any incoming overflow. */
+    if (fe->num_overflow_samps) {
+        int offset = fe->frame_size - fe->num_overflow_samps;
+
+        /* Append start of spch to overflow samples to make a full frame. */
+        memcpy(fe->overflow_samps + fe->num_overflow_samps,
+               *inout_spch, offset * sizeof(**inout_spch));
+        fe_read_frame(fe, fe->overflow_samps, fe->frame_size);
+        /* Update input-output pointers and counters. */
+        *inout_spch += offset;
+        *inout_nsamps -= offset;
+        fe->num_overflow_samps -= fe->frame_shift;
+    } else {
+        fe_read_frame(fe, *inout_spch, fe->frame_size);
+        /* Update input-output pointers and counters. */
+        *inout_spch += fe->frame_size;
+        *inout_nsamps -= fe->frame_size;
+    }
+
+    fe_write_frame(fe, buf_cep[outidx]);
+
+    if (!fe->vad_data->state_changed && fe->vad_data->global_state) {
+        outidx++;
+        (*inout_nframes)--;
+    }
+    if (fe->vad_data->state_changed && fe->vad_data->global_state) {
+        /* previous frame triggered vad into speech state
+         * dumping prespeech buffer */
+        while ((*inout_nframes) > 0 && fe_prespch_read_cep(fe->vad_data->prespch_buf, buf_cep[outidx]) > 0) {
+            outidx++;
+            (*inout_nframes)--;
+        }
+
+        /* Sets the start frame for the returned data so that caller can update timings */
+        if (out_frameidx) {
+            *out_frameidx = (fe->sample_counter + orig_nsamps - *inout_nsamps) / fe->frame_shift - fe->prespch_len;
+        }
+    }
+
+    /* Process all remaining frames. */
+    while (*inout_nframes > 0 && *inout_nsamps >= (size_t)fe->frame_shift) {
+        fe_shift_frame(fe, *inout_spch, fe->frame_shift);
+        fe_write_frame(fe, buf_cep[outidx]);
+        if (!fe->vad_data->state_changed && fe->vad_data->global_state) {
+            (*inout_nframes)--;
+            outidx++;
+        }
+        /* Update input-output pointers and counters. */
+        *inout_spch += fe->frame_shift;
+        *inout_nsamps -= fe->frame_shift;
+        /* Amount of data behind the original input which is still needed. */
+        if (fe->num_overflow_samps > 0)
+            fe->num_overflow_samps -= fe->frame_shift;
+
+        if (fe->vad_data->state_changed && fe->vad_data->global_state) {
+            /* previous frame triggered vad into speech state */
+            while (*inout_nframes > 0 && fe_prespch_read_cep(fe->vad_data->prespch_buf, buf_cep[outidx]) != 0) {
+                (*inout_nframes)--;
+                outidx++;
+            }
+        }
+    }
+
+    /* How many relevant overflow samples are there left? */
+    if (fe->num_overflow_samps <= 0) {
+        /* Maximum number of overflow samples past *inout_spch to save. */
+        n_overflow = *inout_nsamps;
+        if (n_overflow > fe->frame_shift)
+            n_overflow = fe->frame_shift;
+        fe->num_overflow_samps = fe->frame_size - fe->frame_shift;
+        /* Make sure this isn't an illegal read! */
+        if (fe->num_overflow_samps > *inout_spch - orig_spch)
+            fe->num_overflow_samps = *inout_spch - orig_spch;
+        fe->num_overflow_samps += n_overflow;
+        if (fe->num_overflow_samps > 0) {
+            memcpy(fe->overflow_samps,
+                   *inout_spch - (fe->frame_size - fe->frame_shift),
+                   fe->num_overflow_samps * sizeof(**inout_spch));
+            /* Update the input pointer to cover this stuff. */
+            *inout_spch += n_overflow;
+            *inout_nsamps -= n_overflow;
+        }
+    } else {
+        /* There is still some relevant data left in the overflow buffer. */
+        /* Shift existing data to the beginning. */
+        memmove(fe->overflow_samps,
+                fe->overflow_samps + orig_n_overflow - fe->num_overflow_samps,
+                fe->num_overflow_samps * sizeof(*fe->overflow_samps));
+        /* Copy in whatever we had in the original speech buffer. */
+        n_overflow = *inout_spch - orig_spch + *inout_nsamps;
+        if (n_overflow > fe->frame_size - fe->num_overflow_samps)
+            n_overflow = fe->frame_size - fe->num_overflow_samps;
+        memcpy(fe->overflow_samps + fe->num_overflow_samps,
+               orig_spch, n_overflow * sizeof(*orig_spch));
+        fe->num_overflow_samps += n_overflow;
+        /* Advance the input pointers. */
+        if (n_overflow > *inout_spch - orig_spch) {
+            n_overflow -= (*inout_spch - orig_spch);
+            *inout_spch += n_overflow;
+            *inout_nsamps -= n_overflow;
+        }
+    }
+
+    /* Finally update the frame counter with the number of frames
+     * and global sample counter with number of samples we procesed*/
+    *inout_nframes = outidx; /* FIXME: Not sure why I wrote it this way... */
+    fe->sample_counter += orig_nsamps - *inout_nsamps;
+    return 0;
+}
+
+int 
+fe_process_frames_ext(fe_t *fe,
+                  int16 const **inout_spch,
+                  size_t *inout_nsamps,
+                  mfcc_t **buf_cep,
+                  int32 *inout_nframes,
+                  int16 **voiced_spch,
+                  int32 *voiced_spch_nsamps,
+                  int32 *out_frameidx)
+{
+    int proc_result;
+
+    fe_prespch_extend_pcm(fe->vad_data->prespch_buf, *inout_nframes);
+
+    fe->vad_data->store_pcm = TRUE;
+    proc_result = fe_process_frames(fe, inout_spch, inout_nsamps, buf_cep, inout_nframes, out_frameidx);
+    fe->vad_data->store_pcm = FALSE;
+
+    if (fe->vad_data->global_state)
+        fe_prespch_read_pcm(fe->vad_data->prespch_buf, voiced_spch, voiced_spch_nsamps);
+    else
+	*voiced_spch_nsamps = 0;
+
+    return proc_result;
+}
+
+int
+fe_process_utt(fe_t * fe, int16 const * spch, size_t nsamps,
+               mfcc_t *** cep_block, int32 * nframes)
+{
+    mfcc_t **cep;
+    int rv;
+
+    /* Figure out how many frames we will need. */
+    fe_process_frames(fe, NULL, &nsamps, NULL, nframes, NULL);
+    /* Create the output buffer (it has to exist, even if there are no output frames). */
+    if (*nframes)
+        cep = (mfcc_t **)ckd_calloc_2d(*nframes, fe->feature_dimension, sizeof(**cep));
+    else
+        cep = (mfcc_t **)ckd_calloc_2d(1, fe->feature_dimension, sizeof(**cep));
+    /* Now just call fe_process_frames() with the allocated buffer. */
+    rv = fe_process_frames(fe, &spch, &nsamps, cep, nframes, NULL);
+    *cep_block = cep;
+
+    return rv;
+}
+
+
+int32
+fe_end_utt(fe_t * fe, mfcc_t * cepvector, int32 * nframes)
+{
+    /* Process any remaining data. */
+    *nframes = 0;
+    if (fe->num_overflow_samps > 0) {
+        fe_read_frame(fe, fe->overflow_samps, fe->num_overflow_samps);
+        fe_write_frame(fe, cepvector);
+        if (!fe->vad_data->state_changed && fe->vad_data->global_state)
+            (*nframes)++;
+    }
+
+    /* reset overflow buffers... */
+    fe->num_overflow_samps = 0;
+    fe->start_flag = 0;
+
+    return 0;
+}
+
+fe_t *
+fe_retain(fe_t *fe)
+{
+    ++fe->refcount;
+    return fe;
+}
+
+int
+fe_free(fe_t * fe)
+{
+    if (fe == NULL)
+        return 0;
+    if (--fe->refcount > 0)
+        return fe->refcount;
+
+    /* kill FE instance - free everything... */
+    if (fe->mel_fb) {
+        if (fe->mel_fb->mel_cosine)
+            fe_free_2d((void *) fe->mel_fb->mel_cosine);
+        ckd_free(fe->mel_fb->lifter);
+        ckd_free(fe->mel_fb->spec_start);
+        ckd_free(fe->mel_fb->filt_start);
+        ckd_free(fe->mel_fb->filt_width);
+        ckd_free(fe->mel_fb->filt_coeffs);
+        ckd_free(fe->mel_fb);
+    }
+    ckd_free(fe->spch);
+    ckd_free(fe->frame);
+    ckd_free(fe->ccc);
+    ckd_free(fe->sss);
+    ckd_free(fe->spec);
+    ckd_free(fe->mfspec);
+    ckd_free(fe->overflow_samps);
+    ckd_free(fe->hamming_window);
+
+    if (fe->noise_stats)
+        fe_free_noisestats(fe->noise_stats);
+
+    if (fe->vad_data) {
+        fe_prespch_free(fe->vad_data->prespch_buf);
+	ckd_free(fe->vad_data);
+    }
+
+    cmd_ln_free_r(fe->config);
+    ckd_free(fe);
+
+    return 0;
+}
+
+/**
+ * Convert a block of mfcc_t to float32 (can be done in-place)
+ **/
+int32
+fe_mfcc_to_float(fe_t * fe,
+                 mfcc_t ** input, float32 ** output, int32 nframes)
+{
+    int32 i;
+
+#ifndef FIXED_POINT
+    if ((void *) input == (void *) output)
+        return nframes * fe->feature_dimension;
+#endif
+    for (i = 0; i < nframes * fe->feature_dimension; ++i)
+        output[0][i] = MFCC2FLOAT(input[0][i]);
+
+    return i;
+}
+
+/**
+ * Convert a block of float32 to mfcc_t (can be done in-place)
+ **/
+int32
+fe_float_to_mfcc(fe_t * fe,
+                 float32 ** input, mfcc_t ** output, int32 nframes)
+{
+    int32 i;
+
+#ifndef FIXED_POINT
+    if ((void *) input == (void *) output)
+        return nframes * fe->feature_dimension;
+#endif
+    for (i = 0; i < nframes * fe->feature_dimension; ++i)
+        output[0][i] = FLOAT2MFCC(input[0][i]);
+
+    return i;
+}
+
+int32
+fe_logspec_to_mfcc(fe_t * fe, const mfcc_t * fr_spec, mfcc_t * fr_cep)
+{
+#ifdef FIXED_POINT
+    fe_spec2cep(fe, fr_spec, fr_cep);
+#else                           /* ! FIXED_POINT */
+    powspec_t *powspec;
+    int32 i;
+
+    powspec = ckd_malloc(fe->mel_fb->num_filters * sizeof(powspec_t));
+    for (i = 0; i < fe->mel_fb->num_filters; ++i)
+        powspec[i] = (powspec_t) fr_spec[i];
+    fe_spec2cep(fe, powspec, fr_cep);
+    ckd_free(powspec);
+#endif                          /* ! FIXED_POINT */
+    return 0;
+}
+
+int32
+fe_logspec_dct2(fe_t * fe, const mfcc_t * fr_spec, mfcc_t * fr_cep)
+{
+#ifdef FIXED_POINT
+    fe_dct2(fe, fr_spec, fr_cep, 0);
+#else                           /* ! FIXED_POINT */
+    powspec_t *powspec;
+    int32 i;
+
+    powspec = ckd_malloc(fe->mel_fb->num_filters * sizeof(powspec_t));
+    for (i = 0; i < fe->mel_fb->num_filters; ++i)
+        powspec[i] = (powspec_t) fr_spec[i];
+    fe_dct2(fe, powspec, fr_cep, 0);
+    ckd_free(powspec);
+#endif                          /* ! FIXED_POINT */
+    return 0;
+}
+
+int32
+fe_mfcc_dct3(fe_t * fe, const mfcc_t * fr_cep, mfcc_t * fr_spec)
+{
+#ifdef FIXED_POINT
+    fe_dct3(fe, fr_cep, fr_spec);
+#else                           /* ! FIXED_POINT */
+    powspec_t *powspec;
+    int32 i;
+
+    powspec = ckd_malloc(fe->mel_fb->num_filters * sizeof(powspec_t));
+    fe_dct3(fe, fr_cep, powspec);
+    for (i = 0; i < fe->mel_fb->num_filters; ++i)
+        fr_spec[i] = (mfcc_t) powspec[i];
+    ckd_free(powspec);
+#endif                          /* ! FIXED_POINT */
+    return 0;
+}

media/sphinxbase/src/libsphinxbase/fe/fe_internal.h

@@ -0,0 +1,216 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1996-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+#ifndef __FE_INTERNAL_H__
+#define __FE_INTERNAL_H__
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "sphinxbase/fe.h"
+#include "sphinxbase/fixpoint.h"
+
+#include "fe_noise.h"
+#include "fe_prespch_buf.h"
+#include "fe_type.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#if 0
+/* Fool Emacs. */
+}
+#endif
+
+/* Values for the 'logspec' field. */
+enum {
+	RAW_LOG_SPEC = 1,
+	SMOOTH_LOG_SPEC = 2
+};
+
+/* Values for the 'transform' field. */
+enum {
+	LEGACY_DCT = 0,
+	DCT_II = 1,
+        DCT_HTK = 2
+};
+
+typedef struct melfb_s melfb_t;
+/** Base Struct to hold all structure for MFCC computation. */
+struct melfb_s {
+    float32 sampling_rate;
+    int32 num_cepstra;
+    int32 num_filters;
+    int32 fft_size;
+    float32 lower_filt_freq;
+    float32 upper_filt_freq;
+    /* DCT coefficients. */
+    mfcc_t **mel_cosine;
+    /* Filter coefficients. */
+    mfcc_t *filt_coeffs;
+    int16 *spec_start;
+    int16 *filt_start;
+    int16 *filt_width;
+    /* Luxury mobile home. */
+    int32 doublewide;
+    char const *warp_type;
+    char const *warp_params;
+    uint32 warp_id;
+    /* Precomputed normalization constants for unitary DCT-II/DCT-III */
+    mfcc_t sqrt_inv_n, sqrt_inv_2n;
+    /* Value and coefficients for HTK-style liftering */
+    int32 lifter_val;
+    mfcc_t *lifter;
+    /* Normalize filters to unit area */
+    int32 unit_area;
+    /* Round filter frequencies to DFT points (hurts accuracy, but is
+       useful for legacy purposes) */
+    int32 round_filters;
+};
+
+typedef struct ringbuf_s {
+	powspec_t** bufs;
+	int16 buf_num;
+	int32 buf_len;
+	int16 start;
+	int16 end;
+	int32 recs;
+} ringbuf_t;
+
+/* sqrt(1/2), also used for unitary DCT-II/DCT-III */
+#define SQRT_HALF FLOAT2MFCC(0.707106781186548)
+
+typedef struct vad_data_s {
+    uint8 global_state;
+    uint8 state_changed;
+    uint8 store_pcm;
+    int16 prespch_num;
+    int16 postspch_num;
+    prespch_buf_t* prespch_buf;	
+} vad_data_t;
+
+/** Structure for the front-end computation. */
+struct fe_s {
+    cmd_ln_t *config;
+    int refcount;
+
+    int16 prespch_len;
+    int16 postspch_len;
+    float32 vad_threshold;
+
+    float32 sampling_rate;
+    int16 frame_rate;
+    int16 frame_shift;
+
+    float32 window_length;
+    int16 frame_size;
+    int16 fft_size;
+
+    uint8 fft_order;
+    uint8 feature_dimension;
+    uint8 num_cepstra;
+    uint8 remove_dc;
+    uint8 log_spec;
+    uint8 swap;
+    uint8 dither;
+    uint8 transform;
+    uint8 remove_noise;
+    uint8 remove_silence;
+
+    float32 pre_emphasis_alpha;
+    int32 seed;
+
+    size_t sample_counter;
+    uint8 start_flag;
+    uint8 reserved;
+
+    /* Twiddle factors for FFT. */
+    frame_t *ccc, *sss;
+    /* Mel filter parameters. */
+    melfb_t *mel_fb;
+    /* Half of a Hamming Window. */
+    window_t *hamming_window;
+    /* Storage for noise removal  */
+    noise_stats_t *noise_stats;
+
+    /* Storage for VAD variables */
+    vad_data_t *vad_data;
+
+    /* Temporary buffers for processing. */
+    /* FIXME: too many of these. */
+    int16 *spch;
+    frame_t *frame;
+    powspec_t *spec, *mfspec;
+    int16 *overflow_samps;
+    int16 num_overflow_samps;    
+    int16 prior;
+};
+
+void fe_init_dither(int32 seed);
+
+/* Apply 1/2 bit noise to a buffer of audio. */
+int32 fe_dither(int16 *buffer, int32 nsamps);
+
+/* Load a frame of data into the fe. */
+int fe_read_frame(fe_t *fe, int16 const *in, int32 len);
+
+/* Shift the input buffer back and read more data. */
+int fe_shift_frame(fe_t *fe, int16 const *in, int32 len);
+
+/* Process a frame of data into features. */
+void fe_write_frame(fe_t *fe, mfcc_t *fea);
+
+/* Initialization functions. */
+int32 fe_build_melfilters(melfb_t *MEL_FB);
+int32 fe_compute_melcosine(melfb_t *MEL_FB);
+void fe_create_hamming(window_t *in, int32 in_len);
+void fe_create_twiddle(fe_t *fe);
+
+fixed32 fe_log_add(fixed32 x, fixed32 y);
+fixed32 fe_log_sub(fixed32 x, fixed32 y);
+
+/* Miscellaneous processing functions. */
+void fe_spec2cep(fe_t * fe, const powspec_t * mflogspec, mfcc_t * mfcep);
+void fe_dct2(fe_t *fe, const powspec_t *mflogspec, mfcc_t *mfcep, int htk);
+void fe_dct3(fe_t *fe, const mfcc_t *mfcep, powspec_t *mflogspec);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __FE_INTERNAL_H__ */

media/sphinxbase/src/libsphinxbase/fe/fe_noise.c

@@ -0,0 +1,425 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2013 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+/* This noise removal algorithm is inspired by the following papers
+ * Computationally Efficient Speech Enchancement by Spectral Minina Tracking
+ * by G. Doblinger
+ *
+ * Power-Normalized Cepstral Coefficients (PNCC) for Robust Speech Recognition
+ * by C. Kim.
+ *
+ * For the recent research and state of art see papers about IMRCA and
+ * A Minimum-Mean-Square-Error Noise Reduction Algorithm On Mel-Frequency
+ * Cepstra For Robust Speech Recognition by Dong Yu and others
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+
+#include "sphinxbase/prim_type.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/strfuncs.h"
+#include "sphinxbase/err.h"
+
+#include "fe_noise.h"
+#include "fe_internal.h"
+
+/* Noise supression constants */
+#define SMOOTH_WINDOW 4
+#define LAMBDA_POWER 0.7
+#define LAMBDA_A 0.995
+#define LAMBDA_B 0.5
+#define LAMBDA_T 0.85
+#define MU_T 0.2
+#define MAX_GAIN 20
+
+struct noise_stats_s {
+    /* Smoothed power */
+    powspec_t *power;
+    /* Noise estimate */
+    powspec_t *noise;
+    /* Signal floor estimate */
+    powspec_t *floor;
+    /* Peak for temporal masking */
+    powspec_t *peak;
+
+    /* Initialize it next time */
+    uint8 undefined;
+    /* Number of items to process */
+    uint32 num_filters;
+
+    /* Precomputed constants */
+    powspec_t lambda_power;
+    powspec_t comp_lambda_power;
+    powspec_t lambda_a;
+    powspec_t comp_lambda_a;
+    powspec_t lambda_b;
+    powspec_t comp_lambda_b;
+    powspec_t lambda_t;
+    powspec_t mu_t;
+    powspec_t max_gain;
+    powspec_t inv_max_gain;
+
+    powspec_t smooth_scaling[2 * SMOOTH_WINDOW + 3];
+};
+
+static void
+fe_lower_envelope(noise_stats_t *noise_stats, powspec_t * buf, powspec_t * floor_buf, int32 num_filt)
+{
+    int i;
+
+    for (i = 0; i < num_filt; i++) {
+#ifndef FIXED_POINT
+        if (buf[i] >= floor_buf[i]) {
+            floor_buf[i] =
+                noise_stats->lambda_a * floor_buf[i] + noise_stats->comp_lambda_a * buf[i];
+        }
+        else {
+            floor_buf[i] =
+                noise_stats->lambda_b * floor_buf[i] + noise_stats->comp_lambda_b * buf[i];
+        }
+#else
+        if (buf[i] >= floor_buf[i]) {
+            floor_buf[i] = fe_log_add(noise_stats->lambda_a + floor_buf[i],
+                                  noise_stats->comp_lambda_a + buf[i]);
+        }
+        else {
+            floor_buf[i] = fe_log_add(noise_stats->lambda_b + floor_buf[i],
+                                  noise_stats->comp_lambda_b + buf[i]);
+        }
+#endif
+    }
+}
+
+/* temporal masking */
+static void
+fe_temp_masking(noise_stats_t *noise_stats, powspec_t * buf, powspec_t * peak, int32 num_filt)
+{
+    powspec_t cur_in;
+    int i;
+
+    for (i = 0; i < num_filt; i++) {
+        cur_in = buf[i];
+
+#ifndef FIXED_POINT
+        peak[i] *= noise_stats->lambda_t;
+        if (buf[i] < noise_stats->lambda_t * peak[i])
+            buf[i] = peak[i] * noise_stats->mu_t;
+#else
+        peak[i] += noise_stats->lambda_t;
+        if (buf[i] < noise_stats->lambda_t + peak[i])
+            buf[i] = peak[i] + noise_stats->mu_t;
+#endif
+
+        if (cur_in > peak[i])
+            peak[i] = cur_in;
+    }
+}
+
+/* spectral weight smoothing */
+static void
+fe_weight_smooth(noise_stats_t *noise_stats, powspec_t * buf, powspec_t * coefs, int32 num_filt)
+{
+    int i, j;
+    int l1, l2;
+    powspec_t coef;
+
+    for (i = 0; i < num_filt; i++) {
+        l1 = ((i - SMOOTH_WINDOW) > 0) ? (i - SMOOTH_WINDOW) : 0;
+        l2 = ((i + SMOOTH_WINDOW) <
+              (num_filt - 1)) ? (i + SMOOTH_WINDOW) : (num_filt - 1);
+
+#ifndef FIXED_POINT
+        coef = 0;
+        for (j = l1; j <= l2; j++) {
+            coef += coefs[j];
+        }
+        buf[i] = buf[i] * (coef / (l2 - l1 + 1));
+#else
+        coef = MIN_FIXLOG;
+        for (j = l1; j <= l2; j++) {
+            coef = fe_log_add(coef, coefs[j]);
+        }        
+        buf[i] = buf[i] + coef - noise_stats->smooth_scaling[l2 - l1 + 1];
+#endif
+
+    }
+}
+
+noise_stats_t *
+fe_init_noisestats(int num_filters)
+{
+    int i;
+    noise_stats_t *noise_stats;
+
+    noise_stats = (noise_stats_t *) ckd_calloc(1, sizeof(noise_stats_t));
+
+    noise_stats->power =
+        (powspec_t *) ckd_calloc(num_filters, sizeof(powspec_t));
+    noise_stats->noise =
+        (powspec_t *) ckd_calloc(num_filters, sizeof(powspec_t));
+    noise_stats->floor =
+        (powspec_t *) ckd_calloc(num_filters, sizeof(powspec_t));
+    noise_stats->peak =
+        (powspec_t *) ckd_calloc(num_filters, sizeof(powspec_t));
+
+    noise_stats->undefined = TRUE;
+    noise_stats->num_filters = num_filters;
+
+#ifndef FIXED_POINT
+    noise_stats->lambda_power = LAMBDA_POWER;
+    noise_stats->comp_lambda_power = 1 - LAMBDA_POWER;
+    noise_stats->lambda_a = LAMBDA_A;
+    noise_stats->comp_lambda_a = 1 - LAMBDA_A;
+    noise_stats->lambda_b = LAMBDA_B;
+    noise_stats->comp_lambda_b = 1 - LAMBDA_B;
+    noise_stats->lambda_t = LAMBDA_T;
+    noise_stats->mu_t = MU_T;
+    noise_stats->max_gain = MAX_GAIN;
+    noise_stats->inv_max_gain = 1.0 / MAX_GAIN;
+    
+    for (i = 1; i < 2 * SMOOTH_WINDOW + 1; i++) {
+        noise_stats->smooth_scaling[i] = 1.0 / i;
+    }
+#else
+    noise_stats->lambda_power = FLOAT2FIX(log(LAMBDA_POWER));
+    noise_stats->comp_lambda_power = FLOAT2FIX(log(1 - LAMBDA_POWER));
+    noise_stats->lambda_a = FLOAT2FIX(log(LAMBDA_A));
+    noise_stats->comp_lambda_a = FLOAT2FIX(log(1 - LAMBDA_A));
+    noise_stats->lambda_b = FLOAT2FIX(log(LAMBDA_B));
+    noise_stats->comp_lambda_b = FLOAT2FIX(log(1 - LAMBDA_B));
+    noise_stats->lambda_t = FLOAT2FIX(log(LAMBDA_T));
+    noise_stats->mu_t = FLOAT2FIX(log(MU_T));
+    noise_stats->max_gain = FLOAT2FIX(log(MAX_GAIN));
+    noise_stats->inv_max_gain = FLOAT2FIX(log(1.0 / MAX_GAIN));
+
+    for (i = 1; i < 2 * SMOOTH_WINDOW + 3; i++) {
+        noise_stats->smooth_scaling[i] = FLOAT2FIX(log(i));
+    }
+#endif
+
+    return noise_stats;
+}
+
+void
+fe_reset_noisestats(noise_stats_t * noise_stats)
+{
+    if (noise_stats)
+        noise_stats->undefined = TRUE;
+}
+
+void
+fe_free_noisestats(noise_stats_t * noise_stats)
+{
+    ckd_free(noise_stats->power);
+    ckd_free(noise_stats->noise);
+    ckd_free(noise_stats->floor);
+    ckd_free(noise_stats->peak);
+    ckd_free(noise_stats);
+}
+
+/**
+ * For fixed point we are doing the computation in a fixlog domain,
+ * so we have to add many processing cases.
+ */
+void
+fe_track_snr(fe_t * fe, int32 *in_speech)
+{
+    powspec_t *signal;
+    powspec_t *gain;
+    noise_stats_t *noise_stats;
+    powspec_t *mfspec;
+    int32 i, num_filts;
+    powspec_t lrt, snr, max_signal, log_signal;
+
+    if (!(fe->remove_noise || fe->remove_silence)) {
+        *in_speech = TRUE;
+        return;
+    }
+
+    noise_stats = fe->noise_stats;
+    mfspec = fe->mfspec;
+    num_filts = noise_stats->num_filters;
+
+    signal = (powspec_t *) ckd_calloc(num_filts, sizeof(powspec_t));
+
+    if (noise_stats->undefined) {
+        for (i = 0; i < num_filts; i++) {
+            noise_stats->power[i] = mfspec[i];
+            noise_stats->noise[i] = mfspec[i];
+#ifndef FIXED_POINT
+            noise_stats->floor[i] = mfspec[i] / noise_stats->max_gain;
+            noise_stats->peak[i] = 0.0;       
+#else
+            noise_stats->floor[i] = mfspec[i] - noise_stats->max_gain;
+            noise_stats->peak[i] = MIN_FIXLOG;
+#endif
+        }
+        noise_stats->undefined = FALSE;
+    }
+
+    /* Calculate smoothed power */
+    for (i = 0; i < num_filts; i++) {
+#ifndef FIXED_POINT
+        noise_stats->power[i] =
+            noise_stats->lambda_power * noise_stats->power[i] + noise_stats->comp_lambda_power * mfspec[i];   
+#else
+        noise_stats->power[i] = fe_log_add(noise_stats->lambda_power + noise_stats->power[i],
+            noise_stats->comp_lambda_power + mfspec[i]);
+#endif
+    }
+
+    /* Noise estimation and vad decision */
+    fe_lower_envelope(noise_stats, noise_stats->power, noise_stats->noise, num_filts);
+
+    lrt = FLOAT2FIX(0.0f);
+    max_signal = FLOAT2FIX(0.0f);
+    for (i = 0; i < num_filts; i++) {
+#ifndef FIXED_POINT
+        signal[i] = noise_stats->power[i] - noise_stats->noise[i];
+        if (signal[i] < 1.0)
+            signal[i] = 1.0;
+        snr = log(noise_stats->power[i] / noise_stats->noise[i]);
+        log_signal = log(signal[i]);
+#else
+        signal[i] = fe_log_sub(noise_stats->power[i], noise_stats->noise[i]);
+        snr = noise_stats->power[i] - noise_stats->noise[i];
+        log_signal = signal[i];
+#endif    
+        if (snr > lrt) {
+            lrt = snr;
+            if (log_signal > max_signal) {
+		max_signal = log_signal;
+    	    }
+    	}
+    }
+
+#ifndef FIXED_POINT
+    if (fe->remove_silence && (lrt < fe->vad_threshold || max_signal < fe->vad_threshold)) {
+#else
+    if (fe->remove_silence && (lrt < FLOAT2FIX(fe->vad_threshold) || max_signal < FLOAT2FIX(fe->vad_threshold))) {
+#endif
+        *in_speech = FALSE;
+    } else {
+	*in_speech = TRUE;
+    }
+
+    fe_lower_envelope(noise_stats, signal, noise_stats->floor, num_filts);
+
+    fe_temp_masking(noise_stats, signal, noise_stats->peak, num_filts);
+
+    if (!fe->remove_noise) {
+        //no need for further calculations if noise cancellation disabled
+        ckd_free(signal);
+        return;
+    }
+
+    for (i = 0; i < num_filts; i++) {
+        if (signal[i] < noise_stats->floor[i])
+            signal[i] = noise_stats->floor[i];
+    }
+
+    gain = (powspec_t *) ckd_calloc(num_filts, sizeof(powspec_t));
+#ifndef FIXED_POINT
+    for (i = 0; i < num_filts; i++) {
+        if (signal[i] < noise_stats->max_gain * noise_stats->power[i])
+            gain[i] = signal[i] / noise_stats->power[i];
+        else
+            gain[i] = noise_stats->max_gain;
+        if (gain[i] < noise_stats->inv_max_gain)
+            gain[i] = noise_stats->inv_max_gain;
+    }
+#else
+    for (i = 0; i < num_filts; i++) {
+        gain[i] = signal[i] - noise_stats->power[i];
+        if (gain[i] > noise_stats->max_gain)
+            gain[i] = noise_stats->max_gain;
+        if (gain[i] < noise_stats->inv_max_gain)
+            gain[i] = noise_stats->inv_max_gain;
+    }
+#endif
+
+    /* Weight smoothing and time frequency normalization */
+    fe_weight_smooth(noise_stats, mfspec, gain, num_filts);
+
+    ckd_free(gain);
+    ckd_free(signal);
+}
+
+void
+fe_vad_hangover(fe_t * fe, mfcc_t * fea, int32 is_speech)
+{
+    /* track vad state and deal with cepstrum prespeech buffer */
+    fe->vad_data->state_changed = 0;
+    if (is_speech) {
+        fe->vad_data->postspch_num = 0;
+        if (!fe->vad_data->global_state) {
+            fe->vad_data->prespch_num++;
+            fe_prespch_write_cep(fe->vad_data->prespch_buf, fea);
+            /* check for transition sil->speech */
+            if (fe->vad_data->prespch_num >= fe->prespch_len) {
+                fe->vad_data->prespch_num = 0;
+                fe->vad_data->global_state = 1;
+                /* transition silence->speech occurred */
+                fe->vad_data->state_changed = 1;
+            }
+        }
+    } else {
+        fe->vad_data->prespch_num = 0;
+        fe_prespch_reset_cep(fe->vad_data->prespch_buf);
+        if (fe->vad_data->global_state) {
+            fe->vad_data->postspch_num++;
+            /* check for transition speech->sil */
+            if (fe->vad_data->postspch_num >= fe->postspch_len) {
+                fe->vad_data->postspch_num = 0;
+                fe->vad_data->global_state = 0;
+                /* transition speech->silence occurred */
+                fe->vad_data->state_changed = 1;
+            }
+        }
+    }
+
+    if (fe->vad_data->store_pcm) {
+        if (is_speech || fe->vad_data->global_state)
+            fe_prespch_write_pcm(fe->vad_data->prespch_buf, fe->spch);
+        if (!is_speech && !fe->vad_data->global_state)
+            fe_prespch_reset_pcm(fe->vad_data->prespch_buf);
+    }
+}

media/sphinxbase/src/libsphinxbase/fe/fe_noise.h

@@ -0,0 +1,66 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2013 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+#ifndef FE_NOISE_H
+#define FE_NOISE_H
+
+#include "sphinxbase/fe.h"
+#include "sphinxbase/fixpoint.h"
+#include "fe_type.h"
+
+typedef struct noise_stats_s noise_stats_t;
+
+/* Creates noisestats object */
+noise_stats_t *fe_init_noisestats(int num_filters);
+
+/* Resets collected noise statistics */
+void fe_reset_noisestats(noise_stats_t * noise_stats);
+
+/* Frees allocated data */
+void fe_free_noisestats(noise_stats_t * noise_stats);
+
+/**
+ * Process frame, update noise statistics, remove noise components if needed, 
+ * and return local vad decision.
+ */
+void fe_track_snr(fe_t *fe, int32 *in_speech);
+
+/**
+ * Updates global state based on local VAD state smoothing the estimate.
+ */
+void fe_vad_hangover(fe_t *fe, mfcc_t *fea, int32 is_speech);
+
+#endif                          /* FE_NOISE_H */

media/sphinxbase/src/libsphinxbase/fe/fe_prespch_buf.c

@@ -0,0 +1,182 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+* Copyright (c) 2013 Carnegie Mellon University.  All rights 
+* reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions
+* are met:
+*
+* 1. Redistributions of source code must retain the above copyright
+*    notice, this list of conditions and the following disclaimer. 
+*
+* 2. Redistributions in binary form must reproduce the above copyright
+*    notice, this list of conditions and the following disclaimer in
+*    the documentation and/or other materials provided with the
+*    distribution.
+*
+* This work was supported in part by funding from the Defense Advanced 
+* Research Projects Agency and the National Science Foundation of the 
+* United States of America, and the CMU Sphinx Speech Consortium.
+*
+* THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+* ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+* NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+* ====================================================================
+*
+*/
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/err.h"
+
+#include "fe_prespch_buf.h"
+
+struct prespch_buf_s {
+    /* saved mfcc frames */
+    mfcc_t **cep_buf;
+    /* saved pcm audio */
+    int16 *pcm_buf;
+    /* flag for pcm buffer initialization */
+    int16 cep_write_ptr;
+    /* read pointer for cep buffer */
+    int16 cep_read_ptr;
+    /* write pointer for pcm buffer */
+    int16 pcm_write_ptr;
+    /* frames amount in cep buffer */
+    int16 num_frames_cep;
+    /* frames amount in pcm buffer */
+    int16 num_frames_pcm;
+    /* filters amount */
+    int16 num_cepstra;
+    /* amount of fresh samples in frame */
+    int16 num_samples;
+};
+
+prespch_buf_t *
+fe_prespch_init(int num_frames, int num_cepstra, int num_samples)
+{
+    prespch_buf_t *prespch_buf;
+
+    prespch_buf = (prespch_buf_t *) ckd_calloc(1, sizeof(prespch_buf_t));
+
+    prespch_buf->num_cepstra = num_cepstra;
+    prespch_buf->num_frames_cep = num_frames;
+    prespch_buf->num_samples = num_samples;
+    prespch_buf->num_frames_pcm = 0;
+    prespch_buf->cep_write_ptr = 0;
+    prespch_buf->cep_read_ptr = 0;
+    prespch_buf->pcm_write_ptr = 0;
+
+    prespch_buf->cep_buf = (mfcc_t **)
+        ckd_calloc_2d(num_frames, num_cepstra,
+                      sizeof(**prespch_buf->cep_buf));
+
+    return prespch_buf;
+}
+
+void 
+fe_prespch_extend_pcm(prespch_buf_t* prespch_buf, int num_frames_pcm)
+{
+    num_frames_pcm += prespch_buf->num_frames_cep + 1;
+    if (num_frames_pcm > prespch_buf->num_frames_pcm) {
+        prespch_buf->num_frames_pcm = num_frames_pcm;
+        prespch_buf->pcm_buf = (int16 *) ckd_realloc(prespch_buf->pcm_buf, prespch_buf->num_frames_pcm * prespch_buf->num_samples * sizeof(int16));
+    }
+}
+
+int
+fe_prespch_read_cep(prespch_buf_t * prespch_buf, mfcc_t * fea)
+{
+    if (prespch_buf->cep_read_ptr >= prespch_buf->num_frames_cep)
+        return 0;
+    if (prespch_buf->cep_read_ptr >= prespch_buf->cep_write_ptr)
+        return 0;
+    memcpy(fea, prespch_buf->cep_buf[prespch_buf->cep_read_ptr],
+           sizeof(mfcc_t) * prespch_buf->num_cepstra);
+    prespch_buf->cep_read_ptr++;
+    return 1;
+}
+
+void
+fe_prespch_write_cep(prespch_buf_t * prespch_buf, mfcc_t * fea)
+{
+    assert(prespch_buf->cep_write_ptr < prespch_buf->num_frames_cep);
+    memcpy(prespch_buf->cep_buf[prespch_buf->cep_write_ptr], fea,
+           sizeof(mfcc_t) * prespch_buf->num_cepstra);
+    prespch_buf->cep_write_ptr++;
+}
+
+void
+fe_prespch_read_pcm(prespch_buf_t * prespch_buf, int16 ** samples,
+                    int32 * samples_num)
+{
+    if (!prespch_buf->pcm_buf) {
+        /* pcm prespch buffer isn't initialized yet */
+        samples = NULL;
+        *samples_num = 0;
+        return;
+    }
+    *samples = prespch_buf->pcm_buf;
+    *samples_num = prespch_buf->pcm_write_ptr * prespch_buf->num_samples;
+    prespch_buf->pcm_write_ptr = 0;
+}
+
+void
+fe_prespch_write_pcm(prespch_buf_t * prespch_buf, int16 * samples)
+{
+    int32 sample_ptr;
+
+    assert(prespch_buf->pcm_write_ptr < prespch_buf->num_frames_pcm);
+    sample_ptr = prespch_buf->pcm_write_ptr * prespch_buf->num_samples;
+    memcpy(&prespch_buf->pcm_buf[sample_ptr], samples,
+           prespch_buf->num_samples * sizeof(int16));
+    prespch_buf->pcm_write_ptr++;
+}
+
+void
+fe_prespch_reset_cep(prespch_buf_t * prespch_buf)
+{
+    prespch_buf->cep_read_ptr = 0;
+    prespch_buf->cep_write_ptr = 0;
+}
+
+void
+fe_prespch_reset_pcm(prespch_buf_t * prespch_buf)
+{
+    prespch_buf->pcm_write_ptr = 0;
+}
+
+void
+fe_prespch_free(prespch_buf_t * prespch_buf)
+{
+    if (!prespch_buf)
+	return;
+    if (prespch_buf->cep_buf)
+        ckd_free_2d((void **) prespch_buf->cep_buf);
+    if (prespch_buf->pcm_buf)
+        ckd_free(prespch_buf->pcm_buf);
+    ckd_free(prespch_buf);
+}
+
+int32 
+fe_prespch_ncep(prespch_buf_t * prespch_buf)
+{
+    return prespch_buf->cep_write_ptr - prespch_buf->cep_read_ptr;
+}

media/sphinxbase/src/libsphinxbase/fe/fe_prespch_buf.h

@@ -0,0 +1,79 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2013 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+/* Buffer that maintains both features and raw audio for the VAD implementation */
+
+#ifndef FE_INTERNAL_H
+#define FE_INTERNAL_H
+
+#include "sphinxbase/fe.h"
+
+typedef struct prespch_buf_s prespch_buf_t;
+
+/* Creates prespeech buffer */
+prespch_buf_t *fe_prespch_init(int num_frames, int num_cepstra,
+                               int num_samples);
+
+/* Extends pcm prespeech buffer with specified amount of frames */
+void fe_prespch_extend_pcm(prespch_buf_t* prespch_buf, int num_frames_pcm);
+
+/* Reads mfcc frame from prespeech buffer */
+int fe_prespch_read_cep(prespch_buf_t * prespch_buf, mfcc_t * fea);
+
+/* Writes mfcc frame to prespeech buffer */
+void fe_prespch_write_cep(prespch_buf_t * prespch_buf, mfcc_t * fea);
+
+/* Reads pcm frame from prespeech buffer */
+void fe_prespch_read_pcm(prespch_buf_t * prespch_buf, int16 ** samples,
+                         int32 * samples_num);
+
+/* Writes pcm frame to prespeech buffer */
+void fe_prespch_write_pcm(prespch_buf_t * prespch_buf, int16 * samples);
+
+/* Resets read/write pointers for cepstrum buffer */
+void fe_prespch_reset_cep(prespch_buf_t * prespch_buf);
+
+/* Resets read/write pointer for pcm audio buffer */
+void fe_prespch_reset_pcm(prespch_buf_t * prespch_buf);
+
+/* Releases prespeech buffer */
+void fe_prespch_free(prespch_buf_t * prespch_buf);
+
+/* Returns number of accumulated frames */
+int32 fe_prespch_ncep(prespch_buf_t * prespch_buf);
+
+#endif                          /* FE_INTERNAL_H */

media/sphinxbase/src/libsphinxbase/fe/fe_type.h

@@ -0,0 +1,65 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2013 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+#ifndef FE_TYPE_H
+#define FE_TYPE_H
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "sphinxbase/fe.h"
+#include "sphinxbase/fixpoint.h"
+
+#ifdef FIXED16
+/* Q15 format */
+typedef int16 frame_t;
+typedef int16 window_t;
+typedef int32 powspec_t;
+typedef struct { int16 r, i; } complex;
+#elif defined(FIXED_POINT)
+typedef fixed32 frame_t;
+typedef int32 powspec_t;
+typedef fixed32 window_t;
+typedef struct { fixed32 r, i; } complex;
+#else /* FIXED_POINT */
+typedef float64 frame_t;
+typedef float64 powspec_t;
+typedef float64 window_t;
+typedef struct { float64 r, i; } complex;
+#endif /* FIXED_POINT */
+
+#endif /* FE_TYPE_H */

media/sphinxbase/src/libsphinxbase/fe/fe_warp.c

@@ -0,0 +1,252 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2006 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*********************************************************************
+ *
+ * File: fe_warp.c
+ * 
+ * Description: 
+ *      Allows a caller to choose a warping function.
+ *********************************************************************/
+
+/* static char rcsid[] = "@(#)$Id: fe_warp.c,v 1.2 2006/02/17 00:31:34 egouvea Exp $";*/
+
+#include "fe_warp_inverse_linear.h"
+#include "fe_warp_affine.h"
+#include "fe_warp_piecewise_linear.h"
+#include "fe_warp.h"
+
+#include "sphinxbase/err.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <stdlib.h>
+
+/* This is for aliases for each of the entries below. Currently not
+   used.
+*/
+static char *__name2id[] = {
+    "inverse",
+    "linear",
+    "piecewise",
+    NULL
+};
+
+static char *name2id[] = {
+    "inverse_linear",
+    "affine",
+    "piecewise_linear",
+    NULL
+};
+
+static fe_warp_conf_t fe_warp_conf[FE_WARP_ID_MAX + 1] = {
+    {fe_warp_inverse_linear_set_parameters,
+     fe_warp_inverse_linear_doc,
+     fe_warp_inverse_linear_id,
+     fe_warp_inverse_linear_n_param,
+     fe_warp_inverse_linear_warped_to_unwarped,
+     fe_warp_inverse_linear_unwarped_to_warped,
+     fe_warp_inverse_linear_print},     /* Inverse linear warping */
+    {fe_warp_affine_set_parameters,
+     fe_warp_affine_doc,
+     fe_warp_affine_id,
+     fe_warp_affine_n_param,
+     fe_warp_affine_warped_to_unwarped,
+     fe_warp_affine_unwarped_to_warped,
+     fe_warp_affine_print},     /* Affine warping */
+    {fe_warp_piecewise_linear_set_parameters,
+     fe_warp_piecewise_linear_doc,
+     fe_warp_piecewise_linear_id,
+     fe_warp_piecewise_linear_n_param,
+     fe_warp_piecewise_linear_warped_to_unwarped,
+     fe_warp_piecewise_linear_unwarped_to_warped,
+     fe_warp_piecewise_linear_print},   /* Piecewise_Linear warping */
+};
+
+int
+fe_warp_set(melfb_t *mel, const char *id_name)
+{
+    uint32 i;
+
+    for (i = 0; name2id[i]; i++) {
+        if (strcmp(id_name, name2id[i]) == 0) {
+            mel->warp_id = i;
+            break;
+        }
+    }
+
+    if (name2id[i] == NULL) {
+        for (i = 0; __name2id[i]; i++) {
+            if (strcmp(id_name, __name2id[i]) == 0) {
+                mel->warp_id = i;
+                break;
+            }
+        }
+        if (__name2id[i] == NULL) {
+            E_ERROR("Unimplemented warping function %s\n", id_name);
+            E_ERROR("Implemented functions are:\n");
+            for (i = 0; name2id[i]; i++) {
+                fprintf(stderr, "\t%s\n", name2id[i]);
+            }
+            mel->warp_id = FE_WARP_ID_NONE;
+
+            return FE_START_ERROR;
+        }
+    }
+
+    return FE_SUCCESS;
+}
+
+void
+fe_warp_set_parameters(melfb_t *mel, char const *param_str, float sampling_rate)
+{
+    if (mel->warp_id <= FE_WARP_ID_MAX) {
+        fe_warp_conf[mel->warp_id].set_parameters(param_str, sampling_rate);
+    }
+    else if (mel->warp_id == FE_WARP_ID_NONE) {
+        E_FATAL("feat module must be configured w/ a valid ID\n");
+    }
+    else {
+        E_FATAL
+            ("fe_warp module misconfigured with invalid fe_warp_id %u\n",
+             mel->warp_id);
+    }
+}
+
+const char *
+fe_warp_doc(melfb_t *mel)
+{
+    if (mel->warp_id <= FE_WARP_ID_MAX) {
+        return fe_warp_conf[mel->warp_id].doc();
+    }
+    else if (mel->warp_id == FE_WARP_ID_NONE) {
+        E_FATAL("fe_warp module must be configured w/ a valid ID\n");
+    }
+    else {
+        E_FATAL
+            ("fe_warp module misconfigured with invalid fe_warp_id %u\n",
+             mel->warp_id);
+    }
+
+    return NULL;
+}
+
+uint32
+fe_warp_id(melfb_t *mel)
+{
+    if (mel->warp_id <= FE_WARP_ID_MAX) {
+        assert(mel->warp_id == fe_warp_conf[mel->warp_id].id());
+        return mel->warp_id;
+    }
+    else if (mel->warp_id != FE_WARP_ID_NONE) {
+        E_FATAL
+            ("fe_warp module misconfigured with invalid fe_warp_id %u\n",
+             mel->warp_id);
+    }
+
+    return FE_WARP_ID_NONE;
+}
+
+uint32
+fe_warp_n_param(melfb_t *mel)
+{
+    if (mel->warp_id <= FE_WARP_ID_MAX) {
+        return fe_warp_conf[mel->warp_id].n_param();
+    }
+    else if (mel->warp_id == FE_WARP_ID_NONE) {
+        E_FATAL("fe_warp module must be configured w/ a valid ID\n");
+    }
+    else {
+        E_FATAL
+            ("fe_warp module misconfigured with invalid fe_warp_id %u\n",
+             mel->warp_id);
+    }
+
+    return 0;
+}
+
+float
+fe_warp_warped_to_unwarped(melfb_t *mel, float nonlinear)
+{
+    if (mel->warp_id <= FE_WARP_ID_MAX) {
+        return fe_warp_conf[mel->warp_id].warped_to_unwarped(nonlinear);
+    }
+    else if (mel->warp_id == FE_WARP_ID_NONE) {
+        E_FATAL("fe_warp module must be configured w/ a valid ID\n");
+    }
+    else {
+        E_FATAL
+            ("fe_warp module misconfigured with invalid fe_warp_id %u\n",
+             mel->warp_id);
+    }
+
+    return 0;
+}
+
+float
+fe_warp_unwarped_to_warped(melfb_t *mel,float linear)
+{
+    if (mel->warp_id <= FE_WARP_ID_MAX) {
+        return fe_warp_conf[mel->warp_id].unwarped_to_warped(linear);
+    }
+    else if (mel->warp_id == FE_WARP_ID_NONE) {
+        E_FATAL("fe_warp module must be configured w/ a valid ID\n");
+    }
+    else {
+        E_FATAL
+            ("fe_warp module misconfigured with invalid fe_warp_id %u\n",
+             mel->warp_id);
+    }
+
+    return 0;
+}
+
+void
+fe_warp_print(melfb_t *mel, const char *label)
+{
+    if (mel->warp_id <= FE_WARP_ID_MAX) {
+        fe_warp_conf[mel->warp_id].print(label);
+    }
+    else if (mel->warp_id == FE_WARP_ID_NONE) {
+        E_FATAL("fe_warp module must be configured w/ a valid ID\n");
+    }
+    else {
+        E_FATAL
+            ("fe_warp module misconfigured with invalid fe_warp_id %u\n",
+             mel->warp_id);
+    }
+}

media/sphinxbase/src/libsphinxbase/fe/fe_warp.h

@@ -0,0 +1,90 @@
+/* ====================================================================
+ * Copyright (c) 2006 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+#ifndef FE_WARP_H
+#define FE_WARP_H
+
+#include "fe_internal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#if 0
+/* Fool Emacs. */
+}
+#endif
+
+#define FE_WARP_ID_INVERSE_LINEAR	0
+#define FE_WARP_ID_AFFINE	        1
+#define FE_WARP_ID_PIECEWISE_LINEAR	2
+#define FE_WARP_ID_EIDE_GISH		3
+#define FE_WARP_ID_MAX		        2
+#define FE_WARP_ID_NONE	       0xffffffff
+
+typedef struct {
+    void (*set_parameters)(char const *param_str, float sampling_rate);
+    const char * (*doc)(void);
+    uint32 (*id)(void);
+    uint32 (*n_param)(void);
+    float (*warped_to_unwarped)(float nonlinear);
+    float (*unwarped_to_warped)(float linear);
+    void (*print)(const char *label);
+} fe_warp_conf_t;
+
+int fe_warp_set(melfb_t *mel, const char *id_name);
+
+uint32 fe_warp_id(melfb_t *mel);
+
+const char * fe_warp_doc(melfb_t *mel);
+
+void fe_warp_set_parameters(melfb_t *mel, char const *param_str, float sampling_rate);
+
+uint32 fe_warp_n_param(melfb_t *mel);
+
+float fe_warp_warped_to_unwarped(melfb_t *mel, float nonlinear);
+
+float fe_warp_unwarped_to_warped(melfb_t *mel, float linear);
+
+void fe_warp_print(melfb_t *mel, const char *label);
+
+#define FE_WARP_NO_SIZE	0xffffffff
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* FE_WARP_H */ 

media/sphinxbase/src/libsphinxbase/fe/fe_warp_affine.c

@@ -0,0 +1,181 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2006 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*********************************************************************
+ *
+ * File: fe_warp_affine.c
+ * 
+ * Description: 
+ * 	Warp the frequency axis according to an affine function, i.e.:
+ *
+ *		w' = a * w + b
+ *	
+ *********************************************************************/
+
+/* static char rcsid[] = "@(#)$Id: fe_warp_affine.c,v 1.2 2006/02/17 00:31:34 egouvea Exp $"; */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4996)
+#endif
+
+#include "sphinxbase/strfuncs.h"
+#include "sphinxbase/err.h"
+
+#include "fe_warp.h"
+#include "fe_warp_affine.h"
+
+#define N_PARAM		2
+#define YES             1
+#define NO              0
+
+/*
+ * params[0] : a
+ * params[1] : b
+ */
+static float params[N_PARAM] = { 1.0f, 0.0f };
+static int32 is_neutral = YES;
+static char p_str[256] = "";
+static float nyquist_frequency = 0.0f;
+
+
+const char *
+fe_warp_affine_doc()
+{
+    return "affine :== < w' = a * x + b >";
+}
+
+uint32
+fe_warp_affine_id()
+{
+    return FE_WARP_ID_AFFINE;
+}
+
+uint32
+fe_warp_affine_n_param()
+{
+    return N_PARAM;
+}
+
+void
+fe_warp_affine_set_parameters(char const *param_str, float sampling_rate)
+{
+    char *tok;
+    char *seps = " \t";
+    char temp_param_str[256];
+    int param_index = 0;
+
+    nyquist_frequency = sampling_rate / 2;
+    if (param_str == NULL) {
+        is_neutral = YES;
+        return;
+    }
+    /* The new parameters are the same as the current ones, so do nothing. */
+    if (strcmp(param_str, p_str) == 0) {
+        return;
+    }
+    is_neutral = NO;
+    strcpy(temp_param_str, param_str);
+    memset(params, 0, N_PARAM * sizeof(float));
+    strcpy(p_str, param_str);
+    /* FIXME: strtok() is not re-entrant... */
+    tok = strtok(temp_param_str, seps);
+    while (tok != NULL) {
+        params[param_index++] = (float) atof_c(tok);
+        tok = strtok(NULL, seps);
+        if (param_index >= N_PARAM) {
+            break;
+        }
+    }
+    if (tok != NULL) {
+        E_INFO
+            ("Affine warping takes up to two arguments, %s ignored.\n",
+             tok);
+    }
+    if (params[0] == 0) {
+        is_neutral = YES;
+        E_INFO
+            ("Affine warping cannot have slope zero, warping not applied.\n");
+    }
+}
+
+float
+fe_warp_affine_warped_to_unwarped(float nonlinear)
+{
+    if (is_neutral) {
+        return nonlinear;
+    }
+    else {
+        /* linear = (nonlinear - b) / a */
+        float temp = nonlinear - params[1];
+        temp /= params[0];
+        if (temp > nyquist_frequency) {
+            E_WARN
+                ("Warp factor %g results in frequency (%.1f) higher than Nyquist (%.1f)\n",
+                 params[0], temp, nyquist_frequency);
+        }
+        return temp;
+    }
+}
+
+float
+fe_warp_affine_unwarped_to_warped(float linear)
+{
+    if (is_neutral) {
+        return linear;
+    }
+    else {
+        /* nonlinear = a * linear - b */
+        float temp = linear * params[0];
+        temp += params[1];
+        return temp;
+    }
+}
+
+void
+fe_warp_affine_print(const char *label)
+{
+    uint32 i;
+
+    for (i = 0; i < N_PARAM; i++) {
+        printf("%s[%04u]: %6.3f ", label, i, params[i]);
+    }
+    printf("\n");
+}

media/sphinxbase/src/libsphinxbase/fe/fe_warp_affine.h

@@ -0,0 +1,76 @@
+/* ====================================================================
+ * Copyright (c) 2006 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+#ifndef FE_WARP_AFFINE_H
+#define FE_WARP_AFFINE_H
+
+#include "sphinxbase/fe.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#if 0
+/* Fool Emacs. */
+}
+#endif
+
+const char *
+fe_warp_affine_doc(void);
+
+uint32
+fe_warp_affine_id(void);
+
+uint32
+fe_warp_affine_n_param(void);
+
+void
+fe_warp_affine_set_parameters(char const *param_str, float sampling_rate);
+
+float
+fe_warp_affine_warped_to_unwarped(float nonlinear);
+
+float
+fe_warp_affine_unwarped_to_warped(float linear);
+
+void
+fe_warp_affine_print(const char *label);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* FE_WARP_AFFINE_H */ 

media/sphinxbase/src/libsphinxbase/fe/fe_warp_inverse_linear.c

@@ -0,0 +1,178 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2006 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*********************************************************************
+ *
+ * File: fe_warp_inverse_linear.c
+ * 
+ * Description: 
+ * 	Warp the frequency axis according to an inverse_linear function, i.e.:
+ *
+ *		w' = w / a
+ *	
+ *********************************************************************/
+
+/* static char rcsid[] = "@(#)$Id: fe_warp_inverse_linear.c,v 1.3 2006/02/23 19:40:11 eht Exp $"; */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4996)
+#endif
+
+#include "sphinxbase/strfuncs.h"
+#include "sphinxbase/err.h"
+
+#include "fe_warp.h"
+#include "fe_warp_inverse_linear.h"
+
+#define N_PARAM		1
+#define YES             1
+#define NO              0
+
+/*
+ * params[0] : a
+ */
+static float params[N_PARAM] = { 1.0f };
+static int32 is_neutral = YES;
+static char p_str[256] = "";
+static float nyquist_frequency = 0.0f;
+
+
+const char *
+fe_warp_inverse_linear_doc()
+{
+    return "inverse_linear :== < w' = x / a >";
+}
+
+uint32
+fe_warp_inverse_linear_id()
+{
+    return FE_WARP_ID_INVERSE_LINEAR;
+}
+
+uint32
+fe_warp_inverse_linear_n_param()
+{
+    return N_PARAM;
+}
+
+void
+fe_warp_inverse_linear_set_parameters(char const *param_str, float sampling_rate)
+{
+    char *tok;
+    char *seps = " \t";
+    char temp_param_str[256];
+    int param_index = 0;
+
+    nyquist_frequency = sampling_rate / 2;
+    if (param_str == NULL) {
+        is_neutral = YES;
+        return;
+    }
+    /* The new parameters are the same as the current ones, so do nothing. */
+    if (strcmp(param_str, p_str) == 0) {
+        return;
+    }
+    is_neutral = NO;
+    strcpy(temp_param_str, param_str);
+    memset(params, 0, N_PARAM * sizeof(float));
+    strcpy(p_str, param_str);
+    /* FIXME: strtok() is not re-entrant... */
+    tok = strtok(temp_param_str, seps);
+    while (tok != NULL) {
+        params[param_index++] = (float) atof_c(tok);
+        tok = strtok(NULL, seps);
+        if (param_index >= N_PARAM) {
+            break;
+        }
+    }
+    if (tok != NULL) {
+        E_INFO
+            ("Inverse linear warping takes only one argument, %s ignored.\n",
+             tok);
+    }
+    if (params[0] == 0) {
+        is_neutral = YES;
+        E_INFO
+            ("Inverse linear warping cannot have slope zero, warping not applied.\n");
+    }
+}
+
+float
+fe_warp_inverse_linear_warped_to_unwarped(float nonlinear)
+{
+    if (is_neutral) {
+        return nonlinear;
+    }
+    else {
+        /* linear = nonlinear * a */
+        float temp = nonlinear * params[0];
+        if (temp > nyquist_frequency) {
+            E_WARN
+                ("Warp factor %g results in frequency (%.1f) higher than Nyquist (%.1f)\n",
+                 params[0], temp, nyquist_frequency);
+        }
+        return temp;
+    }
+}
+
+float
+fe_warp_inverse_linear_unwarped_to_warped(float linear)
+{
+    if (is_neutral) {
+        return linear;
+    }
+    else {
+        /* nonlinear = a / linear */
+        float temp = linear / params[0];
+        return temp;
+    }
+}
+
+void
+fe_warp_inverse_linear_print(const char *label)
+{
+    uint32 i;
+
+    for (i = 0; i < N_PARAM; i++) {
+        printf("%s[%04u]: %6.3f ", label, i, params[i]);
+    }
+    printf("\n");
+}

media/sphinxbase/src/libsphinxbase/fe/fe_warp_inverse_linear.h

@@ -0,0 +1,77 @@
+/* ====================================================================
+ * Copyright (c) 2006 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+#ifndef FE_WARP_inverse_linear_H
+#define FE_WARP_inverse_linear_H
+
+#include "sphinxbase/fe.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#if 0
+/* Fool Emacs. */
+}
+#endif
+
+const char *
+fe_warp_inverse_linear_doc(void);
+
+uint32
+fe_warp_inverse_linear_id(void);
+
+uint32
+fe_warp_inverse_linear_n_param(void);
+
+void
+fe_warp_inverse_linear_set_parameters(char const *param_str, float sampling_rate);
+
+float
+fe_warp_inverse_linear_warped_to_unwarped(float nonlinear);
+
+float
+fe_warp_inverse_linear_unwarped_to_warped(float linear);
+
+void
+fe_warp_inverse_linear_print(const char *label);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* FE_WARP_inverse_linear_H */ 

media/sphinxbase/src/libsphinxbase/fe/fe_warp_piecewise_linear.c

@@ -0,0 +1,223 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2006 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*********************************************************************
+ *
+ * File: fe_warp_piecewise_linear.c
+ * 
+ * Description: 
+ *
+ * 	Warp the frequency axis according to an piecewise linear
+ * 	function. The function is linear up to a frequency F, where
+ * 	the slope changes so that the Nyquist frequency in the warped
+ * 	axis maps to the Nyquist frequency in the unwarped.
+ *
+ *		w' = a * w, w < F
+ *              w' = a' * w + b, W > F
+ *              w'(0) = 0
+ *              w'(F) = F
+ *              w'(Nyq) = Nyq
+ *	
+ *********************************************************************/
+
+/* static char rcsid[] = "@(#)$Id: fe_warp_piecewise_linear.c,v 1.2 2006/02/17 00:31:34 egouvea Exp $"; */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4996)
+#endif
+
+#include "sphinxbase/strfuncs.h"
+#include "sphinxbase/err.h"
+
+#include "fe_warp.h"
+#include "fe_warp_piecewise_linear.h"
+
+#define N_PARAM		2
+#define YES             1
+#define NO              0
+
+/*
+ * params[0] : a
+ * params[1] : F (the non-differentiable point)
+ */
+static float params[N_PARAM] = { 1.0f, 6800.0f };
+static float final_piece[2];
+static int32 is_neutral = YES;
+static char p_str[256] = "";
+static float nyquist_frequency = 0.0f;
+
+
+const char *
+fe_warp_piecewise_linear_doc()
+{
+    return "piecewise_linear :== < w' = a * w, w < F >";
+}
+
+uint32
+fe_warp_piecewise_linear_id()
+{
+    return FE_WARP_ID_PIECEWISE_LINEAR;
+}
+
+uint32
+fe_warp_piecewise_linear_n_param()
+{
+    return N_PARAM;
+}
+
+void
+fe_warp_piecewise_linear_set_parameters(char const *param_str,
+                                        float sampling_rate)
+{
+    char *tok;
+    char *seps = " \t";
+    char temp_param_str[256];
+    int param_index = 0;
+
+    nyquist_frequency = sampling_rate / 2;
+    if (param_str == NULL) {
+        is_neutral = YES;
+        return;
+    }
+    /* The new parameters are the same as the current ones, so do nothing. */
+    if (strcmp(param_str, p_str) == 0) {
+        return;
+    }
+    is_neutral = NO;
+    strcpy(temp_param_str, param_str);
+    memset(params, 0, N_PARAM * sizeof(float));
+    memset(final_piece, 0, 2 * sizeof(float));
+    strcpy(p_str, param_str);
+    /* FIXME: strtok() is not re-entrant... */
+    tok = strtok(temp_param_str, seps);
+    while (tok != NULL) {
+        params[param_index++] = (float) atof_c(tok);
+        tok = strtok(NULL, seps);
+        if (param_index >= N_PARAM) {
+            break;
+        }
+    }
+    if (tok != NULL) {
+        E_INFO
+            ("Piecewise linear warping takes up to two arguments, %s ignored.\n",
+             tok);
+    }
+    if (params[1] < sampling_rate) {
+        /* Precompute these. These are the coefficients of a
+         * straight line that contains the points (F, aF) and (N,
+         * N), where a = params[0], F = params[1], N = Nyquist
+         * frequency.
+         */
+        if (params[1] == 0) {
+            params[1] = sampling_rate * 0.85f;
+        }
+        final_piece[0] =
+            (nyquist_frequency -
+             params[0] * params[1]) / (nyquist_frequency - params[1]);
+        final_piece[1] =
+            nyquist_frequency * params[1] * (params[0] -
+                                         1.0f) / (nyquist_frequency -
+                                                  params[1]);
+    }
+    else {
+        memset(final_piece, 0, 2 * sizeof(float));
+    }
+    if (params[0] == 0) {
+        is_neutral = YES;
+        E_INFO
+            ("Piecewise linear warping cannot have slope zero, warping not applied.\n");
+    }
+}
+
+float
+fe_warp_piecewise_linear_warped_to_unwarped(float nonlinear)
+{
+    if (is_neutral) {
+        return nonlinear;
+    }
+    else {
+        /* linear = (nonlinear - b) / a */
+        float temp;
+        if (nonlinear < params[0] * params[1]) {
+            temp = nonlinear / params[0];
+        }
+        else {
+            temp = nonlinear - final_piece[1];
+            temp /= final_piece[0];
+        }
+        if (temp > nyquist_frequency) {
+            E_WARN
+                ("Warp factor %g results in frequency (%.1f) higher than Nyquist (%.1f)\n",
+                 params[0], temp, nyquist_frequency);
+        }
+        return temp;
+    }
+}
+
+float
+fe_warp_piecewise_linear_unwarped_to_warped(float linear)
+{
+    if (is_neutral) {
+        return linear;
+    }
+    else {
+        float temp;
+        /* nonlinear = a * linear - b */
+        if (linear < params[1]) {
+            temp = linear * params[0];
+        }
+        else {
+            temp = final_piece[0] * linear + final_piece[1];
+        }
+        return temp;
+    }
+}
+
+void
+fe_warp_piecewise_linear_print(const char *label)
+{
+    uint32 i;
+
+    for (i = 0; i < N_PARAM; i++) {
+        printf("%s[%04u]: %6.3f ", label, i, params[i]);
+    }
+    printf("\n");
+}

media/sphinxbase/src/libsphinxbase/fe/fe_warp_piecewise_linear.h

@@ -0,0 +1,77 @@
+/* ====================================================================
+ * Copyright (c) 2006 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+#ifndef FE_WARP_PIECEWIDE_LINEAR_H
+#define FE_WARP_PIECEWIDE_LINEAR_H
+
+#include "sphinxbase/fe.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#if 0
+/* Fool Emacs. */
+}
+#endif
+
+const char *
+fe_warp_piecewise_linear_doc(void);
+
+uint32
+fe_warp_piecewise_linear_id(void);
+
+uint32
+fe_warp_piecewise_linear_n_param(void);
+
+void
+fe_warp_piecewise_linear_set_parameters(char const *param_str, float sampling_rate);
+
+float
+fe_warp_piecewise_linear_warped_to_unwarped(float nonlinear);
+
+float
+fe_warp_piecewise_linear_unwarped_to_warped(float linear);
+
+void
+fe_warp_piecewise_linear_print(const char *label);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* FE_WARP_PIECEWIDE_LINEAR_H */ 

media/sphinxbase/src/libsphinxbase/fe/fixlog.c

@@ -0,0 +1,229 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2005 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ * File: fixlog.c
+ *
+ * Description: Fast approximate fixed-point logarithms
+ *
+ * Author: David Huggins-Daines <dhuggins@cs.cmu.edu>
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "sphinxbase/prim_type.h"
+#include "sphinxbase/fixpoint.h"
+
+#include "fe_internal.h"
+
+/* Table of log2(x/128)*(1<<DEFAULT_RADIX) */
+/* perl -e 'for (0..128) {my $x = 1 + $_/128; $y = 1 + ($_ + 1.) / 128;
+                  print "    (uint32)(", (log($x) + log($y))/2/log(2)," *(1<<DEFAULT_RADIX)),\n"}' */
+static uint32 logtable[] = {
+    (uint32)(0.00561362771162706*(1<<DEFAULT_RADIX)),
+    (uint32)(0.0167975342258543*(1<<DEFAULT_RADIX)),
+    (uint32)(0.0278954072829524*(1<<DEFAULT_RADIX)),
+    (uint32)(0.0389085604479519*(1<<DEFAULT_RADIX)),
+    (uint32)(0.0498382774298215*(1<<DEFAULT_RADIX)),
+    (uint32)(0.060685812979481*(1<<DEFAULT_RADIX)),
+    (uint32)(0.0714523937543017*(1<<DEFAULT_RADIX)),
+    (uint32)(0.0821392191505851*(1<<DEFAULT_RADIX)),
+    (uint32)(0.0927474621054331*(1<<DEFAULT_RADIX)),
+    (uint32)(0.103278269869348*(1<<DEFAULT_RADIX)),
+    (uint32)(0.113732764750838*(1<<DEFAULT_RADIX)),
+    (uint32)(0.124112044834237*(1<<DEFAULT_RADIX)),
+    (uint32)(0.13441718467188*(1<<DEFAULT_RADIX)),
+    (uint32)(0.144649235951738*(1<<DEFAULT_RADIX)),
+    (uint32)(0.154809228141536*(1<<DEFAULT_RADIX)),
+    (uint32)(0.164898169110351*(1<<DEFAULT_RADIX)),
+    (uint32)(0.174917045728623*(1<<DEFAULT_RADIX)),
+    (uint32)(0.184866824447476*(1<<DEFAULT_RADIX)),
+    (uint32)(0.194748451858191*(1<<DEFAULT_RADIX)),
+    (uint32)(0.204562855232657*(1<<DEFAULT_RADIX)),
+    (uint32)(0.214310943045556*(1<<DEFAULT_RADIX)),
+    (uint32)(0.223993605479021*(1<<DEFAULT_RADIX)),
+    (uint32)(0.23361171491048*(1<<DEFAULT_RADIX)),
+    (uint32)(0.243166126384332*(1<<DEFAULT_RADIX)),
+    (uint32)(0.252657678068119*(1<<DEFAULT_RADIX)),
+    (uint32)(0.262087191693777*(1<<DEFAULT_RADIX)),
+    (uint32)(0.271455472984569*(1<<DEFAULT_RADIX)),
+    (uint32)(0.280763312068243*(1<<DEFAULT_RADIX)),
+    (uint32)(0.290011483876938*(1<<DEFAULT_RADIX)),
+    (uint32)(0.299200748534365*(1<<DEFAULT_RADIX)),
+    (uint32)(0.308331851730729*(1<<DEFAULT_RADIX)),
+    (uint32)(0.317405525085859*(1<<DEFAULT_RADIX)),
+    (uint32)(0.32642248650099*(1<<DEFAULT_RADIX)),
+    (uint32)(0.335383440499621*(1<<DEFAULT_RADIX)),
+    (uint32)(0.344289078557851*(1<<DEFAULT_RADIX)),
+    (uint32)(0.353140079424581*(1<<DEFAULT_RADIX)),
+    (uint32)(0.36193710943195*(1<<DEFAULT_RADIX)),
+    (uint32)(0.37068082279637*(1<<DEFAULT_RADIX)),
+    (uint32)(0.379371861910488*(1<<DEFAULT_RADIX)),
+    (uint32)(0.388010857626406*(1<<DEFAULT_RADIX)),
+    (uint32)(0.396598429530472*(1<<DEFAULT_RADIX)),
+    (uint32)(0.405135186209943*(1<<DEFAULT_RADIX)),
+    (uint32)(0.4136217255118*(1<<DEFAULT_RADIX)),
+    (uint32)(0.422058634793998*(1<<DEFAULT_RADIX)),
+    (uint32)(0.430446491169411*(1<<DEFAULT_RADIX)),
+    (uint32)(0.438785861742727*(1<<DEFAULT_RADIX)),
+    (uint32)(0.447077303840529*(1<<DEFAULT_RADIX)),
+    (uint32)(0.455321365234813*(1<<DEFAULT_RADIX)),
+    (uint32)(0.463518584360147*(1<<DEFAULT_RADIX)),
+    (uint32)(0.471669490524698*(1<<DEFAULT_RADIX)),
+    (uint32)(0.479774604115327*(1<<DEFAULT_RADIX)),
+    (uint32)(0.487834436796966*(1<<DEFAULT_RADIX)),
+    (uint32)(0.49584949170644*(1<<DEFAULT_RADIX)),
+    (uint32)(0.503820263640951*(1<<DEFAULT_RADIX)),
+    (uint32)(0.511747239241369*(1<<DEFAULT_RADIX)),
+    (uint32)(0.519630897170528*(1<<DEFAULT_RADIX)),
+    (uint32)(0.527471708286662*(1<<DEFAULT_RADIX)),
+    (uint32)(0.535270135812172*(1<<DEFAULT_RADIX)),
+    (uint32)(0.543026635497834*(1<<DEFAULT_RADIX)),
+    (uint32)(0.550741655782637*(1<<DEFAULT_RADIX)),
+    (uint32)(0.558415637949355*(1<<DEFAULT_RADIX)),
+    (uint32)(0.56604901627601*(1<<DEFAULT_RADIX)),
+    (uint32)(0.573642218183348*(1<<DEFAULT_RADIX)),
+    (uint32)(0.581195664378452*(1<<DEFAULT_RADIX)),
+    (uint32)(0.588709768994618*(1<<DEFAULT_RADIX)),
+    (uint32)(0.596184939727604*(1<<DEFAULT_RADIX)),
+    (uint32)(0.603621577968369*(1<<DEFAULT_RADIX)),
+    (uint32)(0.61102007893241*(1<<DEFAULT_RADIX)),
+    (uint32)(0.618380831785792*(1<<DEFAULT_RADIX)),
+    (uint32)(0.625704219767993*(1<<DEFAULT_RADIX)),
+    (uint32)(0.632990620311629*(1<<DEFAULT_RADIX)),
+    (uint32)(0.640240405159187*(1<<DEFAULT_RADIX)),
+    (uint32)(0.647453940476827*(1<<DEFAULT_RADIX)),
+    (uint32)(0.654631586965362*(1<<DEFAULT_RADIX)),
+    (uint32)(0.661773699968486*(1<<DEFAULT_RADIX)),
+    (uint32)(0.668880629578336*(1<<DEFAULT_RADIX)),
+    (uint32)(0.675952720738471*(1<<DEFAULT_RADIX)),
+    (uint32)(0.682990313344332*(1<<DEFAULT_RADIX)),
+    (uint32)(0.689993742341272*(1<<DEFAULT_RADIX)),
+    (uint32)(0.696963337820209*(1<<DEFAULT_RADIX)),
+    (uint32)(0.703899425110987*(1<<DEFAULT_RADIX)),
+    (uint32)(0.710802324873503*(1<<DEFAULT_RADIX)),
+    (uint32)(0.717672353186654*(1<<DEFAULT_RADIX)),
+    (uint32)(0.724509821635192*(1<<DEFAULT_RADIX)),
+    (uint32)(0.731315037394519*(1<<DEFAULT_RADIX)),
+    (uint32)(0.738088303313493*(1<<DEFAULT_RADIX)),
+    (uint32)(0.744829917995304*(1<<DEFAULT_RADIX)),
+    (uint32)(0.751540175876464*(1<<DEFAULT_RADIX)),
+    (uint32)(0.758219367303974*(1<<DEFAULT_RADIX)),
+    (uint32)(0.764867778610703*(1<<DEFAULT_RADIX)),
+    (uint32)(0.77148569218905*(1<<DEFAULT_RADIX)),
+    (uint32)(0.778073386562917*(1<<DEFAULT_RADIX)),
+    (uint32)(0.784631136458046*(1<<DEFAULT_RADIX)),
+    (uint32)(0.791159212870769*(1<<DEFAULT_RADIX)),
+    (uint32)(0.797657883135205*(1<<DEFAULT_RADIX)),
+    (uint32)(0.804127410988954*(1<<DEFAULT_RADIX)),
+    (uint32)(0.810568056637321*(1<<DEFAULT_RADIX)),
+    (uint32)(0.816980076816112*(1<<DEFAULT_RADIX)),
+    (uint32)(0.823363724853051*(1<<DEFAULT_RADIX)),
+    (uint32)(0.829719250727828*(1<<DEFAULT_RADIX)),
+    (uint32)(0.836046901130843*(1<<DEFAULT_RADIX)),
+    (uint32)(0.84234691952066*(1<<DEFAULT_RADIX)),
+    (uint32)(0.848619546180216*(1<<DEFAULT_RADIX)),
+    (uint32)(0.854865018271815*(1<<DEFAULT_RADIX)),
+    (uint32)(0.861083569890926*(1<<DEFAULT_RADIX)),
+    (uint32)(0.867275432118842*(1<<DEFAULT_RADIX)),
+    (uint32)(0.873440833074202*(1<<DEFAULT_RADIX)),
+    (uint32)(0.879579997963421*(1<<DEFAULT_RADIX)),
+    (uint32)(0.88569314913005*(1<<DEFAULT_RADIX)),
+    (uint32)(0.891780506103101*(1<<DEFAULT_RADIX)),
+    (uint32)(0.897842285644346*(1<<DEFAULT_RADIX)),
+    (uint32)(0.903878701794633*(1<<DEFAULT_RADIX)),
+    (uint32)(0.90988996591924*(1<<DEFAULT_RADIX)),
+    (uint32)(0.915876286752278*(1<<DEFAULT_RADIX)),
+    (uint32)(0.921837870440188*(1<<DEFAULT_RADIX)),
+    (uint32)(0.927774920584334*(1<<DEFAULT_RADIX)),
+    (uint32)(0.933687638282728*(1<<DEFAULT_RADIX)),
+    (uint32)(0.939576222170905*(1<<DEFAULT_RADIX)),
+    (uint32)(0.945440868461959*(1<<DEFAULT_RADIX)),
+    (uint32)(0.951281770985776*(1<<DEFAULT_RADIX)),
+    (uint32)(0.957099121227478*(1<<DEFAULT_RADIX)),
+    (uint32)(0.962893108365084*(1<<DEFAULT_RADIX)),
+    (uint32)(0.968663919306429*(1<<DEFAULT_RADIX)),
+    (uint32)(0.974411738725344*(1<<DEFAULT_RADIX)),
+    (uint32)(0.980136749097113*(1<<DEFAULT_RADIX)),
+    (uint32)(0.985839130733238*(1<<DEFAULT_RADIX)),
+    (uint32)(0.991519061815512*(1<<DEFAULT_RADIX)),
+    (uint32)(0.997176718429429*(1<<DEFAULT_RADIX)),
+    (uint32)(1.00281227459694*(1<<DEFAULT_RADIX)),
+};
+
+int32
+fixlog2(uint32 x)
+{
+    uint32 y;
+
+    if (x == 0)
+        return MIN_FIXLOG2;
+
+    /* Get the exponent. */
+#if ((defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__) || \
+      defined(__ARM_ARCH_5TE__) || defined(__ARM_ARCH_7A__)) && !defined(__thumb__))
+  __asm__("clz %0, %1\n": "=r"(y):"r"(x));
+    x <<= y;
+    y = 31 - y;
+#elif defined(__ppc__)
+  __asm__("cntlzw %0, %1\n": "=r"(y):"r"(x));
+    x <<= y;
+    y = 31 - y;
+#elif __GNUC__ >= 4
+    y = __builtin_clz(x);
+    x <<= y;
+    y = (31 - y);
+#else
+    for (y = 31; y > 0; --y) {
+        if (x & 0x80000000)
+            break;
+        x <<= 1;
+    }
+#endif
+    y <<= DEFAULT_RADIX;
+    /* Do a table lookup for the MSB of the mantissa. */
+    x = (x >> 24) & 0x7f;
+    return y + logtable[x];
+}
+
+int
+fixlog(uint32 x)
+{
+    int32 y;
+    y = fixlog2(x);
+    return FIXMUL(y, FIXLN_2);
+}

media/sphinxbase/src/libsphinxbase/fe/yin.c

@@ -0,0 +1,412 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ * Copyright (c) 2008 Beyond Access, Inc.  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY BEYOND ACCESS, INC. ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL BEYOND ACCESS, INC.  NOR
+ * ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @file yin.c Implementation of pitch extraction.
+ * @author David Huggins-Daines <dhuggins@cs.cmu.edu>
+ */
+
+/* This implements part of the YIN algorithm:
+ *
+ * "YIN, a fundamental frequency estimator for speech and music".
+ * Alain de Cheveigné and Hideki Kawahara.  Journal of the Acoustical
+ * Society of America, 111 (4), April 2002.
+ */
+
+#include "sphinxbase/prim_type.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/fixpoint.h"
+
+#include "sphinxbase/yin.h"
+
+#include <stdio.h>
+#include <string.h>
+
+struct yin_s {
+    uint16 frame_size;       /** Size of analysis frame. */
+#ifndef FIXED_POINT
+    float search_threshold; /**< Threshold for finding period */
+    float search_range;     /**< Range around best local estimate to search */
+#else
+    uint16 search_threshold; /**< Threshold for finding period, in Q15 */
+    uint16 search_range;     /**< Range around best local estimate to search, in Q15 */
+#endif
+    uint16 nfr;              /**< Number of frames read so far. */
+
+    unsigned char wsize;    /**< Size of smoothing window. */
+    unsigned char wstart;   /**< First frame in window. */
+    unsigned char wcur;     /**< Current frame of analysis. */
+    unsigned char endut;    /**< Hoch Hech! Are we at the utterance end? */
+
+#ifndef FIXED_POINT
+    float **diff_window;    /**< Window of difference function outputs. */
+#else
+    fixed32 **diff_window;  /**< Window of difference function outputs. */
+#endif
+    uint16 *period_window;  /**< Window of best period estimates. */
+    int16 *frame;           /**< Storage for frame */
+};
+
+/**
+ * The core of YIN: cumulative mean normalized difference function.
+ */
+#ifndef FIXED_POINT
+static void
+cmn_diff(int16 const *signal, float *out_diff, int ndiff)
+{
+    double cum;
+    int t, j;
+
+    cum = 0.0f;
+    out_diff[0] = 1.0f;
+
+    for (t = 1; t < ndiff; ++t) {
+        float dd;
+        dd = 0.0f;
+        for (j = 0; j < ndiff; ++j) {
+             int diff = signal[j] - signal[t + j];
+             dd += (diff * diff);
+        }
+        cum += dd;
+        out_diff[t] = (float)(dd * t / cum);
+    }
+}
+#else
+static void
+cmn_diff(int16 const *signal, int32 *out_diff, int ndiff)
+{
+    uint32 cum, cshift;
+    int32 t, tscale;
+
+    out_diff[0] = 32768;
+    cum = 0;
+    cshift = 0;
+
+    /* Determine how many bits we can scale t up by below. */
+    for (tscale = 0; tscale < 32; ++tscale)
+        if (ndiff & (1<<(31-tscale)))
+            break;
+    --tscale; /* Avoid teh overflowz. */
+    /* printf("tscale is %d (ndiff - 1) << tscale is %d\n",
+       tscale, (ndiff-1) << tscale); */
+
+    /* Somewhat elaborate block floating point implementation.
+     * The fp implementation of this is really a lot simpler. */
+    for (t = 1; t < ndiff; ++t) {
+        uint32 dd, dshift, norm;
+        int j;
+
+        dd = 0;
+        dshift = 0;
+        for (j = 0; j < ndiff; ++j) {
+            int diff = signal[j] - signal[t + j];
+            /* Guard against overflows. */
+            if (dd > (1UL<<tscale)) {
+                dd >>= 1;
+                ++dshift;
+            }
+            dd += (diff * diff) >> dshift;
+        }
+        /* Make sure the diffs and cum are shifted to the same
+         * scaling factor (usually dshift will be zero) */
+        if (dshift > cshift) {
+            cum += dd << (dshift-cshift);
+        }
+        else {
+            cum += dd >> (cshift-dshift);
+        }
+
+        /* Guard against overflows and also ensure that (t<<tscale) > cum. */
+        while (cum > (1UL<<tscale)) {
+            cum >>= 1;
+            ++cshift;
+        }
+        /* Avoid divide-by-zero! */
+        if (cum == 0) cum = 1;
+        /* Calculate the normalizer in high precision. */
+        norm = (t << tscale) / cum;
+        /* Do a long multiply and shift down to Q15. */
+        out_diff[t] = (int32)(((long long)dd * norm)
+                              >> (tscale - 15 + cshift - dshift));
+        /* printf("dd %d cshift %d dshift %d scaledt %d cum %d norm %d cmn %d\n",
+           dd, cshift, dshift, (t<<tscale), cum, norm, out_diff[t]); */
+    }
+}
+#endif
+
+yin_t *
+yin_init(int frame_size, float search_threshold,
+         float search_range, int smooth_window)
+{
+    yin_t *pe;
+
+    pe = ckd_calloc(1, sizeof(*pe));
+    pe->frame_size = frame_size;
+#ifndef FIXED_POINT
+    pe->search_threshold = search_threshold;
+    pe->search_range = search_range;
+#else
+    pe->search_threshold = (uint16)(search_threshold * 32768);
+    pe->search_range = (uint16)(search_range * 32768);
+#endif
+    pe->wsize = smooth_window * 2 + 1;
+    pe->diff_window = ckd_calloc_2d(pe->wsize,
+                                    pe->frame_size / 2,
+                                    sizeof(**pe->diff_window));
+    pe->period_window = ckd_calloc(pe->wsize,
+                                   sizeof(*pe->period_window));
+    pe->frame = ckd_calloc(pe->frame_size, sizeof(*pe->frame));
+    return pe;
+}
+
+void
+yin_free(yin_t *pe)
+{
+    ckd_free_2d(pe->diff_window);
+    ckd_free(pe->period_window);
+    ckd_free(pe);
+}
+
+void
+yin_start(yin_t *pe)
+{
+    /* Reset the circular window pointers. */
+    pe->wstart = pe->endut = 0;
+    pe->nfr = 0;
+}
+
+void
+yin_end(yin_t *pe)
+{
+    pe->endut = 1;
+}
+
+int
+#ifndef FIXED_POINT
+thresholded_search(float *diff_window, float threshold, int start, int end)
+#else
+thresholded_search(int32 *diff_window, fixed32 threshold, int start, int end)
+#endif
+{
+    int i, argmin;
+#ifndef FIXED_POINT
+    float min;
+#else
+    int min;
+#endif
+
+    min = diff_window[start];
+    argmin = start;
+    for (i = start + 1; i < end; ++i) {
+#ifndef FIXED_POINT
+        float diff = diff_window[i];
+#else
+        int diff = diff_window[i];
+#endif
+
+        if (diff < threshold) {
+            min = diff;
+            argmin = i;
+            break;
+        }
+        if (diff < min) {
+            min = diff;
+            argmin = i;
+        }
+    }
+    return argmin;
+}
+
+void 
+yin_store(yin_t *pe, int16 const *frame)
+{
+    memcpy(pe->frame, frame, pe->frame_size * sizeof(*pe->frame));
+}
+
+void
+yin_write(yin_t *pe, int16 const *frame)
+{
+    int outptr, difflen;
+
+    /* Rotate the window one frame forward. */
+    ++pe->wstart;
+    /* Fill in the frame before wstart. */
+    outptr = pe->wstart - 1;
+    /* Wrap around the window pointer. */
+    if (pe->wstart == pe->wsize)
+        pe->wstart = 0;
+
+    /* Now calculate normalized difference function. */
+    difflen = pe->frame_size / 2;
+    cmn_diff(frame, pe->diff_window[outptr], difflen);
+
+    /* Find the first point under threshold.  If not found, then
+     * use the absolute minimum. */
+    pe->period_window[outptr]
+        = thresholded_search(pe->diff_window[outptr],
+                             pe->search_threshold, 0, difflen);
+
+    /* Increment total number of frames. */
+    ++pe->nfr;
+}
+
+void 
+yin_write_stored(yin_t *pe)
+{
+    yin_write(pe, pe->frame);
+}
+
+int
+yin_read(yin_t *pe, uint16 *out_period, float *out_bestdiff)
+{
+    int wstart, wlen, half_wsize, i;
+    int best, search_width, low_period, high_period;
+#ifndef FIXED_POINT
+    float best_diff;
+#else
+    int best_diff;
+#endif
+
+    half_wsize = (pe->wsize-1)/2;
+    /* Without any smoothing, just return the current value (don't
+     * need to do anything to the current poitner either). */
+    if (half_wsize == 0) {
+        if (pe->endut)
+            return 0;
+        *out_period = pe->period_window[0];
+#ifndef FIXED_POINT
+        *out_bestdiff = pe->diff_window[0][pe->period_window[0]];
+#else
+        *out_bestdiff = pe->diff_window[0][pe->period_window[0]] / 32768.0f;
+#endif
+        return 1;
+    }
+
+    /* We can't do anything unless we have at least (wsize-1)/2 + 1
+     * frames, unless we're at the end of the utterance. */
+    if (pe->endut == 0 && pe->nfr < half_wsize + 1) {
+        /* Don't increment the current pointer either. */
+        return 0;
+    }
+
+    /* Establish the smoothing window. */
+    /* End of utterance. */
+    if (pe->endut) {
+        /* We are done (no more data) when pe->wcur = pe->wstart. */
+        if (pe->wcur == pe->wstart)
+            return 0;
+        /* I.e. pe->wcur (circular minus) half_wsize. */
+        wstart = (pe->wcur + pe->wsize - half_wsize) % pe->wsize;
+        /* Number of frames from wstart up to pe->wstart. */
+        wlen = pe->wstart - wstart;
+        if (wlen < 0) wlen += pe->wsize;
+        /*printf("ENDUT! ");*/
+    }
+    /* Beginning of utterance. */
+    else if (pe->nfr < pe->wsize) {
+        wstart = 0;
+        wlen = pe->nfr;
+    }
+    /* Normal case, it is what it is. */
+    else {
+        wstart = pe->wstart;
+        wlen = pe->wsize;
+    }
+
+    /* Now (finally) look for the best local estimate. */
+    /* printf("Searching for local estimate in %d frames around %d\n",
+       wlen, pe->nfr + 1 - wlen); */
+    best = pe->period_window[pe->wcur];
+    best_diff = pe->diff_window[pe->wcur][best];
+    for (i = 0; i < wlen; ++i) {
+        int j = wstart + i;
+#ifndef FIXED_POINT
+        float diff;
+#else
+        int diff;
+#endif
+
+        j %= pe->wsize;
+        diff = pe->diff_window[j][pe->period_window[j]];
+        /* printf("%.2f,%.2f ", 1.0 - (double)diff/32768,
+           pe->period_window[j] ? 8000.0/pe->period_window[j] : 8000.0); */
+        if (diff < best_diff) {
+            best_diff = diff;
+            best = pe->period_window[j];
+        }
+    }
+    /* printf("best: %.2f, %.2f\n", 1.0 - (double)best_diff/32768,
+       best ? 8000.0/best : 8000.0); */
+    /* If it's the same as the current one then return it. */
+    if (best == pe->period_window[pe->wcur]) {
+        /* Increment the current pointer. */
+        if (++pe->wcur == pe->wsize)
+            pe->wcur = 0;
+        *out_period = best;
+#ifndef FIXED_POINT
+        *out_bestdiff = best_diff;
+#else
+        *out_bestdiff = best_diff / 32768.0f;
+#endif
+        return 1;
+    }
+    /* Otherwise, redo the search inside a narrower range. */
+#ifndef FIXED_POINT
+    search_width = (int)(best * pe->search_range);
+#else
+    search_width = best * pe->search_range / 32768;
+#endif
+    /* printf("Search width = %d * %.2f = %d\n",
+       best, (double)pe->search_range/32768, search_width); */
+    if (search_width == 0) search_width = 1;
+    low_period = best - search_width;
+    high_period = best + search_width;
+    if (low_period < 0) low_period = 0;
+    if (high_period > pe->frame_size / 2) high_period = pe->frame_size / 2;
+    /* printf("Searching from %d to %d\n", low_period, high_period); */
+    best = thresholded_search(pe->diff_window[pe->wcur],
+                              pe->search_threshold,
+                              low_period, high_period);
+    best_diff = pe->diff_window[pe->wcur][best];
+
+    if (out_period)
+        *out_period = (best > 65535) ? 65535 : best;
+    if (out_bestdiff) {
+#ifndef FIXED_POINT
+        *out_bestdiff = (best_diff > 1.0f) ? 1.0f : best_diff;
+#else
+        *out_bestdiff = (best_diff > 32768) ? 1.0f : best_diff / 32768.0f;
+#endif
+    }
+
+    /* Increment the current pointer. */
+    if (++pe->wcur == pe->wsize)
+        pe->wcur = 0;
+    return 1;
+}

media/sphinxbase/src/libsphinxbase/feat/agc.c

@@ -0,0 +1,227 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * agc.c -- Various forms of automatic gain control (AGC)
+ * 
+ * **********************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 1996 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ * 
+ * HISTORY
+ * $Log$
+ * Revision 1.5  2005/06/21  19:25:41  arthchan2003
+ * 1, Fixed doxygen documentation. 2, Added $ keyword.
+ * 
+ * Revision 1.3  2005/03/30 01:22:46  archan
+ * Fixed mistakes in last updates. Add
+ *
+ * 
+ * 04-Nov-95	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Created.
+ */
+
+#include <string.h>
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "sphinxbase/err.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/agc.h"
+
+/* NOTE!  These must match the enum in agc.h */
+const char *agc_type_str[] = {
+    "none",
+    "max",
+    "emax",
+    "noise"
+};
+static const int n_agc_type_str = sizeof(agc_type_str)/sizeof(agc_type_str[0]);
+
+agc_type_t
+agc_type_from_str(const char *str)
+{
+    int i;
+
+    for (i = 0; i < n_agc_type_str; ++i) {
+        if (0 == strcmp(str, agc_type_str[i]))
+            return (agc_type_t)i;
+    }
+    E_FATAL("Unknown AGC type '%s'\n", str);
+    return AGC_NONE;
+}
+
+agc_t *agc_init(void)
+{
+    agc_t *agc;
+    agc = ckd_calloc(1, sizeof(*agc));
+    agc->noise_thresh = FLOAT2MFCC(2.0);
+    
+    return agc;
+}
+
+void agc_free(agc_t *agc)
+{
+    ckd_free(agc);
+}
+
+/**
+ * Normalize c0 for all frames such that max(c0) = 0.
+ */
+void
+agc_max(agc_t *agc, mfcc_t **mfc, int32 n_frame)
+{
+    int32 i;
+
+    if (n_frame <= 0)
+        return;
+    agc->obs_max = mfc[0][0];
+    for (i = 1; i < n_frame; i++) {
+        if (mfc[i][0] > agc->obs_max) {
+            agc->obs_max = mfc[i][0];
+            agc->obs_frame = 1;
+        }
+    }
+
+    E_INFO("AGCMax: obs=max= %.2f\n", agc->obs_max);
+    for (i = 0; i < n_frame; i++)
+        mfc[i][0] -= agc->obs_max;
+}
+
+void
+agc_emax_set(agc_t *agc, float32 m)
+{
+    agc->max = FLOAT2MFCC(m);
+    E_INFO("AGCEMax: max= %.2f\n", m);
+}
+
+float32
+agc_emax_get(agc_t *agc)
+{
+    return MFCC2FLOAT(agc->max);
+}
+
+void
+agc_emax(agc_t *agc, mfcc_t **mfc, int32 n_frame)
+{
+    int i;
+
+    if (n_frame <= 0)
+        return;
+    for (i = 0; i < n_frame; ++i) {
+        if (mfc[i][0] > agc->obs_max) {
+            agc->obs_max = mfc[i][0];
+            agc->obs_frame = 1;
+        }
+        mfc[i][0] -= agc->max;
+    }
+}
+
+/* Update estimated max for next utterance */
+void
+agc_emax_update(agc_t *agc)
+{
+    if (agc->obs_frame) {            /* Update only if some data observed */
+        agc->obs_max_sum += agc->obs_max;
+        agc->obs_utt++;
+
+        /* Re-estimate max over past history; decay the history */
+        agc->max = agc->obs_max_sum / agc->obs_utt;
+        if (agc->obs_utt == 16) {
+            agc->obs_max_sum /= 2;
+            agc->obs_utt = 8;
+        }
+    }
+    E_INFO("AGCEMax: obs= %.2f, new= %.2f\n", agc->obs_max, agc->max);
+
+    /* Reset the accumulators for the next utterance. */
+    agc->obs_frame = 0;
+    agc->obs_max = FLOAT2MFCC(-1000.0); /* Less than any real C0 value (hopefully!!) */
+}
+
+void
+agc_noise(agc_t *agc,
+          mfcc_t **cep,
+          int32 nfr)
+{
+    mfcc_t min_energy; /* Minimum log-energy */
+    mfcc_t noise_level;        /* Average noise_level */
+    int32 i;           /* frame index */
+    int32 noise_frames;        /* Number of noise frames */
+
+    /* Determine minimum log-energy in utterance */
+    min_energy = cep[0][0];
+    for (i = 0; i < nfr; ++i) {
+        if (cep[i][0] < min_energy)
+            min_energy = cep[i][0];
+    }
+
+    /* Average all frames between min_energy and min_energy + agc->noise_thresh */
+    noise_frames = 0;
+    noise_level = 0;
+    min_energy += agc->noise_thresh;
+    for (i = 0; i < nfr; ++i) {
+        if (cep[i][0] < min_energy) {
+            noise_level += cep[i][0];
+            noise_frames++;
+        }
+    }
+
+    if (noise_frames > 0) {
+        noise_level /= noise_frames;
+        E_INFO("AGC NOISE: max= %6.3f\n", MFCC2FLOAT(noise_level));
+        /* Subtract noise_level from all log_energy values */
+        for (i = 0; i < nfr; i++) {
+            cep[i][0] -= noise_level;
+        }
+    }
+}
+
+void
+agc_set_threshold(agc_t *agc, float32 threshold)
+{
+    agc->noise_thresh = FLOAT2MFCC(threshold);
+}
+
+float32
+agc_get_threshold(agc_t *agc)
+{
+    return FLOAT2MFCC(agc->noise_thresh);
+}

media/sphinxbase/src/libsphinxbase/feat/cmn.c

@@ -0,0 +1,238 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * cmn.c -- Various forms of cepstral mean normalization
+ *
+ * **********************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 1996 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ * 
+ * HISTORY
+ * $Log$
+ * Revision 1.14  2006/02/24  15:57:47  egouvea
+ * Removed cmn = NULL from the cmn_free(), since it's pointless (my bad!).
+ * 
+ * Removed cmn_prior, which was surrounded by #if 0/#endif, since the
+ * function is already in cmn_prior.c
+ * 
+ * Revision 1.13  2006/02/23 03:47:49  arthchan2003
+ * Used Evandro's changes. Resolved conflicts.
+ *
+ *
+ * Revision 1.12  2006/02/23 00:48:23  egouvea
+ * Replaced loops resetting vectors with the more efficient memset()
+ *
+ * Revision 1.11  2006/02/22 23:43:55  arthchan2003
+ * Merged from the branch SPHINX3_5_2_RCI_IRII_BRANCH: Put data structure into the cmn_t structure.
+ *
+ * Revision 1.10.4.2  2005/10/17 04:45:57  arthchan2003
+ * Free stuffs in cmn and feat corectly.
+ *
+ * Revision 1.10.4.1  2005/07/05 06:25:08  arthchan2003
+ * Fixed dox-doc.
+ *
+ * Revision 1.10  2005/06/21 19:28:00  arthchan2003
+ * 1, Fixed doxygen documentation. 2, Added $ keyword.
+ *
+ * Revision 1.3  2005/03/30 01:22:46  archan
+ * Fixed mistakes in last updates. Add
+ *
+ * 
+ * 20.Apr.2001  RAH (rhoughton@mediasite.com, ricky.houghton@cs.cmu.edu)
+ *              Added cmn_free() and moved *mean and *var out global space and named them cmn_mean and cmn_var
+ * 
+ * 28-Apr-1999	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Changed the name norm_mean() to cmn().
+ * 
+ * 19-Jun-1996	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Changed to compute CMN over ALL dimensions of cep instead of 1..12.
+ * 
+ * 04-Nov-1995	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Created.
+ */
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <math.h>
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4244)
+#endif
+
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/cmn.h"
+
+/* NOTE!  These must match the enum in cmn.h */
+const char *cmn_type_str[] = {
+    "none",
+    "current",
+    "prior"
+};
+static const int n_cmn_type_str = sizeof(cmn_type_str)/sizeof(cmn_type_str[0]);
+
+cmn_type_t
+cmn_type_from_str(const char *str)
+{
+    int i;
+
+    for (i = 0; i < n_cmn_type_str; ++i) {
+        if (0 == strcmp(str, cmn_type_str[i]))
+            return (cmn_type_t)i;
+    }
+    E_FATAL("Unknown CMN type '%s'\n", str);
+    return CMN_NONE;
+}
+
+cmn_t *
+cmn_init(int32 veclen)
+{
+    cmn_t *cmn;
+    cmn = (cmn_t *) ckd_calloc(1, sizeof(cmn_t));
+    cmn->veclen = veclen;
+    cmn->cmn_mean = (mfcc_t *) ckd_calloc(veclen, sizeof(mfcc_t));
+    cmn->cmn_var = (mfcc_t *) ckd_calloc(veclen, sizeof(mfcc_t));
+    cmn->sum = (mfcc_t *) ckd_calloc(veclen, sizeof(mfcc_t));
+    /* A front-end dependent magic number */
+    cmn->cmn_mean[0] = FLOAT2MFCC(12.0);
+    cmn->nframe = 0;
+    E_INFO("mean[0]= %.2f, mean[1..%d]= 0.0\n",
+           MFCC2FLOAT(cmn->cmn_mean[0]), veclen - 1);
+
+    return cmn;
+}
+
+
+void
+cmn(cmn_t *cmn, mfcc_t ** mfc, int32 varnorm, int32 n_frame)
+{
+    mfcc_t *mfcp;
+    mfcc_t t;
+    int32 i, f;
+    int32 n_pos_frame;
+
+    assert(mfc != NULL);
+
+    if (n_frame <= 0)
+        return;
+
+    /* If cmn->cmn_mean wasn't NULL, we need to zero the contents */
+    memset(cmn->cmn_mean, 0, cmn->veclen * sizeof(mfcc_t));
+
+    /* Find mean cep vector for this utterance */
+    for (f = 0, n_pos_frame = 0; f < n_frame; f++) {
+        mfcp = mfc[f];
+
+        /* Skip zero energy frames */
+        if (mfcp[0] < 0)
+    	    continue;
+
+        for (i = 0; i < cmn->veclen; i++) {
+            cmn->cmn_mean[i] += mfcp[i];
+        }
+
+        n_pos_frame++;
+    }
+
+    for (i = 0; i < cmn->veclen; i++)
+        cmn->cmn_mean[i] /= n_pos_frame;
+
+    E_INFO("CMN: ");
+    for (i = 0; i < cmn->veclen; i++)
+        E_INFOCONT("%5.2f ", MFCC2FLOAT(cmn->cmn_mean[i]));
+    E_INFOCONT("\n");
+    if (!varnorm) {
+        /* Subtract mean from each cep vector */
+        for (f = 0; f < n_frame; f++) {
+            mfcp = mfc[f];
+            for (i = 0; i < cmn->veclen; i++)
+                mfcp[i] -= cmn->cmn_mean[i];
+        }
+    }
+    else {
+        /* Scale cep vectors to have unit variance along each dimension, and subtract means */
+        /* If cmn->cmn_var wasn't NULL, we need to zero the contents */
+        memset(cmn->cmn_var, 0, cmn->veclen * sizeof(mfcc_t));
+
+        for (f = 0; f < n_frame; f++) {
+            mfcp = mfc[f];
+
+            for (i = 0; i < cmn->veclen; i++) {
+                t = mfcp[i] - cmn->cmn_mean[i];
+                cmn->cmn_var[i] += MFCCMUL(t, t);
+            }
+        }
+        for (i = 0; i < cmn->veclen; i++)
+            /* Inverse Std. Dev, RAH added type case from sqrt */
+            cmn->cmn_var[i] = FLOAT2MFCC(sqrt((float64)n_frame / MFCC2FLOAT(cmn->cmn_var[i])));
+
+        for (f = 0; f < n_frame; f++) {
+            mfcp = mfc[f];
+            for (i = 0; i < cmn->veclen; i++)
+                mfcp[i] = MFCCMUL((mfcp[i] - cmn->cmn_mean[i]), cmn->cmn_var[i]);
+        }
+    }
+}
+
+/* 
+ * RAH, free previously allocated memory
+ */
+void
+cmn_free(cmn_t * cmn)
+{
+    if (cmn != NULL) {
+        if (cmn->cmn_var)
+            ckd_free((void *) cmn->cmn_var);
+
+        if (cmn->cmn_mean)
+            ckd_free((void *) cmn->cmn_mean);
+
+        if (cmn->sum)
+            ckd_free((void *) cmn->sum);
+
+        ckd_free((void *) cmn);
+    }
+}

media/sphinxbase/src/libsphinxbase/feat/cmn_prior.c

@@ -0,0 +1,184 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*************************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 2000 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ * 
+ * 30-Dec-2000  Rita Singh (rsingh@cs.cmu.edu) at Carnegie Mellon University
+ * Created
+ */
+
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4244)
+#endif
+
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/cmn.h"
+
+void
+cmn_prior_set(cmn_t *cmn, mfcc_t const * vec)
+{
+    int32 i;
+
+    E_INFO("cmn_prior_set: from < ");
+    for (i = 0; i < cmn->veclen; i++)
+        E_INFOCONT("%5.2f ", MFCC2FLOAT(cmn->cmn_mean[i]));
+    E_INFOCONT(">\n");
+
+    for (i = 0; i < cmn->veclen; i++) {
+        cmn->cmn_mean[i] = vec[i];
+        cmn->sum[i] = vec[i] * CMN_WIN;
+    }
+    cmn->nframe = CMN_WIN;
+
+    E_INFO("cmn_prior_set: to   < ");
+    for (i = 0; i < cmn->veclen; i++)
+        E_INFOCONT("%5.2f ", MFCC2FLOAT(cmn->cmn_mean[i]));
+    E_INFOCONT(">\n");
+}
+
+void
+cmn_prior_get(cmn_t *cmn, mfcc_t * vec)
+{
+    int32 i;
+
+    for (i = 0; i < cmn->veclen; i++)
+        vec[i] = cmn->cmn_mean[i];
+
+}
+
+static void
+cmn_prior_shiftwin(cmn_t *cmn)
+{
+    mfcc_t sf;
+    int32 i;
+
+    E_INFO("cmn_prior_update: from < ");
+    for (i = 0; i < cmn->veclen; i++)
+        E_INFOCONT("%5.2f ", MFCC2FLOAT(cmn->cmn_mean[i]));
+    E_INFOCONT(">\n");
+
+    sf = FLOAT2MFCC(1.0) / cmn->nframe;
+    for (i = 0; i < cmn->veclen; i++)
+        cmn->cmn_mean[i] = cmn->sum[i] / cmn->nframe; /* sum[i] * sf */
+
+    /* Make the accumulation decay exponentially */
+    if (cmn->nframe >= CMN_WIN_HWM) {
+        sf = CMN_WIN * sf;
+        for (i = 0; i < cmn->veclen; i++)
+            cmn->sum[i] = MFCCMUL(cmn->sum[i], sf);
+        cmn->nframe = CMN_WIN;
+    }
+
+    E_INFO("cmn_prior_update: to   < ");
+    for (i = 0; i < cmn->veclen; i++)
+        E_INFOCONT("%5.2f ", MFCC2FLOAT(cmn->cmn_mean[i]));
+    E_INFOCONT(">\n");
+}
+
+void
+cmn_prior_update(cmn_t *cmn)
+{
+    mfcc_t sf;
+    int32 i;
+
+    if (cmn->nframe <= 0)
+        return;
+
+    E_INFO("cmn_prior_update: from < ");
+    for (i = 0; i < cmn->veclen; i++)
+        E_INFOCONT("%5.2f ", MFCC2FLOAT(cmn->cmn_mean[i]));
+    E_INFOCONT(">\n");
+
+    /* Update mean buffer */
+    sf = FLOAT2MFCC(1.0) / cmn->nframe;
+    for (i = 0; i < cmn->veclen; i++)
+        cmn->cmn_mean[i] = cmn->sum[i] / cmn->nframe; /* sum[i] * sf; */
+
+    /* Make the accumulation decay exponentially */
+    if (cmn->nframe > CMN_WIN_HWM) {
+        sf = CMN_WIN * sf;
+        for (i = 0; i < cmn->veclen; i++)
+            cmn->sum[i] = MFCCMUL(cmn->sum[i], sf);
+        cmn->nframe = CMN_WIN;
+    }
+
+    E_INFO("cmn_prior_update: to   < ");
+    for (i = 0; i < cmn->veclen; i++)
+        E_INFOCONT("%5.2f ", MFCC2FLOAT(cmn->cmn_mean[i]));
+    E_INFOCONT(">\n");
+}
+
+void
+cmn_prior(cmn_t *cmn, mfcc_t **incep, int32 varnorm, int32 nfr)
+{
+    int32 i, j;
+
+    if (nfr <= 0)
+        return;
+
+    if (varnorm)
+        E_FATAL
+            ("Variance normalization not implemented in live mode decode\n");
+
+    for (i = 0; i < nfr; i++) {
+
+	/* Skip zero energy frames */
+	if (incep[i][0] < 0)
+	    continue;
+
+        for (j = 0; j < cmn->veclen; j++) {
+            cmn->sum[j] += incep[i][j];
+            incep[i][j] -= cmn->cmn_mean[j];
+        }
+
+        ++cmn->nframe;
+    }
+
+    /* Shift buffer down if we have more than CMN_WIN_HWM frames */
+    if (cmn->nframe > CMN_WIN_HWM)
+        cmn_prior_shiftwin(cmn);
+}

media/sphinxbase/src/libsphinxbase/feat/lda.c

@@ -0,0 +1,158 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2006 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * lda.c -- Read and apply LDA matrices to features.
+ *
+ * Author: David Huggins-Daines <dhuggins@cs.cmu.edu>
+ */
+
+#include <assert.h>
+#include <string.h>
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4018)
+#endif
+
+#include "sphinxbase/feat.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/bio.h"
+#include "sphinxbase/err.h"
+
+#define MATRIX_FILE_VERSION "0.1"
+
+int32
+feat_read_lda(feat_t *feat, const char *ldafile, int32 dim)
+{
+    FILE *fh;
+    int32 byteswap;
+    uint32 chksum, i, m, n;
+    char **argname, **argval;
+
+    assert(feat);
+    if (feat->n_stream != 1) {
+        E_ERROR("LDA incompatible with multi-stream features (n_stream = %d)\n",
+                feat->n_stream);
+        return -1;
+    }
+
+    if ((fh = fopen(ldafile, "rb")) == NULL) {
+        E_ERROR_SYSTEM("Failed to open transform file '%s' for reading", ldafile);
+        return -1;
+    }
+
+    if (bio_readhdr(fh, &argname, &argval, &byteswap) < 0) {
+        E_ERROR("Failed to read header from transform file '%s'\n", ldafile);
+        fclose(fh);
+        return -1;
+    }
+
+    for (i = 0; argname[i]; i++) {
+        if (strcmp(argname[i], "version") == 0) {
+            if (strcmp(argval[i], MATRIX_FILE_VERSION) != 0)
+                E_WARN("%s: Version mismatch: %s, expecting %s\n",
+                       ldafile, argval[i], MATRIX_FILE_VERSION);
+        }
+    }
+
+    bio_hdrarg_free(argname, argval);
+    argname = argval = NULL;
+
+    chksum = 0;
+
+    if (feat->lda)
+        ckd_free_3d((void ***)feat->lda);
+
+    {
+        /* Use a temporary variable to avoid strict-aliasing problems. */
+        void ***outlda;
+
+        if (bio_fread_3d(&outlda, sizeof(float32),
+                         &feat->n_lda, &m, &n,
+                         fh, byteswap, &chksum) < 0) {
+            E_ERROR_SYSTEM("%s: bio_fread_3d(lda) failed\n", ldafile);
+            fclose(fh);
+            return -1;
+        }
+        feat->lda = (void *)outlda;
+    }
+    fclose(fh);
+    
+#ifdef FIXED_POINT
+    /* FIXME: This is a fragile hack that depends on mfcc_t and
+     * float32 being the same size (which they are, but...) */
+    for (i = 0; i < feat->n_lda * m * n; ++i) {
+        feat->lda[0][0][i] = FLOAT2MFCC(((float *)feat->lda[0][0])[i]);
+    }
+#endif
+
+    /* Note that SphinxTrain stores the eigenvectors as row vectors. */
+    if (n != feat->stream_len[0])
+	E_FATAL("LDA matrix dimension %d doesn't match feature stream size %d\n", n, feat->stream_len[0]);
+    
+    /* Override dim from file if it is 0 or greater than m. */
+    if (dim > m || dim <= 0) {
+        dim = m;
+    }
+    feat->out_dim = dim;
+
+    return 0;
+}
+
+void
+feat_lda_transform(feat_t *fcb, mfcc_t ***inout_feat, uint32 nfr)
+{
+    mfcc_t *tmp;
+    uint32 i, j, k;
+
+    tmp = ckd_calloc(fcb->stream_len[0], sizeof(mfcc_t));
+    for (i = 0; i < nfr; ++i) {
+        /* Do the matrix multiplication inline here since fcb->lda
+         * is transposed (eigenvectors in rows not columns). */
+        /* FIXME: In the future we ought to use the BLAS. */
+        memset(tmp, 0, sizeof(mfcc_t) * fcb->stream_len[0]);
+        for (j = 0; j < feat_dimension(fcb); ++j) {
+            for (k = 0; k < fcb->stream_len[0]; ++k) {
+                tmp[j] += MFCCMUL(inout_feat[i][0][k], fcb->lda[0][j][k]);
+            }
+        }
+        memcpy(inout_feat[i][0], tmp, fcb->stream_len[0] * sizeof(mfcc_t));
+    }
+    ckd_free(tmp);
+}

media/sphinxbase/src/libsphinxbase/lm/fsg_model.c

@@ -0,0 +1,944 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+/* System headers. */
+#ifdef _WIN32_WCE
+/*MC in a debug build it's implicitly included by assert.h
+     but you need this in a release build */
+#include <windows.h>
+#else
+#include <time.h>
+#endif                          /* _WIN32_WCE */
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+/* SphinxBase headers. */
+#include "sphinxbase/err.h"
+#include "sphinxbase/pio.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/prim_type.h"
+#include "sphinxbase/strfuncs.h"
+#include "sphinxbase/hash_table.h"
+#include "sphinxbase/fsg_model.h"
+
+/**
+ * Adjacency list (opaque) for a state in an FSG.
+ *
+ * Actually we use hash tables so that random access is a bit faster.
+ * Plus it allows us to make the lookup code a bit less ugly.
+ */
+
+struct trans_list_s {
+    hash_table_t *null_trans;   /* Null transitions keyed by state. */
+    hash_table_t *trans;        /* Lists of non-null transitions keyed by state. */
+};
+
+/**
+ * Implementation of arc iterator.
+ */
+struct fsg_arciter_s {
+    hash_iter_t *itor, *null_itor;
+    gnode_t *gn;
+};
+
+#define FSG_MODEL_BEGIN_DECL		"FSG_BEGIN"
+#define FSG_MODEL_END_DECL		"FSG_END"
+#define FSG_MODEL_N_DECL			"N"
+#define FSG_MODEL_NUM_STATES_DECL	"NUM_STATES"
+#define FSG_MODEL_S_DECL			"S"
+#define FSG_MODEL_START_STATE_DECL	"START_STATE"
+#define FSG_MODEL_F_DECL			"F"
+#define FSG_MODEL_FINAL_STATE_DECL	"FINAL_STATE"
+#define FSG_MODEL_T_DECL			"T"
+#define FSG_MODEL_TRANSITION_DECL	"TRANSITION"
+#define FSG_MODEL_COMMENT_CHAR		'#'
+
+
+static int32
+nextline_str2words(FILE * fp, int32 * lineno,
+                   char **lineptr, char ***wordptr)
+{
+    for (;;) {
+        size_t len;
+        int32 n;
+
+        ckd_free(*lineptr);
+        if ((*lineptr = fread_line(fp, &len)) == NULL)
+            return -1;
+
+        (*lineno)++;
+
+        if ((*lineptr)[0] == FSG_MODEL_COMMENT_CHAR)
+            continue;           /* Skip comment lines */
+
+        n = str2words(*lineptr, NULL, 0);
+        if (n == 0)
+            continue;           /* Skip blank lines */
+
+        /* Abuse of realloc(), but this doesn't have to be fast. */
+        if (*wordptr == NULL)
+            *wordptr = ckd_calloc(n, sizeof(**wordptr));
+        else
+            *wordptr = ckd_realloc(*wordptr, n * sizeof(**wordptr));
+        return str2words(*lineptr, *wordptr, n);
+    }
+}
+
+void
+fsg_model_trans_add(fsg_model_t * fsg,
+                    int32 from, int32 to, int32 logp, int32 wid)
+{
+    fsg_link_t *link;
+    glist_t gl;
+    gnode_t *gn;
+
+    if (fsg->trans[from].trans == NULL)
+        fsg->trans[from].trans = hash_table_new(5, HASH_CASE_YES);
+
+    /* Check for duplicate link (i.e., link already exists with label=wid) */
+    for (gn = gl = fsg_model_trans(fsg, from, to); gn; gn = gnode_next(gn)) {
+        link = (fsg_link_t *) gnode_ptr(gn);
+        if (link->wid == wid) {
+            if (link->logs2prob < logp)
+                link->logs2prob = logp;
+            return;
+        }
+    }
+
+    /* Create transition object */
+    link = listelem_malloc(fsg->link_alloc);
+    link->from_state = from;
+    link->to_state = to;
+    link->logs2prob = logp;
+    link->wid = wid;
+
+    /* Add it to the list of transitions and update the hash table */
+    gl = glist_add_ptr(gl, (void *) link);
+    hash_table_replace_bkey(fsg->trans[from].trans,
+                            (char const *) &link->to_state,
+                            sizeof(link->to_state), gl);
+}
+
+int32
+fsg_model_tag_trans_add(fsg_model_t * fsg, int32 from, int32 to,
+                        int32 logp, int32 wid)
+{
+    fsg_link_t *link, *link2;
+
+    /* Check for transition probability */
+    if (logp > 0) {
+        E_FATAL("Null transition prob must be <= 1.0 (state %d -> %d)\n",
+                from, to);
+    }
+
+    /* Self-loop null transitions (with prob <= 1.0) are redundant */
+    if (from == to)
+        return -1;
+
+    if (fsg->trans[from].null_trans == NULL)
+        fsg->trans[from].null_trans = hash_table_new(5, HASH_CASE_YES);
+
+    /* Check for a duplicate link; if found, keep the higher prob */
+    link = fsg_model_null_trans(fsg, from, to);
+    if (link) {
+        if (link->logs2prob < logp) {
+            link->logs2prob = logp;
+            return 0;
+        }
+        else
+            return -1;
+    }
+
+    /* Create null transition object */
+    link = listelem_malloc(fsg->link_alloc);
+    link->from_state = from;
+    link->to_state = to;
+    link->logs2prob = logp;
+    link->wid = -1;
+
+    link2 = (fsg_link_t *)
+        hash_table_enter_bkey(fsg->trans[from].null_trans,
+                              (char const *) &link->to_state,
+                              sizeof(link->to_state), link);
+    assert(link == link2);
+
+    return 1;
+}
+
+int32
+fsg_model_null_trans_add(fsg_model_t * fsg, int32 from, int32 to,
+                         int32 logp)
+{
+    return fsg_model_tag_trans_add(fsg, from, to, logp, -1);
+}
+
+glist_t
+fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls)
+{
+    gnode_t *gn1;
+    int updated;
+    fsg_link_t *tl1, *tl2;
+    int32 k, n;
+
+    E_INFO("Computing transitive closure for null transitions\n");
+
+    /* If our caller didn't give us a list of null-transitions,
+       make such a list. Just loop through all the FSG states, 
+       and all the null-transitions in that state (which are kept in
+       their own hash table). */
+    if (nulls == NULL) {
+        int i;
+        for (i = 0; i < fsg->n_state; ++i) {
+            hash_iter_t *itor;
+            hash_table_t *null_trans = fsg->trans[i].null_trans;
+            if (null_trans == NULL)
+                continue;
+            for (itor = hash_table_iter(null_trans);
+                 itor != NULL;
+                 itor = hash_table_iter_next(itor)) {
+                nulls = glist_add_ptr(nulls, hash_entry_val(itor->ent));
+            }
+        }
+    }
+
+    /*
+     * Probably not the most efficient closure implementation, in general, but
+     * probably reasonably efficient for a sparse null transition matrix.
+     */
+    n = 0;
+    do {
+        updated = FALSE;
+
+        for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) {
+            hash_iter_t *itor;
+
+            tl1 = (fsg_link_t *) gnode_ptr(gn1);
+            assert(tl1->wid < 0);
+
+            if (fsg->trans[tl1->to_state].null_trans == NULL)
+                continue;
+
+            for (itor = hash_table_iter(fsg->trans[tl1->to_state].null_trans);
+                 itor; itor = hash_table_iter_next(itor)) {
+
+                tl2 = (fsg_link_t *) hash_entry_val(itor->ent);
+
+                k = fsg_model_null_trans_add(fsg,
+                                             tl1->from_state,
+                                             tl2->to_state,
+                                             tl1->logs2prob +
+                                             tl2->logs2prob);
+                if (k >= 0) {
+                    updated = TRUE;
+                    if (k > 0) {
+                        nulls = glist_add_ptr(nulls, (void *)
+                                              fsg_model_null_trans
+                                              (fsg, tl1->from_state,
+                                               tl2->to_state));
+                        n++;
+                    }
+                }
+            }
+        }
+    } while (updated);
+    
+    E_INFO("%d null transitions added\n", n);
+
+    return nulls;
+}
+
+glist_t
+fsg_model_trans(fsg_model_t * fsg, int32 i, int32 j)
+{
+    void *val;
+
+    if (fsg->trans[i].trans == NULL)
+        return NULL;
+    if (hash_table_lookup_bkey(fsg->trans[i].trans, (char const *) &j,
+                               sizeof(j), &val) < 0)
+        return NULL;
+    return (glist_t) val;
+}
+
+fsg_link_t *
+fsg_model_null_trans(fsg_model_t * fsg, int32 i, int32 j)
+{
+    void *val;
+
+    if (fsg->trans[i].null_trans == NULL)
+        return NULL;
+    if (hash_table_lookup_bkey(fsg->trans[i].null_trans, (char const *) &j,
+                               sizeof(j), &val) < 0)
+        return NULL;
+    return (fsg_link_t *) val;
+}
+
+fsg_arciter_t *
+fsg_model_arcs(fsg_model_t * fsg, int32 i)
+{
+    fsg_arciter_t *itor;
+
+    if (fsg->trans[i].trans == NULL && fsg->trans[i].null_trans == NULL)
+        return NULL;
+    itor = ckd_calloc(1, sizeof(*itor));
+    if (fsg->trans[i].null_trans)
+        itor->null_itor = hash_table_iter(fsg->trans[i].null_trans);
+    if (fsg->trans[i].trans)
+        itor->itor = hash_table_iter(fsg->trans[i].trans);
+    if (itor->itor != NULL)
+        itor->gn = hash_entry_val(itor->itor->ent);
+    return itor;
+}
+
+fsg_link_t *
+fsg_arciter_get(fsg_arciter_t * itor)
+{
+    /* Iterate over non-null arcs first. */
+    if (itor->gn)
+        return (fsg_link_t *) gnode_ptr(itor->gn);
+    else if (itor->null_itor)
+        return (fsg_link_t *) hash_entry_val(itor->null_itor->ent);
+    else
+        return NULL;
+}
+
+fsg_arciter_t *
+fsg_arciter_next(fsg_arciter_t * itor)
+{
+    /* Iterate over non-null arcs first. */
+    if (itor->gn) {
+        itor->gn = gnode_next(itor->gn);
+        /* Move to the next destination arc. */
+        if (itor->gn == NULL) {
+            itor->itor = hash_table_iter_next(itor->itor);
+            if (itor->itor != NULL)
+                itor->gn = hash_entry_val(itor->itor->ent);
+            else if (itor->null_itor == NULL)
+                goto stop_iteration;
+        }
+    }
+    else {
+        if (itor->null_itor == NULL)
+            goto stop_iteration;
+        itor->null_itor = hash_table_iter_next(itor->null_itor);
+        if (itor->null_itor == NULL)
+            goto stop_iteration;
+    }
+    return itor;
+  stop_iteration:
+    fsg_arciter_free(itor);
+    return NULL;
+
+}
+
+void
+fsg_arciter_free(fsg_arciter_t * itor)
+{
+    if (itor == NULL)
+        return;
+    hash_table_iter_free(itor->null_itor);
+    hash_table_iter_free(itor->itor);
+    ckd_free(itor);
+}
+
+int
+fsg_model_word_id(fsg_model_t * fsg, char const *word)
+{
+    int wid;
+
+    /* Search for an existing word matching this. */
+    for (wid = 0; wid < fsg->n_word; ++wid) {
+        if (0 == strcmp(fsg->vocab[wid], word))
+            break;
+    }
+    /* If not found, add this to the vocab. */
+    if (wid == fsg->n_word)
+        return -1;
+    return wid;
+}
+
+int
+fsg_model_word_add(fsg_model_t * fsg, char const *word)
+{
+    int wid, old_size;
+
+    /* Search for an existing word matching this. */
+    wid = fsg_model_word_id(fsg, word);
+    /* If not found, add this to the vocab. */
+    if (wid == -1) {
+        wid = fsg->n_word;
+        if (fsg->n_word == fsg->n_word_alloc) {
+    	    old_size = fsg->n_word_alloc;
+            fsg->n_word_alloc += 10;
+            fsg->vocab = ckd_realloc(fsg->vocab,
+                                     fsg->n_word_alloc *
+                                     sizeof(*fsg->vocab));
+            if (fsg->silwords)
+                fsg->silwords =
+                    bitvec_realloc(fsg->silwords, old_size, fsg->n_word_alloc);
+            if (fsg->altwords)
+                fsg->altwords =
+                    bitvec_realloc(fsg->altwords, old_size, fsg->n_word_alloc);
+        }
+        ++fsg->n_word;
+        fsg->vocab[wid] = ckd_salloc(word);
+    }
+    return wid;
+}
+
+int
+fsg_model_add_silence(fsg_model_t * fsg, char const *silword,
+                      int state, float32 silprob)
+{
+    int32 logsilp;
+    int n_trans, silwid, src;
+
+    E_INFO("Adding silence transitions for %s to FSG\n", silword);
+
+    silwid = fsg_model_word_add(fsg, silword);
+    logsilp = (int32) (logmath_log(fsg->lmath, silprob) * fsg->lw);
+    if (fsg->silwords == NULL)
+        fsg->silwords = bitvec_alloc(fsg->n_word_alloc);
+    bitvec_set(fsg->silwords, silwid);
+
+    n_trans = 0;
+    if (state == -1) {
+        for (src = 0; src < fsg->n_state; src++) {
+            fsg_model_trans_add(fsg, src, src, logsilp, silwid);
+            ++n_trans;
+        }
+    }
+    else {
+        fsg_model_trans_add(fsg, state, state, logsilp, silwid);
+        ++n_trans;
+    }
+
+    E_INFO("Added %d silence word transitions\n", n_trans);
+    return n_trans;
+}
+
+int
+fsg_model_add_alt(fsg_model_t * fsg, char const *baseword,
+                  char const *altword)
+{
+    int i, basewid, altwid;
+    int ntrans;
+
+    /* FIXME: This will get slow, eventually... */
+    for (basewid = 0; basewid < fsg->n_word; ++basewid)
+        if (0 == strcmp(fsg->vocab[basewid], baseword))
+            break;
+    if (basewid == fsg->n_word) {
+        E_ERROR("Base word %s not present in FSG vocabulary!\n", baseword);
+        return -1;
+    }
+    altwid = fsg_model_word_add(fsg, altword);
+    if (fsg->altwords == NULL)
+        fsg->altwords = bitvec_alloc(fsg->n_word_alloc);
+    bitvec_set(fsg->altwords, altwid);
+    if (fsg_model_is_filler(fsg, basewid)) {
+         if (fsg->silwords == NULL)
+	      fsg->silwords = bitvec_alloc(fsg->n_word_alloc);
+         bitvec_set(fsg->silwords, altwid);
+    }
+
+    E_DEBUG(2, ("Adding alternate word transitions (%s,%s) to FSG\n",
+                baseword, altword));
+
+    /* Look for all transitions involving baseword and duplicate them. */
+    /* FIXME: This will also get slow, eventually... */
+    ntrans = 0;
+    for (i = 0; i < fsg->n_state; ++i) {
+        hash_iter_t *itor;
+        if (fsg->trans[i].trans == NULL)
+            continue;
+        for (itor = hash_table_iter(fsg->trans[i].trans); itor;
+             itor = hash_table_iter_next(itor)) {
+            glist_t trans;
+            gnode_t *gn;
+
+            trans = hash_entry_val(itor->ent);
+            for (gn = trans; gn; gn = gnode_next(gn)) {
+                fsg_link_t *fl = gnode_ptr(gn);
+                if (fl->wid == basewid) {
+                    fsg_link_t *link;
+
+                    /* Create transition object */
+                    link = listelem_malloc(fsg->link_alloc);
+                    link->from_state = fl->from_state;
+                    link->to_state = fl->to_state;
+                    link->logs2prob = fl->logs2prob;    /* FIXME!!!??? */
+                    link->wid = altwid;
+
+                    trans = glist_add_ptr(trans, (void *) link);
+                    ++ntrans;
+                }
+            }
+            hash_entry_val(itor->ent) = trans;
+        }
+    }
+
+    E_DEBUG(2, ("Added %d alternate word transitions\n", ntrans));
+    return ntrans;
+}
+
+
+fsg_model_t *
+fsg_model_init(char const *name, logmath_t * lmath, float32 lw,
+               int32 n_state)
+{
+    fsg_model_t *fsg;
+
+    /* Allocate basic stuff. */
+    fsg = ckd_calloc(1, sizeof(*fsg));
+    fsg->refcount = 1;
+    fsg->link_alloc = listelem_alloc_init(sizeof(fsg_link_t));
+    fsg->lmath = lmath;
+    fsg->name = name ? ckd_salloc(name) : NULL;
+    fsg->n_state = n_state;
+    fsg->lw = lw;
+
+    fsg->trans = ckd_calloc(fsg->n_state, sizeof(*fsg->trans));
+
+    return fsg;
+}
+
+fsg_model_t *
+fsg_model_read(FILE * fp, logmath_t * lmath, float32 lw)
+{
+    fsg_model_t *fsg;
+    hash_table_t *vocab;
+    hash_iter_t *itor;
+    int32 lastwid;
+    char **wordptr;
+    char *lineptr;
+    char *fsgname;
+    int32 lineno;
+    int32 n, i, j;
+    int n_state, n_trans, n_null_trans;
+    glist_t nulls;
+    float32 p;
+
+    lineno = 0;
+    vocab = hash_table_new(32, FALSE);
+    wordptr = NULL;
+    lineptr = NULL;
+    nulls = NULL;
+    fsgname = NULL;
+    fsg = NULL;
+
+    /* Scan upto FSG_BEGIN header */
+    for (;;) {
+        n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
+        if (n < 0) {
+            E_ERROR("%s declaration missing\n", FSG_MODEL_BEGIN_DECL);
+            goto parse_error;
+        }
+
+        if ((strcmp(wordptr[0], FSG_MODEL_BEGIN_DECL) == 0)) {
+            if (n > 2) {
+                E_ERROR("Line[%d]: malformed FSG_BEGIN declaration\n",
+                        lineno);
+                goto parse_error;
+            }
+            break;
+        }
+    }
+    /* Save FSG name, or it will get clobbered below :(.
+     * If name is missing, try the default.
+     */
+    if (n == 2) {
+        fsgname = ckd_salloc(wordptr[1]);
+    }
+    else {
+        E_WARN("FSG name is missing\n");
+        fsgname = ckd_salloc("unknown");
+    }
+
+    /* Read #states */
+    n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
+    if ((n != 2)
+        || ((strcmp(wordptr[0], FSG_MODEL_N_DECL) != 0)
+            && (strcmp(wordptr[0], FSG_MODEL_NUM_STATES_DECL) != 0))
+        || (sscanf(wordptr[1], "%d", &n_state) != 1)
+        || (n_state <= 0)) {
+        E_ERROR
+            ("Line[%d]: #states declaration line missing or malformed\n",
+             lineno);
+        goto parse_error;
+    }
+
+    /* Now create the FSG. */
+    fsg = fsg_model_init(fsgname, lmath, lw, n_state);
+    ckd_free(fsgname);
+    fsgname = NULL;
+
+    /* Read start state */
+    n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
+    if ((n != 2)
+        || ((strcmp(wordptr[0], FSG_MODEL_S_DECL) != 0)
+            && (strcmp(wordptr[0], FSG_MODEL_START_STATE_DECL) != 0))
+        || (sscanf(wordptr[1], "%d", &(fsg->start_state)) != 1)
+        || (fsg->start_state < 0)
+        || (fsg->start_state >= fsg->n_state)) {
+        E_ERROR
+            ("Line[%d]: start state declaration line missing or malformed\n",
+             lineno);
+        goto parse_error;
+    }
+
+    /* Read final state */
+    n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
+    if ((n != 2)
+        || ((strcmp(wordptr[0], FSG_MODEL_F_DECL) != 0)
+            && (strcmp(wordptr[0], FSG_MODEL_FINAL_STATE_DECL) != 0))
+        || (sscanf(wordptr[1], "%d", &(fsg->final_state)) != 1)
+        || (fsg->final_state < 0)
+        || (fsg->final_state >= fsg->n_state)) {
+        E_ERROR
+            ("Line[%d]: final state declaration line missing or malformed\n",
+             lineno);
+        goto parse_error;
+    }
+
+    /* Read transitions */
+    lastwid = 0;
+    n_trans = n_null_trans = 0;
+    for (;;) {
+        int32 wid, tprob;
+
+        n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
+        if (n <= 0) {
+            E_ERROR("Line[%d]: transition or FSG_END statement expected\n",
+                    lineno);
+            goto parse_error;
+        }
+
+        if ((strcmp(wordptr[0], FSG_MODEL_END_DECL) == 0)) {
+            break;
+        }
+
+        if ((strcmp(wordptr[0], FSG_MODEL_T_DECL) == 0)
+            || (strcmp(wordptr[0], FSG_MODEL_TRANSITION_DECL) == 0)) {
+
+
+            if (((n != 4) && (n != 5))
+                || (sscanf(wordptr[1], "%d", &i) != 1)
+                || (sscanf(wordptr[2], "%d", &j) != 1)
+                || (i < 0) || (i >= fsg->n_state)
+                || (j < 0) || (j >= fsg->n_state)) {
+                E_ERROR
+                    ("Line[%d]: transition spec malformed; Expecting: from-state to-state trans-prob [word]\n",
+                     lineno);
+                goto parse_error;
+            }
+
+            p = atof_c(wordptr[3]);
+            if ((p <= 0.0) || (p > 1.0)) {
+                E_ERROR
+                    ("Line[%d]: transition spec malformed; Expecting float as transition probability\n",
+                     lineno);
+                goto parse_error;
+            }
+        }
+        else {
+            E_ERROR("Line[%d]: transition or FSG_END statement expected\n",
+                    lineno);
+            goto parse_error;
+        }
+
+        tprob = (int32) (logmath_log(lmath, p) * fsg->lw);
+        /* Add word to "dictionary". */
+        if (n > 4) {
+            if (hash_table_lookup_int32(vocab, wordptr[4], &wid) < 0) {
+                (void) hash_table_enter_int32(vocab,
+                                              ckd_salloc(wordptr[4]),
+                                              lastwid);
+                wid = lastwid;
+                ++lastwid;
+            }
+            fsg_model_trans_add(fsg, i, j, tprob, wid);
+            ++n_trans;
+        }
+        else {
+            if (fsg_model_null_trans_add(fsg, i, j, tprob) == 1) {
+                ++n_null_trans;
+                nulls =
+                    glist_add_ptr(nulls, fsg_model_null_trans(fsg, i, j));
+            }
+        }
+    }
+
+    E_INFO("FSG: %d states, %d unique words, %d transitions (%d null)\n",
+           fsg->n_state, hash_table_inuse(vocab), n_trans, n_null_trans);
+
+
+    /* Now create a string table from the "dictionary" */
+    fsg->n_word = hash_table_inuse(vocab);
+    fsg->n_word_alloc = fsg->n_word + 10;       /* Pad it a bit. */
+    fsg->vocab = ckd_calloc(fsg->n_word_alloc, sizeof(*fsg->vocab));
+    for (itor = hash_table_iter(vocab); itor;
+         itor = hash_table_iter_next(itor)) {
+        char const *word = hash_entry_key(itor->ent);
+        int32 wid = (int32) (long) hash_entry_val(itor->ent);
+        fsg->vocab[wid] = (char *) word;
+    }
+    hash_table_free(vocab);
+
+    /* Do transitive closure on null transitions */
+    nulls = fsg_model_null_trans_closure(fsg, nulls);
+    glist_free(nulls);
+
+    ckd_free(lineptr);
+    ckd_free(wordptr);
+
+    return fsg;
+
+  parse_error:
+    for (itor = hash_table_iter(vocab); itor;
+         itor = hash_table_iter_next(itor))
+        ckd_free((char *) hash_entry_key(itor->ent));
+    glist_free(nulls);
+    hash_table_free(vocab);
+    ckd_free(fsgname);
+    ckd_free(lineptr);
+    ckd_free(wordptr);
+    fsg_model_free(fsg);
+    return NULL;
+}
+
+
+fsg_model_t *
+fsg_model_readfile(const char *file, logmath_t * lmath, float32 lw)
+{
+    FILE *fp;
+    fsg_model_t *fsg;
+
+    if ((fp = fopen(file, "r")) == NULL) {
+        E_ERROR_SYSTEM("Failed to open FSG file '%s' for reading", file);
+        return NULL;
+    }
+    fsg = fsg_model_read(fp, lmath, lw);
+    fclose(fp);
+    return fsg;
+}
+
+fsg_model_t *
+fsg_model_retain(fsg_model_t * fsg)
+{
+    ++fsg->refcount;
+    return fsg;
+}
+
+static void
+trans_list_free(fsg_model_t * fsg, int32 i)
+{
+    hash_iter_t *itor;
+
+    /* FIXME (maybe): FSG links will all get freed when we call
+     * listelem_alloc_free() so don't bother freeing them explicitly
+     * here. */
+    if (fsg->trans[i].trans) {
+        for (itor = hash_table_iter(fsg->trans[i].trans);
+             itor; itor = hash_table_iter_next(itor)) {
+            glist_t gl = (glist_t) hash_entry_val(itor->ent);
+            glist_free(gl);
+        }
+    }
+    hash_table_free(fsg->trans[i].trans);
+    hash_table_free(fsg->trans[i].null_trans);
+}
+
+int
+fsg_model_free(fsg_model_t * fsg)
+{
+    int i;
+
+    if (fsg == NULL)
+        return 0;
+
+    if (--fsg->refcount > 0)
+        return fsg->refcount;
+
+    for (i = 0; i < fsg->n_word; ++i)
+        ckd_free(fsg->vocab[i]);
+    for (i = 0; i < fsg->n_state; ++i)
+        trans_list_free(fsg, i);
+    ckd_free(fsg->trans);
+    ckd_free(fsg->vocab);
+    listelem_alloc_free(fsg->link_alloc);
+    bitvec_free(fsg->silwords);
+    bitvec_free(fsg->altwords);
+    ckd_free(fsg->name);
+    ckd_free(fsg);
+    return 0;
+}
+
+
+void
+fsg_model_write(fsg_model_t * fsg, FILE * fp)
+{
+    int32 i;
+
+    fprintf(fp, "%s %s\n", FSG_MODEL_BEGIN_DECL,
+            fsg->name ? fsg->name : "");
+    fprintf(fp, "%s %d\n", FSG_MODEL_NUM_STATES_DECL, fsg->n_state);
+    fprintf(fp, "%s %d\n", FSG_MODEL_START_STATE_DECL, fsg->start_state);
+    fprintf(fp, "%s %d\n", FSG_MODEL_FINAL_STATE_DECL, fsg->final_state);
+
+    for (i = 0; i < fsg->n_state; i++) {
+        fsg_arciter_t *itor;
+
+        for (itor = fsg_model_arcs(fsg, i); itor;
+             itor = fsg_arciter_next(itor)) {
+            fsg_link_t *tl = fsg_arciter_get(itor);
+
+            fprintf(fp, "%s %d %d %f %s\n", FSG_MODEL_TRANSITION_DECL,
+                    tl->from_state, tl->to_state,
+                    logmath_exp(fsg->lmath,
+                                (int32) (tl->logs2prob / fsg->lw)),
+                    (tl->wid < 0) ? "" : fsg_model_word_str(fsg, tl->wid));
+        }
+    }
+
+    fprintf(fp, "%s\n", FSG_MODEL_END_DECL);
+
+    fflush(fp);
+}
+
+void
+fsg_model_writefile(fsg_model_t * fsg, char const *file)
+{
+    FILE *fp;
+
+    assert(fsg);
+
+    E_INFO("Writing FSG file '%s'\n", file);
+
+    if ((fp = fopen(file, "w")) == NULL) {
+        E_ERROR_SYSTEM("Failed to open FSG file '%s' for reading", file);
+        return;
+    }
+
+    fsg_model_write(fsg, fp);
+
+    fclose(fp);
+}
+
+static void
+fsg_model_write_fsm_trans(fsg_model_t * fsg, int i, FILE * fp)
+{
+    fsg_arciter_t *itor;
+
+    for (itor = fsg_model_arcs(fsg, i); itor;
+         itor = fsg_arciter_next(itor)) {
+        fsg_link_t *tl = fsg_arciter_get(itor);
+        fprintf(fp, "%d %d %s %f\n",
+                tl->from_state, tl->to_state,
+                (tl->wid < 0) ? "<eps>" : fsg_model_word_str(fsg, tl->wid),
+                -logmath_log_to_ln(fsg->lmath, tl->logs2prob / fsg->lw));
+    }
+}
+
+void
+fsg_model_write_fsm(fsg_model_t * fsg, FILE * fp)
+{
+    int i;
+
+    /* Write transitions from initial state first. */
+    fsg_model_write_fsm_trans(fsg, fsg_model_start_state(fsg), fp);
+
+    /* Other states. */
+    for (i = 0; i < fsg->n_state; i++) {
+        if (i == fsg_model_start_state(fsg))
+            continue;
+        fsg_model_write_fsm_trans(fsg, i, fp);
+    }
+
+    /* Final state. */
+    fprintf(fp, "%d 0\n", fsg_model_final_state(fsg));
+
+    fflush(fp);
+}
+
+void
+fsg_model_writefile_fsm(fsg_model_t * fsg, char const *file)
+{
+    FILE *fp;
+
+    assert(fsg);
+
+    E_INFO("Writing FSM file '%s'\n", file);
+
+    if ((fp = fopen(file, "w")) == NULL) {
+        E_ERROR_SYSTEM("Failed to open fsm file '%s' for writing", file);
+        return;
+    }
+
+    fsg_model_write_fsm(fsg, fp);
+
+    fclose(fp);
+}
+
+void
+fsg_model_write_symtab(fsg_model_t * fsg, FILE * file)
+{
+    int i;
+
+    fprintf(file, "<eps> 0\n");
+    for (i = 0; i < fsg_model_n_word(fsg); ++i) {
+        fprintf(file, "%s %d\n", fsg_model_word_str(fsg, i), i + 1);
+    }
+    fflush(file);
+}
+
+void
+fsg_model_writefile_symtab(fsg_model_t * fsg, char const *file)
+{
+    FILE *fp;
+
+    assert(fsg);
+
+    E_INFO("Writing FSM symbol table '%s'\n", file);
+
+    if ((fp = fopen(file, "w")) == NULL) {
+        E_ERROR("Failed to open symbol table '%s' for writing", file);
+        return;
+    }
+
+    fsg_model_write_symtab(fsg, fp);
+
+    fclose(fp);
+}

media/sphinxbase/src/libsphinxbase/lm/jsgf.c

@@ -0,0 +1,943 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+#include <string.h>
+#include <assert.h>
+
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/strfuncs.h"
+#include "sphinxbase/hash_table.h"
+#include "sphinxbase/filename.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/jsgf.h"
+
+#include "jsgf_internal.h"
+#include "jsgf_parser.h"
+#include "jsgf_scanner.h"
+
+extern int yyparse (void* scanner, jsgf_t* jsgf);
+
+/**
+ * \file jsgf.c
+ *
+ * This file implements the data structures for parsing JSGF grammars
+ * into Sphinx finite-state grammars.
+ **/
+
+static int expand_rule(jsgf_t *grammar, jsgf_rule_t *rule, int rule_entry, int rule_exit);
+
+jsgf_atom_t *
+jsgf_atom_new(char *name, float weight)
+{
+    jsgf_atom_t *atom;
+
+    atom = ckd_calloc(1, sizeof(*atom));
+    atom->name = ckd_salloc(name);
+    atom->weight = weight;
+    return atom;
+}
+
+int
+jsgf_atom_free(jsgf_atom_t *atom)
+{
+    if (atom == NULL)
+        return 0;
+    ckd_free(atom->name);
+    ckd_free(atom);
+    return 0;
+}
+
+jsgf_t *
+jsgf_grammar_new(jsgf_t *parent)
+{
+    jsgf_t *grammar;
+
+    grammar = ckd_calloc(1, sizeof(*grammar));
+    /* If this is an imported/subgrammar, then we will share a global
+     * namespace with the parent grammar. */
+    if (parent) {
+        grammar->rules = parent->rules;
+        grammar->imports = parent->imports;
+        grammar->searchpath = parent->searchpath;
+        grammar->parent = parent;
+    }
+    else {
+        grammar->rules = hash_table_new(64, 0);
+        grammar->imports = hash_table_new(16, 0);
+    }
+
+    return grammar;
+}
+
+void
+jsgf_grammar_free(jsgf_t *jsgf)
+{
+    /* FIXME: Probably should just use refcounting instead. */
+    if (jsgf->parent == NULL) {
+        hash_iter_t *itor;
+        gnode_t *gn;
+
+        for (itor = hash_table_iter(jsgf->rules); itor;
+             itor = hash_table_iter_next(itor)) {
+            ckd_free((char *)itor->ent->key);
+            jsgf_rule_free((jsgf_rule_t *)itor->ent->val);
+        }
+        hash_table_free(jsgf->rules);
+        for (itor = hash_table_iter(jsgf->imports); itor;
+             itor = hash_table_iter_next(itor)) {
+            ckd_free((char *)itor->ent->key);
+            jsgf_grammar_free((jsgf_t *)itor->ent->val);
+        }
+        hash_table_free(jsgf->imports);
+        for (gn = jsgf->searchpath; gn; gn = gnode_next(gn))
+            ckd_free(gnode_ptr(gn));
+        glist_free(jsgf->searchpath);
+        for (gn = jsgf->links; gn; gn = gnode_next(gn))
+            ckd_free(gnode_ptr(gn));
+        glist_free(jsgf->links);
+    }
+    ckd_free(jsgf->name);
+    ckd_free(jsgf->version);
+    ckd_free(jsgf->charset);
+    ckd_free(jsgf->locale);
+    ckd_free(jsgf);
+}
+
+static void
+jsgf_rhs_free(jsgf_rhs_t *rhs)
+{
+    gnode_t *gn;
+
+    if (rhs == NULL)
+        return;
+
+    jsgf_rhs_free(rhs->alt);
+    for (gn = rhs->atoms; gn; gn = gnode_next(gn))
+        jsgf_atom_free(gnode_ptr(gn));
+    glist_free(rhs->atoms);
+    ckd_free(rhs);
+}
+
+jsgf_atom_t *
+jsgf_kleene_new(jsgf_t *jsgf, jsgf_atom_t *atom, int plus)
+{
+    jsgf_rule_t *rule;
+    jsgf_atom_t *rule_atom;
+    jsgf_rhs_t *rhs;
+
+    /* Generate an "internal" rule of the form (<NULL> | <name> <g0006>) */
+    /* Or if plus is true, (<name> | <name> <g0006>) */
+    rhs = ckd_calloc(1, sizeof(*rhs));
+    if (plus)
+        rhs->atoms = glist_add_ptr(NULL, jsgf_atom_new(atom->name, 1.0));
+    else
+        rhs->atoms = glist_add_ptr(NULL, jsgf_atom_new("<NULL>", 1.0));
+    rule = jsgf_define_rule(jsgf, NULL, rhs, 0);
+    rule_atom = jsgf_atom_new(rule->name, 1.0);
+    rhs = ckd_calloc(1, sizeof(*rhs));
+    rhs->atoms = glist_add_ptr(NULL, rule_atom);
+    rhs->atoms = glist_add_ptr(rhs->atoms, atom);
+    rule->rhs->alt = rhs;
+
+    return jsgf_atom_new(rule->name, 1.0);
+}
+
+jsgf_rule_t *
+jsgf_optional_new(jsgf_t *jsgf, jsgf_rhs_t *exp)
+{
+    jsgf_rhs_t *rhs = ckd_calloc(1, sizeof(*rhs));
+    jsgf_atom_t *atom = jsgf_atom_new("<NULL>", 1.0);
+    rhs->alt = exp;
+    rhs->atoms = glist_add_ptr(NULL, atom);
+    return jsgf_define_rule(jsgf, NULL, rhs, 0);
+}
+
+void
+jsgf_add_link(jsgf_t *grammar, jsgf_atom_t *atom, int from, int to)
+{
+    jsgf_link_t *link;
+
+    link = ckd_calloc(1, sizeof(*link));
+    link->from = from;
+    link->to = to;
+    link->atom = atom;
+    grammar->links = glist_add_ptr(grammar->links, link);
+}
+
+static char *
+extract_grammar_name(char *rule_name)
+{
+    char* dot_pos;
+    char* grammar_name = ckd_salloc(rule_name + 1);
+    if ((dot_pos = strrchr(grammar_name + 1, '.')) == NULL) {
+        ckd_free(grammar_name);
+        return NULL;
+    }
+    *dot_pos='\0';
+    return grammar_name;
+}
+
+char const *
+jsgf_grammar_name(jsgf_t *jsgf)
+{
+    return jsgf->name;
+}
+
+static char *
+jsgf_fullname(jsgf_t *jsgf, const char *name)
+{
+    char *fullname;
+
+    /* Check if it is already qualified */
+    if (strchr(name + 1, '.'))
+        return ckd_salloc(name);
+
+    /* Skip leading < in name */
+    fullname = ckd_malloc(strlen(jsgf->name) + strlen(name) + 4);
+    sprintf(fullname, "<%s.%s", jsgf->name, name + 1);
+    return fullname;
+}
+
+static char *
+jsgf_fullname_from_rule(jsgf_rule_t *rule, const char *name)
+{
+    char *fullname, *grammar_name;
+
+    /* Check if it is already qualified */
+    if (strchr(name + 1, '.'))
+        return ckd_salloc(name);
+
+    /* Skip leading < in name */
+    if ((grammar_name = extract_grammar_name(rule->name)) == NULL)
+        return ckd_salloc(name);
+    fullname = ckd_malloc(strlen(grammar_name) + strlen(name) + 4);
+    sprintf(fullname, "<%s.%s", grammar_name, name + 1);
+    ckd_free(grammar_name);
+
+    return fullname;
+}
+
+/* Extract as rulename everything after the secondlast dot, if existent. 
+ * Because everything before the secondlast dot is the path-specification. */
+static char *
+importname2rulename(char *importname)
+{
+    char *rulename = ckd_salloc(importname);
+    char *last_dotpos;
+    char *secondlast_dotpos;
+
+    if ((last_dotpos = strrchr(rulename+1, '.')) != NULL) {
+        *last_dotpos='\0';
+        if ((secondlast_dotpos = strrchr(rulename+1, '.')) != NULL) {
+            *last_dotpos='.';
+            *secondlast_dotpos='<';
+            secondlast_dotpos = ckd_salloc(secondlast_dotpos);
+            ckd_free(rulename);
+            return secondlast_dotpos;
+        }
+        else {
+            *last_dotpos='.';
+            return rulename;
+        }
+    }
+    else {
+        return rulename;
+    }
+}
+
+#define NO_NODE -1
+#define RECURSIVE_NODE -2
+
+/**
+ *
+ * Expand a right-hand-side of a rule (i.e. a single alternate).
+ *
+ * @returns the FSG state at the end of this rule, NO_NODE if there's an
+ * error, and RECURSIVE_NODE if the right-hand-side ended in right-recursion (i.e.
+ * a link to an earlier FSG state).
+ */
+static int
+expand_rhs(jsgf_t *grammar, jsgf_rule_t *rule, jsgf_rhs_t *rhs,
+           int rule_entry, int rule_exit)
+{
+    gnode_t *gn;
+    int lastnode;
+
+    /* Last node expanded in this sequence. */
+    lastnode = rule_entry;
+
+    /* Iterate over atoms in rhs and generate links/nodes */
+    for (gn = rhs->atoms; gn; gn = gnode_next(gn)) {
+        jsgf_atom_t *atom = gnode_ptr(gn);
+
+        if (jsgf_atom_is_rule(atom)) {
+            jsgf_rule_t *subrule;
+            char *fullname;
+            gnode_t *subnode;
+            jsgf_rule_stack_t *rule_stack_entry = NULL;
+
+            /* Special case for <NULL> and <VOID> pseudo-rules             
+               If this is the only atom in the rhs, and it's the 
+               first rhs in the rule, then emit a null transition, 
+               creating an exit state if needed. */
+            if (0 == strcmp(atom->name, "<NULL>")) {
+                if (gn == rhs->atoms && gnode_next(gn) == NULL) {
+                    if (rule_exit == NO_NODE) {
+                        jsgf_add_link(grammar, atom,
+                                      lastnode, grammar->nstate);
+                        rule_exit = lastnode = grammar->nstate;
+                        ++grammar->nstate;
+                    } else {
+                        jsgf_add_link(grammar, atom,
+                                      lastnode, rule_exit);
+                    }
+                }
+                continue;
+            }
+            else if (0 == strcmp(atom->name, "<VOID>")) {
+                /* Make this entire RHS unspeakable */
+                return NO_NODE;
+            }
+
+            fullname = jsgf_fullname_from_rule(rule, atom->name);
+            if (hash_table_lookup(grammar->rules, fullname, (void**)&subrule) == -1) {
+                E_ERROR("Undefined rule in RHS: %s\n", fullname);
+                ckd_free(fullname);
+                return NO_NODE;
+            }
+            ckd_free(fullname);
+
+            /* Look for this subrule in the stack of expanded rules */
+            for (subnode = grammar->rulestack; subnode; subnode = gnode_next(subnode)) {
+                rule_stack_entry = (jsgf_rule_stack_t *)gnode_ptr(subnode);
+                if (rule_stack_entry->rule == subrule)
+                    break;
+            }
+
+            if (subnode != NULL) {
+                /* Allow right-recursion only. */
+                if (gnode_next(gn) != NULL) {
+                    E_ERROR("Only right-recursion is permitted (in %s.%s)\n",
+                            grammar->name, rule->name);
+                    return NO_NODE;
+                }
+                /* Add a link back to the beginning of this rule instance */
+                E_INFO("Right recursion %s %d => %d\n", atom->name, lastnode, rule_stack_entry->entry);
+                jsgf_add_link(grammar, atom, lastnode, rule_stack_entry->entry);
+
+                /* Let our caller know that this rhs didn't reach an
+                   end state. */
+                lastnode = RECURSIVE_NODE;
+            }
+            else {
+                /* If this is the last atom in this rhs, link its
+                   expansion to the parent rule's exit state.
+                   Otherwise, create a new exit state for it. */
+                int subruleexit = NO_NODE;
+                if (gnode_next(gn) == NULL && rule_exit >= 0)
+                    subruleexit = rule_exit;
+
+                /* Expand the subrule */
+                lastnode = expand_rule(grammar, subrule, lastnode, subruleexit);
+                
+                if (lastnode == NO_NODE)
+                     return NO_NODE;
+            }
+        }
+        else {
+            /* An exit-state is created if this isn't the last atom
+               in the rhs, or if the containing rule doesn't have an
+               exit state yet.
+               Otherwise, the rhs's exit state becomes the containing
+               rule's exit state. */
+            int exitstate;
+            if (gnode_next(gn) == NULL && rule_exit >= 0) {
+                exitstate = rule_exit;
+            } else {
+                exitstate = grammar->nstate;
+                ++grammar->nstate;
+            }
+
+            /* Add a link for this token */
+            jsgf_add_link(grammar, atom,
+                          lastnode, exitstate);
+            lastnode = exitstate;
+        }
+    }
+
+    return lastnode;
+}
+
+static int
+expand_rule(jsgf_t *grammar, jsgf_rule_t *rule, int rule_entry,
+    int rule_exit)
+{
+    jsgf_rule_stack_t* rule_stack_entry;
+    jsgf_rhs_t *rhs;
+
+    /* Push this rule onto the stack */
+    rule_stack_entry = (jsgf_rule_stack_t*)ckd_calloc(1, sizeof (jsgf_rule_stack_t));
+    rule_stack_entry->rule = rule;
+    rule_stack_entry->entry = rule_entry;
+    grammar->rulestack = glist_add_ptr(grammar->rulestack,
+        rule_stack_entry);
+
+    for (rhs = rule->rhs; rhs; rhs = rhs->alt) {
+        int lastnode;
+
+        lastnode = expand_rhs(grammar, rule, rhs,
+            rule_entry, rule_exit);
+
+        if (lastnode == NO_NODE) {
+            return NO_NODE;
+        } else if (lastnode == RECURSIVE_NODE) {
+            /* The rhs ended with right-recursion, i.e. a transition to
+               an earlier state. Nothing needs to happen at this level. */
+            ;
+        } else if (rule_exit == NO_NODE) {
+            /* If this rule doesn't have an exit state yet, use the exit
+             state of its first right-hand-side.
+             All other right-hand-sides will use this exit state. */
+            assert (lastnode >= 0);
+            rule_exit = lastnode;
+        }
+    }
+
+    /* If no exit-state was created, use the entry-state. */
+    if (rule_exit == NO_NODE) {
+        rule_exit = rule_entry;
+    }
+
+    /* Pop this rule from the rule stack */
+    ckd_free(gnode_ptr(grammar->rulestack));
+    grammar->rulestack = gnode_free(grammar->rulestack, NULL);
+
+    return rule_exit;
+}
+
+jsgf_rule_iter_t *
+jsgf_rule_iter(jsgf_t *grammar)
+{
+    return hash_table_iter(grammar->rules);
+}
+
+jsgf_rule_t *
+jsgf_get_rule(jsgf_t *grammar, char const *name)
+{
+    void *val;
+    char *fullname;
+    
+    fullname = string_join("<", name, ">", NULL);
+    if (hash_table_lookup(grammar->rules, fullname, &val) < 0) {
+	ckd_free(fullname);
+        return NULL;
+    }
+    ckd_free(fullname);
+    return (jsgf_rule_t *)val;
+}
+
+jsgf_rule_t *
+jsgf_get_public_rule(jsgf_t *grammar)
+{
+    jsgf_rule_iter_t *itor;
+    jsgf_rule_t *public_rule = NULL;
+
+    for (itor = jsgf_rule_iter(grammar); itor;
+         itor = jsgf_rule_iter_next(itor)) {
+        jsgf_rule_t *rule = jsgf_rule_iter_rule(itor);
+        if (jsgf_rule_public(rule)) {
+    	    const char *rule_name = jsgf_rule_name(rule);
+    	    char *dot_pos;
+    	    if ((dot_pos = strrchr(rule_name + 1, '.')) == NULL) {
+    		public_rule = rule;
+                jsgf_rule_iter_free(itor);
+                break;
+    	    }
+    	    if (0 == strncmp(rule_name + 1, jsgf_grammar_name(grammar), dot_pos - rule_name - 1)) {
+    		public_rule = rule;
+                jsgf_rule_iter_free(itor);
+                break;    		
+    	    }
+        }
+    }
+    return public_rule;
+}
+
+char const *
+jsgf_rule_name(jsgf_rule_t *rule)
+{
+    return rule->name;
+}
+
+int
+jsgf_rule_public(jsgf_rule_t *rule)
+{
+    return rule->is_public;
+}
+
+static fsg_model_t *
+jsgf_build_fsg_internal(jsgf_t *grammar, jsgf_rule_t *rule,
+                        logmath_t *lmath, float32 lw, int do_closure)
+{
+    fsg_model_t *fsg;
+    glist_t nulls;
+    gnode_t *gn;
+    int rule_entry, rule_exit;
+
+    /* Clear previous links */
+    for (gn = grammar->links; gn; gn = gnode_next(gn)) {
+        ckd_free(gnode_ptr(gn));
+    }
+    glist_free(grammar->links);
+    grammar->links = NULL;
+    grammar->nstate = 0;
+
+    /* Create the top-level entry state, and expand the
+       top-level rule. */
+    rule_entry = grammar->nstate++;
+    rule_exit = expand_rule(grammar, rule, rule_entry, NO_NODE);
+
+    /* If no exit-state was created, create one. */
+    if (rule_exit == NO_NODE) {
+        rule_exit = grammar->nstate++;
+        jsgf_add_link(grammar, NULL, rule_entry, rule_exit);
+    }
+
+    fsg = fsg_model_init(rule->name, lmath, lw, grammar->nstate);
+    fsg->start_state = rule_entry;
+    fsg->final_state = rule_exit;
+    grammar->links = glist_reverse(grammar->links);
+    for (gn = grammar->links; gn; gn = gnode_next(gn)) {
+        jsgf_link_t *link = gnode_ptr(gn);
+
+        if (link->atom) {
+            if (jsgf_atom_is_rule(link->atom)) {
+                fsg_model_null_trans_add(fsg, link->from, link->to,
+                                        logmath_log(lmath, link->atom->weight));
+            }
+            else {
+                int wid = fsg_model_word_add(fsg, link->atom->name);
+                fsg_model_trans_add(fsg, link->from, link->to,
+                                   logmath_log(lmath, link->atom->weight), wid);
+            }
+        }
+        else {
+            fsg_model_null_trans_add(fsg, link->from, link->to, 0);
+        }            
+    }
+    if (do_closure) {
+        nulls = fsg_model_null_trans_closure(fsg, NULL);
+        glist_free(nulls);
+    }
+
+    return fsg;
+}
+
+fsg_model_t *
+jsgf_build_fsg(jsgf_t *grammar, jsgf_rule_t *rule,
+               logmath_t *lmath, float32 lw)
+{
+    return jsgf_build_fsg_internal(grammar, rule, lmath, lw, TRUE);
+}
+
+fsg_model_t *
+jsgf_build_fsg_raw(jsgf_t *grammar, jsgf_rule_t *rule,
+                   logmath_t *lmath, float32 lw)
+{
+    return jsgf_build_fsg_internal(grammar, rule, lmath, lw, FALSE);
+}
+
+fsg_model_t *
+jsgf_read_file(const char *file, logmath_t * lmath, float32 lw)
+{
+    fsg_model_t *fsg;
+    jsgf_rule_t *rule;
+    jsgf_t *jsgf;
+    jsgf_rule_iter_t *itor;
+
+    if ((jsgf = jsgf_parse_file(file, NULL)) == NULL) {
+        E_ERROR("Error parsing file: %s\n", file);
+        return NULL;
+    }
+
+    rule = NULL;
+    for (itor = jsgf_rule_iter(jsgf); itor;
+         itor = jsgf_rule_iter_next(itor)) {
+        rule = jsgf_rule_iter_rule(itor);
+        if (jsgf_rule_public(rule)) {
+            jsgf_rule_iter_free(itor);
+            break;
+        }
+    }
+    if (rule == NULL) {
+        E_ERROR("No public rules found in %s\n", file);
+        return NULL;
+    }
+    fsg = jsgf_build_fsg(jsgf, rule, lmath, lw);
+    jsgf_grammar_free(jsgf);
+    return fsg;
+}
+
+fsg_model_t *
+jsgf_read_string(const char *string, logmath_t * lmath, float32 lw)
+{
+    fsg_model_t *fsg;
+    jsgf_rule_t *rule;
+    jsgf_t *jsgf;
+    jsgf_rule_iter_t *itor;
+
+    if ((jsgf = jsgf_parse_string(string, NULL)) == NULL) {
+        E_ERROR("Error parsing input string\n");
+        return NULL;
+    }
+
+    rule = NULL;
+    for (itor = jsgf_rule_iter(jsgf); itor;
+         itor = jsgf_rule_iter_next(itor)) {
+        rule = jsgf_rule_iter_rule(itor);
+        if (jsgf_rule_public(rule)) {
+            jsgf_rule_iter_free(itor);
+            break;
+        }
+    }
+    if (rule == NULL) {
+        jsgf_grammar_free(jsgf);
+        E_ERROR("No public rules found in input string\n");
+        return NULL;
+    }
+    fsg = jsgf_build_fsg(jsgf, rule, lmath, lw);
+    jsgf_grammar_free(jsgf);
+    return fsg;
+}
+
+
+int
+jsgf_write_fsg(jsgf_t *grammar, jsgf_rule_t *rule, FILE *outfh)
+{
+    fsg_model_t *fsg;
+    logmath_t *lmath = logmath_init(1.0001, 0, 0);
+
+    if ((fsg = jsgf_build_fsg_raw(grammar, rule, lmath, 1.0)) == NULL)
+        goto error_out;
+
+    fsg_model_write(fsg, outfh);
+    logmath_free(lmath);
+    return 0;
+
+error_out:
+    logmath_free(lmath);
+    return -1;
+}
+
+jsgf_rule_t *
+jsgf_define_rule(jsgf_t *jsgf, char *name, jsgf_rhs_t *rhs, int is_public)
+{
+    jsgf_rule_t *rule;
+    void *val;
+
+    if (name == NULL) {
+        name = ckd_malloc(strlen(jsgf->name) + 16);
+        sprintf(name, "<%s.g%05d>", jsgf->name, hash_table_inuse(jsgf->rules));
+    }
+    else {
+        char *newname;
+
+        newname = jsgf_fullname(jsgf, name);
+        name = newname;
+    }
+
+    rule = ckd_calloc(1, sizeof(*rule));
+    rule->refcnt = 1;
+    rule->name = ckd_salloc(name);
+    rule->rhs = rhs;
+    rule->is_public = is_public;
+
+    E_INFO("Defined rule: %s%s\n",
+           rule->is_public ? "PUBLIC " : "",
+           rule->name);
+    val = hash_table_enter(jsgf->rules, name, rule);
+    if (val != (void *)rule) {
+        E_WARN("Multiply defined symbol: %s\n", name);
+    }
+    return rule;
+}
+
+jsgf_rule_t *
+jsgf_rule_retain(jsgf_rule_t *rule)
+{
+    ++rule->refcnt;
+    return rule;
+}
+
+int
+jsgf_rule_free(jsgf_rule_t *rule)
+{
+    if (rule == NULL)
+        return 0;
+    if (--rule->refcnt > 0)
+        return rule->refcnt;
+    jsgf_rhs_free(rule->rhs);
+    ckd_free(rule->name);
+    ckd_free(rule);
+    return 0;
+}
+
+
+/* FIXME: This should go in libsphinxutil */
+static char *
+path_list_search(glist_t paths, char *path)
+{
+    gnode_t *gn;
+
+    for (gn = paths; gn; gn = gnode_next(gn)) {
+        char *fullpath;
+        FILE *tmp;
+
+        fullpath = string_join(gnode_ptr(gn), "/", path, NULL);
+        tmp = fopen(fullpath, "r");
+        if (tmp != NULL) {
+            fclose(tmp);
+            return fullpath;
+        }
+        else {
+            ckd_free(fullpath);
+        }
+    }
+    return NULL;
+}
+
+jsgf_rule_t *
+jsgf_import_rule(jsgf_t *jsgf, char *name)
+{
+    char *c, *path, *newpath;
+    size_t namelen, packlen;
+    void *val;
+    jsgf_t *imp;
+    int import_all;
+
+    /* Trim the leading and trailing <> */
+    namelen = strlen(name);
+    path = ckd_malloc(namelen - 2 + 6); /* room for a trailing .gram */
+    strcpy(path, name + 1);
+    /* Split off the first part of the name */
+    c = strrchr(path, '.');
+    if (c == NULL) {
+        E_ERROR("Imported rule is not qualified: %s\n", name);
+        ckd_free(path);
+        return NULL;
+    }
+    packlen = c - path;
+    *c = '\0';
+
+    /* Look for import foo.* */
+    import_all = (strlen(name) > 2 && 0 == strcmp(name + namelen - 3, ".*>"));
+
+    /* Construct a filename. */
+    for (c = path; *c; ++c)
+        if (*c == '.') *c = '/';
+    strcat(path, ".gram");
+    newpath = path_list_search(jsgf->searchpath, path);
+    if (newpath == NULL) {
+	E_ERROR("Failed to find grammar %s\n", path);
+        ckd_free(path);
+        return NULL;
+    }
+    ckd_free(path);
+
+    path = newpath;
+    E_INFO("Importing %s from %s to %s\n", name, path, jsgf->name);
+
+    /* FIXME: Also, we need to make sure that path is fully qualified
+     * here, by adding any prefixes from jsgf->name to it. */
+    /* See if we have parsed it already */
+    if (hash_table_lookup(jsgf->imports, path, &val) == 0) {
+        E_INFO("Already imported %s\n", path);
+        imp = val;
+        ckd_free(path);
+    }
+    else {
+        /* If not, parse it. */
+        imp = jsgf_parse_file(path, jsgf);
+        val = hash_table_enter(jsgf->imports, path, imp);
+        if (val != (void *)imp) {
+            E_WARN("Multiply imported file: %s\n", path);
+        }
+    }
+    if (imp != NULL) {
+        hash_iter_t *itor;
+        /* Look for public rules matching rulename. */
+        for (itor = hash_table_iter(imp->rules); itor;
+             itor = hash_table_iter_next(itor)) {
+            hash_entry_t *he = itor->ent;
+            jsgf_rule_t *rule = hash_entry_val(he);
+            int rule_matches;
+            char *rule_name = importname2rulename(name);
+
+            if (import_all) {
+                /* Match package name (symbol table is shared) */
+                rule_matches = !strncmp(rule_name, rule->name, packlen + 1);
+            }
+            else {
+                /* Exact match */
+                rule_matches = !strcmp(rule_name, rule->name);
+            }
+            ckd_free(rule_name);
+            if (rule->is_public && rule_matches) {
+                void *val;
+                char *newname;
+
+                /* Link this rule into the current namespace. */
+                c = strrchr(rule->name, '.');
+                assert(c != NULL);
+                newname = jsgf_fullname(jsgf, c);
+
+                E_INFO("Imported %s\n", newname);
+                val = hash_table_enter(jsgf->rules, newname,
+                                       jsgf_rule_retain(rule));
+                if (val != (void *)rule) {
+                    E_WARN("Multiply defined symbol: %s\n", newname);
+                }
+                if (!import_all) {
+                    hash_table_iter_free(itor);
+                    return rule;
+                }
+            }
+        }
+    }
+
+    return NULL;
+}
+
+static void
+jsgf_set_search_path(jsgf_t *jsgf, const char *filename)
+{
+     char *jsgf_path;
+
+#if !defined(_WIN32_WCE)
+     if ((jsgf_path = getenv("JSGF_PATH")) != NULL) {
+	char *word, *c;
+        /* FIXME: This should be a function in libsphinxbase. */
+        word = jsgf_path = ckd_salloc(jsgf_path);
+        while ((c = strchr(word, ':'))) {
+            *c = '\0';
+	     jsgf->searchpath = glist_add_ptr(jsgf->searchpath, word);
+    	     word = c + 1;
+        }
+        jsgf->searchpath = glist_add_ptr(jsgf->searchpath, word);
+        jsgf->searchpath = glist_reverse(jsgf->searchpath);
+        return;
+    }
+#endif
+
+    if (!filename) {
+	jsgf->searchpath = glist_add_ptr(jsgf->searchpath, ckd_salloc("."));
+	return;
+    }
+    
+    jsgf_path = ckd_salloc(filename);
+    path2dirname(filename, jsgf_path);
+    jsgf->searchpath = glist_add_ptr(jsgf->searchpath, jsgf_path);
+}
+
+jsgf_t *
+jsgf_parse_file(const char *filename, jsgf_t *parent)
+{
+    yyscan_t yyscanner;
+    jsgf_t *jsgf;
+    int yyrv;
+    FILE *in = NULL;
+
+    yylex_init(&yyscanner);
+    if (filename == NULL) {
+        yyset_in(stdin, yyscanner);
+    }
+    else {
+        in = fopen(filename, "r");
+        if (in == NULL) {
+            E_ERROR_SYSTEM("Failed to open %s for parsing", filename);
+            return NULL;
+        }
+        yyset_in(in, yyscanner);
+    }
+
+    jsgf = jsgf_grammar_new(parent);
+
+    if (!parent)
+        jsgf_set_search_path(jsgf, filename);
+
+    yyrv = yyparse(yyscanner, jsgf);
+    if (yyrv != 0) {
+        E_ERROR("Failed to parse JSGF grammar from '%s'\n", filename ? filename : "(stdin)");
+        jsgf_grammar_free(jsgf);
+        yylex_destroy(yyscanner);
+        return NULL;
+    }
+    if (in)
+        fclose(in);
+    yylex_destroy(yyscanner);
+
+    return jsgf;
+}
+
+jsgf_t *
+jsgf_parse_string(const char *string, jsgf_t * parent)
+{
+    yyscan_t yyscanner;
+    jsgf_t *jsgf;
+    int yyrv;
+    YY_BUFFER_STATE buf;
+
+    yylex_init(&yyscanner);
+    buf = yy_scan_string(string, yyscanner);
+
+    jsgf = jsgf_grammar_new(parent);
+    if (!parent)
+        jsgf_set_search_path(jsgf, NULL);
+
+    yyrv = yyparse(yyscanner, jsgf);
+    if (yyrv != 0) {
+        E_ERROR("Failed to parse JSGF grammar from input string\n");
+        jsgf_grammar_free(jsgf);
+        yy_delete_buffer(buf, yyscanner);
+        yylex_destroy(yyscanner);
+        return NULL;
+    }
+    yy_delete_buffer(buf, yyscanner);
+    yylex_destroy(yyscanner);
+
+    return jsgf;
+}

media/sphinxbase/src/libsphinxbase/lm/jsgf_internal.h

@@ -0,0 +1,140 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+#ifndef __JSGF_INTERNAL_H__
+#define __JSGF_INTERNAL_H__
+
+/**
+ * @file jsgf_internal.h Internal definitions for JSGF grammar compiler
+ */
+
+#include <stdio.h>
+
+#include <sphinxbase/hash_table.h>
+#include <sphinxbase/glist.h>
+#include <sphinxbase/fsg_model.h>
+#include <sphinxbase/logmath.h>
+#include <sphinxbase/strfuncs.h>
+#include <sphinxbase/jsgf.h>
+
+
+/* Flex uses strdup which is missing on WinCE */
+#if defined(_WIN32) || defined(_WIN32_WCE)
+#define strdup _strdup
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#if 0
+/* Fool Emacs. */
+}
+#endif
+
+#define YY_NO_INPUT /* Silence a compiler warning. */
+
+typedef struct jsgf_rhs_s jsgf_rhs_t;
+typedef struct jsgf_atom_s jsgf_atom_t;
+typedef struct jsgf_link_s jsgf_link_t;
+typedef struct jsgf_rule_stack_s jsgf_rule_stack_t;
+
+struct jsgf_s {
+    char *version;  /**< JSGF version (from header) */
+    char *charset;  /**< JSGF charset (default UTF-8) */
+    char *locale;   /**< JSGF locale (default C) */
+    char *name;     /**< Grammar name */
+
+    hash_table_t *rules;   /**< Defined or imported rules in this grammar. */
+    hash_table_t *imports; /**< Pointers to imported grammars. */
+    jsgf_t *parent;        /**< Parent grammar (if this is an imported one) */
+    glist_t searchpath;    /**< List of directories to search for grammars. */
+
+    /* Scratch variables for FSG conversion. */
+    int nstate;            /**< Number of generated states. */
+    glist_t links;	   /**< Generated FSG links. */
+    glist_t rulestack;     /**< Stack of currently expanded rules. */
+};
+
+/* A type to keep track of the stack of rules currently being expanded. */
+struct jsgf_rule_stack_s {
+    jsgf_rule_t *rule;  /**< The rule being expanded */
+    int entry;          /**< The entry-state for this expansion */
+};
+
+struct jsgf_rule_s {
+    int refcnt;      /**< Reference count. */
+    char *name;      /**< Rule name (NULL for an alternation/grouping) */
+    int is_public;   /**< Is this rule marked 'public'? */
+    jsgf_rhs_t *rhs; /**< Expansion */
+};
+
+struct jsgf_rhs_s {
+    glist_t atoms;   /**< Sequence of items */
+    jsgf_rhs_t *alt; /**< Linked list of alternates */
+};
+
+struct jsgf_atom_s {
+    char *name;        /**< Rule or token name */
+    glist_t tags;      /**< Tags, if any (glist_t of char *) */
+    float weight;      /**< Weight (default 1) */
+};
+
+struct jsgf_link_s {
+    jsgf_atom_t *atom; /**< Name, tags, weight */
+    int from;          /**< From state */
+    int to;            /**< To state */
+};
+
+#define jsgf_atom_is_rule(atom) ((atom)->name[0] == '<')
+
+void jsgf_add_link(jsgf_t *grammar, jsgf_atom_t *atom, int from, int to);
+jsgf_atom_t *jsgf_atom_new(char *name, float weight);
+jsgf_atom_t *jsgf_kleene_new(jsgf_t *jsgf, jsgf_atom_t *atom, int plus);
+jsgf_rule_t *jsgf_optional_new(jsgf_t *jsgf, jsgf_rhs_t *exp);
+jsgf_rule_t *jsgf_define_rule(jsgf_t *jsgf, char *name, jsgf_rhs_t *rhs, int is_public);
+jsgf_rule_t *jsgf_import_rule(jsgf_t *jsgf, char *name);
+
+int jsgf_atom_free(jsgf_atom_t *atom);
+int jsgf_rule_free(jsgf_rule_t *rule);
+jsgf_rule_t *jsgf_rule_retain(jsgf_rule_t *rule);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* __JSGF_H__ */

media/sphinxbase/src/libsphinxbase/lm/jsgf_parser.h

@@ -0,0 +1,90 @@
+
+/* A Bison parser, made by GNU Bison 2.4.1.  */
+
+/* Skeleton interface for Bison's Yacc-like parsers in C
+   
+      Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
+   Free Software Foundation, Inc.
+   
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* As a special exception, you may create a larger work that contains
+   part or all of the Bison parser skeleton and distribute that work
+   under terms of your choice, so long as that work isn't itself a
+   parser generator using the skeleton or a modified version thereof
+   as a parser skeleton.  Alternatively, if you modify or redistribute
+   the parser skeleton itself, you may (at your option) remove this
+   special exception, which will cause the skeleton and the resulting
+   Bison output files to be licensed under the GNU General Public
+   License without this special exception.
+   
+   This special exception was added by the Free Software Foundation in
+   version 2.2 of Bison.  */
+
+
+/* Tokens.  */
+#ifndef YYTOKENTYPE
+# define YYTOKENTYPE
+   /* Put the tokens into the symbol table, so that GDB and other debuggers
+      know about them.  */
+   enum yytokentype {
+     HEADER = 258,
+     GRAMMAR = 259,
+     IMPORT = 260,
+     PUBLIC = 261,
+     TOKEN = 262,
+     RULENAME = 263,
+     TAG = 264,
+     WEIGHT = 265
+   };
+#endif
+/* Tokens.  */
+#define HEADER 258
+#define GRAMMAR 259
+#define IMPORT 260
+#define PUBLIC 261
+#define TOKEN 262
+#define RULENAME 263
+#define TAG 264
+#define WEIGHT 265
+
+
+
+
+#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
+typedef union YYSTYPE
+{
+
+/* Line 1676 of yacc.c  */
+#line 65 "jsgf_parser.y"
+
+       char *name;
+       float weight;
+       jsgf_rule_t *rule;
+       jsgf_rhs_t *rhs;
+       jsgf_atom_t *atom;
+
+
+
+/* Line 1676 of yacc.c  */
+#line 82 "jsgf_parser.h"
+} YYSTYPE;
+# define YYSTYPE_IS_TRIVIAL 1
+# define yystype YYSTYPE /* obsolescent; will be withdrawn */
+# define YYSTYPE_IS_DECLARED 1
+#endif
+
+
+
+

media/sphinxbase/src/libsphinxbase/lm/jsgf_scanner.h

@@ -0,0 +1,352 @@
+#ifndef yyHEADER_H
+#define yyHEADER_H 1
+#define yyIN_HEADER 1
+
+#line 6 "jsgf_scanner.h"
+
+#line 8 "jsgf_scanner.h"
+
+#define  YY_INT_ALIGNED short int
+
+/* A lexical scanner generated by flex */
+
+#define FLEX_SCANNER
+#define YY_FLEX_MAJOR_VERSION 2
+#define YY_FLEX_MINOR_VERSION 5
+#define YY_FLEX_SUBMINOR_VERSION 37
+#if YY_FLEX_SUBMINOR_VERSION > 0
+#define FLEX_BETA
+#endif
+
+/* First, we deal with  platform-specific or compiler-specific issues. */
+
+/* begin standard C headers. */
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <stdlib.h>
+
+/* end standard C headers. */
+
+/* flex integer type definitions */
+
+#ifndef FLEXINT_H
+#define FLEXINT_H
+
+/* C99 systems have <inttypes.h>. Non-C99 systems may or may not. */
+
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+
+/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h,
+ * if you want the limit (max/min) macros for int types. 
+ */
+#ifndef __STDC_LIMIT_MACROS
+#define __STDC_LIMIT_MACROS 1
+#endif
+
+#include <inttypes.h>
+typedef int8_t flex_int8_t;
+typedef uint8_t flex_uint8_t;
+typedef int16_t flex_int16_t;
+typedef uint16_t flex_uint16_t;
+typedef int32_t flex_int32_t;
+typedef uint32_t flex_uint32_t;
+#else
+typedef signed char flex_int8_t;
+typedef short int flex_int16_t;
+typedef int flex_int32_t;
+typedef unsigned char flex_uint8_t; 
+typedef unsigned short int flex_uint16_t;
+typedef unsigned int flex_uint32_t;
+
+/* Limits of integral types. */
+#ifndef INT8_MIN
+#define INT8_MIN               (-128)
+#endif
+#ifndef INT16_MIN
+#define INT16_MIN              (-32767-1)
+#endif
+#ifndef INT32_MIN
+#define INT32_MIN              (-2147483647-1)
+#endif
+#ifndef INT8_MAX
+#define INT8_MAX               (127)
+#endif
+#ifndef INT16_MAX
+#define INT16_MAX              (32767)
+#endif
+#ifndef INT32_MAX
+#define INT32_MAX              (2147483647)
+#endif
+#ifndef UINT8_MAX
+#define UINT8_MAX              (255U)
+#endif
+#ifndef UINT16_MAX
+#define UINT16_MAX             (65535U)
+#endif
+#ifndef UINT32_MAX
+#define UINT32_MAX             (4294967295U)
+#endif
+
+#endif /* ! C99 */
+
+#endif /* ! FLEXINT_H */
+
+#ifdef __cplusplus
+
+/* The "const" storage-class-modifier is valid. */
+#define YY_USE_CONST
+
+#else	/* ! __cplusplus */
+
+/* C99 requires __STDC__ to be defined as 1. */
+#if defined (__STDC__)
+
+#define YY_USE_CONST
+
+#endif	/* defined (__STDC__) */
+#endif	/* ! __cplusplus */
+
+#ifdef YY_USE_CONST
+#define yyconst const
+#else
+#define yyconst
+#endif
+
+/* An opaque pointer. */
+#ifndef YY_TYPEDEF_YY_SCANNER_T
+#define YY_TYPEDEF_YY_SCANNER_T
+typedef void* yyscan_t;
+#endif
+
+/* For convenience, these vars (plus the bison vars far below)
+   are macros in the reentrant scanner. */
+#define yyin yyg->yyin_r
+#define yyout yyg->yyout_r
+#define yyextra yyg->yyextra_r
+#define yyleng yyg->yyleng_r
+#define yytext yyg->yytext_r
+#define yylineno (YY_CURRENT_BUFFER_LVALUE->yy_bs_lineno)
+#define yycolumn (YY_CURRENT_BUFFER_LVALUE->yy_bs_column)
+#define yy_flex_debug yyg->yy_flex_debug_r
+
+/* Size of default input buffer. */
+#ifndef YY_BUF_SIZE
+#define YY_BUF_SIZE 16384
+#endif
+
+#ifndef YY_TYPEDEF_YY_BUFFER_STATE
+#define YY_TYPEDEF_YY_BUFFER_STATE
+typedef struct yy_buffer_state *YY_BUFFER_STATE;
+#endif
+
+#ifndef YY_TYPEDEF_YY_SIZE_T
+#define YY_TYPEDEF_YY_SIZE_T
+typedef size_t yy_size_t;
+#endif
+
+#ifndef YY_STRUCT_YY_BUFFER_STATE
+#define YY_STRUCT_YY_BUFFER_STATE
+struct yy_buffer_state
+	{
+	FILE *yy_input_file;
+
+	char *yy_ch_buf;		/* input buffer */
+	char *yy_buf_pos;		/* current position in input buffer */
+
+	/* Size of input buffer in bytes, not including room for EOB
+	 * characters.
+	 */
+	yy_size_t yy_buf_size;
+
+	/* Number of characters read into yy_ch_buf, not including EOB
+	 * characters.
+	 */
+	yy_size_t yy_n_chars;
+
+	/* Whether we "own" the buffer - i.e., we know we created it,
+	 * and can realloc() it to grow it, and should free() it to
+	 * delete it.
+	 */
+	int yy_is_our_buffer;
+
+	/* Whether this is an "interactive" input source; if so, and
+	 * if we're using stdio for input, then we want to use getc()
+	 * instead of fread(), to make sure we stop fetching input after
+	 * each newline.
+	 */
+	int yy_is_interactive;
+
+	/* Whether we're considered to be at the beginning of a line.
+	 * If so, '^' rules will be active on the next match, otherwise
+	 * not.
+	 */
+	int yy_at_bol;
+
+    int yy_bs_lineno; /**< The line count. */
+    int yy_bs_column; /**< The column count. */
+    
+	/* Whether to try to fill the input buffer when we reach the
+	 * end of it.
+	 */
+	int yy_fill_buffer;
+
+	int yy_buffer_status;
+
+	};
+#endif /* !YY_STRUCT_YY_BUFFER_STATE */
+
+void yyrestart (FILE *input_file ,yyscan_t yyscanner );
+void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
+YY_BUFFER_STATE yy_create_buffer (FILE *file,int size ,yyscan_t yyscanner );
+void yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
+void yy_flush_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
+void yypush_buffer_state (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
+void yypop_buffer_state (yyscan_t yyscanner );
+
+YY_BUFFER_STATE yy_scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanner );
+YY_BUFFER_STATE yy_scan_string (yyconst char *yy_str ,yyscan_t yyscanner );
+YY_BUFFER_STATE yy_scan_bytes (yyconst char *bytes,yy_size_t len ,yyscan_t yyscanner );
+
+void *yyalloc (yy_size_t ,yyscan_t yyscanner );
+void *yyrealloc (void *,yy_size_t ,yyscan_t yyscanner );
+void yyfree (void * ,yyscan_t yyscanner );
+
+/* Begin user sect3 */
+
+#define yywrap(yyscanner) 1
+#define YY_SKIP_YYWRAP
+
+#define yytext_ptr yytext_r
+
+#ifdef YY_HEADER_EXPORT_START_CONDITIONS
+#define INITIAL 0
+#define COMMENT 1
+#define DECL 2
+#define DECLCOMMENT 3
+
+#endif
+
+
+#ifdef HAVE_UNISTD_H
+#ifndef YY_NO_UNISTD_H
+/* Special case for "unistd.h", since it is non-ANSI. We include it way
+ * down here because we want the user's section 1 to have been scanned first.
+ * The user has a chance to override it with an option.
+ */
+#include <unistd.h>
+#endif
+#endif
+
+#ifndef YY_EXTRA_TYPE
+#define YY_EXTRA_TYPE void *
+#endif
+
+int yylex_init (yyscan_t* scanner);
+
+int yylex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner);
+
+/* Accessor methods to globals.
+   These are made visible to non-reentrant scanners for convenience. */
+
+int yylex_destroy (yyscan_t yyscanner );
+
+int yyget_debug (yyscan_t yyscanner );
+
+void yyset_debug (int debug_flag ,yyscan_t yyscanner );
+
+YY_EXTRA_TYPE yyget_extra (yyscan_t yyscanner );
+
+void yyset_extra (YY_EXTRA_TYPE user_defined ,yyscan_t yyscanner );
+
+FILE *yyget_in (yyscan_t yyscanner );
+
+void yyset_in  (FILE * in_str ,yyscan_t yyscanner );
+
+FILE *yyget_out (yyscan_t yyscanner );
+
+void yyset_out  (FILE * out_str ,yyscan_t yyscanner );
+
+yy_size_t yyget_leng (yyscan_t yyscanner );
+
+char *yyget_text (yyscan_t yyscanner );
+
+int yyget_lineno (yyscan_t yyscanner );
+
+void yyset_lineno (int line_number ,yyscan_t yyscanner );
+
+int yyget_column  (yyscan_t yyscanner );
+
+void yyset_column (int column_no ,yyscan_t yyscanner );
+
+YYSTYPE * yyget_lval (yyscan_t yyscanner );
+
+void yyset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner );
+
+/* Macros after this point can all be overridden by user definitions in
+ * section 1.
+ */
+
+#ifndef YY_SKIP_YYWRAP
+#ifdef __cplusplus
+extern "C" int yywrap (yyscan_t yyscanner );
+#else
+extern int yywrap (yyscan_t yyscanner );
+#endif
+#endif
+
+#ifndef yytext_ptr
+static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner);
+#endif
+
+#ifdef YY_NEED_STRLEN
+static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner);
+#endif
+
+#ifndef YY_NO_INPUT
+
+#endif
+
+/* Amount of stuff to slurp up with each read. */
+#ifndef YY_READ_BUF_SIZE
+#define YY_READ_BUF_SIZE 8192
+#endif
+
+/* Number of entries by which start-condition stack grows. */
+#ifndef YY_START_STACK_INCR
+#define YY_START_STACK_INCR 25
+#endif
+
+/* Default declaration of generated scanner - a define so the user can
+ * easily add parameters.
+ */
+#ifndef YY_DECL
+#define YY_DECL_IS_OURS 1
+
+extern int yylex \
+               (YYSTYPE * yylval_param ,yyscan_t yyscanner);
+
+#define YY_DECL int yylex \
+               (YYSTYPE * yylval_param , yyscan_t yyscanner)
+#endif /* !YY_DECL */
+
+/* yy_get_previous_state - get the state just before the EOB char was reached */
+
+#undef YY_NEW_FILE
+#undef YY_FLUSH_BUFFER
+#undef yy_set_bol
+#undef yy_new_buffer
+#undef yy_set_interactive
+#undef YY_DO_BEFORE_ACTION
+
+#ifdef YY_DECL_IS_OURS
+#undef YY_DECL_IS_OURS
+#undef YY_DECL
+#endif
+
+#line 87 "_jsgf_scanner.l"
+
+
+#line 348 "jsgf_scanner.h"
+#undef yyIN_HEADER
+#endif /* yyHEADER_H */

media/sphinxbase/src/libsphinxbase/lm/lm3g_model.c

@@ -0,0 +1,258 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * \file lm3g_model.c Core Sphinx 3-gram code used in
+ * DMP/DMP32/ARPA (for now) model code.
+ *
+ * Author: A cast of thousands, probably.
+ */
+#include <string.h>
+#include <assert.h>
+#include <limits.h>
+
+#include "sphinxbase/listelem_alloc.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/err.h"
+
+#include "lm3g_model.h"
+
+void
+lm3g_tginfo_free(ngram_model_t *base, lm3g_model_t *lm3g)
+{
+	if (lm3g->tginfo == NULL)
+		return;
+        listelem_alloc_free(lm3g->le);
+        ckd_free(lm3g->tginfo);
+}
+
+void
+lm3g_tginfo_reset(ngram_model_t *base, lm3g_model_t *lm3g)
+{
+    if (lm3g->tginfo == NULL)
+        return;
+    listelem_alloc_free(lm3g->le);
+    memset(lm3g->tginfo, 0, base->n_counts[0] * sizeof(tginfo_t *));
+    lm3g->le = listelem_alloc_init(sizeof(tginfo_t));
+}
+
+void
+lm3g_apply_weights(ngram_model_t *base,
+		   lm3g_model_t *lm3g,
+		   float32 lw, float32 wip, float32 uw)
+{
+    int32 log_wip, log_uw, log_uniform_weight;
+    int i;
+
+    /* Precalculate some log values we will like. */
+    log_wip = logmath_log(base->lmath, wip);
+    log_uw = logmath_log(base->lmath, uw);
+    log_uniform_weight = logmath_log(base->lmath, 1.0 - uw);
+
+    for (i = 0; i < base->n_counts[0]; ++i) {
+        int32 prob1, bo_wt, n_used;
+
+        /* Backoff weights just get scaled by the lw. */
+        bo_wt = (int32)(lm3g->unigrams[i].bo_wt1.l / base->lw);
+        /* Unscaling unigram probs is a bit more complicated, so punt
+         * it back to the general code. */
+        prob1 = ngram_ng_prob(base, i, NULL, 0, &n_used);
+        /* Now compute the new scaled probabilities. */
+        lm3g->unigrams[i].bo_wt1.l = (int32)(bo_wt * lw);
+        if (strcmp(base->word_str[i], "<s>") == 0) { /* FIXME: configurable start_sym */
+            /* Apply language weight and WIP */
+            lm3g->unigrams[i].prob1.l = (int32)(prob1 * lw) + log_wip;
+        }
+        else {
+            /* Interpolate unigram probability with uniform. */
+            prob1 += log_uw;
+            prob1 = logmath_add(base->lmath, prob1, base->log_uniform + log_uniform_weight);
+            /* Apply language weight and WIP */
+            lm3g->unigrams[i].prob1.l = (int32)(prob1 * lw) + log_wip;
+        }
+    }
+
+    for (i = 0; i < lm3g->n_prob2; ++i) {
+        int32 prob2;
+        /* Can't just punt this back to general code since it is quantized. */
+        prob2 = (int32)((lm3g->prob2[i].l - base->log_wip) / base->lw);
+        lm3g->prob2[i].l = (int32)(prob2 * lw) + log_wip;
+    }
+
+    if (base->n > 2) {
+        for (i = 0; i < lm3g->n_bo_wt2; ++i) {
+            lm3g->bo_wt2[i].l = (int32)(lm3g->bo_wt2[i].l  / base->lw * lw);
+        }
+        for (i = 0; i < lm3g->n_prob3; i++) {
+            int32 prob3;
+            /* Can't just punt this back to general code since it is quantized. */
+            prob3 = (int32)((lm3g->prob3[i].l - base->log_wip) / base->lw);
+            lm3g->prob3[i].l = (int32)(prob3 * lw) + log_wip;
+        }
+    }
+
+    /* Store updated values in the model. */
+    base->log_wip = log_wip;
+    base->log_uw = log_uw;
+    base->log_uniform_weight = log_uniform_weight;
+    base->lw = lw;
+}
+
+int32
+lm3g_add_ug(ngram_model_t *base,
+            lm3g_model_t *lm3g, int32 wid, int32 lweight)
+{
+    int32 score;
+
+    /* This would be very bad if this happened! */
+    assert(!NGRAM_IS_CLASSWID(wid));
+
+    /* Reallocate unigram array. */
+    lm3g->unigrams = ckd_realloc(lm3g->unigrams,
+                                 sizeof(*lm3g->unigrams) * base->n_1g_alloc);
+    memset(lm3g->unigrams + base->n_counts[0], 0,
+           (base->n_1g_alloc - base->n_counts[0]) * sizeof(*lm3g->unigrams));
+    /* Reallocate tginfo array. */
+    lm3g->tginfo = ckd_realloc(lm3g->tginfo,
+                               sizeof(*lm3g->tginfo) * base->n_1g_alloc);
+    memset(lm3g->tginfo + base->n_counts[0], 0,
+           (base->n_1g_alloc - base->n_counts[0]) * sizeof(*lm3g->tginfo));
+    /* FIXME: we really ought to update base->log_uniform *and*
+     * renormalize all the other unigrams.  This is really slow, so I
+     * will probably just provide a function to renormalize after
+     * adding unigrams, for anyone who really cares. */
+    /* This could be simplified but then we couldn't do it in logmath */
+    score = lweight + base->log_uniform + base->log_uw;
+    score = logmath_add(base->lmath, score,
+                        base->log_uniform + base->log_uniform_weight);
+    lm3g->unigrams[wid].prob1.l = score;
+    /* This unigram by definition doesn't participate in any bigrams,
+     * so its backoff weight and bigram pointer are both undefined. */
+    lm3g->unigrams[wid].bo_wt1.l = 0;
+    lm3g->unigrams[wid].bigrams = 0;
+    /* Finally, increase the unigram count */
+    ++base->n_counts[0];
+    /* FIXME: Note that this can actually be quite bogus due to the
+     * presence of class words.  If wid falls outside the unigram
+     * count, increase it to compensate, at the cost of no longer
+     * really knowing how many unigrams we have :( */
+    if (wid >= base->n_counts[0])
+        base->n_counts[0] = wid + 1;
+
+    return score;
+}
+
+#define INITIAL_SORTED_ENTRIES	MAX_UINT16
+
+void
+init_sorted_list(sorted_list_t * l)
+{
+    l->list = ckd_calloc(INITIAL_SORTED_ENTRIES, sizeof(sorted_entry_t));
+    l->list[0].val.l = INT_MIN;
+    l->list[0].lower = 0;
+    l->list[0].higher = 0;
+    l->free = 1;
+    l->size = INITIAL_SORTED_ENTRIES;
+}
+
+void
+free_sorted_list(sorted_list_t * l)
+{
+    free(l->list);
+}
+
+lmprob_t *
+vals_in_sorted_list(sorted_list_t * l)
+{
+    lmprob_t *vals;
+    int32 i;
+
+    vals = ckd_calloc(l->free, sizeof(lmprob_t));
+    for (i = 0; i < l->free; i++)
+        vals[i] = l->list[i].val;
+    return (vals);
+}
+
+int32
+sorted_id(sorted_list_t * l, int32 *val)
+{
+    int32 i = 0;
+
+    for (;;) {
+        if (*val == l->list[i].val.l)
+            return (i);
+        if (*val < l->list[i].val.l) {
+            if (l->list[i].lower == 0) {
+
+                if (l->free >= l->size) {
+            	    int newsize = l->size + INITIAL_SORTED_ENTRIES;
+            	    l->list = ckd_realloc(l->list, sizeof(sorted_entry_t) * newsize);
+            	    memset(l->list + l->size, 
+            	           0, INITIAL_SORTED_ENTRIES * sizeof(sorted_entry_t));
+            	    l->size = newsize;
+                }
+
+                l->list[i].lower = l->free;
+                (l->free)++;
+                i = l->list[i].lower;
+                l->list[i].val.l = *val;
+                return (i);
+            }
+            else
+                i = l->list[i].lower;
+        }
+        else {
+            if (l->list[i].higher == 0) {
+
+                if (l->free >= l->size) {
+            	    int newsize = l->size + INITIAL_SORTED_ENTRIES;
+            	    l->list = ckd_realloc(l->list, sizeof(sorted_entry_t) * newsize);
+            	    memset(l->list + l->size, 
+            	           0, INITIAL_SORTED_ENTRIES * sizeof(sorted_entry_t));
+            	    l->size = newsize;
+                }
+
+                l->list[i].higher = l->free;
+                (l->free)++;
+                i = l->list[i].higher;
+                l->list[i].val.l = *val;
+                return (i);
+            }
+            else
+                i = l->list[i].higher;
+        }
+    }
+}

media/sphinxbase/src/libsphinxbase/lm/lm3g_model.h

@@ -0,0 +1,177 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * \file lm3g_model.h Core Sphinx 3-gram code used in
+ * DMP/DMP32/ARPA (for now) model code.
+ *
+ * Author: A cast of thousands, probably.
+ */
+
+#ifndef __NGRAM_MODEL_LM3G_H__
+#define __NGRAM_MODEL_LM3G_H__
+
+#include "sphinxbase/listelem_alloc.h"
+
+#include "ngram_model_internal.h"
+
+/**
+ * Type used to store language model probabilities
+ */
+typedef union {
+    float32 f;
+    int32 l;
+} lmprob_t;
+
+/**
+ * Bigram probs and bo-wts, and trigram probs are kept in separate
+ * tables rather than within the bigram_t and trigram_t structures.
+ * These tables hold unique prob and bo-wt values. The following tree
+ * structure is used to construct these tables of unique values.  
+ * Whenever a new value is read from the LM file, the sorted tree 
+ * structure is searched to see if the value already exists, and 
+ * inserted if not found.
+ */
+typedef struct sorted_entry_s {
+    lmprob_t val;               /**< value being kept in this node */
+    uint32 lower;               /**< index of another entry.  All descendants down
+                                   this path have their val < this node's val.
+                                   0 => no son exists (0 is root index) */
+    uint32 higher;              /**< index of another entry.  All descendants down
+                                   this path have their val > this node's val
+                                   0 => no son exists (0 is root index) */
+} sorted_entry_t;
+
+/**
+ * The sorted list.  list is a (64K long) array.  The first entry is the
+ * root of the tree and is created during initialization.
+ */
+typedef struct {
+    sorted_entry_t *list;
+    int32 free;                 /**< first free element in list */
+    int32 size;
+} sorted_list_t;
+
+/**
+ * Unigram structure (common among all lm3g implementations)
+ */
+typedef struct unigram_s {
+    lmprob_t prob1;     /**< Unigram probability. */
+    lmprob_t bo_wt1;    /**< Unigram backoff weight. */
+    int32 bigrams;	/**< Index of 1st entry in lm_t.bigrams[] */
+} unigram_t;
+
+/**
+ * Bigram structure (might be implemented differently)
+ */
+typedef struct bigram_s bigram_t;
+/**
+ * Trigram structure (might be implemented differently)
+ */
+typedef struct trigram_s trigram_t;
+
+
+/*
+ * To conserve space, bigram info is kept in many tables.  Since the number
+ * of distinct values << #bigrams, these table indices can be 16-bit values.
+ * prob2 and bo_wt2 are such indices, but keeping trigram index is less easy.
+ * It is supposed to be the index of the first trigram entry for each bigram.
+ * But such an index cannot be represented in 16-bits, hence the following
+ * segmentation scheme: Partition bigrams into segments of BG_SEG_SZ
+ * consecutive entries, such that #trigrams in each segment <= 2**16 (the
+ * corresponding trigram segment).  The bigram_t.trigrams value is then a
+ * 16-bit relative index within the trigram segment.  A separate table--
+ * lm_t.tseg_base--has the index of the 1st trigram for each bigram segment.
+ */
+#define BG_SEG_SZ	512	/* chosen so that #trigram/segment <= 2**16 */
+#define LOG_BG_SEG_SZ	9
+
+/**
+ * Trigram information cache.
+ *
+ * The following trigram information cache eliminates most traversals of 1g->2g->3g
+ * tree to locate trigrams for a given bigram (lw1,lw2).  The organization is optimized
+ * for locality of access (to the same lw1), given lw2.
+ */
+typedef struct tginfo_s {
+    int32 w1;			/**< lw1 component of bigram lw1,lw2.  All bigrams with
+				   same lw2 linked together (see lm_t.tginfo). */
+    int32 n_tg;			/**< number tg for parent bigram lw1,lw2 */
+    int32 bowt;                 /**< tg bowt for lw1,lw2 */
+    int32 used;			/**< whether used since last lm_reset */
+    trigram_t *tg;		/**< Trigrams for lw1,lw2 */
+    struct tginfo_s *next;      /**< Next lw1 with same parent lw2; NULL if none. */
+} tginfo_t;
+
+/**
+ * Common internal structure for Sphinx 3-gram models.
+ */
+typedef struct lm3g_model_s {
+    unigram_t *unigrams;
+    bigram_t *bigrams;
+    trigram_t *trigrams;
+    lmprob_t *prob2;	     /**< Table of actual bigram probs */
+    int32 n_prob2;	     /**< prob2 size */
+    lmprob_t *bo_wt2;	     /**< Table of actual bigram backoff weights */
+    int32 n_bo_wt2;	     /**< bo_wt2 size */
+    lmprob_t *prob3;	     /**< Table of actual trigram probs */
+    int32 n_prob3;	     /**< prob3 size */
+    int32 *tseg_base;    /**< tseg_base[i>>LOG_BG_SEG_SZ] = index of 1st
+                            trigram for bigram segment (i>>LOG_BG_SEG_SZ) */
+    tginfo_t **tginfo;   /**< tginfo[lw2] is head of linked list of trigram information for
+                            some cached subset of bigrams (*,lw2). */
+    listelem_alloc_t *le; /**< List element allocator for tginfo. */
+} lm3g_model_t;
+
+void lm3g_tginfo_free(ngram_model_t *base, lm3g_model_t *lm3g);
+void lm3g_tginfo_reset(ngram_model_t *base, lm3g_model_t *lm3g);
+void lm3g_apply_weights(ngram_model_t *base,
+			lm3g_model_t *lm3g,
+			float32 lw, float32 wip, float32 uw);
+int32 lm3g_add_ug(ngram_model_t *base,
+                  lm3g_model_t *lm3g, int32 wid, int32 lweight);
+
+
+/**
+ * Initialize sorted list with the 0-th entry = MIN_PROB_F, which may be needed
+ * to replace spurious values in the Darpa LM file.
+ */
+void init_sorted_list(sorted_list_t *l);
+void free_sorted_list(sorted_list_t *l);
+lmprob_t *vals_in_sorted_list(sorted_list_t *l);
+int32 sorted_id(sorted_list_t * l, int32 *val);
+
+#endif /* __NGRAM_MODEL_LM3G_H__ */

media/sphinxbase/src/libsphinxbase/lm/lm3g_templates.c

@@ -0,0 +1,560 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * \file lm3g_templates.c Core Sphinx 3-gram code used in
+ * DMP/DMP32/ARPA (for now) model code.
+ */
+
+#include <assert.h>
+
+/* Locate a specific bigram within a bigram list */
+#define BINARY_SEARCH_THRESH	16
+static int32
+find_bg(bigram_t * bg, int32 n, int32 w)
+{
+    int32 i, b, e;
+
+    /* Binary search until segment size < threshold */
+    b = 0;
+    e = n;
+    while (e - b > BINARY_SEARCH_THRESH) {
+        i = (b + e) >> 1;
+        if (bg[i].wid < w)
+            b = i + 1;
+        else if (bg[i].wid > w)
+            e = i;
+        else
+            return i;
+    }
+
+    /* Linear search within narrowed segment */
+    for (i = b; (i < e) && (bg[i].wid != w); i++);
+    return ((i < e) ? i : -1);
+}
+
+static int32
+lm3g_bg_score(NGRAM_MODEL_TYPE *model,
+              int32 lw1, int32 lw2, int32 *n_used)
+{
+    int32 i, n, b, score;
+    bigram_t *bg;
+
+    if (lw1 < 0 || model->base.n < 2) {
+        *n_used = 1;
+        return model->lm3g.unigrams[lw2].prob1.l;
+    }
+
+    b = FIRST_BG(model, lw1);
+    n = FIRST_BG(model, lw1 + 1) - b;
+    bg = model->lm3g.bigrams + b;
+
+    if ((i = find_bg(bg, n, lw2)) >= 0) {
+        /* Access mode = bigram */
+        *n_used = 2;
+        score = model->lm3g.prob2[bg[i].prob2].l;
+    }
+    else {
+        /* Access mode = unigram */
+        *n_used = 1;
+        score = model->lm3g.unigrams[lw1].bo_wt1.l + model->lm3g.unigrams[lw2].prob1.l;
+    }
+
+    return (score);
+}
+
+static void
+load_tginfo(NGRAM_MODEL_TYPE *model, int32 lw1, int32 lw2)
+{
+    int32 i, n, b, t;
+    bigram_t *bg;
+    tginfo_t *tginfo;
+
+    /* First allocate space for tg information for bg lw1,lw2 */
+    tginfo = (tginfo_t *) listelem_malloc(model->lm3g.le);
+    tginfo->w1 = lw1;
+    tginfo->tg = NULL;
+    tginfo->next = model->lm3g.tginfo[lw2];
+    model->lm3g.tginfo[lw2] = tginfo;
+
+    /* Locate bigram lw1,lw2 */
+    b = model->lm3g.unigrams[lw1].bigrams;
+    n = model->lm3g.unigrams[lw1 + 1].bigrams - b;
+    bg = model->lm3g.bigrams + b;
+
+    if ((n > 0) && ((i = find_bg(bg, n, lw2)) >= 0)) {
+        tginfo->bowt = model->lm3g.bo_wt2[bg[i].bo_wt2].l;
+
+        /* Find t = Absolute first trigram index for bigram lw1,lw2 */
+        b += i;                 /* b = Absolute index of bigram lw1,lw2 on disk */
+        t = FIRST_TG(model, b);
+
+        tginfo->tg = model->lm3g.trigrams + t;
+
+        /* Find #tg for bigram w1,w2 */
+        tginfo->n_tg = FIRST_TG(model, b + 1) - t;
+    }
+    else {                      /* No bigram w1,w2 */
+        tginfo->bowt = 0;
+        tginfo->n_tg = 0;
+    }
+}
+
+/* Similar to find_bg */
+static int32
+find_tg(trigram_t * tg, int32 n, uint32 w)
+{
+    int32 i, b, e;
+
+    b = 0;
+    e = n;
+    while (e - b > BINARY_SEARCH_THRESH) {
+        i = (b + e) >> 1;
+        if (tg[i].wid < w)
+            b = i + 1;
+        else if (tg[i].wid > w)
+            e = i;
+        else
+            return i;
+    }
+
+    for (i = b; (i < e) && (tg[i].wid != w); i++);
+    return ((i < e) ? i : -1);
+}
+
+static int32
+lm3g_tg_score(NGRAM_MODEL_TYPE *model, int32 lw1,
+              int32 lw2, int32 lw3, int32 *n_used)
+{
+    ngram_model_t *base = &model->base;
+    int32 i, n, score;
+    trigram_t *tg;
+    tginfo_t *tginfo, *prev_tginfo;
+
+    if ((base->n < 3) || (lw1 < 0) || (lw2 < 0))
+        return (lm3g_bg_score(model, lw2, lw3, n_used));
+
+    prev_tginfo = NULL;
+    for (tginfo = model->lm3g.tginfo[lw2]; tginfo; tginfo = tginfo->next) {
+        if (tginfo->w1 == lw1)
+            break;
+        prev_tginfo = tginfo;
+    }
+
+    if (!tginfo) {
+        load_tginfo(model, lw1, lw2);
+        tginfo = model->lm3g.tginfo[lw2];
+    }
+    else if (prev_tginfo) {
+        prev_tginfo->next = tginfo->next;
+        tginfo->next = model->lm3g.tginfo[lw2];
+        model->lm3g.tginfo[lw2] = tginfo;
+    }
+
+    tginfo->used = 1;
+
+    /* Trigrams for w1,w2 now pointed to by tginfo */
+    n = tginfo->n_tg;
+    tg = tginfo->tg;
+    if ((i = find_tg(tg, n, lw3)) >= 0) {
+        /* Access mode = trigram */
+        *n_used = 3;
+        score = model->lm3g.prob3[tg[i].prob3].l;
+    }
+    else {
+        score = tginfo->bowt + lm3g_bg_score(model, lw2, lw3, n_used);
+    }
+
+    return (score);
+}
+
+static int32
+lm3g_template_score(ngram_model_t *base, int32 wid,
+                      int32 *history, int32 n_hist,
+                      int32 *n_used)
+{
+    NGRAM_MODEL_TYPE *model = (NGRAM_MODEL_TYPE *)base;
+    switch (n_hist) {
+    case 0:
+        /* Access mode: unigram */
+        *n_used = 1;
+        return model->lm3g.unigrams[wid].prob1.l;
+    case 1:
+        return lm3g_bg_score(model, history[0], wid, n_used);
+    case 2:
+    default:
+        /* Anything greater than 2 is the same as a trigram for now. */
+        return lm3g_tg_score(model, history[1], history[0], wid, n_used);
+    }
+}
+
+static int32
+lm3g_template_raw_score(ngram_model_t *base, int32 wid,
+                        int32 *history, int32 n_hist,
+                          int32 *n_used)
+{
+    NGRAM_MODEL_TYPE *model = (NGRAM_MODEL_TYPE *)base;
+    int32 score;
+
+    switch (n_hist) {
+    case 0:
+        /* Access mode: unigram */
+        *n_used = 1;
+        /* Undo insertion penalty. */
+        score = model->lm3g.unigrams[wid].prob1.l - base->log_wip;
+        /* Undo language weight. */
+        score = (int32)(score / base->lw);
+        /* Undo unigram interpolation */
+        if (strcmp(base->word_str[wid], "<s>") != 0) { /* FIXME: configurable start_sym */
+            /* This operation is numerically unstable, so try to avoid it
+             * as possible */
+            if (base->log_uniform + base->log_uniform_weight > logmath_get_zero(base->lmath)) {
+               score = logmath_log(base->lmath,
+                            logmath_exp(base->lmath, score)
+                            - logmath_exp(base->lmath, 
+                                          base->log_uniform + base->log_uniform_weight));
+            }
+        }
+        return score;
+    case 1:
+        score = lm3g_bg_score(model, history[0], wid, n_used);
+        break;
+    case 2:
+    default:
+        /* Anything greater than 2 is the same as a trigram for now. */
+        score = lm3g_tg_score(model, history[1], history[0], wid, n_used);
+        break;
+    }
+    /* FIXME (maybe): This doesn't undo unigram weighting in backoff cases. */
+    return (int32)((score - base->log_wip) / base->lw);
+}
+
+static int32
+lm3g_template_add_ug(ngram_model_t *base,
+                       int32 wid, int32 lweight)
+{
+    NGRAM_MODEL_TYPE *model = (NGRAM_MODEL_TYPE *)base;
+    return lm3g_add_ug(base, &model->lm3g, wid, lweight);
+}
+
+static void
+lm3g_template_flush(ngram_model_t *base)
+{
+    NGRAM_MODEL_TYPE *model = (NGRAM_MODEL_TYPE *)base;
+    lm3g_tginfo_reset(base, &model->lm3g);
+}
+
+typedef struct lm3g_iter_s {
+    ngram_iter_t base;
+    unigram_t *ug;
+    bigram_t *bg;
+    trigram_t *tg;
+} lm3g_iter_t;
+
+static ngram_iter_t *
+lm3g_template_iter(ngram_model_t *base, int32 wid,
+                   int32 *history, int32 n_hist)
+{
+    NGRAM_MODEL_TYPE *model = (NGRAM_MODEL_TYPE *)base;
+    lm3g_iter_t *itor = (lm3g_iter_t *)ckd_calloc(1, sizeof(*itor));
+
+    ngram_iter_init((ngram_iter_t *)itor, base, n_hist, FALSE);
+
+    if (n_hist == 0) {
+        /* Unigram is the easiest. */
+        itor->ug = model->lm3g.unigrams + wid;
+        return (ngram_iter_t *)itor;
+    }
+    else if (n_hist == 1) {
+        int32 i, n, b;
+        /* Find the bigram, as in bg_score above (duplicate code...) */
+        itor->ug = model->lm3g.unigrams + history[0];
+        b = FIRST_BG(model, history[0]);
+        n = FIRST_BG(model, history[0] + 1) - b;
+        itor->bg = model->lm3g.bigrams + b;
+        /* If no such bigram exists then fail. */
+        if ((i = find_bg(itor->bg, n, wid)) < 0) {
+            ngram_iter_free((ngram_iter_t *)itor);
+            return NULL;
+        }
+        itor->bg += i;
+        return (ngram_iter_t *)itor;
+    }
+    else if (n_hist == 2) {
+        int32 i, n;
+        tginfo_t *tginfo, *prev_tginfo;
+        /* Find the trigram, as in tg_score above (duplicate code...) */
+        itor->ug = model->lm3g.unigrams + history[1];
+        prev_tginfo = NULL;
+        for (tginfo = model->lm3g.tginfo[history[0]];
+             tginfo; tginfo = tginfo->next) {
+            if (tginfo->w1 == history[1])
+                break;
+            prev_tginfo = tginfo;
+        }
+
+        if (!tginfo) {
+            load_tginfo(model, history[1], history[0]);
+            tginfo = model->lm3g.tginfo[history[0]];
+        }
+        else if (prev_tginfo) {
+            prev_tginfo->next = tginfo->next;
+            tginfo->next = model->lm3g.tginfo[history[0]];
+            model->lm3g.tginfo[history[0]] = tginfo;
+        }
+
+        tginfo->used = 1;
+
+        /* Trigrams for w1,w2 now pointed to by tginfo */
+        n = tginfo->n_tg;
+        itor->tg = tginfo->tg;
+        if ((i = find_tg(itor->tg, n, wid)) >= 0) {
+            itor->tg += i;
+            /* Now advance the bigram pointer accordingly.  FIXME:
+             * Note that we actually already found the relevant bigram
+             * in load_tginfo. */
+            itor->bg = model->lm3g.bigrams;
+            while (FIRST_TG(model, (itor->bg - model->lm3g.bigrams + 1))
+                   <= (itor->tg - model->lm3g.trigrams))
+                ++itor->bg;
+            return (ngram_iter_t *)itor;
+        }
+        else {
+            ngram_iter_free((ngram_iter_t *)itor);
+            return (ngram_iter_t *)NULL;
+        }
+    }
+    else {
+        /* Should not happen. */
+        assert(n_hist == 0); /* Guaranteed to fail. */
+        ngram_iter_free((ngram_iter_t *)itor);
+        return NULL;
+    }
+}
+
+static ngram_iter_t *
+lm3g_template_mgrams(ngram_model_t *base, int m)
+{
+    NGRAM_MODEL_TYPE *model = (NGRAM_MODEL_TYPE *)base;
+    lm3g_iter_t *itor = (lm3g_iter_t *)ckd_calloc(1, sizeof(*itor));
+    ngram_iter_init((ngram_iter_t *)itor, base, m, FALSE);
+
+    itor->ug = model->lm3g.unigrams;
+    itor->bg = model->lm3g.bigrams;
+    itor->tg = model->lm3g.trigrams;
+
+    /* Advance bigram pointer to match first trigram. */
+    if (m > 1 && base->n_counts[1] > 1)  {
+        while (FIRST_TG(model, (itor->bg - model->lm3g.bigrams + 1))
+               <= (itor->tg - model->lm3g.trigrams))
+            ++itor->bg;
+    }
+
+    /* Advance unigram pointer to match first bigram. */
+    if (m > 0 && base->n_counts[0] > 1) {
+        while (itor->ug[1].bigrams <= (itor->bg - model->lm3g.bigrams))
+            ++itor->ug;
+    }
+
+    return (ngram_iter_t *)itor;
+}
+
+static ngram_iter_t *
+lm3g_template_successors(ngram_iter_t *bitor)
+{
+    NGRAM_MODEL_TYPE *model = (NGRAM_MODEL_TYPE *)bitor->model;
+    lm3g_iter_t *from = (lm3g_iter_t *)bitor;
+    lm3g_iter_t *itor = (lm3g_iter_t *)ckd_calloc(1, sizeof(*itor));
+
+    itor->ug = from->ug;
+    switch (bitor->m) {
+    case 0:
+        /* Next itor bigrams is the same as this itor bigram or
+	   itor bigrams is more than total count. This means no successors */
+        if (((itor->ug + 1) - model->lm3g.unigrams < bitor->model->n_counts[0] &&
+	    itor->ug->bigrams == (itor->ug + 1)->bigrams) || 
+	    itor->ug->bigrams == bitor->model->n_counts[1])
+	    goto done;
+	    
+        /* Start iterating from first bigram successor of from->ug. */
+        itor->bg = model->lm3g.bigrams + itor->ug->bigrams;
+        break;
+    case 1:
+        itor->bg = from->bg;
+        
+        /* This indicates no successors */
+        if (((itor->bg + 1) - model->lm3g.bigrams < bitor->model->n_counts[1] &&
+    	    FIRST_TG (model, itor->bg - model->lm3g.bigrams) == 
+    	    FIRST_TG (model, (itor->bg + 1) - model->lm3g.bigrams)) ||
+	    FIRST_TG (model, itor->bg - model->lm3g.bigrams) == bitor->model->n_counts[2])
+    	    goto done;
+    	    
+        /* Start iterating from first trigram successor of from->bg. */
+        itor->tg = (model->lm3g.trigrams 
+                    + FIRST_TG(model, (itor->bg - model->lm3g.bigrams)));
+#if 0
+        printf("%s %s => %d (%s)\n",
+               model->base.word_str[itor->ug - model->lm3g.unigrams],
+               model->base.word_str[itor->bg->wid],
+               FIRST_TG(model, (itor->bg - model->lm3g.bigrams)),
+               model->base.word_str[itor->tg->wid]);
+#endif
+        break;
+    case 2:
+    default:
+        /* All invalid! */
+        goto done;
+    }
+
+    ngram_iter_init((ngram_iter_t *)itor, bitor->model, bitor->m + 1, TRUE);
+    return (ngram_iter_t *)itor;
+    done:
+        ckd_free(itor);
+        return NULL;
+}
+
+static int32 const *
+lm3g_template_iter_get(ngram_iter_t *base,
+                       int32 *out_score, int32 *out_bowt)
+{
+    NGRAM_MODEL_TYPE *model = (NGRAM_MODEL_TYPE *)base->model;
+    lm3g_iter_t *itor = (lm3g_iter_t *)base;
+
+    base->wids[0] = itor->ug - model->lm3g.unigrams;
+    if (itor->bg) base->wids[1] = itor->bg->wid;
+    if (itor->tg) base->wids[2] = itor->tg->wid;
+#if 0
+    printf("itor_get: %d %d %d\n", base->wids[0], base->wids[1], base->wids[2]);
+#endif
+
+    switch (base->m) {
+    case 0:
+        *out_score = itor->ug->prob1.l;
+        *out_bowt = itor->ug->bo_wt1.l;
+        break;
+    case 1:
+        *out_score = model->lm3g.prob2[itor->bg->prob2].l;
+        if (model->lm3g.bo_wt2)
+            *out_bowt = model->lm3g.bo_wt2[itor->bg->bo_wt2].l;
+        else
+            *out_bowt = 0;
+        break;
+    case 2:
+        *out_score = model->lm3g.prob3[itor->tg->prob3].l;
+        *out_bowt = 0;
+        break;
+    default: /* Should not happen. */
+        return NULL;
+    }
+    return base->wids;
+}
+
+static ngram_iter_t *
+lm3g_template_iter_next(ngram_iter_t *base)
+{
+    NGRAM_MODEL_TYPE *model = (NGRAM_MODEL_TYPE *)base->model;
+    lm3g_iter_t *itor = (lm3g_iter_t *)base;
+
+    switch (base->m) {
+    case 0:
+        ++itor->ug;
+        /* Check for end condition. */
+        if (itor->ug - model->lm3g.unigrams >= base->model->n_counts[0])
+            goto done;
+        break;
+    case 1:
+        ++itor->bg;
+        /* Check for end condition. */
+        if (itor->bg - model->lm3g.bigrams >= base->model->n_counts[1])
+            goto done;
+        /* Advance unigram pointer if necessary in order to get one
+         * that points to this bigram. */
+        while (itor->bg - model->lm3g.bigrams >= itor->ug[1].bigrams) {
+            /* Stop if this is a successor iterator, since we don't
+             * want a new unigram. */
+            if (base->successor)
+                goto done;
+            ++itor->ug;
+            if (itor->ug == model->lm3g.unigrams + base->model->n_counts[0]) {
+                E_ERROR("Bigram %d has no valid unigram parent\n",
+                        itor->bg - model->lm3g.bigrams);
+                goto done;
+            }
+        }
+        break;
+    case 2:
+        ++itor->tg;
+        /* Check for end condition. */
+        if (itor->tg - model->lm3g.trigrams >= base->model->n_counts[2])
+            goto done;
+        /* Advance bigram pointer if necessary. */
+        while (itor->tg - model->lm3g.trigrams >=
+            FIRST_TG(model, (itor->bg - model->lm3g.bigrams + 1))) {
+            if (base->successor)
+                goto done;
+            ++itor->bg;
+            if (itor->bg == model->lm3g.bigrams + base->model->n_counts[1]) {
+                E_ERROR("Trigram %d has no valid bigram parent\n",
+                        itor->tg - model->lm3g.trigrams);
+
+               goto done;
+            }
+        }
+        /* Advance unigram pointer if necessary. */
+        while (itor->bg - model->lm3g.bigrams >= itor->ug[1].bigrams) {
+            ++itor->ug;
+            if (itor->ug == model->lm3g.unigrams + base->model->n_counts[0]) {
+                E_ERROR("Trigram %d has no valid unigram parent\n",
+                        itor->tg - model->lm3g.trigrams);
+                goto done;
+            }
+        }
+        break;
+    default: /* Should not happen. */
+        goto done;
+    }
+
+    return (ngram_iter_t *)itor;
+done:
+    ngram_iter_free(base);
+    return NULL;
+}
+
+static void
+lm3g_template_iter_free(ngram_iter_t *base)
+{
+    ckd_free(base);
+}

media/sphinxbase/src/libsphinxbase/lm/ngram_model_arpa.c

@@ -0,0 +1,660 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * \file ngram_model_arpa.c ARPA format language models
+ *
+ * Author: David Huggins-Daines <dhuggins@cs.cmu.edu>
+ */
+
+#include "sphinxbase/ckd_alloc.h"
+#include <string.h>
+#include <limits.h>
+#include <assert.h>
+
+#include "sphinxbase/err.h"
+#include "sphinxbase/pio.h"
+#include "sphinxbase/listelem_alloc.h"
+#include "sphinxbase/strfuncs.h"
+
+#include "ngram_model_arpa.h"
+
+static ngram_funcs_t ngram_model_arpa_funcs;
+
+#define TSEG_BASE(m,b)		((m)->lm3g.tseg_base[(b)>>LOG_BG_SEG_SZ])
+#define FIRST_BG(m,u)		((m)->lm3g.unigrams[u].bigrams)
+#define FIRST_TG(m,b)		(TSEG_BASE((m),(b))+((m)->lm3g.bigrams[b].trigrams))
+
+/*
+ * Read and return #unigrams, #bigrams, #trigrams as stated in input file.
+ */
+static int
+ReadNgramCounts(lineiter_t **li, int32 * n_ug, int32 * n_bg, int32 * n_tg)
+{
+    int32 ngram, ngram_cnt;
+
+    /* skip file until past the '\data\' marker */
+    while (*li) {
+        string_trim((*li)->buf, STRING_BOTH);
+        if (strcmp((*li)->buf, "\\data\\") == 0)
+            break;
+        *li = lineiter_next(*li);
+    }
+    if (*li == NULL || strcmp((*li)->buf, "\\data\\") != 0) {
+        E_INFO("No \\data\\ mark in LM file\n");
+        return -1;
+    }
+
+    *n_ug = *n_bg = *n_tg = 0;
+    while ((*li = lineiter_next(*li))) {
+        if (sscanf((*li)->buf, "ngram %d=%d", &ngram, &ngram_cnt) != 2)
+            break;
+        switch (ngram) {
+        case 1:
+            *n_ug = ngram_cnt;
+            break;
+        case 2:
+            *n_bg = ngram_cnt;
+            break;
+        case 3:
+            *n_tg = ngram_cnt;
+            break;
+        default:
+            E_ERROR("Unknown ngram (%d)\n", ngram);
+            return -1;
+        }
+    }
+    if (*li == NULL) {
+        E_ERROR("EOF while reading ngram counts\n");
+        return -1;
+    }
+
+    /* Position iterator to the unigrams header '\1-grams:\' */
+    while ((*li = lineiter_next(*li))) {
+        string_trim((*li)->buf, STRING_BOTH);
+        if (strcmp((*li)->buf, "\\1-grams:") == 0)
+            break;
+    }
+    if (*li == NULL) {
+        E_ERROR_SYSTEM("Failed to read \\1-grams: mark");
+        return -1;
+    }
+
+    if ((*n_ug <= 0) || (*n_bg < 0) || (*n_tg < 0)) {
+        E_ERROR("Bad or missing ngram count\n");
+        return -1;
+    }
+    return 0;
+}
+
+/*
+ * Read in the unigrams from given file into the LM structure model.
+ * On entry to this procedure, the iterator is positioned to the
+ * header line '\1-grams:'.
+ */
+static int
+ReadUnigrams(lineiter_t **li, ngram_model_arpa_t * model)
+{
+    ngram_model_t *base = &model->base;
+    int32 wcnt;
+    float p1;
+
+    E_INFO("Reading unigrams\n");
+
+    wcnt = 0;
+    while ((*li = lineiter_next(*li))) {
+        char *wptr[3], *name;
+        float32 bo_wt = 0.0f;
+        int n;
+
+        string_trim((*li)->buf, STRING_BOTH);
+        if (strcmp((*li)->buf, "\\2-grams:") == 0
+            || strcmp((*li)->buf, "\\end\\") == 0)
+            break;
+
+        if ((n = str2words((*li)->buf, wptr, 3)) < 2) {
+            if ((*li)->buf[0] != '\0')
+                E_WARN("Format error; unigram ignored: %s\n", (*li)->buf);
+            continue;
+        }
+        else {
+            p1 = (float)atof_c(wptr[0]);
+            name = wptr[1];
+            if (n == 3)
+                bo_wt = (float)atof_c(wptr[2]);
+        }
+
+        if (wcnt >= base->n_counts[0]) {
+            E_ERROR("Too many unigrams\n");
+            return -1;
+        }
+
+        /* Associate name with word id */
+        base->word_str[wcnt] = ckd_salloc(name);
+        if ((hash_table_enter(base->wid, base->word_str[wcnt], (void *)(long)wcnt))
+            != (void *)(long)wcnt) {
+                E_WARN("Duplicate word in dictionary: %s\n", base->word_str[wcnt]);
+        }
+        model->lm3g.unigrams[wcnt].prob1.l = logmath_log10_to_log(base->lmath, p1);
+        model->lm3g.unigrams[wcnt].bo_wt1.l = logmath_log10_to_log(base->lmath, bo_wt);
+        wcnt++;
+    }
+
+    if (base->n_counts[0] != wcnt) {
+        E_WARN("lm_t.ucount(%d) != #unigrams read(%d)\n",
+               base->n_counts[0], wcnt);
+        base->n_counts[0] = wcnt;
+        base->n_words = wcnt;
+    }
+    return 0;
+}
+
+/*
+ * Read bigrams from given file into given model structure.
+ */
+static int
+ReadBigrams(lineiter_t **li, ngram_model_arpa_t * model)
+{
+    ngram_model_t *base = &model->base;
+    int32 w1, w2, prev_w1, bgcount;
+    bigram_t *bgptr;
+
+    E_INFO("Reading bigrams\n");
+
+    bgcount = 0;
+    bgptr = model->lm3g.bigrams;
+    prev_w1 = -1;
+
+    while ((*li = lineiter_next(*li))) {
+        float32 p, bo_wt = 0.0f;
+        int32 p2, bo_wt2;
+        char *wptr[4], *word1, *word2;
+        int n;
+
+        string_trim((*li)->buf, STRING_BOTH);
+        wptr[3] = NULL;
+        if ((n = str2words((*li)->buf, wptr, 4)) < 3) {
+            if ((*li)->buf[0] != '\0')
+                break;
+            continue;
+        }
+        else {
+            p = (float32)atof_c(wptr[0]);
+            word1 = wptr[1];
+            word2 = wptr[2];
+            if (wptr[3])
+                bo_wt = (float32)atof_c(wptr[3]);
+        }
+
+        if ((w1 = ngram_wid(base, word1)) == NGRAM_INVALID_WID) {
+            E_ERROR("Unknown word: %s, skipping bigram (%s %s)\n",
+                    word1, word1, word2);
+            continue;
+        }
+        if ((w2 = ngram_wid(base, word2)) == NGRAM_INVALID_WID) {
+            E_ERROR("Unknown word: %s, skipping bigram (%s %s)\n",
+                    word2, word1, word2);
+            continue;
+        }
+
+        /* FIXME: Should use logmath_t quantization here. */
+        /* HACK!! to quantize probs to 4 decimal digits */
+        p = (float32)((int32)(p * 10000)) / 10000;
+        bo_wt = (float32)((int32)(bo_wt * 10000)) / 10000;
+
+        p2 = logmath_log10_to_log(base->lmath, p);
+        bo_wt2 = logmath_log10_to_log(base->lmath, bo_wt);
+
+        if (bgcount >= base->n_counts[1]) {
+            E_ERROR("Too many bigrams\n");
+            return -1;
+        }
+
+        bgptr->wid = w2;
+        bgptr->prob2 = sorted_id(&model->sorted_prob2, &p2);
+        if (base->n_counts[2] > 0)
+            bgptr->bo_wt2 = sorted_id(&model->sorted_bo_wt2, &bo_wt2);
+
+        if (w1 != prev_w1) {
+            if (w1 < prev_w1) {
+                E_ERROR("Bigram %s %s not in unigram order word id: %d prev word id: %d\n", word1, word2, w1, prev_w1);
+                return -1;
+            }
+
+            for (prev_w1++; prev_w1 <= w1; prev_w1++)
+                model->lm3g.unigrams[prev_w1].bigrams = bgcount;
+            prev_w1 = w1;
+        }
+        bgcount++;
+        bgptr++;
+
+        if ((bgcount & 0x0000ffff) == 0) {
+            E_INFOCONT(".");
+        }
+    }
+    if (*li == NULL || ((strcmp((*li)->buf, "\\end\\") != 0)
+                        && (strcmp((*li)->buf, "\\3-grams:") != 0))) {
+        E_ERROR("Bad bigram: %s\n", (*li)->buf);
+        return -1;
+    }
+
+    for (prev_w1++; prev_w1 <= base->n_counts[0]; prev_w1++)
+        model->lm3g.unigrams[prev_w1].bigrams = bgcount;
+
+    return 0;
+}
+
+/*
+ * Very similar to ReadBigrams.
+ */
+static int
+ReadTrigrams(lineiter_t **li, ngram_model_arpa_t * model)
+{
+    ngram_model_t *base = &model->base;
+    int32 i, w1, w2, w3, prev_w1, prev_w2, tgcount, prev_bg, bg, endbg;
+    int32 seg, prev_seg, prev_seg_lastbg;
+    trigram_t *tgptr;
+    bigram_t *bgptr;
+
+    E_INFO("Reading trigrams\n");
+
+    tgcount = 0;
+    tgptr = model->lm3g.trigrams;
+    prev_w1 = -1;
+    prev_w2 = -1;
+    prev_bg = -1;
+    prev_seg = -1;
+
+    while ((*li = lineiter_next(*li))) {
+        float32 p;
+        int32 p3;
+        char *wptr[4], *word1, *word2, *word3;
+
+        string_trim((*li)->buf, STRING_BOTH);
+        if (str2words((*li)->buf, wptr, 4) != 4) {
+            if ((*li)->buf[0] != '\0')
+                break;
+            continue;
+        }
+        else {
+            p = (float32)atof_c(wptr[0]);
+            word1 = wptr[1];
+            word2 = wptr[2];
+            word3 = wptr[3];
+        }
+
+        if ((w1 = ngram_wid(base, word1)) == NGRAM_INVALID_WID) {
+            E_ERROR("Unknown word: %s, skipping trigram (%s %s %s)\n",
+                    word1, word1, word2, word3);
+            continue;
+        }
+        if ((w2 = ngram_wid(base, word2)) == NGRAM_INVALID_WID) {
+            E_ERROR("Unknown word: %s, skipping trigram (%s %s %s)\n",
+                    word2, word1, word2, word3);
+            continue;
+        }
+        if ((w3 = ngram_wid(base, word3)) == NGRAM_INVALID_WID) {
+            E_ERROR("Unknown word: %s, skipping trigram (%s %s %s)\n",
+                    word3, word1, word2, word3);
+            continue;
+        }
+
+        /* FIXME: Should use logmath_t quantization here. */
+        /* HACK!! to quantize probs to 4 decimal digits */
+        p = (float32)((int32)(p * 10000)) / 10000;
+        p3 = logmath_log10_to_log(base->lmath, p);
+
+        if (tgcount >= base->n_counts[2]) {
+            E_ERROR("Too many trigrams\n");
+            return -1;
+        }
+
+        tgptr->wid = w3;
+        tgptr->prob3 = sorted_id(&model->sorted_prob3, &p3);
+
+        if ((w1 != prev_w1) || (w2 != prev_w2)) {
+            /* Trigram for a new bigram; update tg info for all previous bigrams */
+            if ((w1 < prev_w1) || ((w1 == prev_w1) && (w2 < prev_w2))) {
+                E_ERROR("Trigrams not in bigram order\n");
+                return -1;
+            }
+
+            bg = (w1 !=
+                  prev_w1) ? model->lm3g.unigrams[w1].bigrams : prev_bg + 1;
+            endbg = model->lm3g.unigrams[w1 + 1].bigrams;
+            bgptr = model->lm3g.bigrams + bg;
+            for (; (bg < endbg) && (bgptr->wid != w2); bg++, bgptr++);
+            if (bg >= endbg) {
+                E_ERROR("Missing bigram for trigram: %s", (*li)->buf);
+                return -1;
+            }
+
+            /* bg = bigram entry index for <w1,w2>.  Update tseg_base */
+            seg = bg >> LOG_BG_SEG_SZ;
+            for (i = prev_seg + 1; i <= seg; i++)
+                model->lm3g.tseg_base[i] = tgcount;
+
+            /* Update trigrams pointers for all bigrams until bg */
+            if (prev_seg < seg) {
+                int32 tgoff = 0;
+
+                if (prev_seg >= 0) {
+                    tgoff = tgcount - model->lm3g.tseg_base[prev_seg];
+                    if (tgoff > 65535) {
+                        E_ERROR("Size of trigram segment is bigger than 65535, such a big language models are not supported, use smaller vocabulary\n");
+                        return -1;
+                    }
+                }
+
+                prev_seg_lastbg = ((prev_seg + 1) << LOG_BG_SEG_SZ) - 1;
+                bgptr = model->lm3g.bigrams + prev_bg;
+                for (++prev_bg, ++bgptr; prev_bg <= prev_seg_lastbg;
+                     prev_bg++, bgptr++)
+                    bgptr->trigrams = tgoff;
+
+                for (; prev_bg <= bg; prev_bg++, bgptr++)
+                    bgptr->trigrams = 0;
+            }
+            else {
+                int32 tgoff;
+
+                tgoff = tgcount - model->lm3g.tseg_base[prev_seg];
+                if (tgoff > 65535) {
+                    E_ERROR("Size of trigram segment is bigger than 65535, such a big language models are not supported, use smaller vocabulary\n");
+                    return -1;
+                }
+
+                bgptr = model->lm3g.bigrams + prev_bg;
+                for (++prev_bg, ++bgptr; prev_bg <= bg; prev_bg++, bgptr++)
+                    bgptr->trigrams = tgoff;
+            }
+
+            prev_w1 = w1;
+            prev_w2 = w2;
+            prev_bg = bg;
+            prev_seg = seg;
+        }
+
+        tgcount++;
+        tgptr++;
+
+        if ((tgcount & 0x0000ffff) == 0) {
+            E_INFOCONT(".");
+        }
+    }
+    if (*li == NULL || strcmp((*li)->buf, "\\end\\") != 0) {
+        E_ERROR("Bad trigram: %s\n", (*li)->buf);
+        return -1;
+    }
+
+    for (prev_bg++; prev_bg <= base->n_counts[1]; prev_bg++) {
+        if ((prev_bg & (BG_SEG_SZ - 1)) == 0)
+            model->lm3g.tseg_base[prev_bg >> LOG_BG_SEG_SZ] = tgcount;
+        if ((tgcount - model->lm3g.tseg_base[prev_bg >> LOG_BG_SEG_SZ]) > 65535) {
+            E_ERROR("Size of trigram segment is bigger than 65535, such a big language models are not supported, use smaller vocabulary\n");
+            return -1;
+        }
+        model->lm3g.bigrams[prev_bg].trigrams =
+            tgcount - model->lm3g.tseg_base[prev_bg >> LOG_BG_SEG_SZ];
+    }
+    return 0;
+}
+
+static unigram_t *
+new_unigram_table(int32 n_ug)
+{
+    unigram_t *table;
+    int32 i;
+
+    table = ckd_calloc(n_ug, sizeof(unigram_t));
+    for (i = 0; i < n_ug; i++) {
+        table[i].prob1.l = INT_MIN;
+        table[i].bo_wt1.l = INT_MIN;
+    }
+    return table;
+}
+
+ngram_model_t *
+ngram_model_arpa_read(cmd_ln_t *config,
+		      const char *file_name,
+		      logmath_t *lmath)
+{
+    lineiter_t *li;
+    FILE *fp;
+    int32 is_pipe;
+    int32 n_unigram;
+    int32 n_bigram;
+    int32 n_trigram;
+    int32 n;
+    ngram_model_arpa_t *model;
+    ngram_model_t *base;
+
+    if ((fp = fopen_comp(file_name, "r", &is_pipe)) == NULL) {
+        E_ERROR("File %s not found\n", file_name);
+        return NULL;
+    }
+    li = lineiter_start(fp);
+ 
+    /* Read #unigrams, #bigrams, #trigrams from file */
+    if (ReadNgramCounts(&li, &n_unigram, &n_bigram, &n_trigram) == -1) {
+        lineiter_free(li);
+        fclose_comp(fp, is_pipe);
+        return NULL;
+    }
+    E_INFO("ngrams 1=%d, 2=%d, 3=%d\n", n_unigram, n_bigram, n_trigram);
+
+    /* Allocate space for LM, including initial OOVs and placeholders; initialize it */
+    model = ckd_calloc(1, sizeof(*model));
+    base = &model->base;
+    if (n_trigram > 0)
+        n = 3;
+    else if (n_bigram > 0)
+        n = 2;
+    else
+        n = 1;
+    /* Initialize base model. */
+    ngram_model_init(base, &ngram_model_arpa_funcs, lmath, n, n_unigram);
+    base->n_counts[0] = n_unigram;
+    base->n_counts[1] = n_bigram;
+    base->n_counts[2] = n_trigram;
+    base->writable = TRUE;
+
+    /*
+     * Allocate one extra unigram and bigram entry: sentinels to terminate
+     * followers (bigrams and trigrams, respectively) of previous entry.
+     */
+    model->lm3g.unigrams = new_unigram_table(n_unigram + 1);
+    model->lm3g.bigrams =
+        ckd_calloc(n_bigram + 1, sizeof(bigram_t));
+    if (n_trigram > 0)
+        model->lm3g.trigrams =
+            ckd_calloc(n_trigram, sizeof(trigram_t));
+
+    if (n_trigram > 0) {
+        model->lm3g.tseg_base =
+            ckd_calloc((n_bigram + 1) / BG_SEG_SZ + 1,
+                       sizeof(int32));
+    }
+    if (ReadUnigrams(&li, model) == -1) {
+        fclose_comp(fp, is_pipe);
+        ngram_model_free(base);
+        return NULL;
+    }
+    E_INFO("%8d = #unigrams created\n", base->n_counts[0]);
+
+    if (base->n_counts[2] > 0)
+        init_sorted_list(&model->sorted_bo_wt2);
+
+    if (base->n_counts[1] > 0) {
+        init_sorted_list(&model->sorted_prob2);
+
+        if (ReadBigrams(&li, model) == -1) {
+            fclose_comp(fp, is_pipe);
+            ngram_model_free(base);
+            return NULL;
+        }
+
+        base->n_counts[1] = FIRST_BG(model, base->n_counts[0]);
+        model->lm3g.n_prob2 = model->sorted_prob2.free;
+        model->lm3g.prob2 = vals_in_sorted_list(&model->sorted_prob2);
+        free_sorted_list(&model->sorted_prob2);
+        E_INFO("%8d = #bigrams created\n", base->n_counts[1]);
+        E_INFO("%8d = #prob2 entries\n", model->lm3g.n_prob2);
+    }
+
+    if (base->n_counts[2] > 0) {
+        /* Create trigram bo-wts array */
+        model->lm3g.n_bo_wt2 = model->sorted_bo_wt2.free;
+        model->lm3g.bo_wt2 = vals_in_sorted_list(&model->sorted_bo_wt2);
+        free_sorted_list(&model->sorted_bo_wt2);
+        E_INFO("%8d = #bo_wt2 entries\n", model->lm3g.n_bo_wt2);
+
+        init_sorted_list(&model->sorted_prob3);
+
+        if (ReadTrigrams(&li, model) == -1) {
+            fclose_comp(fp, is_pipe);
+            ngram_model_free(base);
+            return NULL;
+        }
+
+        base->n_counts[2] = FIRST_TG(model, base->n_counts[1]);
+        model->lm3g.n_prob3 = model->sorted_prob3.free;
+        model->lm3g.prob3 = vals_in_sorted_list(&model->sorted_prob3);
+        E_INFO("%8d = #trigrams created\n", base->n_counts[2]);
+        E_INFO("%8d = #prob3 entries\n", model->lm3g.n_prob3);
+
+        free_sorted_list(&model->sorted_prob3);
+
+        /* Initialize tginfo */
+        model->lm3g.tginfo = ckd_calloc(n_unigram, sizeof(tginfo_t *));
+        model->lm3g.le = listelem_alloc_init(sizeof(tginfo_t));
+    }
+
+    lineiter_free(li);
+    fclose_comp(fp, is_pipe);
+    return base;
+}
+
+int
+ngram_model_arpa_write(ngram_model_t *model,
+		       const char *file_name)
+{
+    ngram_iter_t *itor;
+    FILE *fh;
+    int i;
+
+    if ((fh = fopen(file_name, "w")) == NULL) {
+        E_ERROR_SYSTEM("Failed to open %s for writing", file_name);
+        return -1;
+    }
+    fprintf(fh, "This is an ARPA-format language model file, generated by CMU Sphinx\n");
+
+    /* The ARPA format doesn't require any extra information that
+     * N-Gram iterators can't give us, so this is very
+     * straightforward compared with DMP writing. */
+
+    /* Write N-gram counts. */
+    fprintf(fh, "\\data\\\n");
+    for (i = 0; i < model->n; ++i) {
+        fprintf(fh, "ngram %d=%d\n", i+1, model->n_counts[i]);
+    }
+
+    /* Write N-grams */
+    for (i = 0; i < model->n; ++i) {
+        fprintf(fh, "\n\\%d-grams:\n", i + 1);
+        for (itor = ngram_model_mgrams(model, i); itor; itor = ngram_iter_next(itor)) {
+            int32 const *wids;
+            int32 score, bowt;
+            int j;
+
+            wids = ngram_iter_get(itor, &score, &bowt);
+            fprintf(fh, "%.4f ", logmath_log_to_log10(model->lmath, score));
+            for (j = 0; j <= i; ++j) {
+                assert(wids[j] < model->n_counts[0]);
+                fprintf(fh, "%s ", model->word_str[wids[j]]);
+            }
+            if (i < model->n-1)
+                fprintf(fh, "%.4f", logmath_log_to_log10(model->lmath, bowt));
+            fprintf(fh, "\n");
+        }
+    }
+    fprintf(fh, "\n\\end\\\n");
+    return fclose(fh);
+}
+
+static int
+ngram_model_arpa_apply_weights(ngram_model_t *base, float32 lw,
+                              float32 wip, float32 uw)
+{
+    ngram_model_arpa_t *model = (ngram_model_arpa_t *)base;
+    lm3g_apply_weights(base, &model->lm3g, lw, wip, uw);
+    return 0;
+}
+
+/* Lousy "templating" for things that are largely the same in DMP and
+ * ARPA models, except for the bigram and trigram types and some
+ * names. */
+#define NGRAM_MODEL_TYPE ngram_model_arpa_t
+#include "lm3g_templates.c"
+
+static void
+ngram_model_arpa_free(ngram_model_t *base)
+{
+    ngram_model_arpa_t *model = (ngram_model_arpa_t *)base;
+    ckd_free(model->lm3g.unigrams);
+    ckd_free(model->lm3g.bigrams);
+    ckd_free(model->lm3g.trigrams);
+    ckd_free(model->lm3g.prob2);
+    ckd_free(model->lm3g.bo_wt2);
+    ckd_free(model->lm3g.prob3);
+    lm3g_tginfo_free(base, &model->lm3g);
+    ckd_free(model->lm3g.tseg_base);
+}
+
+static ngram_funcs_t ngram_model_arpa_funcs = {
+    ngram_model_arpa_free,          /* free */
+    ngram_model_arpa_apply_weights, /* apply_weights */
+    lm3g_template_score,            /* score */
+    lm3g_template_raw_score,        /* raw_score */
+    lm3g_template_add_ug,           /* add_ug */
+    lm3g_template_flush,            /* flush */
+    lm3g_template_iter,             /* iter */
+    lm3g_template_mgrams,           /* mgrams */
+    lm3g_template_successors,       /* successors */
+    lm3g_template_iter_get,         /* iter_get */
+    lm3g_template_iter_next,        /* iter_next */
+    lm3g_template_iter_free         /* iter_free */
+};

media/sphinxbase/src/libsphinxbase/lm/ngram_model_arpa.h

@@ -0,0 +1,86 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * \file ngram_model_arpa.h ARPABO text format for N-Gram models
+ *
+ * Author: David Huggins-Daines <dhuggins@cs.cmu.edu>
+ */
+
+#ifndef __NGRAM_MODEL_ARPA_H__
+#define __NGRAM_MODEL_ARPA_H__
+
+#include "ngram_model_internal.h"
+#include "lm3g_model.h"
+
+/**
+ * Bigram structure.
+ */
+struct bigram_s {
+    uint32 wid;	/**< Index of unigram entry for this.  (NOT dictionary id.) */
+    uint16 prob2;	/**< Index into array of actual bigram probs */
+    uint16 bo_wt2;	/**< Index into array of actual bigram backoff wts */
+    uint16 trigrams;	/**< Index of 1st entry in lm_t.trigrams[],
+			     RELATIVE TO its segment base (see above) */
+};
+
+/**
+ * Trigram structure.
+ *
+ * As with bigrams, trigram prob info kept in a separate table for conserving
+ * memory space.
+ */
+struct trigram_s {
+    uint32 wid;	  /**< Index of unigram entry for this.  (NOT dictionary id.) */
+    uint16 prob3; /**< Index into array of actual trigram probs */
+};
+
+
+/**
+ * Subclass of ngram_model for ARPA file reading.
+ */
+typedef struct ngram_model_arpa_s {
+    ngram_model_t base;  /**< Base ngram_model_t structure */
+    lm3g_model_t lm3g;  /**< Shared lm3g structure */
+
+    /* Arrays of unique bigram probs and bo-wts, and trigram probs
+     * (these are temporary, actually) */
+    sorted_list_t sorted_prob2;
+    sorted_list_t sorted_bo_wt2;
+    sorted_list_t sorted_prob3;
+} ngram_model_arpa_t;
+
+#endif /* __NGRAM_MODEL_ARPA_H__ */

media/sphinxbase/src/libsphinxbase/lm/ngram_model_dmp.c

@@ -0,0 +1,969 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * \file ngram_model_dmp.c DMP format language models
+ *
+ * Author: David Huggins-Daines <dhuggins@cs.cmu.edu>
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <limits.h>
+
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/pio.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/byteorder.h"
+#include "sphinxbase/listelem_alloc.h"
+
+#include "ngram_model_dmp.h"
+
+static const char darpa_hdr[] = "Darpa Trigram LM";
+static ngram_funcs_t ngram_model_dmp_funcs;
+
+#define TSEG_BASE(m,b)		((m)->lm3g.tseg_base[(b)>>LOG_BG_SEG_SZ])
+#define FIRST_BG(m,u)		((m)->lm3g.unigrams[u].bigrams)
+#define FIRST_TG(m,b)		(TSEG_BASE((m),(b))+((m)->lm3g.bigrams[b].trigrams))
+
+static unigram_t *
+new_unigram_table(int32 n_ug)
+{
+    unigram_t *table;
+    int32 i;
+
+    table = ckd_calloc(n_ug, sizeof(unigram_t));
+    for (i = 0; i < n_ug; i++) {
+        table[i].prob1.f = -99.0;
+        table[i].bo_wt1.f = -99.0;
+    }
+    return table;
+}
+
+ngram_model_t *
+ngram_model_dmp_read(cmd_ln_t *config,
+                     const char *file_name,
+                     logmath_t *lmath)
+{
+    ngram_model_t *base;
+    ngram_model_dmp_t *model;
+    FILE *fp;
+    int do_mmap, do_swap;
+    int32 is_pipe;
+    int32 i, j, k, vn, n, ts;
+    int32 n_unigram;
+    int32 n_bigram;
+    int32 n_trigram;
+    char str[1024];
+    unigram_t *ugptr;
+    bigram_t *bgptr;
+    trigram_t *tgptr;
+    char *tmp_word_str;
+    char *map_base = NULL;
+    size_t offset = 0;
+
+    base = NULL;
+    do_mmap = FALSE;
+    if (config)
+        do_mmap = cmd_ln_boolean_r(config, "-mmap");
+
+    if ((fp = fopen_comp(file_name, "rb", &is_pipe)) == NULL) {
+        E_ERROR("Dump file %s not found\n", file_name);
+        goto error_out;
+    }
+
+    if (is_pipe && do_mmap) {
+        E_WARN("Dump file is compressed, will not use memory-mapped I/O\n");
+        do_mmap = 0;
+    }
+
+    do_swap = FALSE;
+    if (fread(&k, sizeof(k), 1, fp) != 1)
+        goto error_out;
+    if (k != strlen(darpa_hdr)+1) {
+        SWAP_INT32(&k);
+        if (k != strlen(darpa_hdr)+1) {
+            E_ERROR("Wrong magic header size number %x: %s is not a dump file\n", k, file_name);
+            goto error_out;
+        }
+        do_swap = 1;
+    }
+    if (fread(str, 1, k, fp) != (size_t) k) {
+        E_ERROR("Cannot read header\n");
+        goto error_out;
+    }
+    if (strncmp(str, darpa_hdr, k) != 0) {
+        E_ERROR("Wrong header %s: %s is not a dump file\n", darpa_hdr);
+        goto error_out;
+    }
+
+    if (do_mmap) {
+        if (do_swap) {
+            E_INFO
+                ("Byteswapping required, will not use memory-mapped I/O for LM file\n");
+            do_mmap = 0;
+        }
+        else {
+            E_INFO("Will use memory-mapped I/O for LM file\n");
+#ifdef __ADSPBLACKFIN__ /* This is true for both VisualDSP++ and uClinux. */
+            E_FATAL("memory mapping is not supported at the moment.");
+#else
+#endif
+        }
+    }
+
+    if (fread(&k, sizeof(k), 1, fp) != 1)
+        goto error_out;
+    if (do_swap) SWAP_INT32(&k);
+    if (fread(str, 1, k, fp) != (size_t) k) {
+        E_ERROR("Cannot read LM filename in header\n");
+        goto error_out;
+    }
+
+    /* read version#, if present (must be <= 0) */
+    if (fread(&vn, sizeof(vn), 1, fp) != 1)
+        goto error_out;
+    if (do_swap) SWAP_INT32(&vn);
+    if (vn <= 0) {
+        /* read and don't compare timestamps (we don't care) */
+        if (fread(&ts, sizeof(ts), 1, fp) != 1)
+            goto error_out;
+        if (do_swap) SWAP_INT32(&ts);
+
+        /* read and skip format description */
+        for (;;) {
+            if (fread(&k, sizeof(k), 1, fp) != 1)
+                goto error_out;
+            if (do_swap) SWAP_INT32(&k);
+            if (k == 0)
+                break;
+            if (fread(str, 1, k, fp) != (size_t) k) {
+                E_ERROR("Failed to read word\n");
+                goto error_out;
+            }
+        }
+        /* read model->ucount */
+        if (fread(&n_unigram, sizeof(n_unigram), 1, fp) != 1)
+            goto error_out;
+        if (do_swap) SWAP_INT32(&n_unigram);
+    }
+    else {
+        n_unigram = vn;
+    }
+
+    /* read model->bcount, tcount */
+    if (fread(&n_bigram, sizeof(n_bigram), 1, fp) != 1)
+        goto error_out;
+    if (do_swap) SWAP_INT32(&n_bigram);
+    if (fread(&n_trigram, sizeof(n_trigram), 1, fp) != 1)
+        goto error_out;
+    if (do_swap) SWAP_INT32(&n_trigram);
+    E_INFO("ngrams 1=%d, 2=%d, 3=%d\n", n_unigram, n_bigram, n_trigram);
+
+    /* Allocate space for LM, including initial OOVs and placeholders; initialize it */
+    model = ckd_calloc(1, sizeof(*model));
+    base = &model->base;
+    if (n_trigram > 0)
+        n = 3;
+    else if (n_bigram > 0)
+        n = 2;
+    else
+        n = 1;
+    ngram_model_init(base, &ngram_model_dmp_funcs, lmath, n, n_unigram);
+    base->n_counts[0] = n_unigram;
+    base->n_counts[1] = n_bigram;
+    base->n_counts[2] = n_trigram;
+
+    /* read unigrams (always in memory, as they contain dictionary
+     * mappings that can't be precomputed, and also could have OOVs added) */
+    model->lm3g.unigrams = new_unigram_table(n_unigram + 1);
+    ugptr = model->lm3g.unigrams;
+    for (i = 0; i <= n_unigram; ++i) {
+        /* Skip over the mapping ID, we don't care about it. */
+        if (fread(ugptr, sizeof(int32), 1, fp) != 1) {
+            E_ERROR("Failed to read maping id %d\n", i);
+            goto error_out;
+        }
+        /* Read the actual unigram structure. */
+        if (fread(ugptr, sizeof(unigram_t), 1, fp) != 1)  {
+            E_ERROR("Failed to read unigrams data\n");
+            ngram_model_free(base);
+            fclose_comp(fp, is_pipe);
+            return NULL;
+        }
+        /* Byte swap if necessary. */
+        if (do_swap) {
+            SWAP_INT32(&ugptr->prob1.l);
+            SWAP_INT32(&ugptr->bo_wt1.l);
+            SWAP_INT32(&ugptr->bigrams);
+        }
+        /* Convert values to log. */
+        ugptr->prob1.l = logmath_log10_to_log(lmath, ugptr->prob1.f);
+        ugptr->bo_wt1.l = logmath_log10_to_log(lmath, ugptr->bo_wt1.f);
+        E_DEBUG(2, ("ug %d: prob %d bo %d bigrams %d\n",
+                    i, ugptr->prob1.l, ugptr->bo_wt1.l, ugptr->bigrams));
+        ++ugptr;
+    }
+    E_INFO("%8d = LM.unigrams(+trailer) read\n", n_unigram);
+
+    /* Now mmap() the file and read in the rest of the (read-only) stuff. */
+    if (do_mmap) {
+        offset = ftell(fp);
+
+        /* Check for improper word alignment. */
+        if (offset & 0x3) {
+            E_WARN("-mmap specified, but trigram index is not word-aligned.  Will not memory-map.\n");
+            do_mmap = FALSE;
+        }
+        else {
+            model->dump_mmap = mmio_file_read(file_name);
+            if (model->dump_mmap == NULL) {
+                do_mmap = FALSE;
+            }
+            else {
+                map_base = mmio_file_ptr(model->dump_mmap);
+            }
+        }
+    }
+    
+    if (n_bigram > 0) {
+        /* read bigrams */
+	if (do_mmap) {
+    	    model->lm3g.bigrams = (bigram_t *) (map_base + offset);
+    	    offset += (n_bigram + 1) * sizeof(bigram_t);
+	}
+	else {
+    	    model->lm3g.bigrams =
+        	ckd_calloc(n_bigram + 1, sizeof(bigram_t));
+    	    if (fread(model->lm3g.bigrams, sizeof(bigram_t), n_bigram + 1, fp)
+        	!= (size_t) n_bigram + 1) {
+    		E_ERROR("Failed to read bigrams data\n");
+        	goto error_out;
+    	    }
+    	    if (do_swap) {
+        	for (i = 0, bgptr = model->lm3g.bigrams; i <= n_bigram;
+            	     i++, bgptr++) {
+            	    SWAP_INT16(&bgptr->wid);
+            	    SWAP_INT16(&bgptr->prob2);
+            	    SWAP_INT16(&bgptr->bo_wt2);
+            	    SWAP_INT16(&bgptr->trigrams);
+        	}
+    	    }
+	}
+	E_INFO("%8d = LM.bigrams(+trailer) read\n", n_bigram);
+    }
+
+    /* read trigrams */
+    if (n_trigram > 0) {
+        if (do_mmap) {
+            model->lm3g.trigrams = (trigram_t *) (map_base + offset);
+            offset += n_trigram * sizeof(trigram_t);
+        }
+        else {
+            model->lm3g.trigrams =
+                ckd_calloc(n_trigram, sizeof(trigram_t));
+            if (fread
+                (model->lm3g.trigrams, sizeof(trigram_t), n_trigram, fp)
+                != (size_t) n_trigram) {
+                E_ERROR("Failed to read trigrams data\n");
+                goto error_out;
+            }
+            if (do_swap) {
+                for (i = 0, tgptr = model->lm3g.trigrams; i < n_trigram;
+                     i++, tgptr++) {
+                    SWAP_INT16(&tgptr->wid);
+                    SWAP_INT16(&tgptr->prob3);
+                }
+            }
+        }
+        E_INFO("%8d = LM.trigrams read\n", n_trigram);
+        /* Initialize tginfo */
+        model->lm3g.tginfo = ckd_calloc(n_unigram, sizeof(tginfo_t *));
+        model->lm3g.le = listelem_alloc_init(sizeof(tginfo_t));
+    }
+
+    if (n_bigram > 0) {
+        /* read n_prob2 and prob2 array (in memory) */
+	if (do_mmap)
+    	    fseek(fp, offset, SEEK_SET);
+        if (fread(&k, sizeof(k), 1, fp) != 1)
+	    goto error_out;
+        if (do_swap) SWAP_INT32(&k);
+	model->lm3g.n_prob2 = k;
+        model->lm3g.prob2 = ckd_calloc(k, sizeof(*model->lm3g.prob2));
+	if (fread(model->lm3g.prob2, sizeof(*model->lm3g.prob2), k, fp) != (size_t) k) {
+    	    E_ERROR("fread(prob2) failed\n");
+    	    goto error_out;
+	}
+	for (i = 0; i < k; i++) {
+    	    if (do_swap)
+        	SWAP_INT32(&model->lm3g.prob2[i].l);
+    	    /* Convert values to log. */
+    	    model->lm3g.prob2[i].l = logmath_log10_to_log(lmath, model->lm3g.prob2[i].f);
+	}
+	E_INFO("%8d = LM.prob2 entries read\n", k);
+    }
+
+    /* read n_bo_wt2 and bo_wt2 array (in memory) */
+    if (base->n > 2) {
+        if (fread(&k, sizeof(k), 1, fp) != 1)
+            goto error_out;
+        if (do_swap) SWAP_INT32(&k);
+        model->lm3g.n_bo_wt2 = k;
+        model->lm3g.bo_wt2 = ckd_calloc(k, sizeof(*model->lm3g.bo_wt2));
+        if (fread(model->lm3g.bo_wt2, sizeof(*model->lm3g.bo_wt2), k, fp) != (size_t) k) {
+            E_ERROR("Failed to read backoff weights\n");
+            goto error_out;
+        }
+        for (i = 0; i < k; i++) {
+            if (do_swap)
+                SWAP_INT32(&model->lm3g.bo_wt2[i].l);
+            /* Convert values to log. */
+            model->lm3g.bo_wt2[i].l = logmath_log10_to_log(lmath, model->lm3g.bo_wt2[i].f);
+        }
+        E_INFO("%8d = LM.bo_wt2 entries read\n", k);
+    }
+
+    /* read n_prob3 and prob3 array (in memory) */
+    if (base->n > 2) {
+        if (fread(&k, sizeof(k), 1, fp) != 1)
+		goto error_out;
+	if (do_swap) SWAP_INT32(&k);
+    	model->lm3g.n_prob3 = k;
+    	model->lm3g.prob3 = ckd_calloc(k, sizeof(*model->lm3g.prob3));
+    	if (fread(model->lm3g.prob3, sizeof(*model->lm3g.prob3), k, fp) != (size_t) k) {
+    	    E_ERROR("Failed to read trigram probability\n");
+    	    goto error_out;
+    	}
+    	for (i = 0; i < k; i++) {
+    	    if (do_swap)
+                SWAP_INT32(&model->lm3g.prob3[i].l);
+    	    /* Convert values to log. */
+    	    model->lm3g.prob3[i].l = logmath_log10_to_log(lmath, model->lm3g.prob3[i].f);
+    	}
+        E_INFO("%8d = LM.prob3 entries read\n", k);
+    }
+
+    /* read tseg_base size and tseg_base */
+    if (do_mmap)
+        offset = ftell(fp);
+    if (n_trigram > 0) {
+        if (do_mmap) {
+            memcpy(&k, map_base + offset, sizeof(k));
+            offset += sizeof(int32);
+            model->lm3g.tseg_base = (int32 *) (map_base + offset);
+            offset += k * sizeof(int32);
+        }
+        else {
+            k = (n_bigram + 1) / BG_SEG_SZ + 1;
+            if (fread(&k, sizeof(k), 1, fp) != 1)
+                goto error_out;
+            if (do_swap) SWAP_INT32(&k);
+            model->lm3g.tseg_base = ckd_calloc(k, sizeof(int32));
+            if (fread(model->lm3g.tseg_base, sizeof(int32), k, fp) !=
+                (size_t) k) {
+                E_ERROR("Failed to read trigram index\n");
+                goto error_out;
+            }
+            if (do_swap)
+                for (i = 0; i < k; i++)
+                    SWAP_INT32(&model->lm3g.tseg_base[i]);
+        }
+        E_INFO("%8d = LM.tseg_base entries read\n", k);
+    }
+
+    /* read ascii word strings */
+    if (do_mmap) {
+        memcpy(&k, map_base + offset, sizeof(k));
+        offset += sizeof(int32);
+        tmp_word_str = (char *) (map_base + offset);
+        offset += k;
+    }
+    else {
+        base->writable = TRUE;
+        if (fread(&k, sizeof(k), 1, fp) != 1)
+            goto error_out;
+        if (do_swap) SWAP_INT32(&k);
+        tmp_word_str = ckd_calloc(k, 1);
+        if (fread(tmp_word_str, 1, k, fp) != (size_t) k) {
+            E_ERROR("Failed to read words\n");
+            goto error_out;
+        }
+    }
+
+    /* First make sure string just read contains n_counts[0] words (PARANOIA!!) */
+    for (i = 0, j = 0; i < k; i++)
+        if (tmp_word_str[i] == '\0')
+            j++;
+    if (j != n_unigram) {
+        E_ERROR("Error reading word strings (%d doesn't match n_unigrams %d)\n",
+                j, n_unigram);
+        goto error_out;
+    }
+
+    /* Break up string just read into words */
+    if (do_mmap) {
+        j = 0;
+        for (i = 0; i < n_unigram; i++) {
+            base->word_str[i] = tmp_word_str + j;
+            if (hash_table_enter(base->wid, base->word_str[i],
+                                 (void *)(long)i) != (void *)(long)i) {
+                E_WARN("Duplicate word in dictionary: %s\n", base->word_str[i]);
+            }
+            j += strlen(base->word_str[i]) + 1;
+        }
+    }
+    else {
+        j = 0;
+        for (i = 0; i < n_unigram; i++) {
+            base->word_str[i] = ckd_salloc(tmp_word_str + j);
+            if (hash_table_enter(base->wid, base->word_str[i],
+                                 (void *)(long)i) != (void *)(long)i) {
+                E_WARN("Duplicate word in dictionary: %s\n", base->word_str[i]);
+            }
+            j += strlen(base->word_str[i]) + 1;
+        }
+        free(tmp_word_str);
+    }
+    E_INFO("%8d = ascii word strings read\n", i);
+
+    fclose_comp(fp, is_pipe);
+    return base;
+
+error_out:
+    if (fp)
+        fclose_comp(fp, is_pipe);
+    ngram_model_free(base);
+    return NULL;
+}
+
+ngram_model_dmp_t *
+ngram_model_dmp_build(ngram_model_t *base)
+{
+    ngram_model_dmp_t *model;
+    ngram_model_t *newbase;
+    ngram_iter_t *itor;
+    sorted_list_t sorted_prob2;
+    sorted_list_t sorted_bo_wt2;
+    sorted_list_t sorted_prob3;
+    bigram_t *bgptr;
+    trigram_t *tgptr;
+    int i, bgcount, tgcount, seg;
+
+    if (base->funcs == &ngram_model_dmp_funcs) {
+        E_INFO("Using existing DMP model.\n");
+        return (ngram_model_dmp_t *)ngram_model_retain(base);
+    }
+
+    /* Initialize new base model structure with params from base. */
+    E_INFO("Building DMP model...\n");
+    model = ckd_calloc(1, sizeof(*model));
+    newbase = &model->base;
+    ngram_model_init(newbase, &ngram_model_dmp_funcs,
+                     logmath_retain(base->lmath),
+                     base->n, base->n_counts[0]);
+    /* Copy N-gram counts over. */
+    memcpy(newbase->n_counts, base->n_counts,
+           base->n * sizeof(*base->n_counts));
+    /* Make sure word strings are freed. */
+    newbase->writable = TRUE;
+    /* Initialize unigram table and string table. */
+    model->lm3g.unigrams = new_unigram_table(newbase->n_counts[0] + 1);
+    for (itor = ngram_model_mgrams(base, 0); itor;
+         itor = ngram_iter_next(itor)) {
+        int32 prob1, bo_wt1;
+        int32 const *wids;
+
+        /* Can't guarantee they will go in unigram order, so just to
+         * be correct, we do this... */
+        wids = ngram_iter_get(itor, &prob1, &bo_wt1);
+        model->lm3g.unigrams[wids[0]].prob1.l = prob1;
+        model->lm3g.unigrams[wids[0]].bo_wt1.l = bo_wt1;
+        newbase->word_str[wids[0]] = ckd_salloc(ngram_word(base, wids[0]));
+        if ((hash_table_enter_int32(newbase->wid,
+                                    newbase->word_str[wids[0]], wids[0]))
+            != wids[0]) {
+                E_WARN("Duplicate word in dictionary: %s\n", newbase->word_str[wids[0]]);
+        }
+    }
+    E_INFO("%8d = #unigrams created\n", newbase->n_counts[0]);
+		
+    if (newbase->n < 2) 
+        return model;
+			 
+    /* Construct quantized probability table for bigrams and
+     * (optionally) trigrams.  Hesitate to use the "sorted list" thing
+     * since it isn't so useful, but it's there already. */
+    init_sorted_list(&sorted_prob2);
+    if (newbase->n > 2) {
+        init_sorted_list(&sorted_bo_wt2);
+        init_sorted_list(&sorted_prob3);
+    }
+    /* Construct bigram and trigram arrays. */
+    bgptr = model->lm3g.bigrams = ckd_calloc(newbase->n_counts[1] + 1, sizeof(bigram_t));
+    if (newbase->n > 2) {
+        tgptr = model->lm3g.trigrams = ckd_calloc(newbase->n_counts[2], sizeof(trigram_t));
+        model->lm3g.tseg_base =
+            ckd_calloc((newbase->n_counts[1] + 1) / BG_SEG_SZ + 1, sizeof(int32));
+    }
+    else
+        tgptr = NULL;
+    /* Since bigrams and trigrams have to be contiguous with others
+     * with the same N-1-gram, we traverse them in depth-first order
+     * to build the bigram and trigram arrays. */
+    for (i = 0; i < newbase->n_counts[0]; ++i) {
+        ngram_iter_t *uitor;
+        bgcount = bgptr - model->lm3g.bigrams;
+        /* First bigram index (same as next if no bigrams...) */
+        model->lm3g.unigrams[i].bigrams = bgcount;
+        E_DEBUG(2, ("unigram %d: %s => bigram %d\n", i, newbase->word_str[i], bgcount));
+        /* All bigrams corresponding to unigram i */
+        uitor = ngram_ng_iter(base, i, NULL, 0);
+        for (itor = ngram_iter_successors(uitor);
+             itor; ++bgptr, itor = ngram_iter_next(itor)) {
+            int32 prob2, bo_wt2;
+            int32 const *wids;
+            ngram_iter_t *titor;
+
+            wids = ngram_iter_get(itor, &prob2, &bo_wt2);
+
+            assert (bgptr - model->lm3g.bigrams < newbase->n_counts[1]);
+
+            bgptr->wid = wids[1];
+            bgptr->prob2 = sorted_id(&sorted_prob2, &prob2);
+            if (newbase->n > 2) {
+                tgcount = (tgptr - model->lm3g.trigrams);
+	        bgcount = (bgptr - model->lm3g.bigrams);
+
+                /* Backoff weight (only if there are trigrams...) */
+                bgptr->bo_wt2 = sorted_id(&sorted_bo_wt2, &bo_wt2);
+
+                /* Find bigram segment for this bigram (this isn't
+                 * used unless there are trigrams) */
+                seg = bgcount >> LOG_BG_SEG_SZ;
+                /* If we just crossed a bigram segment boundary, then
+                 * point tseg_base for the new segment to the current
+                 * trigram pointer. */
+                if (seg != (bgcount - 1) >> LOG_BG_SEG_SZ)
+                    model->lm3g.tseg_base[seg] = tgcount;
+                /* Now calculate the trigram offset. */
+                bgptr->trigrams = tgcount - model->lm3g.tseg_base[seg];
+                E_DEBUG(2, ("bigram %d %s %s => trigram %d:%d\n",
+                            bgcount,
+                            newbase->word_str[wids[0]],
+                            newbase->word_str[wids[1]],
+                            seg, bgptr->trigrams));
+
+                /* And fill in successors' trigram info. */
+                for (titor = ngram_iter_successors(itor);
+                     titor; ++tgptr, titor = ngram_iter_next(titor)) {
+                    int32 prob3, dummy;
+
+                    assert(tgptr - model->lm3g.trigrams < newbase->n_counts[2]);
+                    wids = ngram_iter_get(titor, &prob3, &dummy);
+                    tgptr->wid = wids[2];
+                    tgptr->prob3 = sorted_id(&sorted_prob3, &prob3);
+                    E_DEBUG(2, ("trigram %d %s %s %s => prob %d\n",
+                                tgcount,
+                                newbase->word_str[wids[0]],
+                                newbase->word_str[wids[1]],
+                                newbase->word_str[wids[2]],
+                                tgptr->prob3));
+                }
+            }
+        }
+        ngram_iter_free(uitor);
+    }
+    /* Add sentinal unigram and bigram records. */
+    bgcount = bgptr - model->lm3g.bigrams;
+    tgcount = tgptr - model->lm3g.trigrams;
+    seg = bgcount >> LOG_BG_SEG_SZ;
+    if (seg != (bgcount - 1) >> LOG_BG_SEG_SZ)
+        model->lm3g.tseg_base[seg] = tgcount;
+    model->lm3g.unigrams[i].bigrams = bgcount;
+    if (newbase->n > 2)
+        bgptr->trigrams = tgcount - model->lm3g.tseg_base[seg];
+
+    /* Now create probability tables. */
+    model->lm3g.n_prob2 = sorted_prob2.free;
+    model->lm3g.prob2 = vals_in_sorted_list(&sorted_prob2);
+    E_INFO("%8d = #bigrams created\n", newbase->n_counts[1]);
+    E_INFO("%8d = #prob2 entries\n", model->lm3g.n_prob2);
+    free_sorted_list(&sorted_prob2);
+    if (newbase->n > 2) {
+        /* Create trigram bo-wts array. */
+        model->lm3g.n_bo_wt2 = sorted_bo_wt2.free;
+        model->lm3g.bo_wt2 = vals_in_sorted_list(&sorted_bo_wt2);
+        free_sorted_list(&sorted_bo_wt2);
+        E_INFO("%8d = #bo_wt2 entries\n", model->lm3g.n_bo_wt2);
+        /* Create trigram probability table. */
+        model->lm3g.n_prob3 = sorted_prob3.free;
+        model->lm3g.prob3 = vals_in_sorted_list(&sorted_prob3);
+        E_INFO("%8d = #trigrams created\n", newbase->n_counts[2]);
+        E_INFO("%8d = #prob3 entries\n", model->lm3g.n_prob3);
+        free_sorted_list(&sorted_prob3);
+        /* Initialize tginfo */
+        model->lm3g.tginfo = ckd_calloc(newbase->n_counts[0], sizeof(tginfo_t *));
+        model->lm3g.le = listelem_alloc_init(sizeof(tginfo_t));
+    }
+
+    return model;
+}
+
+static void
+fwrite_int32(FILE *fh, int32 val)
+{
+    fwrite(&val, 4, 1, fh);
+}
+
+static void
+fwrite_ug(FILE *fh, unigram_t *ug, logmath_t *lmath)
+{
+    int32 bogus = -1;
+    float32 log10val;
+
+    /* Bogus dictionary mapping field. */
+    fwrite(&bogus, 4, 1, fh);
+    /* Convert values to log10. */
+    log10val = logmath_log_to_log10(lmath, ug->prob1.l);
+    fwrite(&log10val, 4, 1, fh);
+    log10val = logmath_log_to_log10(lmath, ug->bo_wt1.l);
+    fwrite(&log10val, 4, 1, fh);
+    fwrite_int32(fh, ug->bigrams);
+}
+
+static void
+fwrite_bg(FILE *fh, bigram_t *bg)
+{
+    fwrite(bg, sizeof(*bg), 1, fh);
+}
+
+static void
+fwrite_tg(FILE *fh, trigram_t *tg)
+{
+    fwrite(tg, sizeof(*tg), 1, fh);
+}
+
+/** Please look at the definition of 
+ */
+static char const *fmtdesc[] = {
+    "BEGIN FILE FORMAT DESCRIPTION",
+    "Header string length (int32) and string (including trailing 0)",
+    "Original LM filename string-length (int32) and filename (including trailing 0)",
+    "(int32) version number (present iff value <= 0)",
+    "(int32) original LM file modification timestamp (iff version# present)",
+    "(int32) string-length and string (including trailing 0) (iff version# present)",
+    "... previous entry continued any number of times (iff version# present)",
+    "(int32) 0 (terminating sequence of strings) (iff version# present)",
+    "(int32) log_bg_seg_sz (present iff different from default value of LOG2_BG_SEG_SZ)",
+    "(int32) lm_t.ucount (must be > 0)",
+    "(int32) lm_t.bcount",
+    "(int32) lm_t.tcount",
+    "lm_t.ucount+1 unigrams (including sentinel)",
+    "lm_t.bcount+1 bigrams (including sentinel 64 bits (bg_t) each if version=-1/-2, 128 bits (bg32_t) each if version=-3",
+    "lm_t.tcount trigrams (present iff lm_t.tcount > 0 32 bits (tg_t) each if version=-1/-2, 64 bits (tg32_t) each if version=-3)",
+    "(int32) lm_t.n_prob2",
+    "(int32) lm_t.prob2[]",
+    "(int32) lm_t.n_bo_wt2 (present iff lm_t.tcount > 0)",
+    "(int32) lm_t.bo_wt2[] (present iff lm_t.tcount > 0)",
+    "(int32) lm_t.n_prob3 (present iff lm_t.tcount > 0)",
+    "(int32) lm_t.prob3[] (present iff lm_t.tcount > 0)",
+    "(int32) (lm_t.bcount+1)/BG_SEG_SZ+1 (present iff lm_t.tcount > 0)",
+    "(int32) lm_t.tseg_base[] (present iff lm_t.tcount > 0)",
+    "(int32) Sum(all word string-lengths, including trailing 0 for each)",
+    "All word strings (including trailing 0 for each)",
+    "END FILE FORMAT DESCRIPTION",
+    NULL,
+};
+
+static void
+ngram_model_dmp_write_header(FILE * fh)
+{
+    int32 k;
+    k = strlen(darpa_hdr) + 1;
+    fwrite_int32(fh, k);
+    fwrite(darpa_hdr, 1, k, fh);
+}
+
+static void
+ngram_model_dmp_write_lm_filename(FILE * fh, const char *lmfile)
+{
+    int32 k;
+
+    k = strlen(lmfile) + 1;
+    fwrite_int32(fh, k);
+    fwrite(lmfile, 1, k, fh);
+}
+
+#define LMDMP_VERSION_TG_16BIT -1 /**< VERSION 1 is the simplest DMP file which
+				     is trigram or lower which used 16 bits in
+				     bigram and trigram.*/
+
+static void
+ngram_model_dmp_write_version(FILE * fh, int32 mtime)
+{
+    fwrite_int32(fh, LMDMP_VERSION_TG_16BIT);   /* version # */
+    fwrite_int32(fh, mtime);
+}
+
+static void
+ngram_model_dmp_write_ngram_counts(FILE * fh, ngram_model_t *model)
+{
+    fwrite_int32(fh, model->n_counts[0]);
+    fwrite_int32(fh, model->n_counts[1]);
+    fwrite_int32(fh, model->n_counts[2]);
+}
+
+static void
+ngram_model_dmp_write_fmtdesc(FILE * fh)
+{
+    int32 i, k;
+    long pos;
+
+    /* Write file format description into header */
+    for (i = 0; fmtdesc[i] != NULL; i++) {
+        k = strlen(fmtdesc[i]) + 1;
+        fwrite_int32(fh, k);
+        fwrite(fmtdesc[i], 1, k, fh);
+    }
+    /* Pad it out in order to achieve 32-bit alignment */
+    pos = ftell(fh);
+    k = pos & 3;
+    if (k) {
+        fwrite_int32(fh, 4-k);
+        fwrite("!!!!", 1, 4-k, fh);
+    }
+    fwrite_int32(fh, 0);
+}
+
+static void
+ngram_model_dmp_write_unigram(FILE *fh, ngram_model_t *model)
+{
+    ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
+    int32 i;
+
+    for (i = 0; i <= model->n_counts[0]; i++) {
+        fwrite_ug(fh, &(lm->lm3g.unigrams[i]), model->lmath);
+    }
+}
+
+
+static void
+ngram_model_dmp_write_bigram(FILE *fh, ngram_model_t *model)
+{
+    ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
+    int32 i;
+
+    for (i = 0; i <= model->n_counts[1]; i++) {
+        fwrite_bg(fh, &(lm->lm3g.bigrams[i]));
+    }
+
+}
+
+static void
+ngram_model_dmp_write_trigram(FILE *fh, ngram_model_t *model)
+{
+    ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
+    int32 i;
+
+    for (i = 0; i < model->n_counts[2]; i++) {
+        fwrite_tg(fh, &(lm->lm3g.trigrams[i]));
+    }
+}
+
+static void
+ngram_model_dmp_write_bgprob(FILE *fh, ngram_model_t *model)
+{
+    ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
+    int32 i;
+
+    fwrite_int32(fh, lm->lm3g.n_prob2);
+    for (i = 0; i < lm->lm3g.n_prob2; i++) {
+        float32 log10val = logmath_log_to_log10(model->lmath, lm->lm3g.prob2[i].l);
+        fwrite(&log10val, 4, 1, fh);
+    }
+}
+
+static void
+ngram_model_dmp_write_tgbowt(FILE *fh, ngram_model_t *model)
+{
+    ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
+    int32 i;
+
+    fwrite_int32(fh, lm->lm3g.n_bo_wt2);
+    for (i = 0; i < lm->lm3g.n_bo_wt2; i++) {
+        float32 log10val = logmath_log_to_log10(model->lmath, lm->lm3g.bo_wt2[i].l);
+        fwrite(&log10val, 4, 1, fh);
+    }
+}
+
+static void
+ngram_model_dmp_write_tgprob(FILE *fh, ngram_model_t *model)
+{
+    ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
+    int32 i;
+
+    fwrite_int32(fh, lm->lm3g.n_prob3);
+    for (i = 0; i < lm->lm3g.n_prob3; i++) {
+        float32 log10val = logmath_log_to_log10(model->lmath, lm->lm3g.prob3[i].l);
+        fwrite(&log10val, 4, 1, fh);
+    }
+}
+
+static void
+ngram_model_dmp_write_tg_segbase(FILE *fh, ngram_model_t *model)
+{
+    ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
+    int32 i, k;
+
+    k = (model->n_counts[1] + 1) / BG_SEG_SZ + 1;
+    fwrite_int32(fh, k);
+    for (i = 0; i < k; i++)
+        fwrite_int32(fh, lm->lm3g.tseg_base[i]);
+}
+
+static void
+ngram_model_dmp_write_wordstr(FILE *fh, ngram_model_t *model)
+{
+    int32 i, k;
+
+    k = 0;
+    for (i = 0; i < model->n_counts[0]; i++)
+        k += strlen(model->word_str[i]) + 1;
+    fwrite_int32(fh, k);
+    for (i = 0; i < model->n_counts[0]; i++)
+        fwrite(model->word_str[i], 1,
+               strlen(model->word_str[i]) + 1, fh);
+}
+
+int
+ngram_model_dmp_write(ngram_model_t *base,
+                      const char *file_name)
+{
+    ngram_model_dmp_t *model;
+    ngram_model_t *newbase;
+    FILE *fh;
+
+    /* First, construct a DMP model from the base model. */
+    model = ngram_model_dmp_build(base);
+    newbase = &model->base;
+
+    /* Now write it, confident in the knowledge that it's the right
+     * kind of language model internally. */
+    if ((fh = fopen(file_name, "wb")) == NULL) {
+        E_ERROR("Cannot create file %s\n", file_name);
+        return -1;
+    }
+    ngram_model_dmp_write_header(fh);
+    ngram_model_dmp_write_lm_filename(fh, file_name);
+    ngram_model_dmp_write_version(fh, 0);
+    ngram_model_dmp_write_fmtdesc(fh);
+    ngram_model_dmp_write_ngram_counts(fh, newbase);
+    ngram_model_dmp_write_unigram(fh, newbase);
+    if (newbase->n > 1) {
+        ngram_model_dmp_write_bigram(fh, newbase);
+	if (newbase->n > 2) {
+	    ngram_model_dmp_write_trigram(fh, newbase);
+	}
+	ngram_model_dmp_write_bgprob(fh, newbase);
+	if (newbase->n > 2) {
+	        ngram_model_dmp_write_tgbowt(fh, newbase);
+	        ngram_model_dmp_write_tgprob(fh, newbase);
+	        ngram_model_dmp_write_tg_segbase(fh, newbase);
+        }
+    }
+    ngram_model_dmp_write_wordstr(fh, newbase);
+    ngram_model_free(newbase);
+
+    return fclose(fh);
+}
+
+static int
+ngram_model_dmp_apply_weights(ngram_model_t *base, float32 lw,
+                              float32 wip, float32 uw)
+{
+    ngram_model_dmp_t *model = (ngram_model_dmp_t *)base;
+    lm3g_apply_weights(base, &model->lm3g, lw, wip, uw);
+    return 0;
+}
+
+/* Lousy "templating" for things that are largely the same in DMP and
+ * ARPA models, except for the bigram and trigram types and some
+ * names. */
+#define NGRAM_MODEL_TYPE ngram_model_dmp_t
+#include "lm3g_templates.c"
+
+static void
+ngram_model_dmp_free(ngram_model_t *base)
+{
+    ngram_model_dmp_t *model = (ngram_model_dmp_t *)base;
+
+    ckd_free(model->lm3g.unigrams);
+    ckd_free(model->lm3g.prob2);
+    if (model->dump_mmap) {
+        mmio_file_unmap(model->dump_mmap);
+    } 
+    else {
+        ckd_free(model->lm3g.bigrams);
+        if (base->n > 2) {
+            ckd_free(model->lm3g.trigrams);
+            ckd_free(model->lm3g.tseg_base);
+        }
+    }
+    if (base->n > 2) {
+        ckd_free(model->lm3g.bo_wt2);
+        ckd_free(model->lm3g.prob3);
+    }
+
+    lm3g_tginfo_free(base, &model->lm3g);
+}
+
+static ngram_funcs_t ngram_model_dmp_funcs = {
+    ngram_model_dmp_free,          /* free */
+    ngram_model_dmp_apply_weights, /* apply_weights */
+    lm3g_template_score,           /* score */
+    lm3g_template_raw_score,       /* raw_score */
+    lm3g_template_add_ug,          /* add_ug */
+    lm3g_template_flush,           /* flush */
+    lm3g_template_iter,             /* iter */
+    lm3g_template_mgrams,          /* mgrams */
+    lm3g_template_successors,      /* successors */
+    lm3g_template_iter_get,        /* iter_get */
+    lm3g_template_iter_next,       /* iter_next */
+    lm3g_template_iter_free        /* iter_free */
+};

media/sphinxbase/src/libsphinxbase/lm/ngram_model_dmp.h

@@ -0,0 +1,92 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * \file ngram_model_dmp.h DMP format for N-Gram models
+ *
+ * Author: David Huggins-Daines <dhuggins@cs.cmu.edu>
+ */
+
+#ifndef __NGRAM_MODEL_DMP_H__
+#define __NGRAM_MODEL_DMP_H__
+
+#include "sphinxbase/mmio.h"
+
+#include "ngram_model_internal.h"
+#include "lm3g_model.h"
+
+/**
+ * On-disk representation of bigrams.
+ */
+struct bigram_s {
+    uint16 wid;	/**< Index of unigram entry for this.  (NOT dictionary id.) */
+    uint16 prob2;	/**< Index into array of actual bigram probs */
+    uint16 bo_wt2;	/**< Index into array of actual bigram backoff wts */
+    uint16 trigrams;	/**< Index of 1st entry in lm_t.trigrams[],
+			   RELATIVE TO its segment base (see lm3g_model.h) */
+};
+
+/**
+ * On-disk representation of trigrams.
+ *
+ * As with bigrams, trigram prob info kept in a separate table for conserving
+ * memory space.
+ */
+struct trigram_s {
+    uint16 wid;	  /**< Index of unigram entry for this.  (NOT dictionary id.) */
+    uint16 prob3; /**< Index into array of actual trigram probs */
+};
+
+/**
+ * Subclass of ngram_model for DMP file reading.
+ */
+typedef struct ngram_model_dmp_s {
+    ngram_model_t base;  /**< Base ngram_model_t structure */
+    lm3g_model_t lm3g;   /**< Common lm3g_model_t structure */
+    mmio_file_t *dump_mmap; /**< mmap() of dump file (or NULL if none) */
+} ngram_model_dmp_t;
+
+/**
+ * Construct a DMP format model from a generic base model.
+ *
+ * Note: If base is already a DMP format model, this just calls
+ * ngram_model_retain(), and any changes will also be made in the base
+ * model.
+ */
+ngram_model_dmp_t *ngram_model_dmp_build(ngram_model_t *base);
+
+
+#endif /*  __NGRAM_MODEL_DMP_H__ */

media/sphinxbase/src/libsphinxbase/lm/ngram_model_internal.h

@@ -0,0 +1,282 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * \file ngram_model_internal.h Internal structures for N-Gram models
+ *
+ * Author: David Huggins-Daines <dhuggins@cs.cmu.edu>
+ */
+
+#ifndef __NGRAM_MODEL_INTERNAL_H__
+#define __NGRAM_MODEL_INTERNAL_H__
+
+#include "sphinxbase/ngram_model.h"
+#include "sphinxbase/hash_table.h"
+
+/**
+ * Common implementation of ngram_model_t.
+ *
+ * The details of bigram, trigram, and higher-order N-gram storage, if any, can
+ * vary somewhat depending on the file format in use.
+ */
+struct ngram_model_s {
+    int refcount;       /**< Reference count */
+    int32 *n_counts;    /**< Counts for 1, 2, 3, ... grams */
+    int32 n_1g_alloc;   /**< Number of allocated word strings (for new word addition) */
+    int32 n_words;      /**< Number of actual word strings (NOT the same as the
+                             number of unigrams, due to class words). */
+    uint8 n;            /**< This is an n-gram model (1, 2, 3, ...). */
+    uint8 n_classes;    /**< Number of classes (maximum 128) */
+    uint8 writable;     /**< Are word strings writable? */
+    uint8 flags;        /**< Any other flags we might care about
+                             (FIXME: Merge this and writable) */
+    logmath_t *lmath;   /**< Log-math object */
+    float32 lw;         /**< Language model scaling factor */
+    int32 log_wip;      /**< Log of word insertion penalty */
+    int32 log_uw;       /**< Log of unigram weight */
+    int32 log_uniform;  /**< Log of uniform (0-gram) probability */
+    int32 log_uniform_weight; /**< Log of uniform weight (i.e. 1 - unigram weight) */
+    int32 log_zero;     /**< Zero probability, cached here for quick lookup */
+    char **word_str;    /**< Unigram names */
+    hash_table_t *wid;  /**< Mapping of unigram names to word IDs. */
+    int32 *tmp_wids;    /**< Temporary array of word IDs for ngram_model_get_ngram() */
+    struct ngram_class_s **classes; /**< Word class definitions. */
+    struct ngram_funcs_s *funcs;   /**< Implementation-specific methods. */
+};
+
+/**
+ * Implementation of ngram_class_t.
+ */
+struct ngram_class_s {
+    int32 tag_wid;  /**< Base word ID for this class tag */
+    int32 start_wid; /**< Starting base word ID for this class' words */
+    int32 n_words;   /**< Number of base words for this class */
+    int32 *prob1;    /**< Probability table for base words */
+    /**
+     * Custom hash table for additional words.
+     */
+    struct ngram_hash_s {
+        int32 wid;    /**< Word ID of this bucket */
+        int32 prob1;  /**< Probability for this word */
+        int32 next;   /**< Index of next bucket (or -1 for no collision) */
+    } *nword_hash;
+    int32 n_hash;       /**< Number of buckets in nword_hash (power of 2) */
+    int32 n_hash_inuse; /**< Number of words in nword_hash */
+};
+
+#define NGRAM_HASH_SIZE 128
+
+#define NGRAM_BASEWID(wid) ((wid)&0xffffff)
+#define NGRAM_CLASSID(wid) (((wid)>>24) & 0x7f)
+#define NGRAM_CLASSWID(wid,classid) (((classid)<<24) | 0x80000000 | (wid))
+#define NGRAM_IS_CLASSWID(wid) ((wid)&0x80000000)
+
+#define UG_ALLOC_STEP 10
+
+/** Implementation-specific functions for operating on ngram_model_t objects */
+typedef struct ngram_funcs_s {
+    /**
+     * Implementation-specific function for freeing an ngram_model_t.
+     */
+    void (*free)(ngram_model_t *model);
+    /**
+     * Implementation-specific function for applying language model weights.
+     */
+    int (*apply_weights)(ngram_model_t *model,
+                         float32 lw,
+                         float32 wip,
+                         float32 uw);
+    /**
+     * Implementation-specific function for querying language model score.
+     */
+    int32 (*score)(ngram_model_t *model,
+                   int32 wid,
+                   int32 *history,
+                   int32 n_hist,
+                   int32 *n_used);
+    /**
+     * Implementation-specific function for querying raw language
+     * model probability.
+     */
+    int32 (*raw_score)(ngram_model_t *model,
+                       int32 wid,
+                       int32 *history,
+                       int32 n_hist,
+                       int32 *n_used);
+    /**
+     * Implementation-specific function for adding unigrams.
+     *
+     * This function updates the internal structures of a language
+     * model to add the given unigram with the given weight (defined
+     * as a log-factor applied to the uniform distribution).  This
+     * includes reallocating or otherwise resizing the set of unigrams.
+     *
+     * @return The language model score (not raw log-probability) of
+     * the new word, or 0 for failure.
+     */
+    int32 (*add_ug)(ngram_model_t *model,
+                    int32 wid, int32 lweight);
+    /**
+     * Implementation-specific function for purging N-Gram cache
+     */
+    void (*flush)(ngram_model_t *model);
+
+    /**
+     * Implementation-specific function for iterating.
+     */
+    ngram_iter_t * (*iter)(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist);
+
+    /**
+     * Implementation-specific function for iterating.
+     */
+    ngram_iter_t * (*mgrams)(ngram_model_t *model, int32 m);
+
+    /**
+     * Implementation-specific function for iterating.
+     */
+    ngram_iter_t * (*successors)(ngram_iter_t *itor);
+
+    /**
+     * Implementation-specific function for iterating.
+     */
+    int32 const * (*iter_get)(ngram_iter_t *itor,
+                              int32 *out_score,
+                              int32 *out_bowt);
+
+    /**
+     * Implementation-specific function for iterating.
+     */
+    ngram_iter_t * (*iter_next)(ngram_iter_t *itor);
+
+    /**
+     * Implementation-specific function for iterating.
+     */
+    void (*iter_free)(ngram_iter_t *itor);
+} ngram_funcs_t;
+
+/**
+ * Base iterator structure for N-grams.
+ */
+struct ngram_iter_s {
+    ngram_model_t *model;
+    int32 *wids;      /**< Scratch space for word IDs. */
+    int16 m;          /**< Order of history. */
+    int16 successor;  /**< Is this a successor iterator? */
+};
+
+/**
+ * One class definition from a classdef file.
+ */
+typedef struct classdef_s {
+    char **words;
+    float32 *weights;
+    int32 n_words;
+} classdef_t;
+
+/**
+ * Initialize the base ngram_model_t structure.
+ */
+int32
+ngram_model_init(ngram_model_t *model,
+                 ngram_funcs_t *funcs,
+                 logmath_t *lmath,
+                 int32 n, int32 n_unigram);
+
+/**
+ * Read an N-Gram model from an ARPABO text file.
+ */
+ngram_model_t *ngram_model_arpa_read(cmd_ln_t *config,
+				     const char *file_name,
+				     logmath_t *lmath);
+/**
+ * Read an N-Gram model from a Sphinx .DMP binary file.
+ */
+ngram_model_t *ngram_model_dmp_read(cmd_ln_t *config,
+				    const char *file_name,
+				    logmath_t *lmath);
+/**
+ * Read an N-Gram model from a Sphinx .DMP32 binary file.
+ */
+ngram_model_t *ngram_model_dmp32_read(cmd_ln_t *config,
+				     const char *file_name,
+				     logmath_t *lmath);
+
+/**
+ * Write an N-Gram model to an ARPABO text file.
+ */
+int ngram_model_arpa_write(ngram_model_t *model,
+			   const char *file_name);
+/**
+ * Write an N-Gram model to a Sphinx .DMP binary file.
+ */
+int ngram_model_dmp_write(ngram_model_t *model,
+			  const char *file_name);
+
+/**
+ * Read a probdef file.
+ */
+int32 read_classdef_file(hash_table_t *classes, const char *classdef_file);
+
+/**
+ * Free a class definition.
+ */
+void classdef_free(classdef_t *classdef);
+
+/**
+ * Allocate and initialize an N-Gram class.
+ */
+ngram_class_t *ngram_class_new(ngram_model_t *model, int32 tag_wid,
+                               int32 start_wid, glist_t classwords);
+
+/**
+ * Deallocate an N-Gram class.
+ */
+void ngram_class_free(ngram_class_t *lmclass);
+
+/**
+ * Get the in-class log probability for a word in an N-Gram class.
+ *
+ * @return This probability, or 1 if word not found.
+ */
+int32 ngram_class_prob(ngram_class_t *lmclass, int32 wid);
+
+/**
+ * Initialize base M-Gram iterator structure.
+ */
+void ngram_iter_init(ngram_iter_t *itor, ngram_model_t *model,
+                     int m, int successor);
+
+#endif /* __NGRAM_MODEL_INTERNAL_H__ */

media/sphinxbase/src/libsphinxbase/lm/ngram_model_set.c

@@ -0,0 +1,870 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2008 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/**
+ * @file ngram_model_set.c Set of language models.
+ * @author David Huggins-Daines <dhuggins@cs.cmu.edu>
+ */
+
+#include <string.h>
+#include <stdlib.h>
+
+#include "sphinxbase/err.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/strfuncs.h"
+#include "sphinxbase/filename.h"
+
+#include "ngram_model_set.h"
+
+static ngram_funcs_t ngram_model_set_funcs;
+
+static int
+my_compare(const void *a, const void *b)
+{
+    /* Make sure <UNK> floats to the beginning. */
+    if (strcmp(*(char * const *)a, "<UNK>") == 0)
+        return -1;
+    else if (strcmp(*(char * const *)b, "<UNK>") == 0)
+        return 1;
+    else
+        return strcmp(*(char * const *)a, *(char * const *)b);
+}
+
+static void
+build_widmap(ngram_model_t *base, logmath_t *lmath, int32 n)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    ngram_model_t **models = set->lms;
+    hash_table_t *vocab;
+    glist_t hlist;
+    gnode_t *gn;
+    int32 i;
+
+    /* Construct a merged vocabulary and a set of word-ID mappings. */
+    vocab = hash_table_new(models[0]->n_words, FALSE);
+    /* Create the set of merged words. */
+    for (i = 0; i < set->n_models; ++i) {
+        int32 j;
+        for (j = 0; j < models[i]->n_words; ++j) {
+            /* Ignore collisions. */
+            (void)hash_table_enter_int32(vocab, models[i]->word_str[j], j);
+        }
+    }
+    /* Create the array of words, then sort it. */
+    if (hash_table_lookup(vocab, "<UNK>", NULL) != 0)
+        (void)hash_table_enter_int32(vocab, "<UNK>", 0);
+    /* Now we know the number of unigrams, initialize the base model. */
+    ngram_model_init(base, &ngram_model_set_funcs, lmath, n, hash_table_inuse(vocab));
+    base->writable = FALSE; /* We will reuse the pointers from the submodels. */
+    i = 0;
+    hlist = hash_table_tolist(vocab, NULL);
+    for (gn = hlist; gn; gn = gnode_next(gn)) {
+        hash_entry_t *ent = gnode_ptr(gn);
+        base->word_str[i++] = (char *)ent->key;
+    }
+    glist_free(hlist);
+    qsort(base->word_str, base->n_words, sizeof(*base->word_str), my_compare);
+
+    /* Now create the word ID mappings. */
+    if (set->widmap)
+        ckd_free_2d((void **)set->widmap);
+    set->widmap = (int32 **) ckd_calloc_2d(base->n_words, set->n_models,
+                                           sizeof(**set->widmap));
+    for (i = 0; i < base->n_words; ++i) {
+        int32 j;
+        /* Also create the master wid mapping. */
+        (void)hash_table_enter_int32(base->wid, base->word_str[i], i);
+        /* printf("%s: %d => ", base->word_str[i], i); */
+        for (j = 0; j < set->n_models; ++j) {
+            set->widmap[i][j] = ngram_wid(models[j], base->word_str[i]);
+            /* printf("%d ", set->widmap[i][j]); */
+        }
+        /* printf("\n"); */
+    }
+    hash_table_free(vocab);
+}
+
+ngram_model_t *
+ngram_model_set_init(cmd_ln_t *config,
+                     ngram_model_t **models,
+                     char **names,
+                     const float32 *weights,
+                     int32 n_models)
+{
+    ngram_model_set_t *model;
+    ngram_model_t *base;
+    logmath_t *lmath;
+    int32 i, n;
+
+    if (n_models == 0) /* WTF */
+        return NULL;
+
+    /* Do consistency checking on the models.  They must all use the
+     * same logbase and shift. */
+    lmath = models[0]->lmath;
+    for (i = 1; i < n_models; ++i) {
+        if (logmath_get_base(models[i]->lmath) != logmath_get_base(lmath)
+            || logmath_get_shift(models[i]->lmath) != logmath_get_shift(lmath)) {
+            E_ERROR("Log-math parameters don't match, will not create LM set\n");
+            return NULL;
+        }
+    }
+
+    /* Allocate the combined model, initialize it. */
+    model = ckd_calloc(1, sizeof(*model));
+    base = &model->base;
+    model->n_models = n_models;
+    model->lms = ckd_calloc(n_models, sizeof(*model->lms));
+    model->names = ckd_calloc(n_models, sizeof(*model->names));
+    /* Initialize weights to a uniform distribution */
+    model->lweights = ckd_calloc(n_models, sizeof(*model->lweights));
+    {
+        int32 uniform = logmath_log(lmath, 1.0/n_models);
+        for (i = 0; i < n_models; ++i)
+            model->lweights[i] = uniform;
+    }
+    /* Default to interpolate if weights were given. */
+    if (weights)
+        model->cur = -1;
+
+    n = 0;
+    for (i = 0; i < n_models; ++i) {
+        model->lms[i] = ngram_model_retain(models[i]);
+        model->names[i] = ckd_salloc(names[i]);
+        if (weights)
+            model->lweights[i] = logmath_log(lmath, weights[i]);
+        /* N is the maximum of all merged models. */
+        if (models[i]->n > n)
+            n = models[i]->n;
+    }
+    /* Allocate the history mapping table. */
+    model->maphist = ckd_calloc(n - 1, sizeof(*model->maphist));
+
+    /* Now build the word-ID mapping and merged vocabulary. */
+    build_widmap(base, lmath, n);
+    return base;
+}
+
+ngram_model_t *
+ngram_model_set_read(cmd_ln_t *config,
+                     const char *lmctlfile,
+                     logmath_t *lmath)
+{
+    FILE *ctlfp;
+    glist_t lms = NULL;
+    glist_t lmnames = NULL;
+    __BIGSTACKVARIABLE__ char str[1024];
+    ngram_model_t *set = NULL;
+    hash_table_t *classes;
+    char *basedir, *c;
+
+    /* Read all the class definition files to accumulate a mapping of
+     * classnames to definitions. */
+    classes = hash_table_new(0, FALSE);
+    if ((ctlfp = fopen(lmctlfile, "r")) == NULL) {
+        E_ERROR_SYSTEM("Failed to open %s", lmctlfile);
+        return NULL;
+    }
+
+    /* Try to find the base directory to append to relative paths in
+     * the lmctl file. */
+    if ((c = strrchr(lmctlfile, '/')) || (c = strrchr(lmctlfile, '\\'))) {
+        /* Include the trailing slash. */
+        basedir = ckd_calloc(c - lmctlfile + 2, 1);
+        memcpy(basedir, lmctlfile, c - lmctlfile + 1);
+    }
+    else {
+        basedir = NULL;
+    }
+    E_INFO("Reading LM control file '%s'\n", lmctlfile);
+    if (basedir)
+        E_INFO("Will prepend '%s' to unqualified paths\n", basedir);
+
+    if (fscanf(ctlfp, "%1023s", str) == 1) {
+        if (strcmp(str, "{") == 0) {
+            /* Load LMclass files */
+            while ((fscanf(ctlfp, "%1023s", str) == 1)
+                   && (strcmp(str, "}") != 0)) {
+                char *deffile;
+                if (basedir && !path_is_absolute(str))
+                    deffile = string_join(basedir, str, NULL);
+                else
+                    deffile = ckd_salloc(str);
+                E_INFO("Reading classdef from '%s'\n", deffile);
+                if (read_classdef_file(classes, deffile) < 0) {
+                    ckd_free(deffile);
+                    goto error_out;
+                }
+                ckd_free(deffile);
+            }
+
+            if (strcmp(str, "}") != 0) {
+                E_ERROR("Unexpected EOF in %s\n", lmctlfile);
+                goto error_out;
+            }
+
+            /* This might be the first LM name. */
+            if (fscanf(ctlfp, "%1023s", str) != 1)
+                str[0] = '\0';
+        }
+    }
+    else
+        str[0] = '\0';
+
+    /* Read in one LM at a time and add classes to them as necessary. */
+    while (str[0] != '\0') {
+        char *lmfile;
+        ngram_model_t *lm;
+
+        if (basedir && str[0] != '/' && str[0] != '\\')
+            lmfile = string_join(basedir, str, NULL);
+        else
+            lmfile = ckd_salloc(str);
+        E_INFO("Reading lm from '%s'\n", lmfile);
+        lm = ngram_model_read(config, lmfile, NGRAM_AUTO, lmath);
+        if (lm == NULL) {
+            ckd_free(lmfile);
+            goto error_out;
+        }
+        if (fscanf(ctlfp, "%1023s", str) != 1) {
+            E_ERROR("LMname missing after LMFileName '%s'\n", lmfile);
+            ckd_free(lmfile);
+            goto error_out;
+        }
+        ckd_free(lmfile);
+        lms = glist_add_ptr(lms, lm);
+        lmnames = glist_add_ptr(lmnames, ckd_salloc(str));
+
+        if (fscanf(ctlfp, "%1023s", str) == 1) {
+            if (strcmp(str, "{") == 0) {
+                /* LM uses classes; read their names */
+                while ((fscanf(ctlfp, "%1023s", str) == 1) &&
+                       (strcmp(str, "}") != 0)) {
+                    void *val;
+                    classdef_t *classdef;
+
+                    if (hash_table_lookup(classes, str, &val) == -1) {
+                        E_ERROR("Unknown class %s in control file\n", str);
+                        goto error_out;
+                    }
+                    classdef = val;
+                    if (ngram_model_add_class(lm, str, 1.0,
+                                              classdef->words, classdef->weights,
+                                              classdef->n_words) < 0) {
+                        goto error_out;
+                    }
+                    E_INFO("Added class %s containing %d words\n",
+                           str, classdef->n_words);
+                }
+                if (strcmp(str, "}") != 0) {
+                    E_ERROR("Unexpected EOF in %s\n", lmctlfile);
+                    goto error_out;
+                }
+                if (fscanf(ctlfp, "%1023s", str) != 1)
+                    str[0] = '\0';
+            }
+        }
+        else
+            str[0] = '\0';
+    }
+    fclose(ctlfp);
+
+    /* Now construct arrays out of lms and lmnames, and build an
+     * ngram_model_set. */
+    lms = glist_reverse(lms);
+    lmnames = glist_reverse(lmnames);
+    {
+        int32 n_models;
+        ngram_model_t **lm_array;
+        char **name_array;
+        gnode_t *lm_node, *name_node;
+        int32 i;
+
+        n_models = glist_count(lms);
+        lm_array = ckd_calloc(n_models, sizeof(*lm_array));
+        name_array = ckd_calloc(n_models, sizeof(*name_array));
+        lm_node = lms;
+        name_node = lmnames;
+        for (i = 0; i < n_models; ++i) {
+            lm_array[i] = gnode_ptr(lm_node);
+            name_array[i] = gnode_ptr(name_node);
+            lm_node = gnode_next(lm_node);
+            name_node = gnode_next(name_node);
+        }
+        set = ngram_model_set_init(config, lm_array, name_array,
+                                   NULL, n_models);
+        ckd_free(lm_array);
+        ckd_free(name_array);
+    }
+error_out:
+    {
+        gnode_t *gn;
+        glist_t hlist;
+
+        if (set == NULL) {
+            for (gn = lms; gn; gn = gnode_next(gn)) {
+                ngram_model_free(gnode_ptr(gn));
+            }
+        }
+        glist_free(lms);
+        for (gn = lmnames; gn; gn = gnode_next(gn)) {
+            ckd_free(gnode_ptr(gn));
+        }
+        glist_free(lmnames);
+        hlist = hash_table_tolist(classes, NULL);
+        for (gn = hlist; gn; gn = gnode_next(gn)) {
+            hash_entry_t *he = gnode_ptr(gn);
+            ckd_free((char *)he->key);
+            classdef_free(he->val);
+        }
+        glist_free(hlist);
+        hash_table_free(classes);
+        ckd_free(basedir);
+    }
+    return set;
+}
+
+int32
+ngram_model_set_count(ngram_model_t *base)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    return set->n_models;
+}
+
+ngram_model_set_iter_t *
+ngram_model_set_iter(ngram_model_t *base)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    ngram_model_set_iter_t *itor;
+
+    if (set == NULL || set->n_models == 0)
+        return NULL;
+    itor = ckd_calloc(1, sizeof(*itor));
+    itor->set = set;
+    return itor;
+}
+
+ngram_model_set_iter_t *
+ngram_model_set_iter_next(ngram_model_set_iter_t *itor)
+{
+    if (++itor->cur == itor->set->n_models) {
+        ngram_model_set_iter_free(itor);
+        return NULL;
+    }
+    return itor;
+}
+
+void
+ngram_model_set_iter_free(ngram_model_set_iter_t *itor)
+{
+    ckd_free(itor);
+}
+
+ngram_model_t *
+ngram_model_set_iter_model(ngram_model_set_iter_t *itor,
+                           char const **lmname)
+{
+    if (lmname) *lmname = itor->set->names[itor->cur];
+    return itor->set->lms[itor->cur];
+}
+
+ngram_model_t *
+ngram_model_set_lookup(ngram_model_t *base,
+                       const char *name)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    int32 i;
+
+    if (name == NULL) {
+        if (set->cur == -1)
+            return NULL;
+        else
+            return set->lms[set->cur];
+    }
+
+    /* There probably won't be very many submodels. */
+    for (i = 0; i < set->n_models; ++i)
+        if (0 == strcmp(set->names[i], name))
+            break;
+    if (i == set->n_models)
+        return NULL;
+    return set->lms[i];
+}
+
+ngram_model_t *
+ngram_model_set_select(ngram_model_t *base,
+                       const char *name)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    int32 i;
+
+    /* There probably won't be very many submodels. */
+    for (i = 0; i < set->n_models; ++i)
+        if (0 == strcmp(set->names[i], name))
+            break;
+    if (i == set->n_models)
+        return NULL;
+    set->cur = i;
+    return set->lms[set->cur];
+}
+
+const char *
+ngram_model_set_current(ngram_model_t *base)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+
+    if (set->cur == -1)
+        return NULL;
+    else
+        return set->names[set->cur];
+}
+
+int32
+ngram_model_set_current_wid(ngram_model_t *base,
+                            int32 set_wid)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+
+    if (set->cur == -1 || set_wid >= base->n_words)
+        return NGRAM_INVALID_WID;
+    else
+        return set->widmap[set_wid][set->cur];
+}
+
+int32
+ngram_model_set_known_wid(ngram_model_t *base,
+                          int32 set_wid)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+
+    if (set_wid >= base->n_words)
+        return FALSE;
+    else if (set->cur == -1) {
+        int32 i;
+        for (i = 0; i < set->n_models; ++i) {
+            if (set->widmap[set_wid][i] != ngram_unknown_wid(set->lms[i]))
+                return TRUE;
+        }
+        return FALSE;
+    }
+    else
+        return (set->widmap[set_wid][set->cur]
+                != ngram_unknown_wid(set->lms[set->cur]));
+}
+
+ngram_model_t *
+ngram_model_set_interp(ngram_model_t *base,
+                       const char **names,
+                       const float32 *weights)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+
+    /* If we have a set of weights here, then set them. */
+    if (names && weights) {
+        int32 i, j;
+
+        /* We hope there aren't many models. */
+        for (i = 0; i < set->n_models; ++i) {
+            for (j = 0; j < set->n_models; ++j)
+                if (0 == strcmp(names[i], set->names[j]))
+                    break;
+            if (j == set->n_models) {
+                E_ERROR("Unknown LM name %s\n", names[i]);
+                return NULL;
+            }
+            set->lweights[j] = logmath_log(base->lmath, weights[i]);
+        }
+    }
+    else if (weights) {
+        memcpy(set->lweights, weights, set->n_models * sizeof(*set->lweights));
+    }
+    /* Otherwise just enable existing weights. */
+    set->cur = -1;
+    return base;
+}
+
+ngram_model_t *
+ngram_model_set_add(ngram_model_t *base,
+                    ngram_model_t *model,
+                    const char *name,
+                    float32 weight,
+                    int reuse_widmap)
+                    
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    float32 fprob;
+    int32 scale, i;
+
+    /* Add it to the array of lms. */
+    ++set->n_models;
+    set->lms = ckd_realloc(set->lms, set->n_models * sizeof(*set->lms));
+    set->lms[set->n_models - 1] = model;
+    set->names = ckd_realloc(set->names, set->n_models * sizeof(*set->names));
+    set->names[set->n_models - 1] = ckd_salloc(name);
+    /* Expand the history mapping table if necessary. */
+    if (model->n > base->n) {
+        base->n = model->n;
+        set->maphist = ckd_realloc(set->maphist,
+                                   (model->n - 1) * sizeof(*set->maphist));
+    }
+
+    /* Renormalize the interpolation weights. */
+    fprob = weight * 1.0 / set->n_models;
+    set->lweights = ckd_realloc(set->lweights,
+                                set->n_models * sizeof(*set->lweights));
+    set->lweights[set->n_models - 1] = logmath_log(base->lmath, fprob);
+    /* Now normalize everything else to fit it in.  This is
+     * accomplished by simply scaling all the other probabilities
+     * by (1-fprob). */
+    scale = logmath_log(base->lmath, 1.0 - fprob);
+    for (i = 0; i < set->n_models - 1; ++i)
+        set->lweights[i] += scale;
+
+    /* Reuse the old word ID mapping if requested. */
+    if (reuse_widmap) {
+        int32 **new_widmap;
+
+        /* Tack another column onto the widmap array. */
+        new_widmap = (int32 **)ckd_calloc_2d(base->n_words, set->n_models,
+                                             sizeof (**new_widmap));
+        for (i = 0; i < base->n_words; ++i) {
+            /* Copy all the existing mappings. */
+            memcpy(new_widmap[i], set->widmap[i],
+                   (set->n_models - 1) * sizeof(**new_widmap));
+            /* Create the new mapping. */
+            new_widmap[i][set->n_models-1] = ngram_wid(model, base->word_str[i]);
+        }
+        ckd_free_2d((void **)set->widmap);
+        set->widmap = new_widmap;
+    }
+    else {
+        build_widmap(base, base->lmath, base->n);
+    }
+    return model;
+}
+
+ngram_model_t *
+ngram_model_set_remove(ngram_model_t *base,
+                       const char *name,
+                       int reuse_widmap)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    ngram_model_t *submodel;
+    int32 lmidx, scale, n, i;
+    float32 fprob;
+
+    for (lmidx = 0; lmidx < set->n_models; ++lmidx)
+        if (0 == strcmp(name, set->names[lmidx]))
+            break;
+    if (lmidx == set->n_models)
+        return NULL;
+    submodel = set->lms[lmidx];
+
+    /* Renormalize the interpolation weights by scaling them by
+     * 1/(1-fprob) */
+    fprob = logmath_exp(base->lmath, set->lweights[lmidx]);
+    scale = logmath_log(base->lmath, 1.0 - fprob);
+
+    /* Remove it from the array of lms, renormalize remaining weights,
+     * and recalcluate n. */
+    --set->n_models;
+    n = 0;
+    ckd_free(set->names[lmidx]);
+    set->names[lmidx] = NULL;
+    for (i = 0; i < set->n_models; ++i) {
+        if (i >= lmidx) {
+            set->lms[i] = set->lms[i+1];
+            set->names[i] = set->names[i+1];
+            set->lweights[i] = set->lweights[i+1];
+        }
+        set->lweights[i] -= scale;
+        if (set->lms[i]->n > n)
+            n = set->lms[i]->n;
+    }
+    /* There's no need to shrink these arrays. */
+    set->lms[set->n_models] = NULL;
+    set->lweights[set->n_models] = base->log_zero;
+    /* No need to shrink maphist either. */
+
+    /* Reuse the existing word ID mapping if requested. */
+    if (reuse_widmap) {
+        /* Just go through and shrink each row. */
+        for (i = 0; i < base->n_words; ++i) {
+            memmove(set->widmap[i] + lmidx, set->widmap[i] + lmidx + 1,
+                    (set->n_models - lmidx) * sizeof(**set->widmap));
+        }
+    }
+    else {
+        build_widmap(base, base->lmath, n);
+    }
+    return submodel;
+}
+
+void
+ngram_model_set_map_words(ngram_model_t *base,
+                          const char **words,
+                          int32 n_words)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    int32 i;
+
+    /* Recreate the word mapping. */
+    if (base->writable) {
+        for (i = 0; i < base->n_words; ++i) {
+            ckd_free(base->word_str[i]);
+        }
+    }
+    ckd_free(base->word_str);
+    ckd_free_2d((void **)set->widmap);
+    base->writable = TRUE;
+    base->n_words = base->n_1g_alloc = n_words;
+    base->word_str = ckd_calloc(n_words, sizeof(*base->word_str));
+    set->widmap = (int32 **)ckd_calloc_2d(n_words, set->n_models, sizeof(**set->widmap));
+    hash_table_empty(base->wid);
+    for (i = 0; i < n_words; ++i) {
+        int32 j;
+        base->word_str[i] = ckd_salloc(words[i]);
+        (void)hash_table_enter_int32(base->wid, base->word_str[i], i);
+        for (j = 0; j < set->n_models; ++j) {
+            set->widmap[i][j] = ngram_wid(set->lms[j], base->word_str[i]);
+        }
+    }
+}
+
+static int
+ngram_model_set_apply_weights(ngram_model_t *base, float32 lw,
+                              float32 wip, float32 uw)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    int32 i;
+
+    /* Apply weights to each sub-model. */
+    for (i = 0; i < set->n_models; ++i)
+        ngram_model_apply_weights(set->lms[i], lw, wip, uw);
+    return 0;
+}
+
+static int32
+ngram_model_set_score(ngram_model_t *base, int32 wid,
+                      int32 *history, int32 n_hist,
+                      int32 *n_used)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    int32 mapwid;
+    int32 score;
+    int32 i;
+
+    /* Truncate the history. */
+    if (n_hist > base->n - 1)
+        n_hist = base->n - 1;
+
+    /* Interpolate if there is no current. */
+    if (set->cur == -1) {
+        score = base->log_zero;
+        for (i = 0; i < set->n_models; ++i) {
+            int32 j;
+            /* Map word and history IDs for each model. */
+            mapwid = set->widmap[wid][i];
+            for (j = 0; j < n_hist; ++j) {
+                if (history[j] == NGRAM_INVALID_WID)
+                    set->maphist[j] = NGRAM_INVALID_WID;
+                else
+                    set->maphist[j] = set->widmap[history[j]][i];
+            }
+            score = logmath_add(base->lmath, score,
+                                set->lweights[i] + 
+                                ngram_ng_score(set->lms[i],
+                                               mapwid, set->maphist, n_hist, n_used));
+        }
+    }
+    else {
+        int32 j;
+        /* Map word and history IDs (FIXME: do this in a function?) */
+        mapwid = set->widmap[wid][set->cur];
+        for (j = 0; j < n_hist; ++j) {
+            if (history[j] == NGRAM_INVALID_WID)
+                set->maphist[j] = NGRAM_INVALID_WID;
+            else
+                set->maphist[j] = set->widmap[history[j]][set->cur];
+        }
+        score = ngram_ng_score(set->lms[set->cur],
+                               mapwid, set->maphist, n_hist, n_used);
+    }
+
+    return score;
+}
+
+static int32
+ngram_model_set_raw_score(ngram_model_t *base, int32 wid,
+                          int32 *history, int32 n_hist,
+                          int32 *n_used)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    int32 mapwid;
+    int32 score;
+    int32 i;
+
+    /* Truncate the history. */
+    if (n_hist > base->n - 1)
+        n_hist = base->n - 1;
+
+    /* Interpolate if there is no current. */
+    if (set->cur == -1) {
+        score = base->log_zero;
+        for (i = 0; i < set->n_models; ++i) {
+            int32 j;
+            /* Map word and history IDs for each model. */
+            mapwid = set->widmap[wid][i];
+            for (j = 0; j < n_hist; ++j) {
+                if (history[j] == NGRAM_INVALID_WID)
+                    set->maphist[j] = NGRAM_INVALID_WID;
+                else
+                    set->maphist[j] = set->widmap[history[j]][i];
+            }
+            score = logmath_add(base->lmath, score,
+                                set->lweights[i] + 
+                                ngram_ng_prob(set->lms[i],
+                                              mapwid, set->maphist, n_hist, n_used));
+        }
+    }
+    else {
+        int32 j;
+        /* Map word and history IDs (FIXME: do this in a function?) */
+        mapwid = set->widmap[wid][set->cur];
+        for (j = 0; j < n_hist; ++j) {
+            if (history[j] == NGRAM_INVALID_WID)
+                set->maphist[j] = NGRAM_INVALID_WID;
+            else
+                set->maphist[j] = set->widmap[history[j]][set->cur];
+        }
+        score = ngram_ng_prob(set->lms[set->cur],
+                              mapwid, set->maphist, n_hist, n_used);
+    }
+
+    return score;
+}
+
+static int32
+ngram_model_set_add_ug(ngram_model_t *base,
+                       int32 wid, int32 lweight)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    int32 *newwid;
+    int32 i, prob;
+
+    /* At this point the word has already been added to the master
+       model and we have a new word ID for it.  Add it to active
+       submodels and track the word IDs. */
+    newwid = ckd_calloc(set->n_models, sizeof(*newwid));
+    prob = base->log_zero;
+    for (i = 0; i < set->n_models; ++i) {
+        int32 wprob, n_hist;
+
+        /* Only add to active models. */
+        if (set->cur == -1 || set->cur == i) {
+            /* Did this word already exist? */
+            newwid[i] = ngram_wid(set->lms[i], base->word_str[wid]);
+            if (newwid[i] == NGRAM_INVALID_WID) {
+                /* Add it to the submodel. */
+                newwid[i] = ngram_model_add_word(set->lms[i], base->word_str[wid],
+                                                 logmath_exp(base->lmath, lweight));
+                if (newwid[i] == NGRAM_INVALID_WID) {
+                    ckd_free(newwid);
+                    return base->log_zero;
+                }
+            }
+            /* Now get the unigram probability for the new word and either
+             * interpolate it or use it (if this is the current model). */
+            wprob = ngram_ng_prob(set->lms[i], newwid[i], NULL, 0, &n_hist);
+            if (set->cur == i)
+                prob = wprob;
+            else if (set->cur == -1)
+                prob = logmath_add(base->lmath, prob, set->lweights[i] + wprob);
+        }
+        else {
+            newwid[i] = NGRAM_INVALID_WID;
+        }
+    }
+    /* Okay we have the word IDs for this in all the submodels.  Now
+       do some complicated memory mangling to add this to the
+       widmap. */
+    set->widmap = ckd_realloc(set->widmap, base->n_words * sizeof(*set->widmap));
+    set->widmap[0] = ckd_realloc(set->widmap[0],
+                                 base->n_words
+                                 * set->n_models
+                                 * sizeof(**set->widmap));
+    for (i = 0; i < base->n_words; ++i)
+        set->widmap[i] = set->widmap[0] + i * set->n_models;
+    memcpy(set->widmap[wid], newwid, set->n_models * sizeof(*newwid));
+    ckd_free(newwid);
+    return prob;
+}
+
+static void
+ngram_model_set_free(ngram_model_t *base)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    int32 i;
+
+    for (i = 0; i < set->n_models; ++i)
+        ngram_model_free(set->lms[i]);
+    ckd_free(set->lms);
+    for (i = 0; i < set->n_models; ++i)
+        ckd_free(set->names[i]);
+    ckd_free(set->names);
+    ckd_free(set->lweights);
+    ckd_free(set->maphist);
+    ckd_free_2d((void **)set->widmap);
+}
+
+static void
+ngram_model_set_flush(ngram_model_t *base)
+{
+    ngram_model_set_t *set = (ngram_model_set_t *)base;
+    int32 i;
+
+    for (i = 0; i < set->n_models; ++i)
+        ngram_model_flush(set->lms[i]);
+}
+
+static ngram_funcs_t ngram_model_set_funcs = {
+    ngram_model_set_free,          /* free */
+    ngram_model_set_apply_weights, /* apply_weights */
+    ngram_model_set_score,         /* score */
+    ngram_model_set_raw_score,     /* raw_score */
+    ngram_model_set_add_ug,        /* add_ug */
+    ngram_model_set_flush          /* flush */
+};

media/sphinxbase/src/libsphinxbase/lm/ngram_model_set.h

@@ -0,0 +1,71 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/**
+ * @file ngram_model_set.h Set of language models.
+ * @author David Huggins-Daines <dhuggins@cs.cmu.edu>
+ */
+
+#ifndef __NGRAM_MODEL_SET_H__
+#define __NGRAM_MODEL_SET_H__
+
+#include "ngram_model_internal.h"
+#include "lm3g_model.h"
+
+/**
+ * Subclass of ngram_model for grouping language models.
+ */
+typedef struct ngram_model_set_s {
+    ngram_model_t base;  /**< Base ngram_model_t structure. */
+
+    int32 n_models;      /**< Number of models in this set. */
+    int32 cur;           /**< Currently selected model, or -1 for none. */
+    ngram_model_t **lms; /**< Language models in this set. */
+    char **names;        /**< Names for language models. */
+    int32 *lweights;     /**< Log interpolation weights. */
+    int32 **widmap;      /**< Word ID mapping for submodels. */
+    int32 *maphist;      /**< Word ID mapping for N-Gram history. */
+} ngram_model_set_t;
+
+/**
+ * Iterator over a model set.
+ */
+struct ngram_model_set_iter_s {
+    ngram_model_set_t *set;
+    int32 cur;
+};
+
+#endif /* __NGRAM_MODEL_SET_H__ */

media/sphinxbase/src/libsphinxbase/util/bio.c

@@ -0,0 +1,644 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * bio.c -- Sphinx-3 binary file I/O functions.
+ * 
+ * **********************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 1996 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ * 
+ * HISTORY
+ * $Log$
+ * Revision 1.4  2005/06/21  20:40:46  arthchan2003
+ * 1, Fixed doxygen documentation, 2, Add the $ keyword.
+ * 
+ * Revision 1.3  2005/03/30 01:22:46  archan
+ * Fixed mistakes in last updates. Add
+ *
+ * 
+ * 02-Jul-1997	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Bugfix: Added byteswapping in bio_verify_chksum().
+ * 
+ * 18-Dec-1996	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Created.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4996)
+#endif
+
+#include "sphinxbase/bio.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/ckd_alloc.h"
+
+
+#define BIO_HDRARG_MAX	32
+#define END_COMMENT	"*end_comment*\n"
+
+
+static void
+bcomment_read(FILE * fp)
+{
+    __BIGSTACKVARIABLE__ char iline[16384];
+
+    while (fgets(iline, sizeof(iline), fp) != NULL) {
+        if (strcmp(iline, END_COMMENT) == 0)
+            return;
+    }
+    E_FATAL("Missing %s marker\n", END_COMMENT);
+}
+
+
+static int32
+swap_check(FILE * fp)
+{
+    uint32 magic;
+
+    if (fread(&magic, sizeof(uint32), 1, fp) != 1) {
+        E_ERROR("Cannot read BYTEORDER MAGIC NO.\n");
+        return -1;
+    }
+
+    if (magic != BYTE_ORDER_MAGIC) {
+        /* either need to swap or got bogus magic number */
+        SWAP_INT32(&magic);
+
+        if (magic == BYTE_ORDER_MAGIC)
+            return 1;
+
+        SWAP_INT32(&magic);
+        E_ERROR("Bad BYTEORDER MAGIC NO: %08x, expecting %08x\n",
+                magic, BYTE_ORDER_MAGIC);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+void
+bio_hdrarg_free(char **argname, char **argval)
+{
+    int32 i;
+
+    if (argname == NULL)
+        return;
+    for (i = 0; argname[i]; i++) {
+        ckd_free(argname[i]);
+        ckd_free(argval[i]);
+    }
+    ckd_free(argname);
+    ckd_free(argval);
+}
+
+
+int32
+bio_writehdr_version(FILE * fp, char *version)
+{
+    uint32 b;
+
+    fprintf(fp, "s3\n");
+    fprintf(fp, "version %s\n", version);
+    fprintf(fp, "endhdr\n");
+    fflush(fp);
+
+    b = (uint32) BYTE_ORDER_MAGIC;
+    fwrite(&b, sizeof(uint32), 1, fp);
+    fflush(fp);
+
+    return 0;
+}
+
+
+int32
+bio_writehdr(FILE *fp, ...)
+{
+    char const *key;
+    va_list args;
+    uint32 b;
+
+    fprintf(fp, "s3\n");
+    va_start(args, fp);
+    while ((key = va_arg(args, char const *)) != NULL) {
+        char const *val = va_arg(args, char const *);
+        if (val == NULL) {
+            E_ERROR("Wrong number of arguments\n");
+            va_end(args);
+            return -1;
+        }
+        fprintf(fp, "%s %s\n", key, val);
+    }
+    va_end(args);
+
+    fprintf(fp, "endhdr\n");
+    fflush(fp);
+
+    b = (uint32) BYTE_ORDER_MAGIC;
+    if (fwrite(&b, sizeof(uint32), 1, fp) != 1)
+        return -1;
+    fflush(fp);
+
+    return 0;
+}
+
+
+int32
+bio_readhdr(FILE * fp, char ***argname, char ***argval, int32 * swap)
+{
+    __BIGSTACKVARIABLE__ char line[16384], word[4096];
+    int32 i, l;
+    int32 lineno;
+
+    *argname = (char **) ckd_calloc(BIO_HDRARG_MAX + 1, sizeof(char *));
+    *argval = (char **) ckd_calloc(BIO_HDRARG_MAX, sizeof(char *));
+
+    lineno = 0;
+    if (fgets(line, sizeof(line), fp) == NULL){
+        E_ERROR("Premature EOF, line %d\n", lineno);
+        goto error_out;
+    }
+    lineno++;
+
+    if ((line[0] == 's') && (line[1] == '3') && (line[2] == '\n')) {
+        /* New format (post Dec-1996, including checksums); read argument-value pairs */
+        for (i = 0;;) {
+            if (fgets(line, sizeof(line), fp) == NULL) {
+                E_ERROR("Premature EOF, line %d\n", lineno);
+                goto error_out;
+            }
+            lineno++;
+
+            if (sscanf(line, "%s%n", word, &l) != 1) {
+                E_ERROR("Header format error, line %d\n", lineno);
+                goto error_out;
+            }
+            if (strcmp(word, "endhdr") == 0)
+                break;
+            if (word[0] == '#') /* Skip comments */
+                continue;
+
+            if (i >= BIO_HDRARG_MAX) {
+                E_ERROR
+                    ("Max arg-value limit(%d) exceeded; increase BIO_HDRARG_MAX\n",
+                     BIO_HDRARG_MAX);
+                goto error_out;
+            }
+
+            (*argname)[i] = ckd_salloc(word);
+            if (sscanf(line + l, "%s", word) != 1) {      /* Multi-word values not allowed */
+                E_ERROR("Header format error, line %d\n", lineno);
+                goto error_out;
+            }
+            (*argval)[i] = ckd_salloc(word);
+            i++;
+        }
+    }
+    else {
+        /* Old format (without checksums); the first entry must be the version# */
+        if (sscanf(line, "%s", word) != 1) {
+            E_ERROR("Header format error, line %d\n", lineno);
+            goto error_out;
+        }
+
+        (*argname)[0] = ckd_salloc("version");
+        (*argval)[0] = ckd_salloc(word);
+        i = 1;
+
+        bcomment_read(fp);
+    }
+    (*argname)[i] = NULL;
+
+    if ((*swap = swap_check(fp)) < 0) {
+        E_ERROR("swap_check failed\n");
+        goto error_out;
+    }
+
+    return 0;
+error_out:
+    bio_hdrarg_free(*argname, *argval);
+    *argname = *argval = NULL;
+    return -1;
+}
+
+
+static uint32
+chksum_accum(const void *buf, int32 el_sz, int32 n_el, uint32 sum)
+{
+    int32 i;
+    uint8 *i8;
+    uint16 *i16;
+    uint32 *i32;
+
+    switch (el_sz) {
+    case 1:
+        i8 = (uint8 *) buf;
+        for (i = 0; i < n_el; i++)
+            sum = (sum << 5 | sum >> 27) + i8[i];
+        break;
+    case 2:
+        i16 = (uint16 *) buf;
+        for (i = 0; i < n_el; i++)
+            sum = (sum << 10 | sum >> 22) + i16[i];
+        break;
+    case 4:
+        i32 = (uint32 *) buf;
+        for (i = 0; i < n_el; i++)
+            sum = (sum << 20 | sum >> 12) + i32[i];
+        break;
+    default:
+        E_FATAL("Unsupported elemsize for checksum: %d\n", el_sz);
+        break;
+    }
+
+    return sum;
+}
+
+
+static void
+swap_buf(void *buf, int32 el_sz, int32 n_el)
+{
+    int32 i;
+    uint16 *buf16;
+    uint32 *buf32;
+
+    switch (el_sz) {
+    case 1:
+        break;
+    case 2:
+        buf16 = (uint16 *) buf;
+        for (i = 0; i < n_el; i++)
+            SWAP_INT16(buf16 + i);
+        break;
+    case 4:
+        buf32 = (uint32 *) buf;
+        for (i = 0; i < n_el; i++)
+            SWAP_INT32(buf32 + i);
+        break;
+    default:
+        E_FATAL("Unsupported elemsize for byteswapping: %d\n", el_sz);
+        break;
+    }
+}
+
+
+int32
+bio_fread(void *buf, int32 el_sz, int32 n_el, FILE * fp, int32 swap,
+          uint32 * chksum)
+{
+    if (fread(buf, el_sz, n_el, fp) != (size_t) n_el)
+        return -1;
+
+    if (swap)
+        swap_buf(buf, el_sz, n_el);
+
+    if (chksum)
+        *chksum = chksum_accum(buf, el_sz, n_el, *chksum);
+
+    return n_el;
+}
+
+int32
+bio_fwrite(const void *buf, int32 el_sz, int32 n_el, FILE *fp,
+           int32 swap, uint32 *chksum)
+{
+    if (chksum)
+        *chksum = chksum_accum(buf, el_sz, n_el, *chksum);
+    if (swap) {
+        void *nbuf;
+        int rv;
+
+        nbuf = ckd_calloc(n_el, el_sz);
+        memcpy(nbuf, buf, n_el * el_sz);
+        swap_buf(nbuf, el_sz, n_el);
+        rv = fwrite(nbuf, el_sz, n_el, fp);
+        ckd_free(nbuf);
+        return rv;
+    }
+    else {
+        return fwrite(buf, el_sz, n_el, fp);
+    }
+}
+
+int32
+bio_fread_1d(void **buf, size_t el_sz, uint32 * n_el, FILE * fp,
+             int32 sw, uint32 * ck)
+{
+    /* Read 1-d array size */
+    if (bio_fread(n_el, sizeof(int32), 1, fp, sw, ck) != 1)
+        E_FATAL("fread(arraysize) failed\n");
+    if (*n_el <= 0)
+        E_FATAL("Bad arraysize: %d\n", *n_el);
+
+    /* Allocate memory for array data */
+    *buf = (void *) ckd_calloc(*n_el, el_sz);
+
+    /* Read array data */
+    if (bio_fread(*buf, el_sz, *n_el, fp, sw, ck) != *n_el)
+        E_FATAL("fread(arraydata) failed\n");
+
+    return *n_el;
+}
+
+int32
+bio_fread_2d(void ***arr,
+             size_t e_sz,
+             uint32 *d1,
+             uint32 *d2,
+             FILE *fp,
+             uint32 swap,
+             uint32 *chksum)
+{
+    uint32 l_d1, l_d2;
+    uint32 n;
+    size_t ret;
+    void *raw;
+    
+    ret = bio_fread(&l_d1, sizeof(uint32), 1, fp, swap, chksum);
+    if (ret != 1) {
+	if (ret == 0) {
+	    E_ERROR_SYSTEM("Unable to read complete data");
+	}
+	else {
+	    E_ERROR_SYSTEM("OS error in bio_fread_2d");
+	}
+	return -1;
+    }
+    ret = bio_fread(&l_d2, sizeof(uint32), 1, fp, swap, chksum);
+    if (ret != 1) {
+	if (ret == 0) {
+	    E_ERROR_SYSTEM("Unable to read complete data");
+	}
+	else {
+	    E_ERROR_SYSTEM("OS error in bio_fread_2d");
+	}
+	return -1;
+    }
+    if (bio_fread_1d(&raw, e_sz, &n, fp, swap, chksum) != n)
+	return -1;
+
+    assert(n == l_d1*l_d2);
+
+    *d1 = l_d1;
+    *d2 = l_d2;
+    *arr = ckd_alloc_2d_ptr(l_d1, l_d2, raw, e_sz);
+
+    return n;
+}
+
+int32
+bio_fread_3d(void ****arr,
+             size_t e_sz,
+             uint32 *d1,
+             uint32 *d2,
+             uint32 *d3,
+             FILE *fp,
+             uint32 swap,
+             uint32 *chksum)
+{
+    uint32 l_d1;
+    uint32 l_d2;
+    uint32 l_d3;
+    uint32 n;
+    void *raw;
+    size_t ret;
+
+    ret = bio_fread(&l_d1, sizeof(uint32), 1, fp, swap, chksum);
+    if (ret != 1) {
+	if (ret == 0) {
+	    E_ERROR_SYSTEM("Unable to read complete data");
+	}
+	else {
+	    E_ERROR_SYSTEM("OS error in bio_fread_3d");
+	}
+	return -1;
+    }
+    ret = bio_fread(&l_d2, sizeof(uint32), 1, fp, swap, chksum);
+    if (ret != 1) {
+	if (ret == 0) {
+	    E_ERROR_SYSTEM("Unable to read complete data");
+	}
+	else {
+	    E_ERROR_SYSTEM("OS error in bio_fread_3d");
+	}
+	return -1;
+    }
+    ret = bio_fread(&l_d3, sizeof(uint32), 1, fp, swap, chksum);
+    if (ret != 1) {
+	if (ret == 0) {
+	    E_ERROR_SYSTEM("Unable to read complete data");
+	}
+	else {
+	    E_ERROR_SYSTEM("OS error in bio_fread_3d");
+	}
+	return -1;
+    }
+
+    if (bio_fread_1d(&raw, e_sz, &n, fp, swap, chksum) != n) {
+	return -1;
+    }
+
+    assert(n == l_d1 * l_d2 * l_d3);
+
+    *arr = ckd_alloc_3d_ptr(l_d1, l_d2, l_d3, raw, e_sz);
+    *d1 = l_d1;
+    *d2 = l_d2;
+    *d3 = l_d3;
+    
+    return n;
+}
+
+void
+bio_verify_chksum(FILE * fp, int32 byteswap, uint32 chksum)
+{
+    uint32 file_chksum;
+
+    if (fread(&file_chksum, sizeof(uint32), 1, fp) != 1)
+        E_FATAL("fread(chksum) failed\n");
+    if (byteswap)
+        SWAP_INT32(&file_chksum);
+    if (file_chksum != chksum)
+        E_FATAL
+            ("Checksum error; file-checksum %08x, computed %08x\n",
+             file_chksum, chksum);
+}
+
+int
+bio_fwrite_3d(void ***arr,
+	   size_t e_sz,
+	   uint32 d1,
+	   uint32 d2,
+	   uint32 d3,
+	   FILE *fp,
+	   uint32 *chksum)
+{
+    size_t ret;
+
+    /* write out first dimension 1 */
+    ret = bio_fwrite(&d1, sizeof(uint32), 1, fp, 0, chksum);
+    if (ret != 1) {
+	if (ret == 0) {
+	    E_ERROR_SYSTEM("Unable to write complete data");
+	}
+	else {
+	    E_ERROR_SYSTEM("OS error in bio_fwrite_3d");
+	}
+	return -1;
+    }
+
+    /* write out first dimension 2 */
+    ret = bio_fwrite(&d2, sizeof(uint32), 1, fp, 0, chksum);
+    if (ret != 1) {
+	if (ret == 0) {
+	    E_ERROR_SYSTEM("Unable to write complete data");
+	}
+	else {
+	    E_ERROR_SYSTEM("OS error in bio_fwrite_3d");
+	}
+	return -1;
+    }
+
+    /* write out first dimension 3 */
+    ret = bio_fwrite(&d3, sizeof(uint32), 1, fp, 0, chksum);
+    if (ret != 1) {
+	if (ret == 0) {
+	    E_ERROR_SYSTEM("Unable to write complete data");
+	}
+	else {
+	    E_ERROR_SYSTEM("OS error in bio_fwrite_3d");
+	}
+	return -1;
+    }
+
+    /* write out the data in the array as one big block */
+    return bio_fwrite_1d(arr[0][0], e_sz, d1 * d2 * d3, fp, chksum);
+}
+
+int
+bio_fwrite_1d(void *arr,
+	   size_t e_sz,
+	   uint32 d1,
+	   FILE *fp,
+	   uint32 *chksum)
+{
+    size_t ret;
+    ret = bio_fwrite(&d1, sizeof(uint32), 1, fp, 0, chksum);
+    if (ret != 1) {
+	if (ret == 0) {
+	    E_ERROR_SYSTEM("Unable to write complete data");
+	}
+	else {
+	    E_ERROR_SYSTEM("OS error in bio_fwrite_1d");
+	}
+	return -1;
+    }
+
+    ret = bio_fwrite(arr, e_sz, d1, fp, 0, chksum);
+    if (ret != d1) {
+	if (ret == 0) {
+	    E_ERROR_SYSTEM("Unable to write complete data");
+	}
+	else {
+	    E_ERROR_SYSTEM("OS error in bio_fwrite_1d");
+	}
+
+	return -1;
+    }
+
+    return ret;
+}
+
+int16*
+bio_read_wavfile(char const *directory,
+		 char const *filename,
+		 char const *extension,
+		 int32 header,
+		 int32 endian,
+		 size_t *nsamps)
+{
+    FILE *uttfp;
+    char *inputfile;
+    size_t n, l;
+    int16 *data;
+
+    n = strlen(extension);
+    l = strlen(filename);
+    if ((n <= l) && (0 == strcmp(filename + l - n, extension)))
+        extension = "";
+    inputfile = ckd_calloc(strlen(directory) + l + n + 2, 1);
+    if (directory) {
+        sprintf(inputfile, "%s/%s%s", directory, filename, extension);
+    } else {
+        sprintf(inputfile, "%s%s", filename, extension);
+    }
+
+    if ((uttfp = fopen(inputfile, "rb")) == NULL) {
+        E_FATAL_SYSTEM("Failed to open file '%s' for reading", inputfile);
+    }
+    fseek(uttfp, 0, SEEK_END);
+    n = ftell(uttfp);
+    fseek(uttfp, 0, SEEK_SET);
+    if (header > 0) {
+        if (fseek(uttfp, header, SEEK_SET) < 0) {
+            E_ERROR_SYSTEM("Failed to move to an offset %d in a file '%s'", header, inputfile);
+            fclose(uttfp);
+            ckd_free(inputfile);
+            return NULL;
+        }
+        n -= header;
+    }
+    n /= sizeof(int16);
+    data = ckd_calloc(n, sizeof(*data));
+    if ((l = fread(data, sizeof(int16), n, uttfp)) < n) {
+        E_ERROR_SYSTEM("Failed to read %d samples from %s: %d", n, inputfile, l);
+        ckd_free(data);
+        ckd_free(inputfile);
+        fclose(uttfp);
+        return NULL;
+    }
+    ckd_free(inputfile);
+    fclose(uttfp);
+    if (nsamps) *nsamps = n;
+
+    return data;
+}

media/sphinxbase/src/libsphinxbase/util/bitvec.c

@@ -0,0 +1,101 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * bitvec.c -- Bit vector type.
+ *
+ * **********************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 1999 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ * 
+ * HISTORY
+ * $Log: bitvec.c,v $
+ * Revision 1.4  2005/06/22 02:58:22  arthchan2003
+ * Added  keyword
+ *
+ * Revision 1.3  2005/03/30 01:22:48  archan
+ * Fixed mistakes in last updates. Add
+ *
+ * 
+ * 05-Mar-1999	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon
+ * 		Started.
+ */
+
+
+#include "sphinxbase/bitvec.h"
+
+bitvec_t *
+bitvec_realloc(bitvec_t *vec,
+	       size_t old_len,
+               size_t new_len)
+{
+    bitvec_t *new_vec;
+    size_t old_size = bitvec_size(old_len);
+    size_t new_size = bitvec_size(new_len);
+    
+    new_vec = ckd_realloc(vec, new_size * sizeof(bitvec_t));
+    if (new_size > old_size)
+	memset(new_vec + old_size, 0, (new_size - old_size) * sizeof(bitvec_t));
+
+    return new_vec;
+}
+
+size_t
+bitvec_count_set(bitvec_t *vec, size_t len)
+{
+    size_t words, bits, w, b, n;
+    bitvec_t *v;
+
+    words = len / BITVEC_BITS;
+    bits = len % BITVEC_BITS;
+    v = vec;
+    n = 0;
+    for (w = 0; w < words; ++w, ++v) {
+        if (*v == 0)
+            continue;
+        for (b = 0; b < BITVEC_BITS; ++b)
+            if (*v & (1<<b))
+                ++n;
+    }
+    for (b = 0; b < bits; ++b)
+        if (*v & (1<<b))
+            ++n;
+
+    return n;
+}

media/sphinxbase/src/libsphinxbase/util/case.c

@@ -0,0 +1,141 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * case.c -- Upper/lower case conversion routines
+ *
+ * **********************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 1999 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ * 
+ * HISTORY
+ * $Log: case.c,v $
+ * Revision 1.7  2005/06/22 02:58:54  arthchan2003
+ * Added  keyword
+ *
+ * Revision 1.3  2005/03/30 01:22:48  archan
+ * Fixed mistakes in last updates. Add
+ *
+ * 
+ * 18-Jun-97	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon
+ * 		Added strcmp_nocase.  Moved UPPER_CASE and LOWER_CASE definitions to .h.
+ * 
+ * 16-Feb-97	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon
+ * 		Created.
+ */
+
+
+#include <stdlib.h>
+#include <assert.h>
+
+#include "sphinxbase/case.h"
+#include "sphinxbase/err.h"
+
+
+void
+lcase(register char *cp)
+{
+    if (cp) {
+        while (*cp) {
+            *cp = LOWER_CASE(*cp);
+            cp++;
+        }
+    }
+}
+
+void
+ucase(register char *cp)
+{
+    if (cp) {
+        while (*cp) {
+            *cp = UPPER_CASE(*cp);
+            cp++;
+        }
+    }
+}
+
+int32
+strcmp_nocase(const char *str1, const char *str2)
+{
+    char c1, c2;
+
+    if (str1 == str2)
+        return 0;
+    if (str1 && str2) {
+        for (;;) {
+            c1 = *(str1++);
+            c1 = UPPER_CASE(c1);
+            c2 = *(str2++);
+            c2 = UPPER_CASE(c2);
+            if (c1 != c2)
+                return (c1 - c2);
+            if (c1 == '\0')
+                return 0;
+        }
+    }
+    else
+        return (str1 == NULL) ? -1 : 1;
+
+    return 0;
+}
+
+int32
+strncmp_nocase(const char *str1, const char *str2, size_t len)
+{
+    char c1, c2;
+
+    if (str1 && str2) {
+        size_t n;
+
+        for (n = 0; n < len; ++n) {
+            c1 = *(str1++);
+            c1 = UPPER_CASE(c1);
+            c2 = *(str2++);
+            c2 = UPPER_CASE(c2);
+            if (c1 != c2)
+                return (c1 - c2);
+            if (c1 == '\0')
+                return 0;
+        }
+    }
+    else
+        return (str1 == NULL) ? -1 : 1;
+
+    return 0;
+}

media/sphinxbase/src/libsphinxbase/util/ckd_alloc.c

@@ -0,0 +1,427 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced
+ * Research Projects Agency and the National Science Foundation of the
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * ckd_alloc.c -- Memory allocation package.
+ *
+ * **********************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 1999 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ *
+ * HISTORY
+ * $Log: ckd_alloc.c,v $
+ * Revision 1.6  2005/06/22 02:59:25  arthchan2003
+ * Added  keyword
+ *
+ * Revision 1.3  2005/03/30 01:22:48  archan
+ * Fixed mistakes in last updates. Add
+ *
+ *
+ * 19-Jun-97	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Removed file,line arguments from free functions.
+ * 		Removed debugging stuff.
+ *
+ * 01-Jan-96	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Created.
+ */
+
+
+/*********************************************************************
+ *
+ * $Header: /cvsroot/cmusphinx/sphinx3/src/libutil/ckd_alloc.c,v 1.6 2005/06/22 02:59:25 arthchan2003 Exp $
+ *
+ * Carnegie Mellon ARPA Speech Group
+ *
+ * Copyright (c) 1994 Carnegie Mellon University.
+ * All rights reserved.
+ *
+ *********************************************************************
+ *
+ * file: ckd_alloc.c
+ *
+ * traceability:
+ *
+ * description:
+ *
+ * author:
+ *
+ *********************************************************************/
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <stdarg.h>
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4996)
+#endif
+
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/err.h"
+
+/**
+ * Target for longjmp() on failure.
+ *
+ * FIXME: This should be in thread-local storage.
+ */
+static jmp_buf *ckd_target;
+static int jmp_abort;
+
+jmp_buf *
+ckd_set_jump(jmp_buf *env, int abort)
+{
+    jmp_buf *old;
+
+    if (abort)
+        jmp_abort = 1;
+
+    old = ckd_target;
+    ckd_target = env;
+    return old;
+}
+
+void
+ckd_fail(char *format, ...)
+{
+    va_list args;
+
+    va_start(args, format);
+    vfprintf(stderr, format, args);
+    va_end(args);
+
+    if (jmp_abort)
+        /* abort() doesn't exist in Windows CE */
+        #if defined(_WIN32_WCE)
+        exit(-1);
+        #else
+        abort();
+        #endif
+    else if (ckd_target)
+        longjmp(*ckd_target, 1);
+    else
+        exit(-1);
+}
+
+void *
+__ckd_calloc__(size_t n_elem, size_t elem_size,
+               const char *caller_file, int caller_line)
+{
+    void *mem;
+
+#if defined(__ADSPBLACKFIN__) && !defined(__linux__)
+    if ((mem = heap_calloc(heap_lookup(1),n_elem, elem_size)) == NULL)
+    	if ((mem = heap_calloc(heap_lookup(0),n_elem, elem_size)) == NULL)
+    	{
+        	ckd_fail("calloc(%d,%d) failed from %s(%d), free space: %d\n", n_elem,
+                elem_size, caller_file, caller_line,space_unused());
+    	}
+#else
+    if ((mem = calloc(n_elem, elem_size)) == NULL) {
+        ckd_fail("calloc(%d,%d) failed from %s(%d)\n", n_elem,
+                elem_size, caller_file, caller_line);
+	}
+#endif
+
+
+    return mem;
+}
+
+
+void *
+__ckd_malloc__(size_t size, const char *caller_file, int caller_line)
+{
+    void *mem;
+
+#if defined(__ADSPBLACKFIN__) && !defined(__linux__)
+    if ((mem = heap_malloc(heap_lookup(0),size)) == NULL)
+       	if ((mem = heap_malloc(heap_lookup(1),size)) == NULL)
+#else
+    if ((mem = malloc(size)) == NULL)
+#endif
+	        ckd_fail("malloc(%d) failed from %s(%d)\n", size,
+                caller_file, caller_line);
+
+    return mem;
+}
+
+
+void *
+__ckd_realloc__(void *ptr, size_t new_size,
+                const char *caller_file, int caller_line)
+{
+    void *mem;
+#if defined(__ADSPBLACKFIN__) && !defined(__linux__)
+    if ((mem = heap_realloc(heap_lookup(0),ptr, new_size)) == NULL) {
+#else
+    if ((mem = realloc(ptr, new_size)) == NULL) {
+#endif
+        ckd_fail("malloc(%d) failed from %s(%d)\n", new_size,
+                caller_file, caller_line);
+    }
+
+    return mem;
+}
+
+
+char *
+__ckd_salloc__(const char *orig, const char *caller_file,
+               int caller_line)
+{
+    size_t len;
+    char *buf;
+
+    if (!orig)
+        return NULL;
+
+    len = strlen(orig) + 1;
+    buf = (char *) __ckd_malloc__(len, caller_file, caller_line);
+
+    strcpy(buf, orig);
+    return (buf);
+}
+
+
+void *
+__ckd_calloc_2d__(size_t d1, size_t d2, size_t elemsize,
+                  const char *caller_file, int caller_line)
+{
+    char **ref, *mem;
+    size_t i, offset;
+
+    mem =
+        (char *) __ckd_calloc__(d1 * d2, elemsize, caller_file,
+                                caller_line);
+    ref =
+        (char **) __ckd_malloc__(d1 * sizeof(void *), caller_file,
+                                 caller_line);
+
+    for (i = 0, offset = 0; i < d1; i++, offset += d2 * elemsize)
+        ref[i] = mem + offset;
+
+    return ref;
+}
+
+
+void
+ckd_free(void *ptr)
+{
+#if defined(__ADSPBLACKFIN__) && !defined(__linux__)
+    if (ptr)
+        heap_free(0,ptr);
+#else
+    free(ptr);
+#endif
+}
+
+void
+ckd_free_2d(void *tmpptr)
+{
+    void **ptr = (void **)tmpptr;
+    if (ptr)
+        ckd_free(ptr[0]);
+    ckd_free(ptr);
+}
+
+
+void *
+__ckd_calloc_3d__(size_t d1, size_t d2, size_t d3, size_t elemsize,
+                  const char *caller_file, int caller_line)
+{
+    char ***ref1, **ref2, *mem;
+    size_t i, j, offset;
+
+    mem =
+        (char *) __ckd_calloc__(d1 * d2 * d3, elemsize, caller_file,
+                                caller_line);
+    ref1 =
+        (char ***) __ckd_malloc__(d1 * sizeof(void **), caller_file,
+                                  caller_line);
+    ref2 =
+        (char **) __ckd_malloc__(d1 * d2 * sizeof(void *), caller_file,
+                                 caller_line);
+
+    for (i = 0, offset = 0; i < d1; i++, offset += d2)
+        ref1[i] = ref2 + offset;
+
+    offset = 0;
+    for (i = 0; i < d1; i++) {
+        for (j = 0; j < d2; j++) {
+            ref1[i][j] = mem + offset;
+            offset += d3 * elemsize;
+        }
+    }
+
+    return ref1;
+}
+
+
+void
+ckd_free_3d(void *inptr)
+{
+    void ***ptr = (void ***)inptr;
+
+    if (ptr && ptr[0])
+        ckd_free(ptr[0][0]);
+    if (ptr)
+        ckd_free(ptr[0]);
+    ckd_free(ptr);
+}
+
+
+void ****
+__ckd_calloc_4d__(size_t d1,
+		  size_t d2,
+		  size_t d3,
+		  size_t d4,
+		  size_t elem_size,
+		  char *file,
+		  int line)
+{
+    void *store;
+    void **tmp1;
+    void ***tmp2;
+    void ****out;
+    size_t i, j;
+
+    store = calloc(d1 * d2 * d3 * d4, elem_size);
+    if (store == NULL) {
+	E_FATAL("ckd_calloc_4d failed for caller at %s(%d) at %s(%d)\n",
+		file, line, __FILE__, __LINE__);
+    }
+
+    tmp1 = calloc(d1 * d2 * d3, sizeof(void *));
+    if (tmp1 == NULL) {
+	E_FATAL("ckd_calloc_4d failed for caller at %s(%d) at %s(%d)\n",
+		file, line, __FILE__, __LINE__);
+    }
+
+    tmp2 = ckd_calloc(d1 * d2, sizeof(void **));
+    if (tmp2 == NULL) {
+	E_FATAL("ckd_calloc_4d failed for caller at %s(%d) at %s(%d)\n",
+		file, line, __FILE__, __LINE__);
+    }
+
+    out = ckd_calloc(d1, sizeof(void ***));
+    if (out == NULL) {
+	E_FATAL("ckd_calloc_4d failed for caller at %s(%d) at %s(%d)\n",
+		file, line, __FILE__, __LINE__);
+    }
+
+    for (i = 0, j = 0; i < d1*d2*d3; i++, j += d4) {
+	tmp1[i] = &((char *)store)[j*elem_size];
+    }
+
+    for (i = 0, j = 0; i < d1*d2; i++, j += d3) {
+	tmp2[i] = &tmp1[j];
+    }
+
+    for (i = 0, j = 0; i < d1; i++, j += d2) {
+	out[i] = &tmp2[j];
+    }
+
+    return out;
+}
+
+void
+ckd_free_4d(void *inptr)
+{
+    void ****ptr = (void ****)inptr;
+    if (ptr == NULL)
+	return;
+    /* free the underlying store */
+    ckd_free(ptr[0][0][0]);
+
+    /* free the access overhead */
+    ckd_free(ptr[0][0]);
+    ckd_free(ptr[0]);
+    ckd_free(ptr);
+}
+
+/* Layers a 3d array access structure over a preallocated storage area */
+void *
+__ckd_alloc_3d_ptr(size_t d1,
+		   size_t d2,
+		   size_t d3,
+		   void *store,
+		   size_t elem_size,
+		   char *file,
+		   int line)
+{
+    void **tmp1;
+    void ***out;
+    size_t i, j;
+
+    tmp1 = __ckd_calloc__(d1 * d2, sizeof(void *), file, line);
+
+    out  = __ckd_calloc__(d1, sizeof(void **), file, line);
+
+    for (i = 0, j = 0; i < d1*d2; i++, j += d3) {
+	tmp1[i] = &((char *)store)[j*elem_size];
+    }
+
+    for (i = 0, j = 0; i < d1; i++, j += d2) {
+	out[i] = &tmp1[j];
+    }
+
+    return out;
+}
+
+void *
+__ckd_alloc_2d_ptr(size_t d1,
+		   size_t d2,
+		   void *store,
+		   size_t elem_size,
+		   char *file,
+		   int line)
+{
+    void **out;
+    size_t i, j;
+
+    out = __ckd_calloc__(d1, sizeof(void *), file, line);
+
+    for (i = 0, j = 0; i < d1; i++, j += d2) {
+	out[i] = &((char *)store)[j*elem_size];
+    }
+
+    return out;
+}
+
+/* vim: set ts=4 sw=4: */

media/sphinxbase/src/libsphinxbase/util/err.c

@@ -0,0 +1,297 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced
+ * Research Projects Agency and the National Science Foundation of the
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/**
+ * @file err.c
+ * @brief Somewhat antiquated logging and error interface.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <errno.h>
+
+#include "sphinxbase/err.h"
+#include "sphinxbase/prim_type.h"
+#include "sphinxbase/filename.h"
+#include "sphinxbase/ckd_alloc.h"
+
+static FILE*  logfp = NULL;
+static int    logfp_disabled = FALSE;
+
+static int sphinx_debug_level;
+
+#if defined(__ANDROID__)
+#include <android/log.h>
+static void
+err_logcat_cb(void* user_data, err_lvl_t level, const char *fmt, ...);
+#elif defined(_WIN32_WCE)
+#include <windows.h>
+#define vsnprintf _vsnprintf
+static void
+err_wince_cb(void* user_data, err_lvl_t level, const char *fmt, ...);
+#endif
+
+#if defined(__ANDROID__)
+static err_cb_f err_cb = err_logcat_cb;
+#elif defined(_WIN32_WCE)
+static err_cb_f err_cb = err_wince_cb;
+#else
+static err_cb_f err_cb = err_logfp_cb;
+#endif
+static void* err_user_data;
+
+void
+err_msg(err_lvl_t lvl, const char *path, long ln, const char *fmt, ...)
+{
+    static const char *err_prefix[ERR_MAX] = {
+        "DEBUG", "INFO", "INFOCONT", "WARN", "ERROR", "FATAL"
+    };
+
+    char msg[1024];
+    va_list ap;
+
+    if (!err_cb)
+        return;
+
+    va_start(ap, fmt);
+    vsnprintf(msg, sizeof(msg), fmt, ap);
+    va_end(ap);
+
+    if (path) {
+        const char *fname = path2basename(path);
+        if (lvl == ERR_INFOCONT)
+    	    err_cb(err_user_data, lvl, "%s(%ld): %s", fname, ln, msg);
+        else if (lvl == ERR_INFO)
+            err_cb(err_user_data, lvl, "%s: %s(%ld): %s", err_prefix[lvl], fname, ln, msg);
+        else
+    	    err_cb(err_user_data, lvl, "%s: \"%s\", line %ld: %s", err_prefix[lvl], fname, ln, msg);
+    } else {
+        err_cb(err_user_data, lvl, "%s", msg);
+    }
+}
+
+#ifdef _WIN32_WCE /* No strerror for WinCE, so a separate implementation */
+void
+err_msg_system(err_lvl_t lvl, const char *path, long ln, const char *fmt, ...)
+{
+    static const char *err_prefix[ERR_MAX] = {
+        "DEBUG", "INFO", "INFOCONT", "WARN", "ERROR", "FATAL"
+    };
+
+    va_list ap;
+    LPVOID error_wstring;
+    DWORD error;
+    char msg[1024];
+    char error_string[1024];
+
+    if (!err_cb)
+        return;
+
+    error = GetLastError();
+    FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | 
+                  FORMAT_MESSAGE_FROM_SYSTEM | 
+                  FORMAT_MESSAGE_IGNORE_INSERTS,
+                  NULL,
+                  error,
+                  0, // Default language
+                  (LPTSTR) &error_wstring,
+                  0,
+                  NULL);
+    wcstombs(error_string, error_wstring, 1023);
+    LocalFree(error_wstring);
+
+    va_start(ap, fmt);
+    vsnprintf(msg, sizeof(msg), fmt, ap);
+    va_end(ap);
+
+    if (path) {
+        const char *fname = path2basename(path);
+        if (lvl == ERR_INFOCONT)
+    	    err_cb(err_user_data, lvl, "%s(%ld): %s: %s\n", fname, ln, msg, error_string);
+        else if (lvl == ERR_INFO)
+            err_cb(err_user_data, lvl, "%s: %s(%ld): %s: %s\n", err_prefix[lvl], fname, ln, msg, error_string);
+        else
+    	    err_cb(err_user_data, lvl, "%s: \"%s\", line %ld: %s: %s\n", err_prefix[lvl], fname, ln, msg, error_string);
+    } else {
+        err_cb(err_user_data, lvl, "%s: %s\n", msg, error_string);
+    }
+}
+#else
+void
+err_msg_system(err_lvl_t lvl, const char *path, long ln, const char *fmt, ...)
+{
+    int local_errno = errno;
+    
+    static const char *err_prefix[ERR_MAX] = {
+        "DEBUG", "INFO", "INFOCONT", "WARN", "ERROR", "FATAL"
+    };
+
+    char msg[1024];
+    va_list ap;
+
+    if (!err_cb)
+        return;
+
+    va_start(ap, fmt);
+    vsnprintf(msg, sizeof(msg), fmt, ap);
+    va_end(ap);
+
+    if (path) {
+        const char *fname = path2basename(path);
+        if (lvl == ERR_INFOCONT)
+    	    err_cb(err_user_data, lvl, "%s(%ld): %s: %s\n", fname, ln, msg, strerror(local_errno));
+        else if (lvl == ERR_INFO)
+            err_cb(err_user_data, lvl, "%s: %s(%ld): %s: %s\n", err_prefix[lvl], fname, ln, msg, strerror(local_errno));
+        else
+    	    err_cb(err_user_data, lvl, "%s: \"%s\", line %ld: %s: %s\n", err_prefix[lvl], fname, ln, msg, strerror(local_errno));
+    } else {
+        err_cb(err_user_data, lvl, "%s: %s\n", msg, strerror(local_errno));
+    }
+}
+#endif
+
+#if defined(__ANDROID__)
+static void
+err_logcat_cb(void *user_data, err_lvl_t lvl, const char *fmt, ...)
+{
+    static const int android_level[ERR_MAX] = {ANDROID_LOG_DEBUG, ANDROID_LOG_INFO,
+         ANDROID_LOG_INFO, ANDROID_LOG_WARN, ANDROID_LOG_ERROR, ANDROID_LOG_ERROR};
+
+    va_list ap;
+    va_start(ap, fmt);
+    __android_log_vprint(android_level[lvl], "cmusphinx", fmt, ap);
+    va_end(ap);
+}
+#elif defined(_WIN32_WCE)
+static void
+err_wince_cb(void *user_data, err_lvl_t lvl, const char *fmt, ...)
+{
+    char msg[1024];
+    WCHAR *wmsg;
+    size_t size;
+    va_list ap;
+
+    va_start(ap, fmt);
+    _vsnprintf(msg, sizeof(msg), fmt, ap);
+    va_end(ap);
+
+    size = mbstowcs(NULL, msg, 0) + 1;
+    wmsg = ckd_calloc(size, sizeof(*wmsg));
+    mbstowcs(wmsg, msg, size);
+
+    OutputDebugStringW(wmsg);
+    ckd_free(wmsg);
+}
+#else
+void
+err_logfp_cb(void *user_data, err_lvl_t lvl, const char *fmt, ...)
+{
+    va_list ap;
+    FILE *fp = err_get_logfp();
+
+    if (!fp)
+        return;
+    
+    va_start(ap, fmt);
+    vfprintf(fp, fmt, ap);
+    va_end(ap);
+}
+#endif
+
+int
+err_set_logfile(const char *path)
+{
+    FILE *newfp, *oldfp;
+
+    if ((newfp = fopen(path, "a")) == NULL)
+        return -1;
+    oldfp = err_get_logfp();
+    err_set_logfp(newfp);
+    if (oldfp != NULL && oldfp != stdout && oldfp != stderr)
+        fclose(oldfp);
+    return 0;
+}
+
+void
+err_set_logfp(FILE *stream)
+{
+    if (stream == NULL) {
+	logfp_disabled = TRUE;
+	logfp = NULL;
+	return;
+    }    
+    logfp_disabled = FALSE;
+    logfp = stream;
+    return;
+}
+
+FILE *
+err_get_logfp(void)
+{
+    if (logfp_disabled)
+	return NULL;
+    if (logfp == NULL)
+	return stderr;
+
+    return logfp;
+}
+
+int
+err_set_debug_level(int level)
+{
+    int prev = sphinx_debug_level;
+    sphinx_debug_level = level;
+    return prev;
+}
+
+int
+err_get_debug_level(void)
+{
+    return sphinx_debug_level;
+}
+
+void
+err_set_callback(err_cb_f cb, void* user_data)
+{
+    err_cb = cb;
+    err_user_data= user_data;
+}

media/sphinxbase/src/libsphinxbase/util/errno.c

@@ -0,0 +1,51 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ */
+/*********************************************************************
+ *
+ * File: errno.c
+ * 
+ * Description: functions and variables missing from Windows CE standard
+ * library
+ * 
+ * Author: Silvio Moioli <silvio@moioli.net>
+ * 
+ *********************************************************************/
+
+#include <errno.h>
+
+#if defined(_WIN32_WCE)
+int errno;
+#endif

media/sphinxbase/src/libsphinxbase/util/f2c_lite.c

@@ -0,0 +1,551 @@
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include "sphinxbase/f2c.h"
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4244)
+#endif
+
+
+extern void
+s_wsfe(cilist * f)
+{;
+}
+extern void
+e_wsfe(void)
+{;
+}
+extern void
+do_fio(integer * c, char *s, ftnlen l)
+{;
+}
+
+/* You'll want this if you redo the *_lite.c files with the -C option
+ * to f2c for checking array subscripts. (It's not suggested you do that
+ * for production use, of course.) */
+extern int
+s_rnge(char *var, int index, char *routine, int lineno)
+{
+    fprintf(stderr,
+            "array index out-of-bounds for %s[%d] in routine %s:%d\n", var,
+            index, routine, lineno);
+    fflush(stderr);
+	assert(2+2 == 5);
+	return 0;
+}
+
+
+#ifdef KR_headers
+extern double sqrt();
+float
+f__cabs(real, imag)
+float real, imag;
+#else
+#undef abs
+
+float
+f__cabs(float real, float imag)
+#endif
+{
+    float temp;
+
+    if (real < 0)
+        real = -real;
+    if (imag < 0)
+        imag = -imag;
+    if (imag > real) {
+        temp = real;
+        real = imag;
+        imag = temp;
+    }
+    if ((imag + real) == real)
+        return ((float) real);
+
+    temp = imag / real;
+    temp = real * sqrt(1.0 + temp * temp);      /*overflow!! */
+    return (temp);
+}
+
+
+VOID
+#ifdef KR_headers
+s_cnjg(r, z)
+complex *r, *z;
+#else
+s_cnjg(complex * r, complex * z)
+#endif
+{
+    r->r = z->r;
+    r->i = -z->i;
+}
+
+
+#ifdef KR_headers
+float
+r_imag(z)
+complex *z;
+#else
+float
+r_imag(complex * z)
+#endif
+{
+    return (z->i);
+}
+
+
+#define log10e 0.43429448190325182765
+
+#ifdef KR_headers
+double log();
+float
+r_lg10(x)
+real *x;
+#else
+#undef abs
+
+float
+r_lg10(real * x)
+#endif
+{
+    return (log10e * log(*x));
+}
+
+
+#ifdef KR_headers
+float
+r_sign(a, b)
+real *a, *b;
+#else
+float
+r_sign(real * a, real * b)
+#endif
+{
+    float x;
+    x = (*a >= 0 ? *a : -*a);
+    return (*b >= 0 ? x : -x);
+}
+
+
+#ifdef KR_headers
+double floor();
+integer
+i_dnnt(x)
+real *x;
+#else
+#undef abs
+
+integer
+i_dnnt(real * x)
+#endif
+{
+    return ((*x) >= 0 ? floor(*x + .5) : -floor(.5 - *x));
+}
+
+
+#ifdef KR_headers
+double pow();
+double
+pow_dd(ap, bp)
+doublereal *ap, *bp;
+#else
+#undef abs
+
+double
+pow_dd(doublereal * ap, doublereal * bp)
+#endif
+{
+    return (pow(*ap, *bp));
+}
+
+
+#ifdef KR_headers
+float
+pow_ri(ap, bp)
+real *ap;
+integer *bp;
+#else
+float
+pow_ri(real * ap, integer * bp)
+#endif
+{
+    float pow, x;
+    integer n;
+    unsigned long u;
+
+    pow = 1;
+    x = *ap;
+    n = *bp;
+
+    if (n != 0) {
+        if (n < 0) {
+            n = -n;
+            x = 1 / x;
+        }
+        for (u = n;;) {
+            if (u & 01)
+                pow *= x;
+            if (u >>= 1)
+                x *= x;
+            else
+                break;
+        }
+    }
+    return (pow);
+}
+
+/* Unless compiled with -DNO_OVERWRITE, this variant of s_cat allows the
+ * target of a concatenation to appear on its right-hand side (contrary
+ * to the Fortran 77 Standard, but in accordance with Fortran 90).
+ */
+#define NO_OVERWRITE
+
+
+#ifndef NO_OVERWRITE
+
+#undef abs
+#ifdef KR_headers
+extern char *F77_aloc();
+extern void free();
+extern void exit_();
+#else
+
+extern char *F77_aloc(ftnlen, char *);
+#endif
+
+#endif                          /* NO_OVERWRITE */
+
+VOID
+#ifdef KR_headers
+s_cat(lp, rpp, rnp, np, ll)
+char *lp, *rpp[];
+ftnlen rnp[], *np, ll;
+#else
+s_cat(char *lp, char *rpp[], ftnlen rnp[], ftnlen * np, ftnlen ll)
+#endif
+{
+    ftnlen i, nc;
+    char *rp;
+    ftnlen n = *np;
+#ifndef NO_OVERWRITE
+    ftnlen L, m;
+    char *lp0, *lp1;
+
+    lp0 = 0;
+    lp1 = lp;
+    L = ll;
+    i = 0;
+    while (i < n) {
+        rp = rpp[i];
+        m = rnp[i++];
+        if (rp >= lp1 || rp + m <= lp) {
+            if ((L -= m) <= 0) {
+                n = i;
+                break;
+            }
+            lp1 += m;
+            continue;
+        }
+        lp0 = lp;
+        lp = lp1 = F77_aloc(L = ll, "s_cat");
+        break;
+    }
+    lp1 = lp;
+#endif                          /* NO_OVERWRITE */
+    for (i = 0; i < n; ++i) {
+        nc = ll;
+        if (rnp[i] < nc)
+            nc = rnp[i];
+        ll -= nc;
+        rp = rpp[i];
+        while (--nc >= 0)
+            *lp++ = *rp++;
+    }
+    while (--ll >= 0)
+        *lp++ = ' ';
+#ifndef NO_OVERWRITE
+    if (lp0) {
+        memmove(lp0, lp1, L);
+        free(lp1);
+    }
+#endif
+}
+
+
+/* compare two strings */
+
+#ifdef KR_headers
+integer
+s_cmp(a0, b0, la, lb)
+char *a0, *b0;
+ftnlen la, lb;
+#else
+integer
+s_cmp(char *a0, char *b0, ftnlen la, ftnlen lb)
+#endif
+{
+    register unsigned char *a, *aend, *b, *bend;
+    a = (unsigned char *) a0;
+    b = (unsigned char *) b0;
+    aend = a + la;
+    bend = b + lb;
+
+    if (la <= lb) {
+        while (a < aend)
+            if (*a != *b)
+                return (*a - *b);
+            else {
+                ++a;
+                ++b;
+            }
+
+        while (b < bend)
+            if (*b != ' ')
+                return (' ' - *b);
+            else
+                ++b;
+    }
+
+    else {
+        while (b < bend)
+            if (*a == *b) {
+                ++a;
+                ++b;
+            }
+            else
+                return (*a - *b);
+        while (a < aend)
+            if (*a != ' ')
+                return (*a - ' ');
+            else
+                ++a;
+    }
+    return (0);
+}
+
+/* Unless compiled with -DNO_OVERWRITE, this variant of s_copy allows the
+ * target of an assignment to appear on its right-hand side (contrary
+ * to the Fortran 77 Standard, but in accordance with Fortran 90),
+ * as in  a(2:5) = a(4:7) .
+ */
+
+
+
+/* assign strings:  a = b */
+
+#ifdef KR_headers
+VOID
+s_copy(a, b, la, lb)
+register char *a, *b;
+ftnlen la, lb;
+#else
+void
+s_copy(register char *a, register char *b, ftnlen la, ftnlen lb)
+#endif
+{
+    register char *aend, *bend;
+
+    aend = a + la;
+
+    if (la <= lb)
+#ifndef NO_OVERWRITE
+        if (a <= b || a >= b + la)
+#endif
+            while (a < aend)
+                *a++ = *b++;
+#ifndef NO_OVERWRITE
+        else
+            for (b += la; a < aend;)
+                *--aend = *--b;
+#endif
+
+    else {
+        bend = b + lb;
+#ifndef NO_OVERWRITE
+        if (a <= b || a >= bend)
+#endif
+            while (b < bend)
+                *a++ = *b++;
+#ifndef NO_OVERWRITE
+        else {
+            a += lb;
+            while (b < bend)
+                *--a = *--bend;
+            a += lb;
+        }
+#endif
+        while (a < aend)
+            *a++ = ' ';
+    }
+}
+
+
+#ifdef KR_headers
+float f__cabs();
+float
+z_abs(z)
+complex *z;
+#else
+float f__cabs(float, float);
+float
+z_abs(complex * z)
+#endif
+{
+    return (f__cabs(z->r, z->i));
+}
+
+
+#ifdef KR_headers
+extern void sig_die();
+VOID
+z_div(c, a, b)
+complex *a, *b, *c;
+#else
+extern void sig_die(char *, int);
+void
+z_div(complex * c, complex * a, complex * b)
+#endif
+{
+    float ratio, den;
+    float abr, abi;
+
+    if ((abr = b->r) < 0.)
+        abr = -abr;
+    if ((abi = b->i) < 0.)
+        abi = -abi;
+    if (abr <= abi) {
+        /*Let IEEE Infinties handle this ;( */
+        /*if(abi == 0)
+           sig_die("complex division by zero", 1); */
+        ratio = b->r / b->i;
+        den = b->i * (1 + ratio * ratio);
+        c->r = (a->r * ratio + a->i) / den;
+        c->i = (a->i * ratio - a->r) / den;
+    }
+
+    else {
+        ratio = b->i / b->r;
+        den = b->r * (1 + ratio * ratio);
+        c->r = (a->r + a->i * ratio) / den;
+        c->i = (a->i - a->r * ratio) / den;
+    }
+
+}
+
+
+#ifdef KR_headers
+double sqrt();
+double f__cabs();
+VOID
+z_sqrt(r, z)
+complex *r, *z;
+#else
+#undef abs
+
+extern float f__cabs(float, float);
+void
+z_sqrt(complex * r, complex * z)
+#endif
+{
+    float mag;
+
+    if ((mag = f__cabs(z->r, z->i)) == 0.)
+        r->r = r->i = 0.;
+    else if (z->r > 0) {
+        r->r = sqrt(0.5 * (mag + z->r));
+        r->i = z->i / r->r / 2;
+    }
+    else {
+        r->i = sqrt(0.5 * (mag - z->r));
+        if (z->i < 0)
+            r->i = -r->i;
+        r->r = z->i / r->i / 2;
+    }
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef KR_headers
+    integer pow_ii(ap, bp) integer *ap, *bp;
+#else
+    integer pow_ii(integer * ap, integer * bp)
+#endif
+    {
+        integer pow, x, n;
+        unsigned long u;
+
+         x = *ap;
+         n = *bp;
+
+        if (n <= 0) {
+            if (n == 0 || x == 1)
+                return 1;
+            if (x != -1)
+                return x != 0 ? 1 / x : 0;
+            n = -n;
+        } u = n;
+        for (pow = 1;;) {
+            if (u & 01)
+                pow *= x;
+            if (u >>= 1)
+                x *= x;
+            else
+                break;
+        }
+        return (pow);
+    }
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef KR_headers
+extern void f_exit();
+VOID
+s_stop(s, n)
+char *s;
+ftnlen n;
+#else
+#undef abs
+#undef min
+#undef max
+#ifdef __cplusplus
+extern "C" {
+#endif
+#ifdef __cplusplus
+    extern "C" {
+#endif
+        void f_exit(void);
+
+        int s_stop(char *s, ftnlen n)
+#endif
+        {
+            int i;
+
+            if (n > 0) {
+                fprintf(stderr, "STOP ");
+                for (i = 0; i < n; ++i)
+                    putc(*s++, stderr);
+                fprintf(stderr, " statement executed\n");
+            }
+#ifdef NO_ONEXIT
+            f_exit();
+#endif
+             exit(0);
+
+/* We cannot avoid (useless) compiler diagnostics here:		*/
+/* some compilers complain if there is no return statement,	*/
+/* and others complain that this one cannot be reached.		*/
+
+             return 0;          /* NOT REACHED */
+        }
+#ifdef __cplusplus
+    }
+#endif
+#ifdef __cplusplus
+}
+#endif

media/sphinxbase/src/libsphinxbase/util/filename.c

@@ -0,0 +1,120 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * filename.c -- File and path name operations.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include "sphinxbase/filename.h"
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4996)
+#endif
+
+const char *
+path2basename(const char *path)
+{
+    const char *result;
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+    result = strrchr(path, '\\');
+#else
+    result = strrchr(path, '/');
+#endif
+
+    return (result == NULL ? path : result + 1);
+}
+
+/* Return all leading pathname components */
+void
+path2dirname(const char *path, char *dir)
+{
+    size_t i, l;
+
+    l = strlen(path);
+#if defined(_WIN32) || defined(__CYGWIN__)
+    for (i = l - 1; (i > 0) && !(path[i] == '/' || path[i] == '\\'); --i);
+#else
+    for (i = l - 1; (i > 0) && !(path[i] == '/'); --i);
+#endif
+    if (i == 0) {
+        dir[0] = '.';
+        dir[1] = '\0';
+    } else {
+        memcpy(dir, path, i);
+        dir[i] = '\0';
+    }
+}
+
+
+/* Strip off the shortest trailing .xyz suffix */
+void
+strip_fileext(const char *path, char *root)
+{
+    size_t i, l;
+
+    l = strlen(path);
+    for (i = l - 1; (i > 0) && (path[i] != '.'); --i);
+    if (i == 0) {
+        strcpy(root, path);     /* Didn't find a . */
+    } else {
+        strncpy(root, path, i);
+    }
+}
+
+/* Test if this path is absolute. */
+int
+path_is_absolute(const char *path)
+{
+#if defined(_WIN32) && !defined(_WIN32_WCE) /* FIXME: Also SymbianOS */
+    return /* Starts with drive letter : \ or / */
+        (strlen(path) >= 3
+         &&
+         ((path[0] >= 'A' && path[0] <= 'Z')
+          || (path[0] >= 'a' && path[0] <= 'z'))
+         && path[1] == ':'
+         && (path[2] == '/' || path[2] == '\\'));
+#elif defined(_WIN32_WCE)
+    return path[0] == '\\' || path[0] == '/';
+#else /* Assume Unix */
+    return path[0] == '/';
+#endif
+}

media/sphinxbase/src/libsphinxbase/util/genrand.c

@@ -0,0 +1,234 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/* 
+   A C-program for MT19937, with initialization improved 2002/1/26.
+   Coded by Takuji Nishimura and Makoto Matsumoto.
+
+   Before using, initialize the state by using init_genrand(seed)  
+   or init_by_array(init_key, key_length).
+
+   Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+   All rights reserved.                          
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+     1. Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+
+     2. Redistributions in binary form must reproduce the above copyright
+`        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+
+     3. The names of its contributors may not be used to endorse or promote 
+        products derived from this software without specific prior written 
+        permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+   Any feedback is very welcome.
+   http://www.math.keio.ac.jp/matumoto/emt.html
+   email: matumoto@math.keio.ac.jp
+*/
+
+
+/*
+ * randgen.c   : a portable random generator 
+ * 
+ *
+ * **********************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 1999 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ * 
+ * HISTORY
+ * $Log: genrand.c,v $
+ * Revision 1.2  2005/06/22 03:01:50  arthchan2003
+ * Added  keyword
+ *
+ * Revision 1.3  2005/03/30 01:22:48  archan
+ * Fixed mistakes in last updates. Add
+ *
+ * 
+ * 18-Nov-04 ARCHAN (archan@cs.cmu.edu) at Carnegie Mellon University
+ *              First incorporated from the Mersenne Twister Random
+ *              Number Generator package. It was chosen because it is
+ *              in BSD-license and its performance is quite
+ *              reasonable. Of course if you look at the inventors's
+ *              page.  This random generator can actually gives
+ *              19937-bits period.  This is already far from we need. 
+ *              This will possibly good enough for the next 10 years. 
+ *
+ *              I also downgrade the code a little bit to avoid Sphinx's
+ *              developers misused it. 
+ */
+
+
+
+#include <stdio.h>
+
+#include "sphinxbase/genrand.h"
+
+/* Period parameters */
+#define N 624
+#define M 397
+#define MATRIX_A 0x9908b0dfUL   /* constant vector a */
+#define UPPER_MASK 0x80000000UL /* most significant w-r bits */
+#define LOWER_MASK 0x7fffffffUL /* least significant r bits */
+
+void init_genrand(unsigned long s);
+
+void
+genrand_seed(unsigned long s)
+{
+    init_genrand(s);
+}
+
+
+static unsigned long mt[N];     /* the array for the state vector  */
+static int mti = N + 1;         /* mti==N+1 means mt[N] is not initialized */
+
+/* initializes mt[N] with a seed */
+void
+init_genrand(unsigned long s)
+{
+    mt[0] = s & 0xffffffffUL;
+    for (mti = 1; mti < N; mti++) {
+        mt[mti] =
+            (1812433253UL * (mt[mti - 1] ^ (mt[mti - 1] >> 30)) + mti);
+        /* See Knuth TAOCP Vol2. 3rd Ed. P.106 for multiplier. */
+        /* In the previous versions, MSBs of the seed affect   */
+        /* only MSBs of the array mt[].                        */
+        /* 2002/01/09 modified by Makoto Matsumoto             */
+        mt[mti] &= 0xffffffffUL;
+        /* for >32 bit machines */
+    }
+}
+
+/* generates a random number on [0,0xffffffff]-interval */
+unsigned long
+genrand_int32(void)
+{
+    unsigned long y;
+    static unsigned long mag01[2] = { 0x0UL, MATRIX_A };
+    /* mag01[x] = x * MATRIX_A  for x=0,1 */
+
+    if (mti >= N) {             /* generate N words at one time */
+        int kk;
+
+        if (mti == N + 1)       /* if init_genrand() has not been called, */
+            init_genrand(5489UL);       /* a default initial seed is used */
+
+        for (kk = 0; kk < N - M; kk++) {
+            y = (mt[kk] & UPPER_MASK) | (mt[kk + 1] & LOWER_MASK);
+            mt[kk] = mt[kk + M] ^ (y >> 1) ^ mag01[y & 0x1UL];
+        }
+        for (; kk < N - 1; kk++) {
+            y = (mt[kk] & UPPER_MASK) | (mt[kk + 1] & LOWER_MASK);
+            mt[kk] = mt[kk + (M - N)] ^ (y >> 1) ^ mag01[y & 0x1UL];
+        }
+        y = (mt[N - 1] & UPPER_MASK) | (mt[0] & LOWER_MASK);
+        mt[N - 1] = mt[M - 1] ^ (y >> 1) ^ mag01[y & 0x1UL];
+
+        mti = 0;
+    }
+
+    y = mt[mti++];
+
+    /* Tempering */
+    y ^= (y >> 11);
+    y ^= (y << 7) & 0x9d2c5680UL;
+    y ^= (y << 15) & 0xefc60000UL;
+    y ^= (y >> 18);
+
+    return y;
+}
+
+/* generates a random number on [0,0x7fffffff]-interval */
+long
+genrand_int31(void)
+{
+    return (long) (genrand_int32() >> 1);
+}
+
+/* generates a random number on [0,1]-real-interval */
+double
+genrand_real1(void)
+{
+    return genrand_int32() * (1.0 / 4294967295.0);
+    /* divided by 2^32-1 */
+}
+
+/* generates a random number on [0,1)-real-interval */
+double
+genrand_real2(void)
+{
+    return genrand_int32() * (1.0 / 4294967296.0);
+    /* divided by 2^32 */
+}
+
+/* generates a random number on (0,1)-real-interval */
+double
+genrand_real3(void)
+{
+    return (((double) genrand_int32()) + 0.5) * (1.0 / 4294967296.0);
+    /* divided by 2^32 */
+}
+
+/* generates a random number on [0,1) with 53-bit resolution*/
+double
+genrand_res53(void)
+{
+    unsigned long a = genrand_int32() >> 5, b = genrand_int32() >> 6;
+    return (a * 67108864.0 + b) * (1.0 / 9007199254740992.0);
+}
+
+/* These real versions are due to Isaku Wada, 2002/01/09 added */

media/sphinxbase/src/libsphinxbase/util/glist.c

@@ -0,0 +1,271 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * glist.h -- Module for maintaining a generic, linear linked-list structure.
+ *
+ * **********************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 1999 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ * 
+ * HISTORY
+ * $Log: glist.c,v $
+ * Revision 1.8  2005/06/22 03:02:51  arthchan2003
+ * 1, Fixed doxygen documentation, 2, add  keyword.
+ *
+ * Revision 1.3  2005/03/30 01:22:48  archan
+ * Fixed mistakes in last updates. Add
+ *
+ * 
+ * 09-Mar-1999	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Added glist_chkdup_*().
+ * 
+ * 13-Feb-1999	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Created from earlier version.
+ */
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include "sphinxbase/glist.h"
+#include "sphinxbase/ckd_alloc.h"
+
+
+glist_t
+glist_add_ptr(glist_t g, void *ptr)
+{
+    gnode_t *gn;
+
+    gn = (gnode_t *) ckd_calloc(1, sizeof(gnode_t));
+    gn->data.ptr = ptr;
+    gn->next = g;
+    return ((glist_t) gn);      /* Return the new head of the list */
+}
+
+
+glist_t
+glist_add_int32(glist_t g, int32 val)
+{
+    gnode_t *gn;
+
+    gn = (gnode_t *) ckd_calloc(1, sizeof(gnode_t));
+    gn->data.i = (long)val;
+    gn->next = g;
+    return ((glist_t) gn);      /* Return the new head of the list */
+}
+
+
+glist_t
+glist_add_uint32(glist_t g, uint32 val)
+{
+    gnode_t *gn;
+
+    gn = (gnode_t *) ckd_calloc(1, sizeof(gnode_t));
+    gn->data.ui = (unsigned long)val;
+    gn->next = g;
+    return ((glist_t) gn);      /* Return the new head of the list */
+}
+
+
+glist_t
+glist_add_float32(glist_t g, float32 val)
+{
+    gnode_t *gn;
+
+    gn = (gnode_t *) ckd_calloc(1, sizeof(gnode_t));
+    gn->data.fl = (double)val;
+    gn->next = g;
+    return ((glist_t) gn);      /* Return the new head of the list */
+}
+
+
+glist_t
+glist_add_float64(glist_t g, float64 val)
+{
+    gnode_t *gn;
+
+    gn = (gnode_t *) ckd_calloc(1, sizeof(gnode_t));
+    gn->data.fl = (double)val;
+    gn->next = g;
+    return ((glist_t) gn);      /* Return the new head of the list */
+}
+
+void
+glist_free(glist_t g)
+{
+    gnode_t *gn;
+
+    while (g) {
+        gn = g;
+        g = gn->next;
+        ckd_free((void *) gn);
+    }
+}
+
+int32
+glist_count(glist_t g)
+{
+    gnode_t *gn;
+    int32 n;
+
+    for (gn = g, n = 0; gn; gn = gn->next, n++);
+    return n;
+}
+
+
+gnode_t *
+glist_tail(glist_t g)
+{
+    gnode_t *gn;
+
+    if (!g)
+        return NULL;
+
+    for (gn = g; gn->next; gn = gn->next);
+    return gn;
+}
+
+
+glist_t
+glist_reverse(glist_t g)
+{
+    gnode_t *gn, *nextgn;
+    gnode_t *rev;
+
+    rev = NULL;
+    for (gn = g; gn; gn = nextgn) {
+        nextgn = gn->next;
+
+        gn->next = rev;
+        rev = gn;
+    }
+
+    return rev;
+}
+
+
+gnode_t *
+glist_insert_ptr(gnode_t * gn, void *ptr)
+{
+    gnode_t *newgn;
+
+    newgn = (gnode_t *) ckd_calloc(1, sizeof(gnode_t));
+    newgn->data.ptr = ptr;
+    newgn->next = gn->next;
+    gn->next = newgn;
+
+    return newgn;
+}
+
+
+gnode_t *
+glist_insert_int32(gnode_t * gn, int32 val)
+{
+    gnode_t *newgn;
+
+    newgn = (gnode_t *) ckd_calloc(1, sizeof(gnode_t));
+    newgn->data.i = val;
+    newgn->next = gn->next;
+    gn->next = newgn;
+
+    return newgn;
+}
+
+
+gnode_t *
+glist_insert_uint32(gnode_t * gn, uint32 val)
+{
+    gnode_t *newgn;
+
+    newgn = (gnode_t *) ckd_calloc(1, sizeof(gnode_t));
+    newgn->data.ui = val;
+    newgn->next = gn->next;
+
+    gn->next = newgn;
+
+    return newgn;
+}
+
+
+gnode_t *
+glist_insert_float32(gnode_t * gn, float32 val)
+{
+    gnode_t *newgn;
+
+    newgn = (gnode_t *) ckd_calloc(1, sizeof(gnode_t));
+    newgn->data.fl = (double)val;
+    newgn->next = gn->next;
+    gn->next = newgn;
+
+    return newgn;
+}
+
+
+gnode_t *
+glist_insert_float64(gnode_t * gn, float64 val)
+{
+    gnode_t *newgn;
+
+    newgn = (gnode_t *) ckd_calloc(1, sizeof(gnode_t));
+    newgn->data.fl = (double)val;
+    newgn->next = gn->next;
+    gn->next = newgn;
+
+    return newgn;
+}
+
+gnode_t *
+gnode_free(gnode_t * gn, gnode_t * pred)
+{
+    gnode_t *next;
+
+    next = gn->next;
+    if (pred) {
+        assert(pred->next == gn);
+
+        pred->next = next;
+    }
+
+    ckd_free((char *) gn);
+
+    return next;
+}

media/sphinxbase/src/libsphinxbase/util/hash_table.c

@@ -0,0 +1,713 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * hash.c -- Hash table module.
+ *
+ * **********************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 1999 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ * 
+ * HISTORY
+ * $Log: hash.c,v $
+ * Revision 1.5  2005/06/22 03:04:01  arthchan2003
+ * 1, Implemented hash_delete and hash_display, 2, Fixed doxygen documentation, 3, Added  keyword.
+ *
+ * Revision 1.9  2005/05/25 06:17:53  archan
+ * Delete the test code in cmd_ln.c and fixed platform specific code of hash.c
+ *
+ * Revision 1.8  2005/05/24 01:10:54  archan
+ * Fix a bug when the value only appear in the hash but there is no chain.   Also make sure that prev was initialized to NULL. All success cases were tested, but not tested with the deletion is tested.
+ *
+ * Revision 1.6  2005/05/24 00:00:45  archan
+ * Added basic functionalities to hash_t: 1, display and 2, delete a key from a hash. \n
+ *
+ * Revision 1.5  2005/05/11 07:01:38  archan
+ * Added comments on the usage of the current implementation of hash tables.
+ *
+ * Revision 1.4  2005/05/03 04:09:11  archan
+ * Implemented the heart of word copy search. For every ci-phone, every word end, a tree will be allocated to preserve its pathscore.  This is different from 3.5 or below, only the best score for a particular ci-phone, regardless of the word-ends will be preserved at every frame.  The graph propagation will not collect unused word tree at this point. srch_WST_propagate_wd_lv2 is also as the most stupid in the century.  But well, after all, everything needs a start.  I will then really get the results from the search and see how it looks.
+ *
+ * Revision 1.3  2005/03/30 01:22:48  archan
+ * Fixed mistakes in last updates. Add
+ *
+ * 
+ * 05-May-1999	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon
+ * 		Removed hash_key2hash().  Added hash_enter_bkey() and hash_lookup_bkey(),
+ * 		and len attribute to hash_entry_t.
+ * 
+ * 30-Apr-1999	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon
+ * 		Added hash_key2hash().
+ * 
+ * 18-Jun-97	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon
+ * 		Included case sensitive/insensitive option.  Removed local, static
+ * 		maintenance of all hash tables.
+ * 
+ * 31-Jul-95	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon
+ * 		Created.
+ */
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4018)
+#endif
+
+#include "sphinxbase/hash_table.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/case.h"
+
+
+#if 0
+static void
+prime_sieve(int32 max)
+{
+    char *notprime;
+    int32 p, pp;
+
+    notprime = (char *) ckd_calloc(max + 1, 1);
+    p = 2;
+    for (;;) {
+        printf("%d\n", p);
+        for (pp = p + p; pp <= max; pp += p)
+            notprime[pp] = 1;
+        for (++p; (p <= max) && notprime[p]; p++);
+        if (p > max)
+            break;
+    }
+}
+#endif
+
+
+/*
+ * HACK!!  Initial hash table size is restricted by this set of primes.  (Of course,
+ * collision resolution by chaining will accommodate more entries indefinitely, but
+ * efficiency will drop.)
+ */
+const int32 prime[] = {
+    101, 211, 307, 401, 503, 601, 701, 809, 907,
+    1009, 1201, 1601, 2003, 2411, 3001, 4001, 5003, 6007, 7001, 8009,
+    9001,
+    10007, 12007, 16001, 20011, 24001, 30011, 40009, 50021, 60013,
+    70001, 80021, 90001,
+    100003, 120011, 160001, 200003, 240007, 300007, 400009, 500009,
+    600011, 700001, 800011, 900001,
+    -1
+};
+
+
+/**
+ * This function returns a very large prime. 
+ */
+static int32
+prime_size(int32 size)
+{
+    int32 i;
+
+    for (i = 0; (prime[i] > 0) && (prime[i] < size); i++);
+    if (prime[i] <= 0) {
+        E_WARN("Very large hash table requested (%d entries)\n", size);
+        --i;
+    }
+    return (prime[i]);
+}
+
+
+hash_table_t *
+hash_table_new(int32 size, int32 casearg)
+{
+    hash_table_t *h;
+
+    h = (hash_table_t *) ckd_calloc(1, sizeof(hash_table_t));
+    h->size = prime_size(size + (size >> 1));
+    h->nocase = (casearg == HASH_CASE_NO);
+    h->table = (hash_entry_t *) ckd_calloc(h->size, sizeof(hash_entry_t));
+    /* The above calloc clears h->table[*].key and .next to NULL, i.e. an empty table */
+
+    return h;
+}
+
+
+/*
+ * Compute hash value for given key string.
+ * Somewhat tuned for English text word strings.
+ */
+static uint32
+key2hash(hash_table_t * h, const char *key)
+{
+
+    register const char *cp;
+
+    /** ARCHAN 20050712: 
+	[1236322] libutil\str2words special character bgu
+	HACK Apply suggested hack of fixing the hash table such that
+	it can work with extended ascii code . This is a hack because
+	the best way to solve it is to make sure all character
+	representation is unsigned character in the first place. (or
+	better unicode.)
+    **/
+
+    /*register char c; */
+    register unsigned char c;
+    register int32 s;
+    register uint32 hash;
+
+    hash = 0;
+    s = 0;
+
+    if (h->nocase) {
+        for (cp = key; *cp; cp++) {
+            c = *cp;
+            c = UPPER_CASE(c);
+            hash += c << s;
+            s += 5;
+            if (s >= 25)
+                s -= 24;
+        }
+    }
+    else {
+        for (cp = key; *cp; cp++) {
+            hash += (*cp) << s;
+            s += 5;
+            if (s >= 25)
+                s -= 24;
+        }
+    }
+
+    return (hash % h->size);
+}
+
+
+static char *
+makekey(uint8 * data, size_t len, char *key)
+{
+    size_t i, j;
+
+    if (!key)
+        key = (char *) ckd_calloc(len * 2 + 1, sizeof(char));
+
+    for (i = 0, j = 0; i < len; i++, j += 2) {
+        key[j] = 'A' + (data[i] & 0x000f);
+        key[j + 1] = 'J' + ((data[i] >> 4) & 0x000f);
+    }
+    key[j] = '\0';
+
+    return key;
+}
+
+
+static int32
+keycmp_nocase(hash_entry_t * entry, const char *key)
+{
+    char c1, c2;
+    int32 i;
+    const char *str;
+
+    str = entry->key;
+    for (i = 0; i < entry->len; i++) {
+        c1 = *(str++);
+        c1 = UPPER_CASE(c1);
+        c2 = *(key++);
+        c2 = UPPER_CASE(c2);
+        if (c1 != c2)
+            return (c1 - c2);
+    }
+
+    return 0;
+}
+
+
+static int32
+keycmp_case(hash_entry_t * entry, const char *key)
+{
+    char c1, c2;
+    int32 i;
+    const char *str;
+
+    str = entry->key;
+    for (i = 0; i < entry->len; i++) {
+        c1 = *(str++);
+        c2 = *(key++);
+        if (c1 != c2)
+            return (c1 - c2);
+    }
+
+    return 0;
+}
+
+
+/*
+ * Lookup entry with hash-value hash in table h for given key
+ * Return value: hash_entry_t for key
+ */
+static hash_entry_t *
+lookup(hash_table_t * h, uint32 hash, const char *key, size_t len)
+{
+    hash_entry_t *entry;
+
+    entry = &(h->table[hash]);
+    if (entry->key == NULL)
+        return NULL;
+
+    if (h->nocase) {
+        while (entry && ((entry->len != len)
+                         || (keycmp_nocase(entry, key) != 0)))
+            entry = entry->next;
+    }
+    else {
+        while (entry && ((entry->len != len)
+                         || (keycmp_case(entry, key) != 0)))
+            entry = entry->next;
+    }
+
+    return entry;
+}
+
+
+int32
+hash_table_lookup(hash_table_t * h, const char *key, void ** val)
+{
+    hash_entry_t *entry;
+    uint32 hash;
+    size_t len;
+
+    hash = key2hash(h, key);
+    len = strlen(key);
+
+    entry = lookup(h, hash, key, len);
+    if (entry) {
+        if (val)
+            *val = entry->val;
+        return 0;
+    }
+    else
+        return -1;
+}
+
+int32
+hash_table_lookup_int32(hash_table_t * h, const char *key, int32 *val)
+{
+    void *vval;
+    int32 rv;
+
+    rv = hash_table_lookup(h, key, &vval);
+    if (rv != 0)
+        return rv;
+    if (val)
+        *val = (int32)(long)vval;
+    return 0;
+}
+
+
+int32
+hash_table_lookup_bkey(hash_table_t * h, const char *key, size_t len, void ** val)
+{
+    hash_entry_t *entry;
+    uint32 hash;
+    char *str;
+
+    str = makekey((uint8 *) key, len, NULL);
+    hash = key2hash(h, str);
+    ckd_free(str);
+
+    entry = lookup(h, hash, key, len);
+    if (entry) {
+        if (val)
+            *val = entry->val;
+        return 0;
+    }
+    else
+        return -1;
+}
+
+int32
+hash_table_lookup_bkey_int32(hash_table_t * h, const char *key, size_t len, int32 *val)
+{
+    void *vval;
+    int32 rv;
+
+    rv = hash_table_lookup_bkey(h, key, len, &vval);
+    if (rv != 0)
+        return rv;
+    if (val)
+        *val = (int32)(long)vval;
+    return 0;
+}
+
+
+static void *
+enter(hash_table_t * h, uint32 hash, const char *key, size_t len, void *val, int32 replace)
+{
+    hash_entry_t *cur, *new;
+
+    if ((cur = lookup(h, hash, key, len)) != NULL) {
+        void *oldval;
+        /* Key already exists. */
+        oldval = cur->val;
+        if (replace) {
+            /* Replace the pointer if replacement is requested,
+             * because this might be a different instance of the same
+             * string (this verges on magic, sorry) */
+            cur->key = key;
+            cur->val = val;
+        }
+        return oldval;
+    }
+
+    cur = &(h->table[hash]);
+    if (cur->key == NULL) {
+        /* Empty slot at hashed location; add this entry */
+        cur->key = key;
+        cur->len = len;
+        cur->val = val;
+
+        /* Added by ARCHAN at 20050515. This allows deletion could work. */
+        cur->next = NULL;
+
+    }
+    else {
+        /* Key collision; create new entry and link to hashed location */
+        new = (hash_entry_t *) ckd_calloc(1, sizeof(hash_entry_t));
+        new->key = key;
+        new->len = len;
+        new->val = val;
+        new->next = cur->next;
+        cur->next = new;
+    }
+    ++h->inuse;
+
+    return val;
+}
+
+/* 20050523 Added by ARCHAN  to delete a key from a hash table */
+static void *
+delete(hash_table_t * h, uint32 hash, const char *key, size_t len)
+{
+    hash_entry_t *entry, *prev;
+    void *val;
+
+    prev = NULL;
+    entry = &(h->table[hash]);
+    if (entry->key == NULL)
+        return NULL;
+
+    if (h->nocase) {
+        while (entry && ((entry->len != len)
+                         || (keycmp_nocase(entry, key) != 0))) {
+            prev = entry;
+            entry = entry->next;
+        }
+    }
+    else {
+        while (entry && ((entry->len != len)
+                         || (keycmp_case(entry, key) != 0))) {
+            prev = entry;
+            entry = entry->next;
+        }
+    }
+
+    if (entry == NULL)
+        return NULL;
+
+    /* At this point, entry will be the one required to be deleted, prev
+       will contain the previous entry
+     */
+    val = entry->val;
+
+    if (prev == NULL) {
+        /* That is to say the entry in the hash table (not the chain) matched the key. */
+        /* We will then copy the things from the next entry to the hash table */
+        prev = entry;
+        if (entry->next) {      /* There is a next entry, great, copy it. */
+            entry = entry->next;
+            prev->key = entry->key;
+            prev->len = entry->len;
+            prev->val = entry->val;
+            prev->next = entry->next;
+            ckd_free(entry);
+        }
+        else {                  /* There is not a next entry, just set the key to null */
+            prev->key = NULL;
+            prev->len = 0;
+            prev->next = NULL;
+        }
+
+    }
+    else {                      /* This case is simple */
+        prev->next = entry->next;
+        ckd_free(entry);
+    }
+
+    /* Do wiring and free the entry */
+
+    --h->inuse;
+
+    return val;
+}
+
+void
+hash_table_empty(hash_table_t *h)
+{
+    hash_entry_t *e, *e2;
+    int32 i;
+
+    for (i = 0; i < h->size; i++) {
+        /* Free collision lists. */
+        for (e = h->table[i].next; e; e = e2) {
+            e2 = e->next;
+            ckd_free((void *) e);
+        }
+        memset(&h->table[i], 0, sizeof(h->table[i]));
+    }
+    h->inuse = 0;
+}
+
+
+void *
+hash_table_enter(hash_table_t * h, const char *key, void *val)
+{
+    uint32 hash;
+    size_t len;
+
+    hash = key2hash(h, key);
+    len = strlen(key);
+    return (enter(h, hash, key, len, val, 0));
+}
+
+void *
+hash_table_replace(hash_table_t * h, const char *key, void *val)
+{
+    uint32 hash;
+    size_t len;
+
+    hash = key2hash(h, key);
+    len = strlen(key);
+    return (enter(h, hash, key, len, val, 1));
+}
+
+void *
+hash_table_delete(hash_table_t * h, const char *key)
+{
+    uint32 hash;
+    size_t len;
+
+    hash = key2hash(h, key);
+    len = strlen(key);
+
+    return (delete(h, hash, key, len));
+}
+
+void *
+hash_table_enter_bkey(hash_table_t * h, const char *key, size_t len, void *val)
+{
+    uint32 hash;
+    char *str;
+
+    str = makekey((uint8 *) key, len, NULL);
+    hash = key2hash(h, str);
+    ckd_free(str);
+
+    return (enter(h, hash, key, len, val, 0));
+}
+
+void *
+hash_table_replace_bkey(hash_table_t * h, const char *key, size_t len, void *val)
+{
+    uint32 hash;
+    char *str;
+
+    str = makekey((uint8 *) key, len, NULL);
+    hash = key2hash(h, str);
+    ckd_free(str);
+
+    return (enter(h, hash, key, len, val, 1));
+}
+
+void *
+hash_table_delete_bkey(hash_table_t * h, const char *key, size_t len)
+{
+    uint32 hash;
+    char *str;
+
+    str = makekey((uint8 *) key, len, NULL);
+    hash = key2hash(h, str);
+    ckd_free(str);
+
+    return (delete(h, hash, key, len));
+}
+
+void
+hash_table_display(hash_table_t * h, int32 showdisplay)
+{
+    hash_entry_t *e;
+    int i, j;
+    j = 0;
+
+    printf("Hash with chaining representation of the hash table\n");
+
+    for (i = 0; i < h->size; i++) {
+        e = &(h->table[i]);
+        if (e->key != NULL) {
+            printf("|key:");
+            if (showdisplay)
+                printf("%s", e->key);
+            else
+                printf("%p", e->key);
+
+            printf("|len:%zd|val=%ld|->", e->len, (long)e->val);
+            if (e->next == NULL) {
+                printf("NULL\n");
+            }
+            j++;
+
+            for (e = e->next; e; e = e->next) {
+                printf("|key:");
+                if (showdisplay)
+                    printf("%s", e->key);
+
+                printf("|len:%zd|val=%ld|->", e->len, (long)e->val);
+                if (e->next == NULL) {
+                    printf("NULL\n");
+                }
+                j++;
+            }
+        }
+    }
+
+    printf("The total number of keys =%d\n", j);
+}
+
+
+glist_t
+hash_table_tolist(hash_table_t * h, int32 * count)
+{
+    glist_t g;
+    hash_entry_t *e;
+    int32 i, j;
+
+    g = NULL;
+
+    j = 0;
+    for (i = 0; i < h->size; i++) {
+        e = &(h->table[i]);
+
+        if (e->key != NULL) {
+            g = glist_add_ptr(g, (void *) e);
+            j++;
+
+            for (e = e->next; e; e = e->next) {
+                g = glist_add_ptr(g, (void *) e);
+                j++;
+            }
+        }
+    }
+
+    if (count)
+        *count = j;
+
+    return g;
+}
+
+hash_iter_t *
+hash_table_iter(hash_table_t *h)
+{
+	hash_iter_t *itor;
+
+	itor = ckd_calloc(1, sizeof(*itor));
+	itor->ht = h;
+	return hash_table_iter_next(itor);
+}
+
+hash_iter_t *
+hash_table_iter_next(hash_iter_t *itor)
+{
+	/* If there is an entry, walk down its list. */
+	if (itor->ent)
+		itor->ent = itor->ent->next;
+	/* If we got to the end of the chain, or we had no entry, scan
+	 * forward in the table to find the next non-empty bucket. */
+	if (itor->ent == NULL) {
+		while (itor->idx < itor->ht->size
+		       && itor->ht->table[itor->idx].key == NULL) 
+			++itor->idx;
+		/* If we did not find one then delete the iterator and
+		 * return NULL. */
+		if (itor->idx == itor->ht->size) {
+			hash_table_iter_free(itor);
+			return NULL;
+		}
+		/* Otherwise use this next entry. */
+		itor->ent = itor->ht->table + itor->idx;
+		/* Increase idx for the next time around. */
+		++itor->idx;
+	}
+	return itor;
+}
+
+void
+hash_table_iter_free(hash_iter_t *itor)
+{
+	ckd_free(itor);
+}
+
+void
+hash_table_free(hash_table_t * h)
+{
+    hash_entry_t *e, *e2;
+    int32 i;
+
+    if (h == NULL)
+        return;
+
+    /* Free additional entries created for key collision cases */
+    for (i = 0; i < h->size; i++) {
+        for (e = h->table[i].next; e; e = e2) {
+            e2 = e->next;
+            ckd_free((void *) e);
+        }
+    }
+
+    ckd_free((void *) h->table);
+    ckd_free((void *) h);
+}

media/sphinxbase/src/libsphinxbase/util/heap.c

@@ -0,0 +1,292 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * heap.c -- Generic heap structure for inserting in any and popping in sorted
+ * 		order.
+ *
+ * **********************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 1999 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ * 
+ * HISTORY
+ * $Log: heap.c,v $
+ * Revision 1.4  2005/06/22 03:05:49  arthchan2003
+ * 1, Fixed doxygen documentation, 2, Add  keyword.
+ *
+ * Revision 1.3  2005/03/30 01:22:48  archan
+ * Fixed mistakes in last updates. Add
+ *
+ * 
+ * 05-Mar-99	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Fixed bug in heap_destroy() (in while loop exit condition).
+ * 
+ * 23-Dec-96	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Started.
+ */
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include "sphinxbase/heap.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/ckd_alloc.h"
+
+/**
+ * One node on the heap
+ */
+typedef struct heapnode_s {
+    void *data;                 /**< Application data at this node */
+    int32 val;                  /**< Associated with above application data; according to which
+                                   heap is sorted (in ascending order) */
+    int32 nl, nr;               /**< #left/right descendants of this node (for balancing heap) */
+    struct heapnode_s *l;       /**< Root of left descendant heap */
+    struct heapnode_s *r;       /**< Root of right descendant heap */
+} heapnode_t;
+
+/**
+ * Internal heap data structure.
+ */
+struct heap_s {
+    heapnode_t *top;
+};
+
+
+#if 0
+static void
+heap_dump(heapnode_t * top, int32 level)
+{
+    int32 i;
+
+    if (!top)
+        return;
+
+    for (i = 0; i < level; i++)
+        printf("  ");
+    /* print top info */
+    heap_dump(top->l, level + 1);
+    heap_dump(top->r, level + 1);
+}
+#endif
+
+
+heap_t *
+heap_new(void)
+{
+    heap_t *h = ckd_calloc(1, sizeof(*h));
+    return h;
+}
+
+
+static heapnode_t *
+subheap_insert(heapnode_t * root, void *data, int32 val)
+{
+    heapnode_t *h;
+    void *tmpdata;
+    int32 tmpval;
+
+    if (!root) {
+        h = (heapnode_t *) ckd_calloc(1, sizeof(heapnode_t));
+        h->data = data;
+        h->val = val;
+        h->l = h->r = NULL;
+        h->nl = h->nr = 0;
+        return h;
+    }
+
+    /* Root already exists; if new value is less, replace root node */
+    if (root->val > val) {
+        tmpdata = root->data;
+        tmpval = root->val;
+        root->data = data;
+        root->val = val;
+        data = tmpdata;
+        val = tmpval;
+    }
+
+    /* Insert new or old (replaced) node in right or left subtree; keep them balanced */
+    if (root->nl > root->nr) {
+        root->r = subheap_insert(root->r, data, val);
+        root->nr++;
+    }
+    else {
+        root->l = subheap_insert(root->l, data, val);
+        root->nl++;
+    }
+
+    return root;
+}
+
+
+int
+heap_insert(heap_t *heap, void *data, int32 val)
+{
+    heap->top = subheap_insert(heap->top, data, val);
+    return 0;
+}
+
+
+static heapnode_t *
+subheap_pop(heapnode_t * root)
+{
+    heapnode_t *l, *r;
+
+    /* Propagate best value from below into root, if any */
+    l = root->l;
+    r = root->r;
+
+    if (!l) {
+        if (!r) {
+            ckd_free((char *) root);
+            return NULL;
+        }
+        else {
+            root->data = r->data;
+            root->val = r->val;
+            root->r = subheap_pop(r);
+            root->nr--;
+        }
+    }
+    else {
+        if ((!r) || (l->val < r->val)) {
+            root->data = l->data;
+            root->val = l->val;
+            root->l = subheap_pop(l);
+            root->nl--;
+        }
+        else {
+            root->data = r->data;
+            root->val = r->val;
+            root->r = subheap_pop(r);
+            root->nr--;
+        }
+    }
+
+    return root;
+}
+
+
+int
+heap_pop(heap_t *heap, void **data, int32 * val)
+{
+    if (heap->top == NULL)
+        return 0;
+    *data = heap->top->data;
+    *val = heap->top->val;
+    heap->top = subheap_pop(heap->top);
+    return 1;
+}
+
+
+int
+heap_top(heap_t *heap, void **data, int32 * val)
+{
+    if (heap->top == NULL)
+        return 0;
+    *data = heap->top->data;
+    *val = heap->top->val;
+    return 1;
+}
+
+static int
+heap_remove_one(heap_t *heap, heapnode_t *top, void *data)
+{
+    if (top == NULL)
+        return -1;
+    else if (top->data == data) {
+        assert(top == heap->top);
+        heap->top = subheap_pop(heap->top);
+        return 0;
+    }
+    if (top->l) {
+        if (top->l->data == data) {
+            top->l = subheap_pop(top->l);
+            --top->nl;
+            return 0;
+        }
+        else if (heap_remove_one(heap, top->l, data) == 0) {
+            --top->nl;
+            return 0;
+        }
+    }
+    if (top->r) {
+        if (top->r->data == data) {
+            top->r = subheap_pop(top->r);
+            --top->nr;
+            return 0;
+        }
+        else if (heap_remove_one(heap, top->r, data) == 0) {
+            --top->nr;
+            return 0;
+        }
+    }
+    return -1;
+}
+
+int
+heap_remove(heap_t *heap, void *data)
+{
+    return heap_remove_one(heap, heap->top, data);
+}
+
+
+size_t
+heap_size(heap_t *heap)
+{
+    if (heap->top == NULL)
+        return 0;
+    return heap->top->nl + heap->top->nr + 1;
+}
+
+int
+heap_destroy(heap_t *heap)
+{
+    void *data;
+    int32 val;
+
+    /* Empty the heap and free it */
+    while (heap_pop(heap, &data, &val) > 0)
+        ;
+    ckd_free(heap);
+
+    return 0;
+}

media/sphinxbase/src/libsphinxbase/util/huff_code.c

@@ -0,0 +1,651 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2009 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+#include <string.h>
+
+#include "sphinxbase/huff_code.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/hash_table.h"
+#include "sphinxbase/byteorder.h"
+#include "sphinxbase/heap.h"
+#include "sphinxbase/pio.h"
+#include "sphinxbase/err.h"
+
+typedef struct huff_node_s {
+    int nbits;
+    struct huff_node_s *l;
+    union {
+        int32 ival;
+        char *sval;
+        struct huff_node_s *r;
+    } r;
+} huff_node_t;
+
+typedef struct huff_codeword_s {
+    union {
+        int32 ival;
+        char *sval;
+    } r;
+    uint32 nbits, codeword;
+} huff_codeword_t;
+
+enum {
+    HUFF_CODE_INT,
+    HUFF_CODE_STR
+};
+
+struct huff_code_s {
+    int16 refcount;
+    uint8 maxbits;
+    uint8 type;
+    uint32 *firstcode;
+    uint32 *numl;
+    huff_codeword_t **syms;
+    hash_table_t *codewords;
+    FILE *fh;
+    bit_encode_t *be;
+    int boff;
+};
+
+static huff_node_t *
+huff_node_new_int(int32 val)
+{
+    huff_node_t *hn = ckd_calloc(1, sizeof(*hn));
+    hn->r.ival = val;
+    return hn;
+}
+
+static huff_node_t *
+huff_node_new_str(char const *val)
+{
+    huff_node_t *hn = ckd_calloc(1, sizeof(*hn));
+    hn->r.sval = ckd_salloc(val);
+    return hn;
+}
+
+static huff_node_t *
+huff_node_new_parent(huff_node_t *l, huff_node_t *r)
+{
+    huff_node_t *hn = ckd_calloc(1, sizeof(*hn));
+    hn->l = l;
+    hn->r.r = r;
+    /* Propagate maximum bit length. */
+    if (r->nbits > l->nbits)
+        hn->nbits = r->nbits + 1;
+    else
+        hn->nbits = l->nbits + 1;
+    return hn;
+}
+
+static void
+huff_node_free_int(huff_node_t *root)
+{
+    if (root->l) {
+        huff_node_free_int(root->l);
+        huff_node_free_int(root->r.r);
+    }
+    ckd_free(root);
+}
+
+static void
+huff_node_free_str(huff_node_t *root, int freestr)
+{
+    if (root->l) {
+        huff_node_free_str(root->l, freestr);
+        huff_node_free_str(root->r.r, freestr);
+    }
+    else {
+        if (freestr)
+            ckd_free(root->r.sval);
+    }
+    ckd_free(root);
+}
+
+static huff_node_t *
+huff_code_build_tree(heap_t *q)
+{
+    huff_node_t *root = NULL;
+    int32 rf;
+
+    while (heap_size(q) > 1) {
+        huff_node_t *l, *r, *p;
+        int32 lf, rf;
+
+        heap_pop(q, (void *)&l, &lf);
+        heap_pop(q, (void *)&r, &rf);
+        p = huff_node_new_parent(l, r);
+        heap_insert(q, p, lf + rf);
+    }
+    heap_pop(q, (void **)&root, &rf);
+    return root;
+}
+
+static void
+huff_code_canonicalize(huff_code_t *hc, huff_node_t *root)
+{
+    glist_t agenda;
+    uint32 *nextcode;
+    int i, ncw;
+
+    hc->firstcode = ckd_calloc(hc->maxbits+1, sizeof(*hc->firstcode));
+    hc->syms = ckd_calloc(hc->maxbits+1, sizeof(*hc->syms));
+    hc->numl = ckd_calloc(hc->maxbits+1, sizeof(*nextcode));
+    nextcode = ckd_calloc(hc->maxbits+1, sizeof(*nextcode));
+
+    /* Traverse the tree, annotating it with the actual bit
+     * lengths, and histogramming them in numl. */
+    root->nbits = 0;
+    ncw = 0;
+    agenda = glist_add_ptr(NULL, root);
+    while (agenda) {
+        huff_node_t *node = gnode_ptr(agenda);
+        agenda = gnode_free(agenda, NULL);
+        if (node->l) {
+            node->l->nbits = node->nbits + 1;
+            agenda = glist_add_ptr(agenda, node->l);
+            node->r.r->nbits = node->nbits + 1;
+            agenda = glist_add_ptr(agenda, node->r.r);
+        }
+        else {
+            hc->numl[node->nbits]++;
+            ncw++;
+        }
+    }
+    /* Create starting codes and symbol tables for each bit length. */
+    hc->syms[hc->maxbits] = ckd_calloc(hc->numl[hc->maxbits], sizeof(**hc->syms));
+    for (i = hc->maxbits - 1; i > 0; --i) {
+        hc->firstcode[i] = (hc->firstcode[i+1] + hc->numl[i+1]) / 2;
+        hc->syms[i] = ckd_calloc(hc->numl[i], sizeof(**hc->syms));
+    }
+    memcpy(nextcode, hc->firstcode, (hc->maxbits + 1) * sizeof(*nextcode));
+    /* Traverse the tree again to produce the codebook itself. */
+    hc->codewords = hash_table_new(ncw, HASH_CASE_YES);
+    agenda = glist_add_ptr(NULL, root);
+    while (agenda) {
+        huff_node_t *node = gnode_ptr(agenda);
+        agenda = gnode_free(agenda, NULL);
+        if (node->l) {
+            agenda = glist_add_ptr(agenda, node->l);
+            agenda = glist_add_ptr(agenda, node->r.r);
+        }
+        else {
+            /* Initialize codebook entry, which also retains symbol pointer. */
+            huff_codeword_t *cw;
+            uint32 codeword = nextcode[node->nbits] & ((1 << node->nbits) - 1);
+            cw = hc->syms[node->nbits] + (codeword - hc->firstcode[node->nbits]);
+            cw->nbits = node->nbits;
+            cw->r.sval = node->r.sval; /* Will copy ints too... */
+            cw->codeword = codeword;
+            if (hc->type == HUFF_CODE_INT) {
+                hash_table_enter_bkey(hc->codewords,
+                                      (char const *)&cw->r.ival,
+                                      sizeof(cw->r.ival),
+                                      (void *)cw);
+            }
+            else {
+                hash_table_enter(hc->codewords, cw->r.sval, (void *)cw);
+            }
+            ++nextcode[node->nbits];
+        }
+    }
+    ckd_free(nextcode);
+}
+
+huff_code_t *
+huff_code_build_int(int32 const *values, int32 const *frequencies, int nvals)
+{
+    huff_code_t *hc;
+    huff_node_t *root;
+    heap_t *q;
+    int i;
+
+    hc = ckd_calloc(1, sizeof(*hc));
+    hc->refcount = 1;
+    hc->type = HUFF_CODE_INT;
+
+    /* Initialize the heap with nodes for each symbol. */
+    q = heap_new();
+    for (i = 0; i < nvals; ++i) {
+        heap_insert(q,
+                    huff_node_new_int(values[i]),
+                    frequencies[i]);
+    }
+
+    /* Now build the tree, which gives us codeword lengths. */
+    root = huff_code_build_tree(q);
+    heap_destroy(q);
+    if (root == NULL || root->nbits > 32) {
+        E_ERROR("Huffman trees currently limited to 32 bits\n");
+        huff_node_free_int(root);
+        huff_code_free(hc);
+        return NULL;
+    }
+
+    /* Build a canonical codebook. */
+    hc->maxbits = root->nbits;
+    huff_code_canonicalize(hc, root);
+
+    /* Tree no longer needed. */
+    huff_node_free_int(root);
+
+    return hc;
+}
+
+huff_code_t *
+huff_code_build_str(char * const *values, int32 const *frequencies, int nvals)
+{
+    huff_code_t *hc;
+    huff_node_t *root;
+    heap_t *q;
+    int i;
+
+    hc = ckd_calloc(1, sizeof(*hc));
+    hc->refcount = 1;
+    hc->type = HUFF_CODE_STR;
+
+    /* Initialize the heap with nodes for each symbol. */
+    q = heap_new();
+    for (i = 0; i < nvals; ++i) {
+        heap_insert(q,
+                    huff_node_new_str(values[i]),
+                    frequencies[i]);
+    }
+
+    /* Now build the tree, which gives us codeword lengths. */
+    root = huff_code_build_tree(q);
+    heap_destroy(q);
+    if (root == NULL || root->nbits > 32) {
+        E_ERROR("Huffman trees currently limited to 32 bits\n");
+        huff_node_free_str(root, TRUE);
+        huff_code_free(hc);
+        return NULL;
+    }
+
+    /* Build a canonical codebook. */
+    hc->maxbits = root->nbits;
+    huff_code_canonicalize(hc, root);
+
+    /* Tree no longer needed (note we retain pointers to its strings). */
+    huff_node_free_str(root, FALSE);
+
+    return hc;
+}
+
+huff_code_t *
+huff_code_read(FILE *infh)
+{
+    huff_code_t *hc;
+    int i, j;
+
+    hc = ckd_calloc(1, sizeof(*hc));
+    hc->refcount = 1;
+
+    hc->maxbits = fgetc(infh);
+    hc->type = fgetc(infh);
+
+    /* Two bytes of padding. */
+    fgetc(infh);
+    fgetc(infh);
+
+    /* Allocate stuff. */
+    hc->firstcode = ckd_calloc(hc->maxbits + 1, sizeof(*hc->firstcode));
+    hc->numl = ckd_calloc(hc->maxbits + 1, sizeof(*hc->numl));
+    hc->syms = ckd_calloc(hc->maxbits + 1, sizeof(*hc->syms));
+
+    /* Read the symbol tables. */
+    hc->codewords = hash_table_new(hc->maxbits, HASH_CASE_YES);
+    for (i = 1; i <= hc->maxbits; ++i) {
+        if (fread(&hc->firstcode[i], 4, 1, infh) != 1)
+            goto error_out;
+        SWAP_BE_32(&hc->firstcode[i]);
+        if (fread(&hc->numl[i], 4, 1, infh) != 1)
+            goto error_out;
+        SWAP_BE_32(&hc->numl[i]);
+        hc->syms[i] = ckd_calloc(hc->numl[i], sizeof(**hc->syms));
+        for (j = 0; j < hc->numl[i]; ++j) {
+            huff_codeword_t *cw = &hc->syms[i][j];
+            cw->nbits = i;
+            cw->codeword = hc->firstcode[i] + j;
+            if (hc->type == HUFF_CODE_INT) {
+                if (fread(&cw->r.ival, 4, 1, infh) != 1)
+                    goto error_out;
+                SWAP_BE_32(&cw->r.ival);
+                hash_table_enter_bkey(hc->codewords,
+                                      (char const *)&cw->r.ival,
+                                      sizeof(cw->r.ival),
+                                      (void *)cw);
+            }
+            else {
+                size_t len;
+                cw->r.sval = fread_line(infh, &len);
+                cw->r.sval[len-1] = '\0';
+                hash_table_enter(hc->codewords, cw->r.sval, (void *)cw);
+            }
+        }
+    }
+
+    return hc;
+error_out:
+    huff_code_free(hc);
+    return NULL;
+}
+
+int
+huff_code_write(huff_code_t *hc, FILE *outfh)
+{
+    int i, j;
+
+    /* Maximum codeword length */
+    fputc(hc->maxbits, outfh);
+    /* Symbol type */
+    fputc(hc->type, outfh);
+    /* Two extra bytes (for future use and alignment) */
+    fputc(0, outfh);
+    fputc(0, outfh);
+    /* For each codeword length: */
+    for (i = 1; i <= hc->maxbits; ++i) {
+        uint32 val;
+
+        /* Starting code, number of codes. */
+        val = hc->firstcode[i];
+        /* Canonically big-endian (like the data itself) */
+        SWAP_BE_32(&val);
+        fwrite(&val, 4, 1, outfh);
+        val = hc->numl[i];
+        SWAP_BE_32(&val);
+        fwrite(&val, 4, 1, outfh);
+
+        /* Symbols for each code (FIXME: Should compress these too) */
+        for (j = 0; j < hc->numl[i]; ++j) {
+            if (hc->type == HUFF_CODE_INT) {
+                int32 val = hc->syms[i][j].r.ival;
+                SWAP_BE_32(&val);
+                fwrite(&val, 4, 1, outfh);
+            }
+            else {
+                /* Write them all separated by newlines, so that
+                 * fgets() will read them for us. */
+                fprintf(outfh, "%s\n", hc->syms[i][j].r.sval);
+            }
+        }
+    }
+    return 0;
+}
+
+int
+huff_code_dump_codebits(FILE *dumpfh, uint32 nbits, uint32 codeword)
+{
+    uint32 i;
+
+    for (i = 0; i < nbits; ++i)
+        fputc((codeword & (1<<(nbits-i-1))) ? '1' : '0', dumpfh);
+    return 0;
+}
+
+int
+huff_code_dump(huff_code_t *hc, FILE *dumpfh)
+{
+    int i, j;
+
+    /* Print out all codewords. */
+    fprintf(dumpfh, "Maximum codeword length: %d\n", hc->maxbits);
+    fprintf(dumpfh, "Symbols are %s\n", (hc->type == HUFF_CODE_STR) ? "strings" : "ints");
+    fprintf(dumpfh, "Codewords:\n");
+    for (i = 1; i <= hc->maxbits; ++i) {
+        for (j = 0; j < hc->numl[i]; ++j) {
+            if (hc->type == HUFF_CODE_STR)
+                fprintf(dumpfh, "%-30s", hc->syms[i][j].r.sval);
+            else
+                fprintf(dumpfh, "%-30d", hc->syms[i][j].r.ival);
+            huff_code_dump_codebits(dumpfh, hc->syms[i][j].nbits,
+                                    hc->syms[i][j].codeword);
+            fprintf(dumpfh, "\n");
+        }
+    }
+    return 0;
+}
+
+huff_code_t *
+huff_code_retain(huff_code_t *hc)
+{
+    ++hc->refcount;
+    return hc;
+}
+
+int
+huff_code_free(huff_code_t *hc)
+{
+    int i;
+
+    if (hc == NULL)
+        return 0;
+    if (--hc->refcount > 0)
+        return hc->refcount;
+    for (i = 0; i <= hc->maxbits; ++i) {
+        int j;
+        for (j = 0; j < hc->numl[i]; ++j) {
+            if (hc->type == HUFF_CODE_STR)
+                ckd_free(hc->syms[i][j].r.sval);
+        }
+        ckd_free(hc->syms[i]);
+    }
+    ckd_free(hc->firstcode);
+    ckd_free(hc->numl);
+    ckd_free(hc->syms);
+    hash_table_free(hc->codewords);
+    ckd_free(hc);
+    return 0;
+}
+
+FILE *
+huff_code_attach(huff_code_t *hc, FILE *fh, char const *mode)
+{
+    FILE *oldfh = huff_code_detach(hc);
+
+    hc->fh = fh;
+    if (mode[0] == 'w')
+        hc->be = bit_encode_attach(hc->fh);
+    return oldfh;
+}
+
+FILE *
+huff_code_detach(huff_code_t *hc)
+{
+    FILE *oldfh = hc->fh;
+	
+    if (hc->be) {
+        bit_encode_flush(hc->be);
+        bit_encode_free(hc->be);
+        hc->be = NULL;
+    }
+    hc->fh = NULL;
+    return oldfh;
+}
+
+int
+huff_code_encode_int(huff_code_t *hc, int32 sym, uint32 *outcw)
+{
+    huff_codeword_t *cw;
+
+    if (hash_table_lookup_bkey(hc->codewords,
+                               (char const *)&sym,
+                               sizeof(sym),
+                               (void **)&cw) < 0)
+        return 0;
+    if (hc->be)
+        bit_encode_write_cw(hc->be, cw->codeword, cw->nbits);
+    if (outcw) *outcw = cw->codeword;
+    return cw->nbits;
+}
+
+int
+huff_code_encode_str(huff_code_t *hc, char const *sym, uint32 *outcw)
+{
+    huff_codeword_t *cw;
+
+    if (hash_table_lookup(hc->codewords,
+                          sym,
+                          (void **)&cw) < 0)
+        return 0;
+    if (hc->be)
+        bit_encode_write_cw(hc->be, cw->codeword, cw->nbits);
+    if (outcw) *outcw = cw->codeword;
+    return cw->nbits;
+}
+
+static huff_codeword_t *
+huff_code_decode_data(huff_code_t *hc, char const **inout_data,
+                      size_t *inout_data_len, int *inout_offset)
+{
+    char const *data = *inout_data;
+    char const *end = data + *inout_data_len;
+    int offset = *inout_offset;
+    uint32 cw;
+    int cwlen;
+    int byte;
+
+    if (data == end)
+        return NULL;
+    byte = *data++;
+    cw = !!(byte & (1 << (7-offset++)));
+    cwlen = 1;
+    /* printf("%.*x ", cwlen, cw); */
+    while (cwlen <= hc->maxbits && cw < hc->firstcode[cwlen]) {
+        ++cwlen;
+        cw <<= 1;
+        if (offset > 7) {
+            if (data == end)
+                return NULL;
+            byte = *data++;
+            offset = 0;
+        }
+        cw |= !!(byte & (1 << (7-offset++)));
+        /* printf("%.*x ", cwlen, cw); */
+    }
+    if (cwlen > hc->maxbits) /* FAIL: invalid data */
+        return NULL;
+
+    /* Put the last byte back if there are bits left over. */
+    if (offset < 8)
+        --data;
+    else
+        offset = 0;
+
+    /* printf("%.*x\n", cwlen, cw); */
+    *inout_data_len = end - data;
+    *inout_data = data;
+    *inout_offset = offset;
+    return hc->syms[cwlen] + (cw - hc->firstcode[cwlen]);
+}
+
+static huff_codeword_t *
+huff_code_decode_fh(huff_code_t *hc)
+{
+    uint32 cw;
+    int cwlen;
+    int byte;
+
+    if ((byte = fgetc(hc->fh)) == EOF)
+        return NULL;
+    cw = !!(byte & (1 << (7-hc->boff++)));
+    cwlen = 1;
+    /* printf("%.*x ", cwlen, cw); */
+    while (cwlen <= hc->maxbits && cw < hc->firstcode[cwlen]) {
+        ++cwlen;
+        cw <<= 1;
+        if (hc->boff > 7) {
+            if ((byte = fgetc(hc->fh)) == EOF)
+                return NULL;
+            hc->boff = 0;
+        }
+        cw |= !!(byte & (1 << (7-hc->boff++)));
+        /* printf("%.*x ", cwlen, cw); */
+    }
+    if (cwlen > hc->maxbits) /* FAIL: invalid data */
+        return NULL;
+
+    /* Put the last byte back if there are bits left over. */
+    if (hc->boff < 8)
+        ungetc(byte, hc->fh);
+    else
+        hc->boff = 0;
+
+    /* printf("%.*x\n", cwlen, cw); */
+    return hc->syms[cwlen] + (cw - hc->firstcode[cwlen]);
+}
+
+int
+huff_code_decode_int(huff_code_t *hc, int *outval,
+                     char const **inout_data,
+                     size_t *inout_data_len, int *inout_offset)
+{
+    huff_codeword_t *cw;
+
+    if (inout_data)
+        cw = huff_code_decode_data(hc, inout_data, inout_data_len, inout_offset);
+    else if (hc->fh)
+        cw = huff_code_decode_fh(hc);
+    else
+        return -1;
+
+    if (cw == NULL)
+        return -1;
+    if (outval)
+        *outval = cw->r.ival;
+
+    return 0;
+}
+
+char const *
+huff_code_decode_str(huff_code_t *hc,
+                     char const **inout_data,
+                     size_t *inout_data_len, int *inout_offset)
+{
+    huff_codeword_t *cw;
+
+    if (inout_data)
+        cw = huff_code_decode_data(hc, inout_data, inout_data_len, inout_offset);
+    else if (hc->fh)
+        cw = huff_code_decode_fh(hc);
+    else
+        return NULL;
+
+    if (cw == NULL)
+        return NULL;
+
+    return cw->r.sval;
+}

media/sphinxbase/src/libsphinxbase/util/listelem_alloc.c

@@ -0,0 +1,294 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "sphinxbase/err.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/listelem_alloc.h"
+#include "sphinxbase/glist.h"
+
+/**
+ * Fast linked list allocator.
+ * 
+ * We keep a separate linked list for each element-size.  Element-size
+ * must be a multiple of pointer-size.
+ *
+ * Initially a block of empty elements is allocated, where the first
+ * machine word in each element points to the next available element.
+ * To allocate, we use this pointer to move the freelist to the next
+ * element, then return the current element.
+ *
+ * The last element in the list starts with a NULL pointer, which is
+ * used as a signal to allocate a new block of elements.
+ *
+ * In order to be able to actually release the memory allocated, we
+ * have to add a linked list of block pointers.  This shouldn't create
+ * much overhead since we never access it except when freeing the
+ * allocator.
+ */
+struct listelem_alloc_s {
+    char **freelist;            /**< ptr to first element in freelist */
+    glist_t blocks;             /**< Linked list of blocks allocated. */
+    glist_t blocksize;          /**< Number of elements in each block */
+    size_t elemsize;            /**< Number of (char *) in element */
+    size_t blk_alloc;           /**< Number of alloc operations before increasing blocksize */
+    size_t n_blocks;
+    size_t n_alloc;
+    size_t n_freed;
+};
+
+#define MIN_ALLOC	50      /**< Minimum number of elements to allocate in one block */
+#define BLKID_SHIFT     16      /**< Bit position of block number in element ID */
+#define BLKID_MASK ((1<<BLKID_SHIFT)-1)
+
+/**
+ * Allocate a new block of elements.
+ */
+static void listelem_add_block(listelem_alloc_t *list,
+			       char *caller_file, int caller_line);
+
+listelem_alloc_t *
+listelem_alloc_init(size_t elemsize)
+{
+    listelem_alloc_t *list;
+
+    if ((elemsize % sizeof(void *)) != 0) {
+        size_t rounded = (elemsize + sizeof(void *) - 1) & ~(sizeof(void *)-1);
+        E_WARN
+            ("List item size (%lu) not multiple of sizeof(void *), rounding to %lu\n",
+             (unsigned long)elemsize,
+             (unsigned long)rounded);
+        elemsize = rounded;
+    }
+    list = ckd_calloc(1, sizeof(*list));
+    list->freelist = NULL;
+    list->blocks = NULL;
+    list->elemsize = elemsize;
+    /* Intent of this is to increase block size once we allocate
+     * 256KiB (i.e. 1<<18). If somehow the element size is big enough
+     * to overflow that, just fail, people should use malloc anyway. */
+    list->blk_alloc = (1 << 18) / (MIN_ALLOC * elemsize);
+    if (list->blk_alloc <= 0) {
+        E_ERROR("Element size * block size exceeds 256k, use malloc instead.\n");
+        ckd_free(list);
+        return NULL;
+    }
+    list->n_alloc = 0;
+    list->n_freed = 0;
+
+    /* Allocate an initial block to minimize latency. */
+    listelem_add_block(list, __FILE__, __LINE__);
+    return list;
+}
+
+void
+listelem_alloc_free(listelem_alloc_t *list)
+{
+    gnode_t *gn;
+    if (list == NULL)
+	return;
+    for (gn = list->blocks; gn; gn = gnode_next(gn))
+	ckd_free(gnode_ptr(gn));
+    glist_free(list->blocks);
+    glist_free(list->blocksize);
+    ckd_free(list);
+}
+
+static void
+listelem_add_block(listelem_alloc_t *list, char *caller_file, int caller_line)
+{
+    char **cpp, *cp;
+    size_t j;
+    int32 blocksize;
+
+    blocksize = list->blocksize ? gnode_int32(list->blocksize) : MIN_ALLOC;
+    /* Check if block size should be increased (if many requests for this size) */
+    if (list->blk_alloc == 0) {
+        /* See above.  No sense in allocating blocks bigger than
+         * 256KiB (well, actually, there might be, but we'll worry
+         * about that later). */
+	blocksize <<= 1;
+        if (blocksize * list->elemsize > (1 << 18))
+            blocksize = (1 << 18) / list->elemsize;
+	list->blk_alloc = (1 << 18) / (blocksize * list->elemsize);
+    }
+
+    /* Allocate block */
+    cpp = list->freelist =
+	(char **) __ckd_calloc__(blocksize, list->elemsize,
+				 caller_file, caller_line);
+    list->blocks = glist_add_ptr(list->blocks, cpp);
+    list->blocksize = glist_add_int32(list->blocksize, blocksize);
+    cp = (char *) cpp;
+    /* Link up the blocks via their first machine word. */
+    for (j = blocksize - 1; j > 0; --j) {
+	cp += list->elemsize;
+	*cpp = cp;
+	cpp = (char **) cp;
+    }
+    /* Make sure the last element's forward pointer is NULL */
+    *cpp = NULL;
+    --list->blk_alloc;
+    ++list->n_blocks;
+}
+
+
+void *
+__listelem_malloc__(listelem_alloc_t *list, char *caller_file, int caller_line)
+{
+    char **ptr;
+
+    /* Allocate a new block if list empty */
+    if (list->freelist == NULL)
+	listelem_add_block(list, caller_file, caller_line);
+
+    /* Unlink and return first element in freelist */
+    ptr = list->freelist;
+    list->freelist = (char **) (*(list->freelist));
+    (list->n_alloc)++;
+
+    return (void *)ptr;
+}
+
+void *
+__listelem_malloc_id__(listelem_alloc_t *list, char *caller_file,
+                       int caller_line, int32 *out_id)
+{
+    char **ptr;
+
+    /* Allocate a new block if list empty */
+    if (list->freelist == NULL)
+	listelem_add_block(list, caller_file, caller_line);
+
+    /* Unlink and return first element in freelist */
+    ptr = list->freelist;
+    list->freelist = (char **) (*(list->freelist));
+    (list->n_alloc)++;
+
+    if (out_id) {
+        int32 blksize, blkidx, ptridx;
+        gnode_t *gn, *gn2;
+        char **block;
+
+        gn2 = list->blocksize;
+        block = NULL;
+        blkidx = 0;
+        for (gn = list->blocks; gn; gn = gnode_next(gn)) {
+            block = gnode_ptr(gn);
+            blksize = gnode_int32(gn2) * list->elemsize / sizeof(*block);
+            if (ptr >= block && ptr < block + blksize)
+                break;
+            gn2 = gnode_next(gn2);
+            ++blkidx;
+        }
+        if (gn == NULL) {
+            E_ERROR("Failed to find block index for pointer %p!\n", ptr);
+        }
+        ptridx = (ptr - block) / (list->elemsize / sizeof(*block));
+        E_DEBUG(4,("ptr %p block %p blkidx %d ptridx %d\n",
+                   ptr, block, list->n_blocks - blkidx - 1, ptridx));
+        *out_id = ((list->n_blocks - blkidx - 1) << BLKID_SHIFT) | ptridx;
+    }
+
+    return ptr;
+}
+
+void *
+listelem_get_item(listelem_alloc_t *list, int32 id)
+{
+    int32 blkidx, ptridx, i;
+    gnode_t *gn;
+
+    blkidx = (id >> BLKID_SHIFT) & BLKID_MASK;
+    ptridx = id & BLKID_MASK;
+
+    i = 0;
+    blkidx = list->n_blocks - blkidx;
+    for (gn = list->blocks; gn; gn = gnode_next(gn)) {
+        if (++i == blkidx)
+            break;
+    }
+    if (gn == NULL) {
+        E_ERROR("Failed to find block index %d\n", blkidx);
+        return NULL;
+    }
+
+    return (void *)((char **)gnode_ptr(gn)
+                    + ptridx * (list->elemsize / sizeof(void *)));
+}
+
+void
+__listelem_free__(listelem_alloc_t *list, void *elem,
+                  char *caller_file, int caller_line)
+{
+    char **cpp;
+
+    /*
+     * Insert freed item at head of list.
+     */
+    cpp = (char **) elem;
+    *cpp = (char *) list->freelist;
+    list->freelist = cpp;
+    (list->n_freed)++;
+}
+
+
+void
+listelem_stats(listelem_alloc_t *list)
+{
+    gnode_t *gn, *gn2;
+    char **cpp;
+    size_t n;
+
+    E_INFO("Linklist stats:\n");
+    for (n = 0, cpp = list->freelist; cpp;
+         cpp = (char **) (*cpp), n++);
+    E_INFO
+        ("elemsize %lu, #alloc %lu, #freed %lu, #freelist %lu\n",
+         (unsigned long)list->elemsize,
+         (unsigned long)list->n_alloc,
+         (unsigned long)list->n_freed,
+         (unsigned long)n);
+    E_INFO("Allocated blocks:\n");
+    gn2 = list->blocksize;
+    for (gn = list->blocks; gn; gn = gnode_next(gn)) {
+	E_INFO("%p (%d * %d bytes)\n", gnode_ptr(gn), gnode_int32(gn2), list->elemsize);
+        gn2 = gnode_next(gn2);
+    }
+}

media/sphinxbase/src/libsphinxbase/util/logmath.c

@@ -0,0 +1,483 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2007 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+#include <math.h>
+#include <string.h>
+#include <assert.h>
+
+#include "sphinxbase/logmath.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/mmio.h"
+#include "sphinxbase/bio.h"
+#include "sphinxbase/strfuncs.h"
+
+struct logmath_s {
+    logadd_t t;
+    int refcount;
+    mmio_file_t *filemap;
+    float64 base;
+    float64 log_of_base;
+    float64 log10_of_base;
+    float64 inv_log_of_base;
+    float64 inv_log10_of_base;
+    int32 zero;
+};
+
+logmath_t *
+logmath_init(float64 base, int shift, int use_table)
+{
+    logmath_t *lmath;
+    uint32 maxyx, i;
+    float64 byx;
+    int width;
+
+    /* Check that the base is correct. */
+    if (base <= 1.0) {
+        E_ERROR("Base must be greater than 1.0\n");
+        return NULL;
+    }
+    
+    /* Set up various necessary constants. */
+    lmath = ckd_calloc(1, sizeof(*lmath));
+    lmath->refcount = 1;
+    lmath->base = base;
+    lmath->log_of_base = log(base);
+    lmath->log10_of_base = log10(base);
+    lmath->inv_log_of_base = 1.0/lmath->log_of_base;
+    lmath->inv_log10_of_base = 1.0/lmath->log10_of_base;
+    lmath->t.shift = shift;
+    /* Shift this sufficiently that overflows can be avoided. */
+    lmath->zero = MAX_NEG_INT32 >> (shift + 2);
+
+    if (!use_table)
+        return lmath;
+
+    /* Create a logadd table with the appropriate width */
+    maxyx = (uint32) (log(2.0) / log(base) + 0.5) >> shift;
+    /* Poor man's log2 */
+    if (maxyx < 256) width = 1;
+    else if (maxyx < 65536) width = 2;
+    else width = 4;
+
+    lmath->t.width = width;
+    /* Figure out size of add table required. */
+    byx = 1.0; /* Maximum possible base^{y-x} value - note that this implies that y-x == 0 */
+    for (i = 0;; ++i) {
+        float64 lobyx = log(1.0 + byx) * lmath->inv_log_of_base; /* log_{base}(1 + base^{y-x}); */
+        int32 k = (int32) (lobyx + 0.5 * (1<<shift)) >> shift; /* Round to shift */
+
+        /* base^{y-x} has reached the smallest representable value. */
+        if (k <= 0)
+            break;
+
+        /* This table is indexed by -(y-x), so we multiply byx by
+         * base^{-1} here which is equivalent to subtracting one from
+         * (y-x). */
+        byx /= base;
+    }
+    i >>= shift;
+
+    /* Never produce a table smaller than 256 entries. */
+    if (i < 255) i = 255;
+
+    lmath->t.table = ckd_calloc(i+1, width);
+    lmath->t.table_size = i + 1;
+    /* Create the add table (see above). */
+    byx = 1.0;
+    for (i = 0;; ++i) {
+        float64 lobyx = log(1.0 + byx) * lmath->inv_log_of_base;
+        int32 k = (int32) (lobyx + 0.5 * (1<<shift)) >> shift; /* Round to shift */
+        uint32 prev = 0;
+
+        /* Check any previous value - if there is a shift, we want to
+         * only store the highest one. */
+        switch (width) {
+        case 1:
+            prev = ((uint8 *)lmath->t.table)[i >> shift];
+            break;
+        case 2:
+            prev = ((uint16 *)lmath->t.table)[i >> shift];
+            break;
+        case 4:
+            prev = ((uint32 *)lmath->t.table)[i >> shift];
+            break;
+        }
+        if (prev == 0) {
+            switch (width) {
+            case 1:
+                ((uint8 *)lmath->t.table)[i >> shift] = (uint8) k;
+                break;
+            case 2:
+                ((uint16 *)lmath->t.table)[i >> shift] = (uint16) k;
+                break;
+            case 4:
+                ((uint32 *)lmath->t.table)[i >> shift] = (uint32) k;
+                break;
+            }
+        }
+        if (k <= 0)
+            break;
+
+        /* Decay base^{y-x} exponentially according to base. */
+        byx /= base;
+    }
+
+    return lmath;
+}
+
+logmath_t *
+logmath_read(const char *file_name)
+{
+    logmath_t *lmath;
+    char **argname, **argval;
+    int32 byteswap, i;
+    int chksum_present, do_mmap;
+    uint32 chksum;
+    long pos;
+    FILE *fp;
+
+    E_INFO("Reading log table file '%s'\n", file_name);
+    if ((fp = fopen(file_name, "rb")) == NULL) {
+        E_ERROR_SYSTEM("Failed to open log table file '%s' for reading", file_name);
+        return NULL;
+    }
+
+    /* Read header, including argument-value info and 32-bit byteorder magic */
+    if (bio_readhdr(fp, &argname, &argval, &byteswap) < 0) {
+        E_ERROR("Failed to read the header from the file '%s'\n", file_name);
+        fclose(fp);
+        return NULL;
+    }
+
+    lmath = ckd_calloc(1, sizeof(*lmath));
+    /* Default values. */
+    lmath->t.shift = 0;
+    lmath->t.width = 2;
+    lmath->base = 1.0001;
+
+    /* Parse argument-value list */
+    chksum_present = 0;
+    for (i = 0; argname[i]; i++) {
+        if (strcmp(argname[i], "version") == 0) {
+        }
+        else if (strcmp(argname[i], "chksum0") == 0) {
+            if (strcmp(argval[i], "yes") == 0)
+                chksum_present = 1;
+        }
+        else if (strcmp(argname[i], "width") == 0) {
+            lmath->t.width = atoi(argval[i]);
+        }
+        else if (strcmp(argname[i], "shift") == 0) {
+            lmath->t.shift = atoi(argval[i]);
+        }
+        else if (strcmp(argname[i], "logbase") == 0) {
+            lmath->base = atof_c(argval[i]);
+        }
+    }
+    bio_hdrarg_free(argname, argval);
+    chksum = 0;
+
+    /* Set up various necessary constants. */
+    lmath->log_of_base = log(lmath->base);
+    lmath->log10_of_base = log10(lmath->base);
+    lmath->inv_log_of_base = 1.0/lmath->log_of_base;
+    lmath->inv_log10_of_base = 1.0/lmath->log10_of_base;
+    /* Shift this sufficiently that overflows can be avoided. */
+    lmath->zero = MAX_NEG_INT32 >> (lmath->t.shift + 2);
+
+    /* #Values to follow */
+    if (bio_fread(&lmath->t.table_size, sizeof(int32), 1, fp, byteswap, &chksum) != 1) {
+        E_ERROR("Failed to read values from the file '%s'", file_name);
+        goto error_out;
+    }
+
+    /* Check alignment constraints for memory mapping */
+    do_mmap = 1;
+    pos = ftell(fp);
+    if (pos & ((long)lmath->t.width - 1)) {
+        E_WARN("%s: Data start %ld is not aligned on %d-byte boundary, will not memory map\n",
+                  file_name, pos, lmath->t.width);
+        do_mmap = 0;
+    }
+    /* Check byte order for memory mapping */
+    if (byteswap) {
+        E_WARN("%s: Data is wrong-endian, will not memory map\n", file_name);
+        do_mmap = 0;
+    }
+
+    if (do_mmap) {
+        lmath->filemap = mmio_file_read(file_name);
+        lmath->t.table = (char *)mmio_file_ptr(lmath->filemap) + pos;
+    }
+    else {
+        lmath->t.table = ckd_calloc(lmath->t.table_size, lmath->t.width);
+        if (bio_fread(lmath->t.table, lmath->t.width, lmath->t.table_size,
+                      fp, byteswap, &chksum) != lmath->t.table_size) {
+            E_ERROR("Failed to read data (%d x %d bytes) from the file '%s' failed",
+                    lmath->t.table_size, lmath->t.width, file_name);
+            goto error_out;
+        }
+        if (chksum_present)
+            bio_verify_chksum(fp, byteswap, chksum);
+
+        if (fread(&i, 1, 1, fp) == 1) {
+            E_ERROR("%s: More data than expected\n", file_name);
+            goto error_out;
+        }
+    }
+    fclose(fp);
+
+    return lmath;
+error_out:
+    logmath_free(lmath);
+    return NULL;
+}
+
+int32
+logmath_write(logmath_t *lmath, const char *file_name)
+{
+    FILE *fp;
+    long pos;
+    uint32 chksum;
+
+    if (lmath->t.table == NULL) {
+        E_ERROR("No log table to write!\n");
+        return -1;
+    }
+
+    E_INFO("Writing log table file '%s'\n", file_name);
+    if ((fp = fopen(file_name, "wb")) == NULL) {
+        E_ERROR_SYSTEM("Failed to open logtable file '%s' for writing", file_name);
+        return -1;
+    }
+
+    /* For whatever reason, we have to do this manually at the
+     * moment. */
+    fprintf(fp, "s3\nversion 1.0\nchksum0 yes\n");
+    fprintf(fp, "width %d\n", lmath->t.width);
+    fprintf(fp, "shift %d\n", lmath->t.shift);
+    fprintf(fp, "logbase %f\n", lmath->base);
+    /* Pad it out to ensure alignment. */
+    pos = ftell(fp) + strlen("endhdr\n");
+    if (pos & ((long)lmath->t.width - 1)) {
+        size_t align = lmath->t.width - (pos & ((long)lmath->t.width - 1));
+        assert(lmath->t.width <= 8);
+        fwrite("        " /* 8 spaces */, 1, align, fp);
+    }
+    fprintf(fp, "endhdr\n");
+
+    /* Now write the binary data. */
+    chksum = (uint32)BYTE_ORDER_MAGIC;
+    fwrite(&chksum, sizeof(uint32), 1, fp);
+    chksum = 0;
+    /* #Values to follow */
+    if (bio_fwrite(&lmath->t.table_size, sizeof(uint32),
+                   1, fp, 0, &chksum) != 1) {
+        E_ERROR("Failed to write data to a file '%s'", file_name);
+        goto error_out;
+    }
+
+    if (bio_fwrite(lmath->t.table, lmath->t.width, lmath->t.table_size,
+                   fp, 0, &chksum) != lmath->t.table_size) {
+        E_ERROR("Failed to write data (%d x %d bytes) to the file '%s'",
+                lmath->t.table_size, lmath->t.width, file_name);
+        goto error_out;
+    }
+    if (bio_fwrite(&chksum, sizeof(uint32), 1, fp, 0, NULL) != 1) {
+        E_ERROR("Failed to write checksum to the file '%s'", file_name);
+        goto error_out;
+    }
+
+    fclose(fp);
+    return 0;
+
+error_out:
+    fclose(fp);
+    return -1;
+}
+
+logmath_t *
+logmath_retain(logmath_t *lmath)
+{
+    ++lmath->refcount;
+    return lmath;
+}
+
+int
+logmath_free(logmath_t *lmath)
+{
+    if (lmath == NULL)
+        return 0;
+    if (--lmath->refcount > 0)
+        return lmath->refcount;
+    if (lmath->filemap)
+        mmio_file_unmap(lmath->filemap);
+    else
+        ckd_free(lmath->t.table);
+    ckd_free(lmath);
+    return 0;
+}
+
+int32
+logmath_get_table_shape(logmath_t *lmath, uint32 *out_size,
+                        uint32 *out_width, uint32 *out_shift)
+{
+    if (out_size) *out_size = lmath->t.table_size;
+    if (out_width) *out_width = lmath->t.width;
+    if (out_shift) *out_shift = lmath->t.shift;
+
+    return lmath->t.table_size * lmath->t.width;
+}
+
+float64
+logmath_get_base(logmath_t *lmath)
+{
+    return lmath->base;
+}
+
+int
+logmath_get_zero(logmath_t *lmath)
+{
+    return lmath->zero;
+}
+
+int
+logmath_get_width(logmath_t *lmath)
+{
+    return lmath->t.width;
+}
+
+int
+logmath_get_shift(logmath_t *lmath)
+{
+    return lmath->t.shift;
+}
+
+int
+logmath_add(logmath_t *lmath, int logb_x, int logb_y)
+{
+    logadd_t *t = LOGMATH_TABLE(lmath);
+    int d, r;
+
+    /* handle 0 + x = x case. */
+    if (logb_x <= lmath->zero)
+        return logb_y;
+    if (logb_y <= lmath->zero)
+        return logb_x;
+
+    if (t->table == NULL)
+        return logmath_add_exact(lmath, logb_x, logb_y);
+
+    /* d must be positive, obviously. */
+    if (logb_x > logb_y) {
+        d = (logb_x - logb_y);
+        r = logb_x;
+    }
+    else {
+        d = (logb_y - logb_x);
+        r = logb_y;
+    }
+
+    if (d < 0) {
+        /* Some kind of overflow has occurred, fail gracefully. */
+        return r;
+    }
+    if ((size_t)d >= t->table_size) {
+        /* If this happens, it's not actually an error, because the
+         * last entry in the logadd table is guaranteed to be zero.
+         * Therefore we just return the larger of the two values. */
+        return r;
+    }
+
+    switch (t->width) {
+    case 1:
+        return r + (((uint8 *)t->table)[d]);
+    case 2:
+        return r + (((uint16 *)t->table)[d]);
+    case 4:
+        return r + (((uint32 *)t->table)[d]);
+    }
+    return r;
+}
+
+int
+logmath_add_exact(logmath_t *lmath, int logb_p, int logb_q)
+{
+    return logmath_log(lmath,
+                       logmath_exp(lmath, logb_p)
+                       + logmath_exp(lmath, logb_q));
+}
+
+int
+logmath_log(logmath_t *lmath, float64 p)
+{
+    if (p <= 0) {
+        return lmath->zero;
+    }
+    return (int)(log(p) * lmath->inv_log_of_base) >> lmath->t.shift;
+}
+
+float64
+logmath_exp(logmath_t *lmath, int logb_p)
+{
+    return pow(lmath->base, (float64)(logb_p << lmath->t.shift));
+}
+
+int
+logmath_ln_to_log(logmath_t *lmath, float64 log_p)
+{
+    return (int)(log_p * lmath->inv_log_of_base) >> lmath->t.shift;
+}
+
+float64
+logmath_log_to_ln(logmath_t *lmath, int logb_p)
+{
+    return (float64)(logb_p << lmath->t.shift) * lmath->log_of_base;
+}
+
+int
+logmath_log10_to_log(logmath_t *lmath, float64 log_p)
+{
+    return (int)(log_p * lmath->inv_log10_of_base) >> lmath->t.shift;
+}
+
+float64
+logmath_log_to_log10(logmath_t *lmath, int logb_p)
+{
+    return (float64)(logb_p << lmath->t.shift) * lmath->log10_of_base;
+}

media/sphinxbase/src/libsphinxbase/util/matrix.c

@@ -0,0 +1,313 @@
+/* -*- c-basic-offset: 4 -*- */
+/* ====================================================================
+ * Copyright (c) 1997-2006 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+#include <string.h>
+#include <stdlib.h>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "sphinxbase/clapack_lite.h"
+#include "sphinxbase/matrix.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/ckd_alloc.h"
+
+void
+norm_3d(float32 ***arr,
+	uint32 d1,
+	uint32 d2,
+	uint32 d3)
+{
+    uint32 i, j, k;
+    float64 s;
+
+    for (i = 0; i < d1; i++) {
+	for (j = 0; j < d2; j++) {
+
+	    /* compute sum (i, j) as over all k */
+	    for (k = 0, s = 0; k < d3; k++) {
+		s += arr[i][j][k];
+	    }
+
+	    /* do 1 floating point divide */
+	    s = 1.0 / s;
+
+	    /* divide all k by sum over k */
+	    for (k = 0; k < d3; k++) {
+		arr[i][j][k] *= s;
+	    }
+	}
+    }
+}
+
+void
+accum_3d(float32 ***out,
+	 float32 ***in,
+	 uint32 d1,
+	 uint32 d2,
+	 uint32 d3)
+{
+    uint32 i, j, k;
+
+    for (i = 0; i < d1; i++) {
+	for (j = 0; j < d2; j++) {
+	    for (k = 0; k < d3; k++) {
+		out[i][j][k] += in[i][j][k];
+	    }
+	}
+    }
+}
+
+void
+floor_nz_3d(float32 ***m,
+	    uint32 d1,
+	    uint32 d2,
+	    uint32 d3,
+	    float32 floor)
+{
+    uint32 i, j, k;
+
+    for (i = 0; i < d1; i++) {
+	for (j = 0; j < d2; j++) {
+	    for (k = 0; k < d3; k++) {
+		if ((m[i][j][k] != 0) && (m[i][j][k] < floor))
+		    m[i][j][k] = floor;
+	    }
+	}
+    }
+}
+void
+floor_nz_1d(float32 *v,
+	    uint32 d1,
+	    float32 floor)
+{
+    uint32 i;
+
+    for (i = 0; i < d1; i++) {
+	if ((v[i] != 0) && (v[i] < floor))
+	    v[i] = floor;
+    }
+}
+
+void
+band_nz_1d(float32 *v,
+	   uint32 d1,
+	   float32 band)
+{
+    uint32 i;
+
+    for (i = 0; i < d1; i++) {
+	if (v[i] != 0) {
+	    if ((v[i] > 0) && (v[i] < band)) {
+		v[i] = band;
+	    }
+	    else if ((v[i] < 0) && (v[i] > -band)) {
+		v[i] = -band;
+	    }
+	}
+    }
+}
+
+#ifndef WITH_LAPACK
+float64
+determinant(float32 **a, int32 n)
+{
+    E_FATAL("No LAPACK library available, cannot compute determinant (FIXME)\n");
+    return 0.0;
+}
+int32
+invert(float32 **ainv, float32 **a, int32 n)
+{
+    E_FATAL("No LAPACK library available, cannot compute matrix inverse (FIXME)\n");
+    return 0;
+}
+int32
+solve(float32 **a, float32 *b, float32 *out_x, int32   n)
+{
+    E_FATAL("No LAPACK library available, cannot solve linear equations (FIXME)\n");
+    return 0;
+}
+
+void
+matrixmultiply(float32 ** c, float32 ** a, float32 ** b, int32 n)
+{
+    int32 i, j, k;
+
+    memset(c[0], 0, n*n*sizeof(float32));
+    for (i = 0; i < n; ++i) {
+	for (j = 0; j < n; ++j) {
+	    for (k = 0; k < n; ++k) {
+		c[i][k] += a[i][j] * b[j][k];
+	    }
+	}
+    }
+}
+#else /* WITH_LAPACK */
+/* Find determinant through LU decomposition. */
+float64
+determinant(float32 ** a, int32 n)
+{
+    float32 **tmp_a;
+    float64 det;
+    char uplo;
+    int32 info, i;
+
+    /* a is assumed to be symmetric, so we don't need to switch the
+     * ordering of the data.  But we do need to copy it since it is
+     * overwritten by LAPACK. */
+    tmp_a = (float32 **)ckd_calloc_2d(n, n, sizeof(float32));
+    memcpy(tmp_a[0], a[0], n*n*sizeof(float32));
+
+    uplo = 'L';
+    spotrf_(&uplo, &n, tmp_a[0], &n, &info);
+    det = tmp_a[0][0];
+    /* det = prod(diag(l))^2 */
+    for (i = 1; i < n; ++i)
+	det *= tmp_a[i][i];
+    ckd_free_2d((void **)tmp_a);
+    if (info > 0)
+	return -1.0; /* Generic "not positive-definite" answer */
+    else
+	return det * det;
+}
+
+int32
+solve(float32 **a, /*Input : an n*n matrix A */
+      float32 *b,  /*Input : a n dimesion vector b */
+      float32 *out_x,  /*Output : a n dimesion vector x */
+      int32   n)
+{
+    char uplo;
+    float32 **tmp_a;
+    int32 info, nrhs;
+
+    /* a is assumed to be symmetric, so we don't need to switch the
+     * ordering of the data.  But we do need to copy it since it is
+     * overwritten by LAPACK. */
+    tmp_a = (float32 **)ckd_calloc_2d(n, n, sizeof(float32));
+    memcpy(tmp_a[0], a[0], n*n*sizeof(float32));
+    memcpy(out_x, b, n*sizeof(float32));
+    uplo = 'L';
+    nrhs = 1;
+    sposv_(&uplo, &n, &nrhs, tmp_a[0], &n, out_x, &n, &info);
+    ckd_free_2d((void **)tmp_a);
+
+    if (info != 0)
+	return -1;
+    else
+	return info;
+}
+
+/* Find inverse by solving AX=I. */
+int32
+invert(float32 ** ainv, float32 ** a, int32 n)
+{
+    char uplo;
+    float32 **tmp_a;
+    int32 info, nrhs, i;
+
+    /* Construct an identity matrix. */
+    memset(ainv[0], 0, sizeof(float32) * n * n);
+    for (i = 0; i < n; i++)
+        ainv[i][i] = 1.0;
+    /* a is assumed to be symmetric, so we don't need to switch the
+     * ordering of the data.  But we do need to copy it since it is
+     * overwritten by LAPACK. */
+    tmp_a = (float32 **)ckd_calloc_2d(n, n, sizeof(float32));
+    memcpy(tmp_a[0], a[0], n*n*sizeof(float32));
+    uplo = 'L';
+    nrhs = n;
+    sposv_(&uplo, &n, &nrhs, tmp_a[0], &n, ainv[0], &n, &info);
+    ckd_free_2d((void **)tmp_a);
+
+    if (info != 0)
+	return -1;
+    else
+	return info;
+}
+
+void
+matrixmultiply(float32 ** c, float32 ** a, float32 ** b, int32 n)
+{
+    char side, uplo;
+    float32 alpha;
+
+    side = 'L';
+    uplo = 'L';
+    alpha = 1.0;
+    ssymm_(&side, &uplo, &n, &n, &alpha, a[0], &n, b[0], &n, &alpha, c[0], &n);
+}
+
+#endif /* WITH_LAPACK */
+
+void
+outerproduct(float32 ** a, float32 * x, float32 * y, int32 len)
+{
+    int32 i, j;
+
+    for (i = 0; i < len; ++i) {
+        a[i][i] = x[i] * y[i];
+        for (j = i + 1; j < len; ++j) {
+            a[i][j] = x[i] * y[j];
+            a[j][i] = x[j] * y[i];
+        }
+    }
+}
+
+void
+scalarmultiply(float32 ** a, float32 x, int32 n)
+{
+    int32 i, j;
+
+    for (i = 0; i < n; ++i) {
+	a[i][i] *= x;
+        for (j = i+1; j < n; ++j) {
+            a[i][j] *= x;
+            a[j][i] *= x;
+	}
+    }
+}
+
+void
+matrixadd(float32 ** a, float32 ** b, int32 n)
+{
+    int32 i, j;
+
+    for (i = 0; i < n; ++i)
+        for (j = 0; j < n; ++j)
+            a[i][j] += b[i][j];
+}

media/sphinxbase/src/libsphinxbase/util/mmio.c

@@ -0,0 +1,257 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2005 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*********************************************************************
+ *
+ * File: mmio.c
+ * 
+ * Description: mmap() wrappers for Unix/Windows
+ * 
+ * Author: David Huggins-Daines <dhuggins@cs.cmu.edu>
+ * 
+ *********************************************************************/
+
+#include <string.h>
+#include <stdlib.h>
+
+
+#ifdef GNUWINCE
+# include <sys/wcebase.h>
+# include <sys/wcetypes.h>
+# include <sys/wcememory.h>
+# include <sys/wcefile.h>
+#elif defined(__SYMBIAN32__) /* SYMBIAN32 must be before WIN32 since Symbian SDK defines WIN32 as well */
+# include <unistd.h>
+# include <fcntl.h>
+# include <sys/stat.h>
+# include <sys/mman.h>
+#elif defined(_WIN32)
+# include <windows.h>
+#else
+# include <unistd.h>
+# include <fcntl.h>
+# include <sys/stat.h>
+# include <sys/file.h>
+# include <sys/mman.h>
+#endif
+
+#include "sphinxbase/prim_type.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/mmio.h"
+#include "sphinxbase/ckd_alloc.h"
+
+#if defined(_WIN32_WCE) || defined(GNUWINCE)
+struct mmio_file_s {
+	int dummy;
+};
+
+mmio_file_t *
+mmio_file_read(const char *filename)
+{
+    HANDLE ffm, fd;
+    WCHAR *wfilename;
+    void *rv;
+    int len;
+
+    len = mbstowcs(NULL, filename, 0) + 1;
+    wfilename = malloc(len * sizeof(WCHAR));
+    mbstowcs(wfilename, filename, len);
+
+    if ((ffm =
+         CreateFileForMappingW(wfilename, GENERIC_READ, FILE_SHARE_READ,
+                               NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
+                               NULL)) == INVALID_HANDLE_VALUE) {
+        E_ERROR("Failed to create mapping for the file '%s': %08x\n", filename,
+                GetLastError());
+        return NULL;
+    }
+    if ((fd =
+         CreateFileMappingW(ffm, NULL, PAGE_READONLY, 0, 0, NULL)) == NULL) {
+        E_ERROR("Failed to CreateFileMapping: %08x\n", GetLastError());
+        CloseHandle(ffm);
+        return NULL;
+    }
+    rv = MapViewOfFile(fd, FILE_MAP_READ, 0, 0, 0);
+    free(wfilename);
+    CloseHandle(ffm);
+    CloseHandle(fd);
+
+    return (mmio_file_t *) rv;
+}
+
+void
+mmio_file_unmap(mmio_file_t *mf)
+{
+    if (!UnmapViewOfFile((void *)mf)) {
+        E_ERROR("Failed to UnmapViewOfFile: %08x\n", GetLastError());
+    }
+}
+
+void *
+mmio_file_ptr(mmio_file_t *mf)
+{
+    return (void *)mf;
+}
+
+#elif defined(_WIN32) && !defined(_WIN32_WP) /* !WINCE */
+struct mmio_file_s {
+	int dummy;
+};
+
+mmio_file_t *
+mmio_file_read(const char *filename)
+{
+    HANDLE ffm, fd;
+    void *rv;
+
+    if ((ffm = CreateFile(filename, GENERIC_READ, FILE_SHARE_READ,
+                         NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
+                         NULL)) == INVALID_HANDLE_VALUE) {
+        E_ERROR("Failed to create file '%s': %08x\n",
+                filename, GetLastError());
+        return NULL;
+    }
+    if ((fd = CreateFileMapping(ffm, NULL,
+                                PAGE_READONLY, 0, 0, NULL)) == NULL) {
+        E_ERROR("Failed to CreateFileMapping: %08x\n", GetLastError());
+        CloseHandle(ffm);
+    }
+    rv = MapViewOfFile(fd, FILE_MAP_READ, 0, 0, 0);
+    CloseHandle(ffm);
+    CloseHandle(fd);
+
+    return (mmio_file_t *)rv;
+}
+
+void
+mmio_file_unmap(mmio_file_t *mf)
+{
+    if (!UnmapViewOfFile((void *)mf)) {
+        E_ERROR("Failed to UnmapViewOfFile: %08x\n", GetLastError());
+    }
+}
+
+void *
+mmio_file_ptr(mmio_file_t *mf)
+{
+    return (void *)mf;
+}
+
+#else /* !WIN32, !WINCE */
+#if defined(__ADSPBLACKFIN__) || defined(_WIN32_WP) 
+				/* This is true for both uClinux and VisualDSP++,
+                                 but actually we need a better way to detect it. */
+struct mmio_file_s {
+    int dummy;
+};
+
+mmio_file_t *
+mmio_file_read(const char *filename)
+{
+    E_ERROR("mmio is not implemented on this platform!");
+    return NULL;
+}
+
+void
+mmio_file_unmap(mmio_file_t *mf)
+{
+    E_ERROR("mmio is not implemented on this platform!");
+}
+
+void *
+mmio_file_ptr(mmio_file_t *mf)
+{
+    E_ERROR("mmio is not implemented on this platform!");
+    return NULL;
+}
+#else /* !__ADSPBLACKFIN__ */
+struct mmio_file_s {
+    void *ptr;
+    size_t mapsize;
+};
+
+mmio_file_t *
+mmio_file_read(const char *filename)
+{
+    mmio_file_t *mf;
+    struct stat buf;
+    void *ptr;
+    int fd;
+    size_t pagesize;
+
+    if ((fd = open(filename, O_RDONLY)) == -1) {
+        E_ERROR_SYSTEM("Failed to open %s", filename);
+        return NULL;
+    }
+    if (fstat(fd, &buf) == -1) {
+        E_ERROR_SYSTEM("Failed to stat %s", filename);
+        close(fd);
+        return NULL;
+    }
+    ptr = mmap(NULL, buf.st_size, PROT_READ, MAP_SHARED, fd, 0);
+    if (ptr == (void *)-1) {
+        E_ERROR_SYSTEM("Failed to mmap %lld bytes", (unsigned long long)buf.st_size);
+        close(fd);
+        return NULL;
+    }
+    close(fd);
+    mf = ckd_calloc(1, sizeof(*mf));
+    mf->ptr = ptr;
+    /* Align map size to next page. */
+    pagesize = sysconf(_SC_PAGESIZE);
+    mf->mapsize = (buf.st_size + pagesize - 1) / pagesize * pagesize;
+
+    return mf;
+}
+
+void
+mmio_file_unmap(mmio_file_t *mf)
+{
+    if (mf == NULL)
+        return;
+    if (munmap(mf->ptr, mf->mapsize) < 0) {
+        E_ERROR_SYSTEM("Failed to unmap %ld bytes at %p", mf->mapsize, mf->ptr);
+    }
+    ckd_free(mf);
+}
+
+void *
+mmio_file_ptr(mmio_file_t *mf)
+{
+    return mf->ptr;
+}
+#endif /* !__ADSPBLACKFIN__ */ 
+#endif /* !(WINCE || WIN32) */

media/sphinxbase/src/libsphinxbase/util/pio.c

@@ -0,0 +1,655 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2004 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+#if defined(_WIN32) && !defined(CYGWIN)
+#include <direct.h>
+#endif
+
+#include "sphinxbase/pio.h"
+#include "sphinxbase/filename.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/strfuncs.h"
+#include "sphinxbase/ckd_alloc.h"
+
+#ifndef EXEEXT
+#define EXEEXT ""
+#endif
+
+enum {
+    COMP_NONE,
+    COMP_COMPRESS,
+    COMP_GZIP,
+    COMP_BZIP2
+};
+
+static void
+guess_comptype(char const *file, int32 *ispipe, int32 *isgz)
+{
+    size_t k;
+
+    k = strlen(file);
+    *ispipe = 0;
+    *isgz = COMP_NONE;
+    if ((k > 2)
+        && ((strcmp(file + k - 2, ".Z") == 0)
+            || (strcmp(file + k - 2, ".z") == 0))) {
+        *ispipe = 1;
+        *isgz = COMP_COMPRESS;
+    }
+    else if ((k > 3) && ((strcmp(file + k - 3, ".gz") == 0)
+                        || (strcmp(file + k - 3, ".GZ") == 0))) {
+        *ispipe = 1;
+        *isgz = COMP_GZIP;
+    }
+    else if ((k > 4) && ((strcmp(file + k - 4, ".bz2") == 0)
+                        || (strcmp(file + k - 4, ".BZ2") == 0))) {
+        *ispipe = 1;
+        *isgz = COMP_BZIP2;
+    }
+}
+
+FILE *
+fopen_comp(const char *file, const char *mode, int32 * ispipe)
+{
+    FILE *fp;
+
+#ifndef HAVE_POPEN
+    *ispipe = 0; /* No popen() on WinCE */
+#else /* HAVE_POPEN */
+    int32 isgz;
+    guess_comptype(file, ispipe, &isgz);
+#endif /* HAVE_POPEN */
+
+    if (*ispipe) {
+#ifndef HAVE_POPEN
+        /* Shouldn't get here, anyway */
+        E_FATAL("No popen() on WinCE\n");
+#else
+        if (strcmp(mode, "r") == 0) {
+            char *command;
+            switch (isgz) {
+            case COMP_GZIP:
+                command = string_join("gunzip" EXEEXT, " -c ", file, NULL);
+                break;
+            case COMP_COMPRESS:
+                command = string_join("zcat" EXEEXT, " ", file, NULL);
+                break;
+            case COMP_BZIP2:
+                command = string_join("bunzip2" EXEEXT, " -c ", file, NULL);
+                break;
+            default:
+                command = NULL; /* Make compiler happy. */
+                E_FATAL("Unknown  compression type %d\n", isgz);
+            }
+            if ((fp = popen(command, mode)) == NULL) {
+                E_ERROR_SYSTEM("Failed to open a pipe for a command '%s' mode '%s'", command, mode);
+                ckd_free(command);
+                return NULL;
+            }
+            ckd_free(command);
+        }
+        else if (strcmp(mode, "w") == 0) {
+            char *command;
+            switch (isgz) {
+            case COMP_GZIP:
+                command = string_join("gzip" EXEEXT, " > ", file, NULL);
+                break;
+            case COMP_COMPRESS:
+                command = string_join("compress" EXEEXT, " -c > ", file, NULL);
+                break;
+            case COMP_BZIP2:
+                command = string_join("bzip2" EXEEXT, " > ", file, NULL);
+                break;
+            default:
+                command = NULL; /* Make compiler happy. */
+                E_FATAL("Unknown compression type %d\n", isgz);
+            }
+            if ((fp = popen(command, mode)) == NULL) {
+                E_ERROR_SYSTEM("Failed to open a pipe for a command '%s' mode '%s'", command, mode);
+                ckd_free(command);
+                return NULL;
+            }
+            ckd_free(command);
+        }
+        else {
+            E_ERROR("Compressed file operation for mode %s is not supported", mode);
+            return NULL;
+        }
+#endif /* HAVE_POPEN */
+    }
+    else {
+        fp = fopen(file, mode);
+    }
+
+    return (fp);
+}
+
+
+void
+fclose_comp(FILE * fp, int32 ispipe)
+{
+    if (ispipe) {
+#ifdef HAVE_POPEN
+#if defined(_WIN32) && (!defined(__SYMBIAN32__))
+        _pclose(fp);
+#else
+        pclose(fp);
+#endif
+#endif
+    }
+    else
+        fclose(fp);
+}
+
+
+FILE *
+fopen_compchk(const char *file, int32 * ispipe)
+{
+#ifndef HAVE_POPEN
+    *ispipe = 0; /* No popen() on WinCE */
+    /* And therefore the rest of this function is useless. */
+    return (fopen_comp(file, "r", ispipe));
+#else /* HAVE_POPEN */
+    int32 isgz;
+    FILE *fh;
+
+    /* First just try to fopen_comp() it */
+    if ((fh = fopen_comp(file, "r", ispipe)) != NULL)
+        return fh;
+    else {
+        char *tmpfile;
+        size_t k;
+
+        /* File doesn't exist; try other compressed/uncompressed form, as appropriate */
+        guess_comptype(file, ispipe, &isgz);
+        k = strlen(file);
+        tmpfile = ckd_calloc(k+5, 1);
+        strcpy(tmpfile, file);
+        switch (isgz) {
+        case COMP_GZIP:
+            tmpfile[k - 3] = '\0';
+            break;
+        case COMP_BZIP2:
+            tmpfile[k - 4] = '\0';
+            break;
+        case COMP_COMPRESS:
+            tmpfile[k - 2] = '\0';
+            break;
+        case COMP_NONE:
+            strcpy(tmpfile + k, ".gz");
+            if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) {
+                E_WARN("Using %s instead of %s\n", tmpfile, file);
+                ckd_free(tmpfile);
+                return fh;
+            }
+            strcpy(tmpfile + k, ".bz2");
+            if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) {
+                E_WARN("Using %s instead of %s\n", tmpfile, file);
+                ckd_free(tmpfile);
+                return fh;
+            }
+            strcpy(tmpfile + k, ".Z");
+            if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) {
+                E_WARN("Using %s instead of %s\n", tmpfile, file);
+                ckd_free(tmpfile);
+                return fh;
+            }
+            ckd_free(tmpfile);
+            return NULL;
+        }
+        E_WARN("Using %s instead of %s\n", tmpfile, file);
+        fh = fopen_comp(tmpfile, "r", ispipe);
+        ckd_free(tmpfile);
+        return NULL;
+    }
+#endif /* HAVE_POPEN */
+}
+
+lineiter_t *
+lineiter_start(FILE *fh)
+{
+    lineiter_t *li;
+
+    li = (lineiter_t *)ckd_calloc(1, sizeof(*li));
+    li->buf = (char *)ckd_malloc(128);
+    li->buf[0] = '\0';
+    li->bsiz = 128;
+    li->len = 0;
+    li->fh = fh;
+
+    li = lineiter_next(li);
+    
+    /* Strip the UTF-8 BOM */
+    
+    if (li && 0 == strncmp(li->buf, "\xef\xbb\xbf", 3)) {
+	memmove(li->buf, li->buf + 3, strlen(li->buf + 1));
+	li->len -= 3;
+    }
+    
+    return li;
+}
+
+lineiter_t *
+lineiter_start_clean(FILE *fh)
+{
+    lineiter_t *li;
+    
+    li = lineiter_start(fh);
+    
+    if (li == NULL)
+	return li;
+    
+    li->clean = TRUE;
+    
+    if (li->buf && li->buf[0] == '#') {
+	li = lineiter_next(li);
+    } else {
+	string_trim(li->buf, STRING_BOTH);
+    }
+    
+    return li;
+}
+
+
+static lineiter_t *
+lineiter_next_plain(lineiter_t *li)
+{
+    /* We are reading the next line */
+    li->lineno++;
+    
+    /* Read a line and check for EOF. */
+    if (fgets(li->buf, li->bsiz, li->fh) == NULL) {
+        lineiter_free(li);
+        return NULL;
+    }
+    /* If we managed to read the whole thing, then we are done
+     * (this will be by far the most common result). */
+    li->len = (int32)strlen(li->buf);
+    if (li->len < li->bsiz - 1 || li->buf[li->len - 1] == '\n')
+        return li;
+
+    /* Otherwise we have to reallocate and keep going. */
+    while (1) {
+        li->bsiz *= 2;
+        li->buf = (char *)ckd_realloc(li->buf, li->bsiz);
+        /* If we get an EOF, we are obviously done. */
+        if (fgets(li->buf + li->len, li->bsiz - li->len, li->fh) == NULL) {
+            li->len += strlen(li->buf + li->len);
+            return li;
+        }
+        li->len += strlen(li->buf + li->len);
+        /* If we managed to read the whole thing, then we are done. */
+        if (li->len < li->bsiz - 1 || li->buf[li->len - 1] == '\n')
+            return li;
+    }
+
+    /* Shouldn't get here. */
+    return li;
+}
+
+
+lineiter_t *
+lineiter_next(lineiter_t *li)
+{
+    if (!li->clean)
+	return lineiter_next_plain(li);
+    
+    for (li = lineiter_next_plain(li); li; li = lineiter_next_plain(li)) {
+	if (li->buf && li->buf[0] != '#') {
+	    li->buf = string_trim(li->buf, STRING_BOTH);
+	    break;
+	}
+    }
+    return li;
+}
+
+int lineiter_lineno(lineiter_t *li)
+{
+    return li->lineno;
+}
+
+void
+lineiter_free(lineiter_t *li)
+{
+    if (li == NULL)
+        return;
+    ckd_free(li->buf);
+    ckd_free(li);
+}
+
+char *
+fread_line(FILE *stream, size_t *out_len)
+{
+    char *output, *outptr;
+    char buf[128];
+
+    output = outptr = NULL;
+    while (fgets(buf, sizeof(buf), stream)) {
+        size_t len = strlen(buf);
+        /* Append this data to the buffer. */
+        if (output == NULL) {
+            output = (char *)ckd_malloc(len + 1);
+            outptr = output;
+        }
+        else {
+            size_t cur = outptr - output;
+            output = (char *)ckd_realloc(output, cur + len + 1);
+            outptr = output + cur;
+        }
+        memcpy(outptr, buf, len + 1);
+        outptr += len;
+        /* Stop on a short read or end of line. */
+        if (len < sizeof(buf)-1 || buf[len-1] == '\n')
+            break;
+    }
+    if (out_len) *out_len = outptr - output;
+    return output;
+}
+
+#define FREAD_RETRY_COUNT	60
+
+int32
+fread_retry(void *pointer, int32 size, int32 num_items, FILE * stream)
+{
+    char *data;
+    size_t n_items_read;
+    size_t n_items_rem;
+    uint32 n_retry_rem;
+    int32 loc;
+
+    n_retry_rem = FREAD_RETRY_COUNT;
+
+    data = (char *)pointer;
+    loc = 0;
+    n_items_rem = num_items;
+
+    do {
+        n_items_read = fread(&data[loc], size, n_items_rem, stream);
+
+        n_items_rem -= n_items_read;
+
+        if (n_items_rem > 0) {
+            /* an incomplete read occurred */
+
+            if (n_retry_rem == 0)
+                return -1;
+
+            if (n_retry_rem == FREAD_RETRY_COUNT) {
+                E_ERROR_SYSTEM("fread() failed; retrying...\n");
+            }
+
+            --n_retry_rem;
+
+            loc += n_items_read * size;
+#if !defined(_WIN32) && defined(HAVE_UNISTD_H)
+            sleep(1);
+#endif
+        }
+    } while (n_items_rem > 0);
+
+    return num_items;
+}
+
+
+#ifdef _WIN32_WCE /* No stat() on WinCE */
+int32
+stat_retry(const char *file, struct stat * statbuf)
+{
+    WIN32_FIND_DATAW file_data;
+    HANDLE *h;
+    wchar_t *wfile;
+    size_t len;
+
+    len = mbstowcs(NULL, file, 0) + 1;
+    wfile = ckd_calloc(len, sizeof(*wfile));
+    mbstowcs(wfile, file, len);
+    if ((h = FindFirstFileW(wfile, &file_data)) == INVALID_HANDLE_VALUE) {
+        ckd_free(wfile);
+        return -1;
+    }
+    ckd_free(wfile);
+    memset(statbuf, 0, sizeof(*statbuf));
+    statbuf->st_mtime = file_data.ftLastWriteTime.dwLowDateTime;
+    statbuf->st_size = file_data.nFileSizeLow;
+    FindClose(h);
+
+    return 0;
+}
+
+
+int32
+stat_mtime(const char *file)
+{
+    struct stat statbuf;
+
+    if (stat_retry(file, &statbuf) != 0)
+        return -1;
+
+    return ((int32) statbuf.st_mtime);
+}
+#else
+#define STAT_RETRY_COUNT	10
+int32
+stat_retry(const char *file, struct stat * statbuf)
+{
+    int32 i;
+
+    for (i = 0; i < STAT_RETRY_COUNT; i++) {
+#ifndef HAVE_SYS_STAT_H
+	FILE *fp;
+
+	if ((fp = (FILE *)fopen(file, "r")) != 0) {
+	    fseek(fp, 0, SEEK_END);
+	    statbuf->st_size = ftell(fp);
+	    fclose(fp);
+	    return 0;
+	}
+#else /* HAVE_SYS_STAT_H */
+        if (stat(file, statbuf) == 0)
+            return 0;
+#endif
+        if (i == 0) {
+            E_ERROR_SYSTEM("Failed to stat file '%s'; retrying...", file);
+        }
+#ifdef HAVE_UNISTD_H
+        sleep(1);
+#endif
+    }
+
+    return -1;
+}
+
+int32
+stat_mtime(const char *file)
+{
+    struct stat statbuf;
+
+#ifdef HAVE_SYS_STAT_H
+    if (stat(file, &statbuf) != 0)
+        return -1;
+#else /* HAVE_SYS_STAT_H */
+    if (stat_retry(file, &statbuf) != 0)
+        return -1;
+#endif /* HAVE_SYS_STAT_H */
+
+    return ((int32) statbuf.st_mtime);
+}
+#endif /* !_WIN32_WCE */
+
+struct bit_encode_s {
+    FILE *fh;
+    unsigned char buf, bbits;
+    int16 refcount;
+};
+
+bit_encode_t *
+bit_encode_attach(FILE *outfh)
+{
+    bit_encode_t *be;
+
+    be = (bit_encode_t *)ckd_calloc(1, sizeof(*be));
+    be->refcount = 1;
+    be->fh = outfh;
+    return be;
+}
+
+bit_encode_t *
+bit_encode_retain(bit_encode_t *be)
+{
+    ++be->refcount;
+    return be;
+}
+
+int
+bit_encode_free(bit_encode_t *be)
+{
+    if (be == NULL)
+        return 0;
+    if (--be->refcount > 0)
+        return be->refcount;
+    ckd_free(be);
+
+    return 0;
+}
+
+int
+bit_encode_write(bit_encode_t *be, unsigned char const *bits, int nbits)
+{
+    int tbits;
+
+    tbits = nbits + be->bbits;
+    if (tbits < 8)  {
+        /* Append to buffer. */
+        be->buf |= ((bits[0] >> (8 - nbits)) << (8 - tbits));
+    }
+    else {
+        int i = 0;
+        while (tbits >= 8) {
+            /* Shift bits out of the buffer and splice with high-order bits */
+            fputc(be->buf | ((bits[i]) >> be->bbits), be->fh);
+            /* Put low-order bits back into buffer */
+            be->buf = (bits[i] << (8 - be->bbits)) & 0xff;
+            tbits -= 8;
+            ++i;
+        }
+    }
+    /* tbits contains remaining number of  bits. */
+    be->bbits = tbits;
+
+    return nbits;
+}
+
+int
+bit_encode_write_cw(bit_encode_t *be, uint32 codeword, int nbits)
+{
+    unsigned char bits[4];
+    codeword <<= (32 - nbits);
+    bits[0] = (codeword >> 24) & 0xff;
+    bits[1] = (codeword >> 16) & 0xff;
+    bits[2] = (codeword >> 8) & 0xff;
+    bits[3] = codeword & 0xff;
+    return bit_encode_write(be, bits, nbits);
+}
+
+int
+bit_encode_flush(bit_encode_t *be)
+{
+    if (be->bbits) {
+        fputc(be->buf, be->fh);
+        be->bbits = 0;
+    }
+    return 0;
+}
+
+int
+build_directory(const char *path)
+{
+    int rv;
+
+    /* Utterly failed... */
+    if (strlen(path) == 0)
+        return -1;
+
+#if defined(_WIN32) && !defined(CYGWIN)
+    else if ((rv = _mkdir(path)) == 0)
+        return 0;
+#elif defined(HAVE_SYS_STAT_H) /* Unix, Cygwin, doesn't work on MINGW */
+    else if ((rv = mkdir(path, 0777)) == 0)
+        return 0;
+#endif
+
+    /* Or, it already exists... */
+    else if (errno == EEXIST)
+        return 0;
+    else if (errno != ENOENT) {
+        E_ERROR_SYSTEM("Failed to create %s", path);
+        return -1;
+    }
+    else {
+        char *dirname = ckd_salloc(path);
+        path2dirname(path, dirname);
+        build_directory(dirname);
+        ckd_free(dirname);
+
+#if defined(_WIN32) && !defined(CYGWIN)
+	return _mkdir(path);
+#elif defined(HAVE_SYS_STAT_H) /* Unix, Cygwin, doesn't work on MINGW */
+        return mkdir(path, 0777);
+#endif
+    }
+}

media/sphinxbase/src/libsphinxbase/util/profile.c

@@ -0,0 +1,345 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2001 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * profile.c -- For timing and event counting.
+ *
+ * **********************************************
+ * CMU ARPA Speech Project
+ *
+ * Copyright (c) 1999 Carnegie Mellon University.
+ * ALL RIGHTS RESERVED.
+ * **********************************************
+ * 
+ * HISTORY
+ * $Log: profile.c,v $
+ * Revision 1.7  2005/06/22 03:10:59  arthchan2003
+ * 1, Fixed doxygen documentation, 2, Added  keyword.
+ *
+ * Revision 1.3  2005/03/30 01:22:48  archan
+ * Fixed mistakes in last updates. Add
+ *
+ * 
+ * 11-Mar-1999	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Added ptmr_init().
+ * 
+ * 19-Jun-97	M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University
+ * 		Created.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if defined(_WIN32) && !defined(__SYMBIAN32__)
+# include <windows.h>
+# ifndef _WIN32_WCE
+#  include <time.h>
+# endif
+#elif defined(HAVE_UNISTD_H) /* I know this, this is Unix... */
+# include <unistd.h>
+# include <sys/time.h>
+# include <sys/resource.h>
+#endif
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4996)
+#endif
+
+#include "sphinxbase/profile.h"
+#include "sphinxbase/err.h"
+#include "sphinxbase/ckd_alloc.h"
+
+#if defined(_WIN32_WCE) || defined(_WIN32_WP)
+DWORD unlink(const char *filename)
+{
+	WCHAR *wfilename;
+	DWORD rv;
+	size_t len;
+
+	len = mbstowcs(NULL, filename, 0);
+	wfilename = ckd_calloc(len+1, sizeof(*wfilename));
+	mbstowcs(wfilename, filename, len);
+	rv = DeleteFileW(wfilename);
+	ckd_free(wfilename);
+
+	return rv;
+}
+#endif
+
+pctr_t *
+pctr_new(char *nm)
+{
+    pctr_t *pc;
+
+    pc = ckd_calloc(1, sizeof(pctr_t));
+    pc->name = ckd_salloc(nm);
+    pc->count = 0;
+
+    return pc;
+}
+
+void
+pctr_reset(pctr_t * ctr)
+{
+    ctr->count = 0;
+}
+
+
+void
+pctr_increment(pctr_t * ctr, int32 inc)
+{
+    ctr->count += inc;
+    /*   E_INFO("Name %s, Count %d, inc %d\n",ctr->name, ctr->count, inc); */
+}
+
+void
+pctr_print(FILE * fp, pctr_t * ctr)
+{
+    fprintf(fp, "CTR:");
+    fprintf(fp, "[%d %s]", ctr->count, ctr->name);
+}
+
+void
+pctr_free(pctr_t * pc)
+{
+    if (pc) {
+        if (pc->name)
+            ckd_free(pc->name);
+    }
+    ckd_free(pc);
+}
+
+
+#if defined(_WIN32) && !defined(GNUWINCE) && !defined(__SYMBIAN32__)
+
+#define TM_LOWSCALE	1e-7
+#define TM_HIGHSCALE	(4294967296.0 * TM_LOWSCALE);
+
+static float64
+make_sec(FILETIME * tm)
+{
+    float64 dt;
+
+    dt = tm->dwLowDateTime * TM_LOWSCALE;
+    dt += tm->dwHighDateTime * TM_HIGHSCALE;
+
+    return (dt);
+}
+
+#else /* NOT WINDOWS */
+
+static float64
+make_sec(struct timeval *s)
+{
+    return (s->tv_sec + s->tv_usec * 0.000001);
+}
+
+#endif
+
+
+void
+ptmr_start(ptmr_t * tm)
+{
+#if (! defined(_WIN32)) || defined(GNUWINCE) || defined(__SYMBIAN32__)
+    struct timeval e_start;     /* Elapsed time */
+
+#if (! defined(_HPUX_SOURCE))  && (! defined(__SYMBIAN32__))
+    struct rusage start;        /* CPU time */
+
+    /* Unix but not HPUX */
+    getrusage(RUSAGE_SELF, &start);
+    tm->start_cpu = make_sec(&start.ru_utime) + make_sec(&start.ru_stime);
+#endif
+    /* Unix + HP */
+    gettimeofday(&e_start, 0);
+    tm->start_elapsed = make_sec(&e_start);
+#elif defined(_WIN32_WP)
+    tm->start_cpu = GetTickCount64() / 1000;
+    tm->start_elapsed = GetTickCount64() / 1000;
+#elif defined(_WIN32_WCE)
+    /* No GetProcessTimes() on WinCE.  (Note CPU time will be bogus) */
+    tm->start_cpu = GetTickCount() / 1000;
+    tm->start_elapsed = GetTickCount() / 1000;
+#else
+    HANDLE pid;
+    FILETIME t_create, t_exit, kst, ust;
+
+    /* PC */
+    pid = GetCurrentProcess();
+    GetProcessTimes(pid, &t_create, &t_exit, &kst, &ust);
+    tm->start_cpu = make_sec(&ust) + make_sec(&kst);
+
+    tm->start_elapsed = (float64) clock() / CLOCKS_PER_SEC;
+#endif
+}
+
+
+void
+ptmr_stop(ptmr_t * tm)
+{
+    float64 dt_cpu, dt_elapsed;
+
+#if (! defined(_WIN32)) || defined(GNUWINCE) || defined(__SYMBIAN32__)
+    struct timeval e_stop;      /* Elapsed time */
+
+#if (! defined(_HPUX_SOURCE))  && (! defined(__SYMBIAN32__))
+    struct rusage stop;         /* CPU time */
+
+    /* Unix but not HPUX */
+    getrusage(RUSAGE_SELF, &stop);
+    dt_cpu =
+        make_sec(&stop.ru_utime) + make_sec(&stop.ru_stime) -
+        tm->start_cpu;
+#else
+    dt_cpu = 0.0;
+#endif
+    /* Unix + HP */
+    gettimeofday(&e_stop, 0);
+    dt_elapsed = (make_sec(&e_stop) - tm->start_elapsed);
+#elif defined(_WIN32_WP)
+    dt_cpu = GetTickCount64() / 1000 - tm->start_cpu;
+    dt_elapsed = GetTickCount64() / 1000 - tm->start_elapsed;
+#elif defined(_WIN32_WCE)
+    /* No GetProcessTimes() on WinCE.  (Note CPU time will be bogus) */
+    dt_cpu = GetTickCount() / 1000 - tm->start_cpu;
+    dt_elapsed = GetTickCount() / 1000 - tm->start_elapsed;
+#else
+    HANDLE pid;
+    FILETIME t_create, t_exit, kst, ust;
+
+    /* PC */
+    pid = GetCurrentProcess();
+    GetProcessTimes(pid, &t_create, &t_exit, &kst, &ust);
+    dt_cpu = make_sec(&ust) + make_sec(&kst) - tm->start_cpu;
+    dt_elapsed = ((float64) clock() / CLOCKS_PER_SEC) - tm->start_elapsed;
+#endif
+
+    tm->t_cpu += dt_cpu;
+    tm->t_elapsed += dt_elapsed;
+
+    tm->t_tot_cpu += dt_cpu;
+    tm->t_tot_elapsed += dt_elapsed;
+}
+
+
+void
+ptmr_reset(ptmr_t * tm)
+{
+    tm->t_cpu = 0.0;
+    tm->t_elapsed = 0.0;
+}
+
+
+void
+ptmr_init(ptmr_t * tm)
+{
+    tm->t_cpu = 0.0;
+    tm->t_elapsed = 0.0;
+    tm->t_tot_cpu = 0.0;
+    tm->t_tot_elapsed = 0.0;
+}
+
+
+void
+ptmr_reset_all(ptmr_t * tm)
+{
+    for (; tm->name; tm++)
+        ptmr_reset(tm);
+}
+
+
+void
+ptmr_print_all(FILE * fp, ptmr_t * tm, float64 norm)
+{
+    if (norm != 0.0) {
+        norm = 1.0 / norm;
+        for (; tm->name; tm++)
+            fprintf(fp, "  %6.2fx %s", tm->t_cpu * norm, tm->name);
+    }
+}
+
+
+int32
+host_endian(void)
+{
+    FILE *fp;
+    int32 BYTE_ORDER_MAGIC;
+    char *file;
+    char buf[8];
+    int32 k, endian;
+
+    file = "/tmp/__EnDiAn_TeSt__";
+
+    if ((fp = fopen(file, "wb")) == NULL) {
+        E_ERROR("Failed to open file '%s' for writing", file);
+        return -1;
+    }
+
+    BYTE_ORDER_MAGIC = (int32) 0x11223344;
+
+    k = (int32) BYTE_ORDER_MAGIC;
+    if (fwrite(&k, sizeof(int32), 1, fp) != 1) {
+        E_ERROR("Failed to write to file '%s'\n", file);
+        fclose(fp);
+        unlink(file);
+        return -1;
+    }
+
+    fclose(fp);
+    if ((fp = fopen(file, "rb")) == NULL) {
+        E_ERROR_SYSTEM("Failed to open file '%s' for reading", file);
+        unlink(file);
+        return -1;
+    }
+    if (fread(buf, 1, sizeof(int32), fp) != sizeof(int32)) {
+        E_ERROR("Failed to read from file '%s'\n", file);
+        fclose(fp);
+        unlink(file);
+        return -1;
+    }
+    fclose(fp);
+    unlink(file);
+
+    /* If buf[0] == lsB of BYTE_ORDER_MAGIC, we are little-endian */
+    endian = (buf[0] == (BYTE_ORDER_MAGIC & 0x000000ff)) ? 1 : 0;
+
+    return (endian);
+}

media/sphinxbase/src/libsphinxbase/util/sbthread.c

@@ -0,0 +1,741 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 2008 Carnegie Mellon University.  All rights 
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+
+/**
+ * @file sbthread.c
+ * @brief Simple portable thread functions
+ * @author David Huggins-Daines <dhuggins@cs.cmu.edu>
+ */
+
+#include <string.h>
+
+#include "sphinxbase/sbthread.h"
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/err.h"
+
+/*
+ * Platform-specific parts: threads, mutexes, and signals.
+ */
+#if (defined(_WIN32) || defined(__CYGWIN__)) && !defined(__SYMBIAN32__)
+#ifndef _WIN32_WINNT
+#define _WIN32_WINNT 0x0400
+#endif /* not _WIN32_WINNT */
+#include <windows.h>
+
+struct sbthread_s {
+    cmd_ln_t *config;
+    sbmsgq_t *msgq;
+    sbthread_main func;
+    void *arg;
+    HANDLE th;
+    DWORD tid;
+};
+
+struct sbmsgq_s {
+    /* Ringbuffer for passing messages. */
+    char *data;
+    size_t depth;
+    size_t out;
+    size_t nbytes;
+
+    /* Current message is stored here. */
+    char *msg;
+    size_t msglen;
+    CRITICAL_SECTION mtx;
+    HANDLE evt;
+};
+
+struct sbevent_s {
+    HANDLE evt;
+};
+
+struct sbmtx_s {
+    CRITICAL_SECTION mtx;
+};
+
+DWORD WINAPI
+sbthread_internal_main(LPVOID arg)
+{
+    sbthread_t *th = (sbthread_t *)arg;
+    int rv;
+
+    rv = (*th->func)(th);
+    return (DWORD)rv;
+}
+
+sbthread_t *
+sbthread_start(cmd_ln_t *config, sbthread_main func, void *arg)
+{
+    sbthread_t *th;
+
+    th = ckd_calloc(1, sizeof(*th));
+    th->config = config;
+    th->func = func;
+    th->arg = arg;
+    th->msgq = sbmsgq_init(256);
+    th->th = CreateThread(NULL, 0, sbthread_internal_main, th, 0, &th->tid);
+    if (th->th == NULL) {
+        sbthread_free(th);
+        return NULL;
+    }
+    return th;
+}
+
+int
+sbthread_wait(sbthread_t *th)
+{
+    DWORD rv, exit;
+
+    /* It has already been joined. */
+    if (th->th == NULL)
+        return -1;
+
+    rv = WaitForSingleObject(th->th, INFINITE);
+    if (rv == WAIT_FAILED) {
+        E_ERROR("Failed to join thread: WAIT_FAILED\n");
+        return -1;
+    }
+    GetExitCodeThread(th->th, &exit);
+    CloseHandle(th->th);
+    th->th = NULL;
+    return (int)exit;
+}
+
+static DWORD
+cond_timed_wait(HANDLE cond, int sec, int nsec)
+{
+    DWORD rv;
+    if (sec == -1) {
+        rv = WaitForSingleObject(cond, INFINITE);
+    }
+    else {
+        DWORD ms;
+
+        ms = sec * 1000 + nsec / (1000*1000);
+        rv = WaitForSingleObject(cond, ms);
+    }
+    return rv;
+}
+
+/* Updated to use Unicode */
+sbevent_t *
+sbevent_init(void)
+{
+    sbevent_t *evt;
+
+    evt = ckd_calloc(1, sizeof(*evt));
+    evt->evt = CreateEventW(NULL, FALSE, FALSE, NULL);
+    if (evt->evt == NULL) {
+        ckd_free(evt);
+        return NULL;
+    }
+    return evt;
+}
+
+void
+sbevent_free(sbevent_t *evt)
+{
+    CloseHandle(evt->evt);
+    ckd_free(evt);
+}
+
+int
+sbevent_signal(sbevent_t *evt)
+{
+    return SetEvent(evt->evt) ? 0 : -1;
+}
+
+int
+sbevent_wait(sbevent_t *evt, int sec, int nsec)
+{
+    DWORD rv;
+
+    rv = cond_timed_wait(evt->evt, sec, nsec);
+    return rv;
+}
+
+sbmtx_t *
+sbmtx_init(void)
+{
+    sbmtx_t *mtx;
+
+    mtx = ckd_calloc(1, sizeof(*mtx));
+    InitializeCriticalSection(&mtx->mtx);
+    return mtx;
+}
+
+int
+sbmtx_trylock(sbmtx_t *mtx)
+{
+    return TryEnterCriticalSection(&mtx->mtx) ? 0 : -1;
+}
+
+int
+sbmtx_lock(sbmtx_t *mtx)
+{
+    EnterCriticalSection(&mtx->mtx);
+    return 0;
+}
+
+int
+sbmtx_unlock(sbmtx_t *mtx)
+{
+    LeaveCriticalSection(&mtx->mtx);
+    return 0;
+}
+
+void
+sbmtx_free(sbmtx_t *mtx)
+{
+    DeleteCriticalSection(&mtx->mtx);
+    ckd_free(mtx);
+}
+
+sbmsgq_t *
+sbmsgq_init(size_t depth)
+{
+    sbmsgq_t *msgq;
+
+    msgq = ckd_calloc(1, sizeof(*msgq));
+    msgq->depth = depth;
+    msgq->evt = CreateEventW(NULL, FALSE, FALSE, NULL);
+    if (msgq->evt == NULL) {
+        ckd_free(msgq);
+        return NULL;
+    }
+    InitializeCriticalSection(&msgq->mtx);
+    msgq->data = ckd_calloc(depth, 1);
+    msgq->msg = ckd_calloc(depth, 1);
+    return msgq;
+}
+
+void
+sbmsgq_free(sbmsgq_t *msgq)
+{
+    CloseHandle(msgq->evt);
+    ckd_free(msgq->data);
+    ckd_free(msgq->msg);
+    ckd_free(msgq);
+}
+
+int
+sbmsgq_send(sbmsgq_t *q, size_t len, void const *data)
+{
+    char const *cdata = (char const *)data;
+    size_t in;
+
+    /* Don't allow things bigger than depth to be sent! */
+    if (len + sizeof(len) > q->depth)
+        return -1;
+
+    if (q->nbytes + len + sizeof(len) > q->depth)
+        WaitForSingleObject(q->evt, INFINITE);
+
+    /* Lock things while we manipulate the buffer (FIXME: this
+       actually should have been atomic with the wait above ...) */
+    EnterCriticalSection(&q->mtx);
+    in = (q->out + q->nbytes) % q->depth;
+    /* First write the size of the message. */
+    if (in + sizeof(len) > q->depth) {
+        /* Handle the annoying case where the size field gets wrapped around. */
+        size_t len1 = q->depth - in;
+        memcpy(q->data + in, &len, len1);
+        memcpy(q->data, ((char *)&len) + len1, sizeof(len) - len1);
+        q->nbytes += sizeof(len);
+        in = sizeof(len) - len1;
+    }
+    else {
+        memcpy(q->data + in, &len, sizeof(len));
+        q->nbytes += sizeof(len);
+        in += sizeof(len);
+    }
+
+    /* Now write the message body. */
+    if (in + len > q->depth) {
+        /* Handle wraparound. */
+        size_t len1 = q->depth - in;
+        memcpy(q->data + in, cdata, len1);
+        q->nbytes += len1;
+        cdata += len1;
+        len -= len1;
+        in = 0;
+    }
+    memcpy(q->data + in, cdata, len);
+    q->nbytes += len;
+
+    /* Signal the condition variable. */
+    SetEvent(q->evt);
+    /* Unlock. */
+    LeaveCriticalSection(&q->mtx);
+
+    return 0;
+}
+
+void *
+sbmsgq_wait(sbmsgq_t *q, size_t *out_len, int sec, int nsec)
+{
+    char *outptr;
+    size_t len;
+
+    /* Wait for data to be available. */
+    if (q->nbytes == 0) {
+        if (cond_timed_wait(q->evt, sec, nsec) == WAIT_FAILED)
+            /* Timed out or something... */
+            return NULL;
+    }
+    /* Lock to manipulate the queue (FIXME) */
+    EnterCriticalSection(&q->mtx);
+    /* Get the message size. */
+    if (q->out + sizeof(q->msglen) > q->depth) {
+        /* Handle annoying wraparound case. */
+        size_t len1 = q->depth - q->out;
+        memcpy(&q->msglen, q->data + q->out, len1);
+        memcpy(((char *)&q->msglen) + len1, q->data,
+               sizeof(q->msglen) - len1);
+        q->out = sizeof(q->msglen) - len1;
+    }
+    else {
+        memcpy(&q->msglen, q->data + q->out, sizeof(q->msglen));
+        q->out += sizeof(q->msglen);
+    }
+    q->nbytes -= sizeof(q->msglen);
+    /* Get the message body. */
+    outptr = q->msg;
+    len = q->msglen;
+    if (q->out + q->msglen > q->depth) {
+        /* Handle wraparound. */
+        size_t len1 = q->depth - q->out;
+        memcpy(outptr, q->data + q->out, len1);
+        outptr += len1;
+        len -= len1;
+        q->nbytes -= len1;
+        q->out = 0;
+    }
+    memcpy(outptr, q->data + q->out, len);
+    q->nbytes -= len;
+    q->out += len;
+
+    /* Signal the condition variable. */
+    SetEvent(q->evt);
+    /* Unlock. */
+    LeaveCriticalSection(&q->mtx);
+    if (out_len)
+        *out_len = q->msglen;
+    return q->msg;
+}
+
+#else /* POSIX */
+#include <pthread.h>
+#include <sys/time.h>
+
+struct sbthread_s {
+    cmd_ln_t *config;
+    sbmsgq_t *msgq;
+    sbthread_main func;
+    void *arg;
+    pthread_t th;
+};
+
+struct sbmsgq_s {
+    /* Ringbuffer for passing messages. */
+    char *data;
+    size_t depth;
+    size_t out;
+    size_t nbytes;
+
+    /* Current message is stored here. */
+    char *msg;
+    size_t msglen;
+    pthread_mutex_t mtx;
+    pthread_cond_t cond;
+};
+
+struct sbevent_s {
+    pthread_mutex_t mtx;
+    pthread_cond_t cond;
+    int signalled;
+};
+
+struct sbmtx_s {
+    pthread_mutex_t mtx;
+};
+
+static void *
+sbthread_internal_main(void *arg)
+{
+    sbthread_t *th = (sbthread_t *)arg;
+    int rv;
+
+    rv = (*th->func)(th);
+    return (void *)(long)rv;
+}
+
+sbthread_t *
+sbthread_start(cmd_ln_t *config, sbthread_main func, void *arg)
+{
+    sbthread_t *th;
+    int rv;
+
+    th = ckd_calloc(1, sizeof(*th));
+    th->config = config;
+    th->func = func;
+    th->arg = arg;
+    th->msgq = sbmsgq_init(1024);
+    if ((rv = pthread_create(&th->th, NULL, &sbthread_internal_main, th)) != 0) {
+        E_ERROR("Failed to create thread: %d\n", rv);
+        sbthread_free(th);
+        return NULL;
+    }
+    return th;
+}
+
+int
+sbthread_wait(sbthread_t *th)
+{
+    void *exit;
+    int rv;
+
+    /* It has already been joined. */
+    if (th->th == (pthread_t)-1)
+        return -1;
+
+    rv = pthread_join(th->th, &exit);
+    if (rv != 0) {
+        E_ERROR("Failed to join thread: %d\n", rv);
+        return -1;
+    }
+    th->th = (pthread_t)-1;
+    return (int)(long)exit;
+}
+
+sbmsgq_t *
+sbmsgq_init(size_t depth)
+{
+    sbmsgq_t *msgq;
+
+    msgq = ckd_calloc(1, sizeof(*msgq));
+    msgq->depth = depth;
+    if (pthread_cond_init(&msgq->cond, NULL) != 0) {
+        ckd_free(msgq);
+        return NULL;
+    }
+    if (pthread_mutex_init(&msgq->mtx, NULL) != 0) {
+        pthread_cond_destroy(&msgq->cond);
+        ckd_free(msgq);
+        return NULL;
+    }
+    msgq->data = ckd_calloc(depth, 1);
+    msgq->msg = ckd_calloc(depth, 1);
+    return msgq;
+}
+
+void
+sbmsgq_free(sbmsgq_t *msgq)
+{
+    pthread_mutex_destroy(&msgq->mtx);
+    pthread_cond_destroy(&msgq->cond);
+    ckd_free(msgq->data);
+    ckd_free(msgq->msg);
+    ckd_free(msgq);
+}
+
+int
+sbmsgq_send(sbmsgq_t *q, size_t len, void const *data)
+{
+    size_t in;
+
+    /* Don't allow things bigger than depth to be sent! */
+    if (len + sizeof(len) > q->depth)
+        return -1;
+
+    /* Lock the condition variable while we manipulate the buffer. */
+    pthread_mutex_lock(&q->mtx);
+    if (q->nbytes + len + sizeof(len) > q->depth) {
+        /* Unlock and wait for space to be available. */
+        if (pthread_cond_wait(&q->cond, &q->mtx) != 0) {
+            /* Timed out, don't send anything. */
+            pthread_mutex_unlock(&q->mtx);
+            return -1;
+        }
+        /* Condition is now locked again. */
+    }
+    in = (q->out + q->nbytes) % q->depth;
+
+    /* First write the size of the message. */
+    if (in + sizeof(len) > q->depth) {
+        /* Handle the annoying case where the size field gets wrapped around. */
+        size_t len1 = q->depth - in;
+        memcpy(q->data + in, &len, len1);
+        memcpy(q->data, ((char *)&len) + len1, sizeof(len) - len1);
+        q->nbytes += sizeof(len);
+        in = sizeof(len) - len1;
+    }
+    else {
+        memcpy(q->data + in, &len, sizeof(len));
+        q->nbytes += sizeof(len);
+        in += sizeof(len);
+    }
+
+    /* Now write the message body. */
+    if (in + len > q->depth) {
+        /* Handle wraparound. */
+        size_t len1 = q->depth - in;
+        memcpy(q->data + in, data, len1);
+        q->nbytes += len1;
+        data = (char const *)data + len1;
+        len -= len1;
+        in = 0;
+    }
+    memcpy(q->data + in, data, len);
+    q->nbytes += len;
+
+    /* Signal the condition variable. */
+    pthread_cond_signal(&q->cond);
+    /* Unlock it, we have nothing else to do. */
+    pthread_mutex_unlock(&q->mtx);
+    return 0;
+}
+
+static int
+cond_timed_wait(pthread_cond_t *cond, pthread_mutex_t *mtx, int sec, int nsec)
+{
+    int rv;
+    if (sec == -1) {
+        rv = pthread_cond_wait(cond, mtx);
+    }
+    else {
+        struct timeval now;
+        struct timespec end;
+
+        gettimeofday(&now, NULL);
+        end.tv_sec = now.tv_sec + sec;
+        end.tv_nsec = now.tv_usec * 1000 + nsec;
+        if (end.tv_nsec > (1000*1000*1000)) {
+            sec += end.tv_nsec / (1000*1000*1000);
+            end.tv_nsec = end.tv_nsec % (1000*1000*1000);
+        }
+        rv = pthread_cond_timedwait(cond, mtx, &end);
+    }
+    return rv;
+}
+
+void *
+sbmsgq_wait(sbmsgq_t *q, size_t *out_len, int sec, int nsec)
+{
+    char *outptr;
+    size_t len;
+
+    /* Lock the condition variable while we manipulate nmsg. */
+    pthread_mutex_lock(&q->mtx);
+    if (q->nbytes == 0) {
+        /* Unlock the condition variable and wait for a signal. */
+        if (cond_timed_wait(&q->cond, &q->mtx, sec, nsec) != 0) {
+            /* Timed out or something... */
+            pthread_mutex_unlock(&q->mtx);
+            return NULL;
+        }
+        /* Condition variable is now locked again. */
+    }
+    /* Get the message size. */
+    if (q->out + sizeof(q->msglen) > q->depth) {
+        /* Handle annoying wraparound case. */
+        size_t len1 = q->depth - q->out;
+        memcpy(&q->msglen, q->data + q->out, len1);
+        memcpy(((char *)&q->msglen) + len1, q->data,
+               sizeof(q->msglen) - len1);
+        q->out = sizeof(q->msglen) - len1;
+    }
+    else {
+        memcpy(&q->msglen, q->data + q->out, sizeof(q->msglen));
+        q->out += sizeof(q->msglen);
+    }
+    q->nbytes -= sizeof(q->msglen);
+    /* Get the message body. */
+    outptr = q->msg;
+    len = q->msglen;
+    if (q->out + q->msglen > q->depth) {
+        /* Handle wraparound. */
+        size_t len1 = q->depth - q->out;
+        memcpy(outptr, q->data + q->out, len1);
+        outptr += len1;
+        len -= len1;
+        q->nbytes -= len1;
+        q->out = 0;
+    }
+    memcpy(outptr, q->data + q->out, len);
+    q->nbytes -= len;
+    q->out += len;
+
+    /* Signal the condition variable. */
+    pthread_cond_signal(&q->cond);
+    /* Unlock the condition variable, we are done. */
+    pthread_mutex_unlock(&q->mtx);
+    if (out_len)
+        *out_len = q->msglen;
+    return q->msg;
+}
+
+sbevent_t *
+sbevent_init(void)
+{
+    sbevent_t *evt;
+    int rv;
+
+    evt = ckd_calloc(1, sizeof(*evt));
+    if ((rv = pthread_mutex_init(&evt->mtx, NULL)) != 0) {
+        E_ERROR("Failed to initialize mutex: %d\n", rv);
+        ckd_free(evt);
+        return NULL;
+    }
+    if ((rv = pthread_cond_init(&evt->cond, NULL)) != 0) {
+        E_ERROR_SYSTEM("Failed to initialize mutex: %d\n", rv);
+        pthread_mutex_destroy(&evt->mtx);
+        ckd_free(evt);
+        return NULL;
+    }
+    return evt;
+}
+
+void
+sbevent_free(sbevent_t *evt)
+{
+    pthread_mutex_destroy(&evt->mtx);
+    pthread_cond_destroy(&evt->cond);
+    ckd_free(evt);
+}
+
+int
+sbevent_signal(sbevent_t *evt)
+{
+    int rv;
+
+    pthread_mutex_lock(&evt->mtx);
+    evt->signalled = TRUE;
+    rv = pthread_cond_signal(&evt->cond);
+    pthread_mutex_unlock(&evt->mtx);
+    return rv;
+}
+
+int
+sbevent_wait(sbevent_t *evt, int sec, int nsec)
+{
+    int rv = 0;
+
+    /* Lock the mutex before we check its signalled state. */
+    pthread_mutex_lock(&evt->mtx);
+    /* If it's not signalled, then wait until it is. */
+    if (!evt->signalled)
+        rv = cond_timed_wait(&evt->cond, &evt->mtx, sec, nsec);
+    /* Set its state to unsignalled if we were successful. */
+    if (rv == 0)
+        evt->signalled = FALSE;
+    /* And unlock its mutex. */
+    pthread_mutex_unlock(&evt->mtx);
+
+    return rv;
+}
+
+sbmtx_t *
+sbmtx_init(void)
+{
+    sbmtx_t *mtx;
+
+    mtx = ckd_calloc(1, sizeof(*mtx));
+    if (pthread_mutex_init(&mtx->mtx, NULL) != 0) {
+        ckd_free(mtx);
+        return NULL;
+    }
+    return mtx;
+}
+
+int
+sbmtx_trylock(sbmtx_t *mtx)
+{
+    return pthread_mutex_trylock(&mtx->mtx);
+}
+
+int
+sbmtx_lock(sbmtx_t *mtx)
+{
+    return pthread_mutex_lock(&mtx->mtx);
+}
+
+int
+sbmtx_unlock(sbmtx_t *mtx)
+{
+    return pthread_mutex_unlock(&mtx->mtx);
+}
+
+void
+sbmtx_free(sbmtx_t *mtx)
+{
+    pthread_mutex_destroy(&mtx->mtx);
+    ckd_free(mtx);
+}
+#endif /* not WIN32 */
+
+cmd_ln_t *
+sbthread_config(sbthread_t *th)
+{
+    return th->config;
+}
+
+void *
+sbthread_arg(sbthread_t *th)
+{
+    return th->arg;
+}
+
+sbmsgq_t *
+sbthread_msgq(sbthread_t *th)
+{
+    return th->msgq;
+}
+
+int
+sbthread_send(sbthread_t *th, size_t len, void const *data)
+{
+    return sbmsgq_send(th->msgq, len, data);
+}
+
+void
+sbthread_free(sbthread_t *th)
+{
+    sbthread_wait(th);
+    sbmsgq_free(th->msgq);
+    ckd_free(th);
+}

media/sphinxbase/src/libsphinxbase/util/strfuncs.c

@@ -0,0 +1,194 @@
+/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/* ====================================================================
+ * Copyright (c) 1999-2006 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */
+/*
+ * strfuncs.c -- String functions
+ */
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <stdarg.h>
+
+#include "sphinxbase/ckd_alloc.h"
+#include "sphinxbase/strfuncs.h"
+
+/* Defined in dtoa.c */
+double sb_strtod(const char *s00, char **se);
+
+double
+atof_c(char const *str)
+{
+    return sb_strtod(str, NULL);
+}
+
+/* Locale-independent isspace to avoid different incompatibilities */
+static int
+isspace_c(char ch)
+{
+    if (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')
+	return 1;
+    return 0;
+}
+
+char *
+string_join(const char *base, ...)
+{
+    va_list args;
+    size_t len;
+    const char *c;
+    char *out;
+
+    va_start(args, base);
+    len = strlen(base);
+    while ((c = va_arg(args, const char *)) != NULL) {
+        len += strlen(c);
+    }
+    len++;
+    va_end(args);
+
+    out = ckd_calloc(len, 1);
+    va_start(args, base);
+    strcpy(out, base);
+    while ((c = va_arg(args, const char *)) != NULL) {
+        strcat(out, c);
+    }
+    va_end(args);
+
+    return out;
+}
+
+char *
+string_trim(char *string, enum string_edge_e which)
+{
+    size_t len;
+
+    len = strlen(string);
+    if (which == STRING_START || which == STRING_BOTH) {
+        size_t sub = strspn(string, " \t\n\r\f");
+        if (sub > 0) {
+            memmove(string, string + sub, len + 1 - sub);
+            len -= sub;
+        }
+    }
+    if (which == STRING_END || which == STRING_BOTH) {
+        long sub = len;
+        while (--sub >= 0)
+            if (strchr(" \t\n\r\f", string[sub]) == NULL)
+                break;
+        if (sub == -1)
+            string[0] = '\0';
+        else
+            string[sub+1] = '\0';
+    }
+    return string;
+}
+
+int32
+str2words(char *line, char **ptr, int32 max_ptr)
+{
+    int32 i, n;
+
+    n = 0;                      /* #words found so far */
+    i = 0;                      /* For scanning through the input string */
+    while (1) {
+        /* Skip whitespace before next word */
+        while (line[i] && isspace_c(line[i]))
+            ++i;
+        if (!line[i])
+            break;
+
+        if (ptr != NULL && n >= max_ptr) {
+            /*
+             * Pointer array size insufficient.  Restore NULL chars inserted so far
+             * to space chars.  Not a perfect restoration, but better than nothing.
+             */
+            for (; i >= 0; --i)
+                if (line[i] == '\0')
+                    line[i] = ' ';
+
+            return -1;
+        }
+
+        /* Scan to end of word */
+        if (ptr != NULL)
+            ptr[n] = line + i;
+        ++n;
+        while (line[i] && !isspace_c(line[i]))
+            ++i;
+        if (!line[i])
+            break;
+        if (ptr != NULL)
+            line[i] = '\0';
+        ++i;
+    }
+
+    return n;
+}
+
+
+int32
+nextword(char *line, const char *delim, char **word, char *delimfound)
+{
+    const char *d;
+    char *w;
+
+    /* Skip past any preceding delimiters */
+    for (w = line; *w; w++) {
+        for (d = delim; *d && (*d != *w); d++);
+        if (!*d)
+            break;
+    }
+    if (!*w)
+        return -1;
+
+    *word = w;                  /* Beginning of word */
+
+    /* Skip until first delimiter char */
+    for (w++; *w; w++) {
+        for (d = delim; *d && (*d != *w); d++);
+        if (*d)
+            break;
+    }
+
+    /* Replace delimiter with NULL char, but return the original first */
+    *delimfound = *w;
+    *w = '\0';
+
+    return (w - *word);
+}

media/sphinxbase/src/libsphinxbase/util/utf8.c

@@ -0,0 +1,75 @@
+/* Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de> */
+/* See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. */
+
+/* Slightly modified to use Sphinx types and remove explicit inline. */
+
+#include "sphinxbase/prim_type.h"
+
+#define UTF8_ACCEPT 0
+#define UTF8_REJECT 1
+
+static const uint8 utf8d[] = {
+  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 00..1f */
+  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 20..3f */
+  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 40..5f */
+  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 60..7f */
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, /* 80..9f */
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, /* a0..bf */
+  8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, /* c0..df */
+  0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, /* e0..ef */
+  0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, /* f0..ff */
+  0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, /* s0..s0 */
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, /* s1..s2 */
+  1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, /* s3..s4 */
+  1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, /* s5..s6 */
+  1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* s7..s8 */
+};
+
+uint32
+utf8_decode(uint32 *state, uint32 *codep, uint32 byte) {
+  uint32 type = utf8d[byte];
+
+  *codep = (*state != UTF8_ACCEPT) ?
+    (byte & 0x3fu) | (*codep << 6) :
+    (0xff >> type) & (byte);
+
+  *state = utf8d[256 + *state*16 + type];
+  return *state;
+}
+
+/* CMU code starts here. */
+/* ====================================================================
+ * Copyright (c) 2009 Carnegie Mellon University.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * This work was supported in part by funding from the Defense Advanced 
+ * Research Projects Agency and the National Science Foundation of the 
+ * United States of America, and the CMU Sphinx Speech Consortium.
+ *
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+ * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ */