Example #1
0
bin_mdef_t *
bin_mdef_read(cmd_ln_t *config, const char *filename)
{
    bin_mdef_t *m;
    FILE *fh;
    size_t tree_start;
    int32 val, i, swap, pos, end;
    int32 *sseq_size;
    int do_mmap;

    /* Try to read it as text first. */
    if ((m = bin_mdef_read_text(config, filename)) != NULL)
        return m;

    E_INFO("Reading binary model definition: %s\n", filename);
    if ((fh = fopen(filename, "rb")) == NULL)
        return NULL;

    if (fread(&val, 4, 1, fh) != 1) {
        fclose(fh);
        E_ERROR_SYSTEM("Failed to read byte-order marker from %s\n",
                       filename);
        return NULL;
    }
    swap = 0;
    if (val == BIN_MDEF_OTHER_ENDIAN) {
        swap = 1;
        E_INFO("Must byte-swap %s\n", filename);
    }
    if (fread(&val, 4, 1, fh) != 1) {
        fclose(fh);
        E_ERROR_SYSTEM("Failed to read version from %s\n", filename);
        return NULL;
    }
    if (swap)
        SWAP_INT32(&val);
    if (val > BIN_MDEF_FORMAT_VERSION) {
        E_ERROR("File format version %d for %s is newer than library\n",
                val, filename);
        fclose(fh);
        return NULL;
    }
    if (fread(&val, 4, 1, fh) != 1) {
        fclose(fh);
        E_ERROR_SYSTEM("Failed to read header length from %s\n", filename);
        return NULL;
    }
    if (swap)
        SWAP_INT32(&val);
    /* Skip format descriptor. */
    fseek(fh, val, SEEK_CUR);

    /* Finally allocate it. */
    m = ckd_calloc(1, sizeof(*m));
    m->refcnt = 1;

    /* Check these, to make gcc/glibc shut up. */
#define FREAD_SWAP32_CHK(dest)                                          \
    if (fread((dest), 4, 1, fh) != 1) {                                 \
        fclose(fh);                                                     \
        ckd_free(m);                                                    \
        E_ERROR_SYSTEM("Failed to read %s from %s\n", #dest, filename); \
        return NULL;                                                    \
    }                                                                   \
    if (swap) SWAP_INT32(dest);
    
    FREAD_SWAP32_CHK(&m->n_ciphone);
    FREAD_SWAP32_CHK(&m->n_phone);
    FREAD_SWAP32_CHK(&m->n_emit_state);
    FREAD_SWAP32_CHK(&m->n_ci_sen);
    FREAD_SWAP32_CHK(&m->n_sen);
    FREAD_SWAP32_CHK(&m->n_tmat);
    FREAD_SWAP32_CHK(&m->n_sseq);
    FREAD_SWAP32_CHK(&m->n_ctx);
    FREAD_SWAP32_CHK(&m->n_cd_tree);
    FREAD_SWAP32_CHK(&m->sil);

    /* CI names are first in the file. */
    m->ciname = ckd_calloc(m->n_ciphone, sizeof(*m->ciname));

    /* Decide whether to read in the whole file or mmap it. */
    do_mmap = config ? cmd_ln_boolean_r(config, "-mmap") : TRUE;
    if (swap) {
        E_WARN("-mmap specified, but mdef is other-endian.  Will not memory-map.\n");
        do_mmap = FALSE;
    } 
    /* Actually try to mmap it. */
    if (do_mmap) {
        m->filemap = mmio_file_read(filename);
        if (m->filemap == NULL)
            do_mmap = FALSE;
    }
    pos = ftell(fh);
    if (do_mmap) {
        /* Get the base pointer from the memory map. */
        m->ciname[0] = (char *)mmio_file_ptr(m->filemap) + pos;
        /* Success! */
        m->alloc_mode = BIN_MDEF_ON_DISK;
    }
    else {
        /* Read everything into memory. */
        m->alloc_mode = BIN_MDEF_IN_MEMORY;
        fseek(fh, 0, SEEK_END);
        end = ftell(fh);
        fseek(fh, pos, SEEK_SET);
        m->ciname[0] = ckd_malloc(end - pos);
        if (fread(m->ciname[0], 1, end - pos, fh) != end - pos)
            E_FATAL("Failed to read %d bytes of data from %s\n", end - pos, filename);
    }

    for (i = 1; i < m->n_ciphone; ++i)
        m->ciname[i] = m->ciname[i - 1] + strlen(m->ciname[i - 1]) + 1;

    /* Skip past the padding. */
    tree_start =
        m->ciname[i - 1] + strlen(m->ciname[i - 1]) + 1 - m->ciname[0];
    tree_start = (tree_start + 3) & ~3;
    m->cd_tree = (cd_tree_t *) (m->ciname[0] + tree_start);
    if (swap) {
        for (i = 0; i < m->n_cd_tree; ++i) {
            SWAP_INT16(&m->cd_tree[i].ctx);
            SWAP_INT16(&m->cd_tree[i].n_down);
            SWAP_INT32(&m->cd_tree[i].c.down);
        }
    }
    m->phone = (mdef_entry_t *) (m->cd_tree + m->n_cd_tree);
    if (swap) {
        for (i = 0; i < m->n_phone; ++i) {
            SWAP_INT32(&m->phone[i].ssid);
            SWAP_INT32(&m->phone[i].tmat);
        }
    }
    sseq_size = (int32 *) (m->phone + m->n_phone);
    if (swap)
        SWAP_INT32(sseq_size);
    m->sseq = ckd_calloc(m->n_sseq, sizeof(*m->sseq));
    m->sseq[0] = (uint16 *) (sseq_size + 1);
    if (swap) {
        for (i = 0; i < *sseq_size; ++i)
            SWAP_INT16(m->sseq[0] + i);
    }
    if (m->n_emit_state) {
        for (i = 1; i < m->n_sseq; ++i)
            m->sseq[i] = m->sseq[0] + i * m->n_emit_state;
    }
    else {
        m->sseq_len = (uint8 *) (m->sseq[0] + *sseq_size);
        for (i = 1; i < m->n_sseq; ++i)
            m->sseq[i] = m->sseq[i - 1] + m->sseq_len[i - 1];
    }

    /* Now build the CD-to-CI mappings using the senone sequences.
     * This is the only really accurate way to do it, though it is
     * still inaccurate in the case of heterogeneous topologies or
     * cross-state tying. */
    m->cd2cisen = (int16 *) ckd_malloc(m->n_sen * sizeof(*m->cd2cisen));
    m->sen2cimap = (int16 *) ckd_malloc(m->n_sen * sizeof(*m->sen2cimap));

    /* Default mappings (identity, none) */
    for (i = 0; i < m->n_ci_sen; ++i)
        m->cd2cisen[i] = i;
    for (; i < m->n_sen; ++i)
        m->cd2cisen[i] = -1;
    for (i = 0; i < m->n_sen; ++i)
        m->sen2cimap[i] = -1;
    for (i = 0; i < m->n_phone; ++i) {
        int32 j, ssid = m->phone[i].ssid;

        for (j = 0; j < bin_mdef_n_emit_state_phone(m, i); ++j) {
            int s = bin_mdef_sseq2sen(m, ssid, j);
            int ci = bin_mdef_pid2ci(m, i);
            /* Take the first one and warn if we have cross-state tying. */
            if (m->sen2cimap[s] == -1)
                m->sen2cimap[s] = ci;
            if (m->sen2cimap[s] != ci)
                E_WARN
                    ("Senone %d is shared between multiple base phones\n",
                     s);

            if (j > bin_mdef_n_emit_state_phone(m, ci))
                E_WARN("CD phone %d has fewer states than CI phone %d\n",
                       i, ci);
            else
                m->cd2cisen[s] =
                    bin_mdef_sseq2sen(m, m->phone[ci].ssid, j);
        }
    }

    /* Set the silence phone. */
    m->sil = bin_mdef_ciphone_id(m, S3_SILENCE_CIPHONE);

    E_INFO
        ("%d CI-phone, %d CD-phone, %d emitstate/phone, %d CI-sen, %d Sen, %d Sen-Seq\n",
         m->n_ciphone, m->n_phone - m->n_ciphone, m->n_emit_state,
         m->n_ci_sen, m->n_sen, m->n_sseq);
    fclose(fh);
    return m;
}
Example #2
0
static int32
read_sendump(s2_semi_mgau_t *s, mdef_t *mdef, char const *file)
{
    FILE *fp;
    char line[1000];
    int32 i, n;
    int32 do_swap, do_mmap;
    size_t filesize, offset;
    int n_clust = 256;          /* Number of clusters (if zero, we are just using
                                 * 8-bit quantized weights) */
    int r = s->n_density;
    int c = mdef_n_sen(mdef);

    s->n_sen = c;
    do_mmap = cmd_ln_boolean_r(s->config, "-mmap");

    if ((fp = fopen(file, "rb")) == NULL)
        return -1;

    E_INFO("Loading senones from dump file %s\n", file);
    /* Read title size, title */
    fread(&n, sizeof(int32), 1, fp);
    /* This is extremely bogus */
    do_swap = 0;
    if (n < 1 || n > 999) {
        SWAP_INT32(&n);
        if (n < 1 || n > 999) {
            E_FATAL("Title length %x in dump file %s out of range\n", n, file);
        }
        do_swap = 1;
    }
    if (fread(line, sizeof(char), n, fp) != n)
        E_FATAL("Cannot read title\n");
    if (line[n - 1] != '\0')
        E_FATAL("Bad title in dump file\n");
    E_INFO("%s\n", line);

    /* Read header size, header */
    fread(&n, 1, sizeof(n), fp);
    if (do_swap) SWAP_INT32(&n);
    if (fread(line, sizeof(char), n, fp) != n)
        E_FATAL("Cannot read header\n");
    if (line[n - 1] != '\0')
        E_FATAL("Bad header in dump file\n");

    /* Read other header strings until string length = 0 */
    for (;;) {
        fread(&n, 1, sizeof(n), fp);
        if (do_swap) SWAP_INT32(&n);
        if (n == 0)
            break;
        if (fread(line, sizeof(char), n, fp) != n)
            E_FATAL("Cannot read header\n");
        /* Look for a cluster count, if present */
        if (!strncmp(line, "cluster_count ", strlen("cluster_count "))) {
            n_clust = atoi(line + strlen("cluster_count "));
        }
    }

    /* Read #codewords, #pdfs */
    fread(&r, 1, sizeof(r), fp);
    if (do_swap) SWAP_INT32(&r);
    fread(&c, 1, sizeof(c), fp);
    if (do_swap) SWAP_INT32(&c);
    E_INFO("Rows: %d, Columns: %d\n", r, c);

    if (n_clust) {
	E_ERROR ("Dump file is incompatible with PocketSphinx\n");
	fclose(fp);
	return -1;
    }
    if (do_mmap) {
            E_INFO("Using memory-mapped I/O for senones\n");
    }
    offset = ftell(fp);
    fseek(fp, 0, SEEK_END);
    filesize = ftell(fp);
    fseek(fp, offset, SEEK_SET);

    /* Allocate memory for pdfs (or memory map them) */
    if (do_mmap)
        s->sendump_mmap = mmio_file_read(file);

    /* Otherwise, set up all pointers, etc. */
    if (s->sendump_mmap) {
        s->mixw = ckd_calloc(s->n_feat, sizeof(*s->mixw));
        for (i = 0; i < s->n_feat; i++) {
            /* Pointers into the mmap()ed 2d array */
	    s->mixw[i] = ckd_calloc(r, sizeof(**s->mixw));
        }

        for (n = 0; n < s->n_feat; n++) {
            for (i = 0; i < r; i++) {
                s->mixw[n][i] = ((uint8 *) mmio_file_ptr(s->sendump_mmap)) + offset;
                offset += c;
            }
        }
    }
    else {
        s->mixw = ckd_calloc_3d(s->n_feat, r, c, sizeof(***s->mixw));
        /* Read pdf values and ids */
        for (n = 0; n < s->n_feat; n++) {
            for (i = 0; i < r; i++) {
                if (fread(s->mixw[n][i], sizeof(***s->mixw), c, fp) != (size_t) c) {
                    E_ERROR("Failed to read %d bytes from sendump\n", c);
                    return -1;
                }
            }
        }
    }

    fclose(fp);
    return 0;
}