static void
encode(VMPM *vmpm)
{
Arithcoder *ac;
Arithmodel *am;
Arithmodel *bin_am;
Arithmodel **low_ams;
unsigned int *symbol_to_index;
int i, n, match_found;
unsigned int j;
//debug_message_fn("()\n");
if (!vmpm->outfile) {
debug_message_fnc("outfile is NULL.\n");
return;
}
ac = arithcoder_arith_create();
arithcoder_encode_init(ac, vmpm->outfile);
if (vmpm->nlowbits < 8) {
am = arithmodel_order_zero_create();
arithmodel_encode_init(am, ac);
arithmodel_order_zero_set_update_escape_freq(am, update_escape_freq);
bin_am = arithmodel_order_zero_create();
arithmodel_encode_init(bin_am, ac);
{
Arithmodel_order_zero *am_oz = (Arithmodel_order_zero *)am;
am_oz->bin_am = bin_am;
am_oz->escape_encoded_with_rle = 1;
}
match_found = 0;
for (i = vmpm->I; i >= 1; i--) {
int nsymbols = 0;
for (j = 0; j < vmpm->token_index[i]; j++) {
Token *t = vmpm->token[i][j];
Token_value tv = t->value - 1;
if (nsymbols == tv) {
nsymbols++;
} else {
match_found++;
break;
}
}
if (match_found) {
stat_message(vmpm, "Match found at Level %d\n", i);
break;
}
}
fprintf(vmpm->outfile, "%c", i);
if (match_found) {
for (; i >= 1; i--) {
int nsymbols = 0;
stat_message(vmpm, "Level %d (%d tokens, %d distinct): ", i, vmpm->token_index[i], vmpm->newtoken[i] - 1);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
/* Send the number of distinct symbols. */
arithmodel_encode_cbt(bin_am, vmpm->newtoken[i] - 1, vmpm->token_index[i], 0, 1);
arithmodel_order_zero_reset(am, 0, vmpm->newtoken[i]);
for (j = 0; j < vmpm->token_index[i]; j++) {
Token *t = vmpm->token[i][j];
Token_value tv = t->value - 1;
if (nsymbols == tv) {
stat_message(vmpm, "e ");
nsymbols++;
} else {
stat_message(vmpm, "%d ", tv);
}
arithmodel_encode(am, tv);
}
stat_message(vmpm, "\n");
stat_message(vmpm, "Level %d: %ld bytes\n", i, ftell(vmpm->outfile));
}
}
if ((symbol_to_index = malloc(vmpm->alphabetsize * sizeof(unsigned int))) == NULL)
memory_error(NULL, MEMORY_ERROR);
memset(symbol_to_index, 255, vmpm->alphabetsize * sizeof(unsigned int));
n = 0;
arithmodel_order_zero_reset(am, 0, vmpm->alphabetsize - 1);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
stat_message(vmpm, "Level 0 (%d tokens): ", vmpm->token_index[0]);
for (j = 0; j < vmpm->token_index[0]; j++) {
if (symbol_to_index[(int)vmpm->token[0][j]] == (unsigned int)-1) {
stat_message(vmpm, "e ");
arithmodel_encode(am, n);
symbol_to_index[(int)vmpm->token[0][j]] = n++;
arithmodel_encode_bits(bin_am, (int)vmpm->token[0][j], vmpm->bits_per_symbol, 0, 1);
} else {
stat_message(vmpm, "%d ", symbol_to_index[(int)vmpm->token[0][j]]);
arithmodel_encode(am, symbol_to_index[(int)vmpm->token[0][j]]);
}
}
stat_message(vmpm, "\n");
free(symbol_to_index);
arithmodel_encode_final(bin_am);
arithmodel_encode_final(am);
arithmodel_destroy(bin_am);
arithmodel_destroy(am);
}
stat_message(vmpm, "Higher part: %ld bytes (%s)\n", ftell(vmpm->outfile), vmpm->outfilepath);
if ((low_ams = calloc(1 << (8 - vmpm->nlowbits), sizeof(Arithmodel *))) == NULL)
memory_error(NULL, MEMORY_ERROR);
for (i = 0; i < (1 << (8 - vmpm->nlowbits)); i++) {
low_ams[i] = arithmodel_order_zero_create();
arithmodel_encode_init(low_ams[i], ac);
arithmodel_order_zero_reset(low_ams[i], 0, 0);
for (j = 0; j < (1 << vmpm->nlowbits); j++)
arithmodel_install_symbol(low_ams[i], 1);
}
for (i = 0; i < vmpm->buffer_low_size; i++)
arithmodel_encode(low_ams[vmpm->buffer_high[i]], vmpm->buffer_low[i]);
for (i = 0; i < (1 << (8 - vmpm->nlowbits)); i++)
arithmodel_encode_final(low_ams[i]);
for (i = 0; i < (1 << (8 - vmpm->nlowbits)); i++)
arithmodel_destroy(low_ams[i]);
arithcoder_encode_final(ac);
arithcoder_destroy(ac);
free(low_ams);
}
static void
encode(VMPM *vmpm)
{
Arithcoder *ac;
Arithmodel *am;
Arithmodel *bin_am;
unsigned int j, nsymbols, *symbol_to_index;
int i, match_found;
//debug_message_fn("()\n");
if (!vmpm->outfile) {
debug_message_fnc("outfile is NULL.\n");
return;
}
ac = arithcoder_arith_create();
arithcoder_encode_init(ac, vmpm->outfile);
am = arithmodel_order_zero_create();
arithmodel_encode_init(am, ac);
arithmodel_order_zero_set_update_escape_freq(am, update_escape_freq);
bin_am = arithmodel_order_zero_create();
arithmodel_encode_init(bin_am, ac);
match_found = 0;
for (i = vmpm->I; i >= 1; i--) {
nsymbols = 0;
for (j = 0; j < vmpm->token_index[i]; j++) {
Token_value tv = vmpm->token[i][j]->value - 1;
if (nsymbols == tv) {
nsymbols++;
} else {
match_found++;
break;
}
}
if (match_found) {
stat_message(vmpm, "Match found at Level %d\n", i);
break;
}
}
fputc(i, vmpm->outfile);
if (match_found) {
for (; i >= 1; i--) {
int jj = -1;
nsymbols = 0;
stat_message(vmpm, "Level %d (%d tokens, %d distinct): ", i, vmpm->token_index[i], vmpm->newtoken[i] - 1);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
/* Send the number of distinct symbols. */
arithmodel_encode_cbt(bin_am, vmpm->newtoken[i] - 1, vmpm->token_index[i], 0, 1);
arithmodel_order_zero_reset(am, 0, vmpm->newtoken[i]);
arithmodel_order_zero_reset(bin_am, 0, 0);
/* The first token of each level must be t_0. */
if (vmpm->token[i][0]->value != 1)
generic_error((char *)"Invalid token value.\n", INVALID_TOKEN_VALUE_ERROR);
/* Hence, we don't need to encode it. */
stat_message(vmpm, "e ");
arithmodel_install_symbol(am, 1);
nsymbols++;
for (j = 1; j < vmpm->token_index[i]; j++) {
Token_value tv = vmpm->token[i][j]->value - 1;
if (nsymbols == tv) {
if (nsymbols > vmpm->newtoken[i] - 1)
generic_error((char *)"Internal error\n", INTERNAL_ERROR);
stat_message(vmpm, "e ");
nsymbols++;
arithmodel_encode(am, tv);
if (nsymbols == vmpm->newtoken[i] - 1 && j < vmpm->token_index[i] - 1) {
/* All escapes are emitted. */
jj = j;
arithmodel_order_zero_uninstall_escape(am);
debug_message_fnc("Level %d: Escape is uninstalled: %d/%d.\n", i, j, vmpm->token_index[i] - 1);
}
} else {
stat_message(vmpm, "%d ", tv);
arithmodel_encode(am, tv);
}
}
stat_message(vmpm, "\n");
if (jj != -1)
stat_message(vmpm, "Level %d: Escape is uninstalled at %d\n", i, jj);
stat_message(vmpm, "Level %d: %ld bytes\n", i, ftell(vmpm->outfile));
}
}
if ((symbol_to_index = malloc(vmpm->alphabetsize * sizeof(unsigned int))) == NULL)
memory_error(NULL, MEMORY_ERROR);
memset(symbol_to_index, 255, vmpm->alphabetsize * sizeof(unsigned int));
nsymbols = 0;
arithmodel_order_zero_reset(am, 0, vmpm->alphabetsize - 1);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
stat_message(vmpm, "Level 0 (%d tokens): ", vmpm->token_index[0]);
for (j = 0; j < vmpm->token_index[0]; j++) {
if (symbol_to_index[(int)vmpm->token[0][j]] == (unsigned int)-1) {
if (nsymbols > vmpm->alphabetsize)
generic_error((char *)"Internal error\n", INTERNAL_ERROR);
stat_message(vmpm, "e ");
arithmodel_encode(am, nsymbols);
symbol_to_index[(int)vmpm->token[0][j]] = nsymbols++;
arithmodel_encode_bits(bin_am, (int)vmpm->token[0][j], vmpm->bits_per_symbol, 0, 1);
if (nsymbols == vmpm->alphabetsize) {
arithmodel_order_zero_uninstall_escape(am);
debug_message_fnc("Level 0: Escape is uninstalled: %d/%d.\n", j, vmpm->token_index[0] - 1);
}
} else {
stat_message(vmpm, "%d ", symbol_to_index[(int)vmpm->token[0][j]]);
arithmodel_encode(am, symbol_to_index[(int)vmpm->token[0][j]]);
}
}
stat_message(vmpm, "\n");
free(symbol_to_index);
arithmodel_encode_final(bin_am);
arithmodel_encode_final(am);
arithcoder_encode_final(ac);
arithmodel_destroy(bin_am);
arithmodel_destroy(am);
arithcoder_destroy(ac);
}
static void
encode(VMPM *vmpm)
{
Arithcoder *ac;
Arithmodel *am;
Arithmodel *bin_am;
unsigned int j, nsymbols, *symbol_to_index;
int i, match_found;
//debug_message_fn("()\n");
if (!vmpm->outfile) {
debug_message_fnc("outfile is NULL.\n");
return;
}
ac = arithcoder_arith_create();
arithcoder_encode_init(ac, vmpm->outfile);
am = arithmodel_order_zero_create();
arithmodel_encode_init(am, ac);
arithmodel_order_zero_set_update_escape_freq(am, update_escape_freq);
bin_am = arithmodel_order_zero_create();
arithmodel_encode_init(bin_am, ac);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
match_found = 0;
for (i = vmpm->I; i >= 1; i--) {
nsymbols = 0;
for (j = 0; j < vmpm->token_index[i]; j++) {
Token_value tv = vmpm->token[i][j]->value - 1;
if (nsymbols == tv) {
nsymbols++;
} else {
match_found++;
break;
}
}
if (match_found) {
stat_message(vmpm, "Match found at Level %d\n", i);
break;
}
}
fputc(i, vmpm->outfile);
if (match_found) {
for (; i >= 1; i--) {
stat_message(vmpm, "Level %d (%d tokens, %d distinct): ", i, vmpm->token_index[i], vmpm->newtoken[i] - 1);
/* Encode escape symbols at once. */
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
if (vmpm->token[i][0]->value != 1)
generic_error((char *)"Invalid token value.\n", INVALID_TOKEN_VALUE_ERROR);
/* Hence, we don't need to encode it. */
stat_message(vmpm, "e ");
nsymbols = 1;
#if 0
/* It's simpler to encode simultaneously. */
for (j = 1; j < vmpm->token_index[i]; j++) {
Token_value tv = vmpm->token[i][j]->value - 1;
if (nsymbols == tv) {
stat_message(vmpm, "e ");
nsymbols++;
arithmodel_encode(bin_am, 1);
} else {
stat_message(vmpm, "%d ", tv);
arithmodel_encode(bin_am, 0);
arithmodel_encode_cbt(bin_am, tv, nsymbols - 1, 0, 1);
}
}
#else
/* It's better to encode separately. */
for (j = 1; j < vmpm->token_index[i]; j++) {
Token_value tv = vmpm->token[i][j]->value - 1;
if (nsymbols == tv) {
stat_message(vmpm, "e ");
nsymbols++;
arithmodel_encode(bin_am, 1);
} else {
stat_message(vmpm, "%d ", tv);
arithmodel_encode(bin_am, 0);
}
}
/* Then encode non-escape symbols. */
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
nsymbols = 1;
for (j = 0; j < vmpm->token_index[i]; j++) {
Token_value tv = vmpm->token[i][j]->value - 1;
if (nsymbols == tv) {
nsymbols++;
} else {
arithmodel_encode_cbt(bin_am, tv, nsymbols - 1, 0, 1);
}
}
#endif
stat_message(vmpm, "\n");
stat_message(vmpm, "Level %d: %ld bytes\n", i, ftell(vmpm->outfile));
}
}
if ((symbol_to_index = malloc(vmpm->alphabetsize * sizeof(unsigned int))) == NULL)
memory_error(NULL, MEMORY_ERROR);
memset(symbol_to_index, 255, vmpm->alphabetsize * sizeof(unsigned int));
nsymbols = 0;
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
arithmodel_order_zero_reset(am, 0, vmpm->alphabetsize - 1);
stat_message(vmpm, "Level 0 (%d tokens): ", vmpm->token_index[0]);
for (j = 0; j < vmpm->token_index[0]; j++) {
if (symbol_to_index[(int)vmpm->token[0][j]] == (unsigned int)-1) {
stat_message(vmpm, "e ");
arithmodel_encode(am, nsymbols);
symbol_to_index[(int)vmpm->token[0][j]] = nsymbols++;
arithmodel_encode_bits(bin_am, (int)vmpm->token[0][j], vmpm->bits_per_symbol, 0, 1);
} else {
stat_message(vmpm, "%d ", symbol_to_index[(int)vmpm->token[0][j]]);
arithmodel_encode(am, symbol_to_index[(int)vmpm->token[0][j]]);
}
}
stat_message(vmpm, "\n");
free(symbol_to_index);
arithmodel_encode_final(bin_am);
arithmodel_encode_final(am);
arithcoder_encode_final(ac);
arithmodel_destroy(bin_am);
arithmodel_destroy(am);
arithcoder_destroy(ac);
}
static void
encode(VMPM *vmpm)
{
Arithcoder *ac;
Arithmodel *char_am;
Arithmodel *am;
Arithmodel *bin_am;
unsigned int j, nsymbols, *symbol_to_index;
int i, match_found;
//debug_message_fn("()\n");
if (!vmpm->outfile) {
debug_message_fnc("outfile is NULL.\n");
return;
}
ac = arithcoder_arith_create();
arithcoder_encode_init(ac, vmpm->outfile);
char_am = arithmodel_order_zero_create();
arithmodel_encode_init(char_am, ac);
arithmodel_order_zero_set_update_escape_freq(char_am, update_escape_freq);
am = arithmodel_order_zero_create();
arithmodel_encode_init(am, ac);
bin_am = arithmodel_order_zero_create();
arithmodel_encode_init(bin_am, ac);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
match_found = 0;
for (i = vmpm->I; i >= 1; i--) {
nsymbols = 0;
for (j = 0; j < vmpm->token_index[i]; j++) {
Token *t = vmpm->token[i][j];
Token_value tv = t->value - 1;
if (nsymbols == tv) {
nsymbols++;
} else {
match_found++;
break;
}
}
if (match_found) {
stat_message(vmpm, "Match found at Level %d\n", i);
break;
}
}
fputc(i, vmpm->outfile);
if (match_found) {
for (; i >= 1; i--) {
nsymbols = 0;
stat_message(vmpm, "Level %d (%d tokens, %d distinct): ", i, vmpm->token_index[i], vmpm->newtoken[i] - 1);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
/* Send the number of distinct symbols. */
arithmodel_encode_cbt(bin_am, vmpm->newtoken[i] - 1, vmpm->token_index[i], 0, 1);
arithmodel_order_zero_reset(am, 0, vmpm->newtoken[i]);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
/* The first token of each level must be t_0. */
if (vmpm->token[i][0]->value != 1)
generic_error((char *)"Invalid token value.\n", INVALID_TOKEN_VALUE_ERROR);
/* Hence, we don't need to encode it. */
stat_message(vmpm, "e ");
nsymbols = 1;
arithmodel_order_zero_reset(am, 0, 0);
arithmodel_install_symbol(am, 1);
for (j = 1; j < vmpm->token_index[i]; j++) {
Token_value tv = vmpm->token[i][j]->value - 1;
Arithmodel_order_zero *bin_am_oz = (Arithmodel_order_zero *)bin_am;
/* calculate and assign the escape symbol's probability */
bin_am_oz->freq[0] = vmpm->token_index[i] - (vmpm->newtoken[i] - 1) + nsymbols - (j + 1) + 1;
bin_am_oz->freq[1] = vmpm->token_index[i] - (j + 1) + 1;
//stat_message(vmpm, "freq[0] = %d, freq[1] = %d, nsymbols %d.\n", bin_am_oz->freq[0], bin_am_oz->freq[1], nsymbols);
if (nsymbols == tv) {
stat_message(vmpm, "e ");
nsymbols++;
if (bin_am_oz->freq[0] > 0 && bin_am_oz->freq[1] > bin_am_oz->freq[0])
arithmodel_encode(bin_am, 1);
else if (bin_am_oz->freq[0] != 0) {
stat_message(vmpm, "BUG(1)\n");
}
arithmodel_install_symbol(am, 1);
} else {
stat_message(vmpm, "%d ", tv);
if (bin_am_oz->freq[0] > 0 && bin_am_oz->freq[1] > bin_am_oz->freq[0])
arithmodel_encode(bin_am, 0);
else if (bin_am_oz->freq[0] == 0) {
stat_message(vmpm, "BUG(2)\n");
}
arithmodel_encode(am, tv);
}
}
stat_message(vmpm, "\n");
stat_message(vmpm, "Level %d: %ld bytes\n", i, ftell(vmpm->outfile));
}
}
if ((symbol_to_index = malloc(vmpm->alphabetsize * sizeof(unsigned int))) == NULL)
memory_error(NULL, MEMORY_ERROR);
memset(symbol_to_index, 255, vmpm->alphabetsize * sizeof(unsigned int));
nsymbols = 0;
arithmodel_order_zero_reset(char_am, 0, vmpm->alphabetsize - 1);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
stat_message(vmpm, "Level 0 (%d tokens): ", vmpm->token_index[0]);
for (j = 0; j < vmpm->token_index[0]; j++) {
if (symbol_to_index[(int)vmpm->token[0][j]] == (unsigned int)-1) {
stat_message(vmpm, "e ");
arithmodel_encode(char_am, nsymbols);
symbol_to_index[(int)vmpm->token[0][j]] = nsymbols++;
arithmodel_encode_bits(bin_am, (int)vmpm->token[0][j], vmpm->bits_per_symbol, 0, 1);
} else {
stat_message(vmpm, "%d ", symbol_to_index[(int)vmpm->token[0][j]]);
arithmodel_encode(char_am, symbol_to_index[(int)vmpm->token[0][j]]);
}
}
stat_message(vmpm, "\n");
free(symbol_to_index);
arithmodel_encode_final(bin_am);
arithmodel_encode_final(char_am);
arithcoder_encode_final(ac);
arithmodel_destroy(bin_am);
arithmodel_destroy(char_am);
arithcoder_destroy(ac);
}
