static void
encode(VMPM *vmpm)
{
Arithcoder *ac;
Arithmodel *char_am;
Arithmodel *am;
Arithmodel *bin_am;
unsigned int j, nsymbols, *symbol_to_index;
int i, match_found;
//debug_message_fn("()\n");
if (!vmpm->outfile) {
debug_message_fnc("outfile is NULL.\n");
return;
}
ac = arithcoder_arith_create();
arithcoder_encode_init(ac, vmpm->outfile);
char_am = arithmodel_order_zero_create();
arithmodel_encode_init(char_am, ac);
arithmodel_order_zero_set_update_escape_freq(char_am, update_escape_freq);
am = arithmodel_order_zero_create();
arithmodel_encode_init(am, ac);
bin_am = arithmodel_order_zero_create();
arithmodel_encode_init(bin_am, ac);
match_found = 0;
for (i = vmpm->I; i >= 1; i--) {
nsymbols = 0;
for (j = 0; j < vmpm->token_index[i]; j++) {
Token_value tv = vmpm->token[i][j]->value - 1;
if (nsymbols == tv) {
nsymbols++;
} else {
match_found++;
break;
}
}
if (match_found) {
stat_message(vmpm, "Match found at Level %d\n", i);
break;
}
}
fputc(i, vmpm->outfile);
if (match_found) {
for (; i >= 1; i--) {
stat_message(vmpm, "Level %d (%d tokens, %d distinct): ", i, vmpm->token_index[i], vmpm->newtoken[i] - 1);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
/* The first token of each level must be t_0. */
if (vmpm->token[i][0]->value != 1)
generic_error((char *)"Invalid token value.\n", INVALID_TOKEN_VALUE_ERROR);
/* Hence, we don't need to encode it. */
stat_message(vmpm, "e ");
nsymbols = 1;
arithmodel_order_zero_reset(am, 0, 0);
arithmodel_install_symbol(am, 1);
for (j = 1; j < vmpm->token_index[i]; j++) {
Token_value tv = vmpm->token[i][j]->value - 1;
if (nsymbols == tv) {
stat_message(vmpm, "e ");
nsymbols++;
arithmodel_encode(bin_am, 1);
arithmodel_install_symbol(am, 1);
} else {
stat_message(vmpm, "%d ", tv);
arithmodel_encode(bin_am, 0);
arithmodel_encode(am, tv);
}
}
stat_message(vmpm, "\n");
stat_message(vmpm, "Level %d: %ld bytes\n", i, ftell(vmpm->outfile));
}
}
if ((symbol_to_index = malloc(vmpm->alphabetsize * sizeof(unsigned int))) == NULL)
memory_error(NULL, MEMORY_ERROR);
memset(symbol_to_index, 255, vmpm->alphabetsize * sizeof(unsigned int));
nsymbols = 0;
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
arithmodel_order_zero_reset(char_am, 0, vmpm->alphabetsize - 1);
stat_message(vmpm, "Level 0 (%d tokens): ", vmpm->token_index[0]);
for (j = 0; j < vmpm->token_index[0]; j++) {
if (symbol_to_index[(int)vmpm->token[0][j]] == (unsigned int)-1) {
stat_message(vmpm, "e ");
arithmodel_encode(char_am, nsymbols);
symbol_to_index[(int)vmpm->token[0][j]] = nsymbols++;
arithmodel_encode_bits(bin_am, (int)vmpm->token[0][j], vmpm->bits_per_symbol, 0, 1);
} else {
stat_message(vmpm, "%d ", symbol_to_index[(int)vmpm->token[0][j]]);
arithmodel_encode(char_am, symbol_to_index[(int)vmpm->token[0][j]]);
}
}
stat_message(vmpm, "\n");
free(symbol_to_index);
arithmodel_encode_final(bin_am);
arithmodel_encode_final(char_am);
arithcoder_encode_final(ac);
arithmodel_destroy(bin_am);
arithmodel_destroy(char_am);
arithcoder_destroy(ac);
}
static void
encode(VMPM *vmpm)
{
Arithcoder *ac;
Arithmodel *am;
Arithmodel *bin_am;
Arithmodel **low_ams;
unsigned int *symbol_to_index;
int i, n, match_found;
unsigned int j;
//debug_message_fn("()\n");
if (!vmpm->outfile) {
debug_message_fnc("outfile is NULL.\n");
return;
}
ac = arithcoder_arith_create();
arithcoder_encode_init(ac, vmpm->outfile);
if (vmpm->nlowbits < 8) {
am = arithmodel_order_zero_create();
arithmodel_encode_init(am, ac);
arithmodel_order_zero_set_update_escape_freq(am, update_escape_freq);
bin_am = arithmodel_order_zero_create();
arithmodel_encode_init(bin_am, ac);
{
Arithmodel_order_zero *am_oz = (Arithmodel_order_zero *)am;
am_oz->bin_am = bin_am;
am_oz->escape_encoded_with_rle = 1;
}
match_found = 0;
for (i = vmpm->I; i >= 1; i--) {
int nsymbols = 0;
for (j = 0; j < vmpm->token_index[i]; j++) {
Token *t = vmpm->token[i][j];
Token_value tv = t->value - 1;
if (nsymbols == tv) {
nsymbols++;
} else {
match_found++;
break;
}
}
if (match_found) {
stat_message(vmpm, "Match found at Level %d\n", i);
break;
}
}
fprintf(vmpm->outfile, "%c", i);
if (match_found) {
for (; i >= 1; i--) {
int nsymbols = 0;
stat_message(vmpm, "Level %d (%d tokens, %d distinct): ", i, vmpm->token_index[i], vmpm->newtoken[i] - 1);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
/* Send the number of distinct symbols. */
arithmodel_encode_cbt(bin_am, vmpm->newtoken[i] - 1, vmpm->token_index[i], 0, 1);
arithmodel_order_zero_reset(am, 0, vmpm->newtoken[i]);
for (j = 0; j < vmpm->token_index[i]; j++) {
Token *t = vmpm->token[i][j];
Token_value tv = t->value - 1;
if (nsymbols == tv) {
stat_message(vmpm, "e ");
nsymbols++;
} else {
stat_message(vmpm, "%d ", tv);
}
arithmodel_encode(am, tv);
}
stat_message(vmpm, "\n");
stat_message(vmpm, "Level %d: %ld bytes\n", i, ftell(vmpm->outfile));
}
}
if ((symbol_to_index = malloc(vmpm->alphabetsize * sizeof(unsigned int))) == NULL)
memory_error(NULL, MEMORY_ERROR);
memset(symbol_to_index, 255, vmpm->alphabetsize * sizeof(unsigned int));
n = 0;
arithmodel_order_zero_reset(am, 0, vmpm->alphabetsize - 1);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
stat_message(vmpm, "Level 0 (%d tokens): ", vmpm->token_index[0]);
for (j = 0; j < vmpm->token_index[0]; j++) {
if (symbol_to_index[(int)vmpm->token[0][j]] == (unsigned int)-1) {
stat_message(vmpm, "e ");
arithmodel_encode(am, n);
symbol_to_index[(int)vmpm->token[0][j]] = n++;
arithmodel_encode_bits(bin_am, (int)vmpm->token[0][j], vmpm->bits_per_symbol, 0, 1);
} else {
stat_message(vmpm, "%d ", symbol_to_index[(int)vmpm->token[0][j]]);
arithmodel_encode(am, symbol_to_index[(int)vmpm->token[0][j]]);
}
}
stat_message(vmpm, "\n");
free(symbol_to_index);
arithmodel_encode_final(bin_am);
arithmodel_encode_final(am);
arithmodel_destroy(bin_am);
arithmodel_destroy(am);
}
stat_message(vmpm, "Higher part: %ld bytes (%s)\n", ftell(vmpm->outfile), vmpm->outfilepath);
if ((low_ams = calloc(1 << (8 - vmpm->nlowbits), sizeof(Arithmodel *))) == NULL)
memory_error(NULL, MEMORY_ERROR);
for (i = 0; i < (1 << (8 - vmpm->nlowbits)); i++) {
low_ams[i] = arithmodel_order_zero_create();
arithmodel_encode_init(low_ams[i], ac);
arithmodel_order_zero_reset(low_ams[i], 0, 0);
for (j = 0; j < (1 << vmpm->nlowbits); j++)
arithmodel_install_symbol(low_ams[i], 1);
}
for (i = 0; i < vmpm->buffer_low_size; i++)
arithmodel_encode(low_ams[vmpm->buffer_high[i]], vmpm->buffer_low[i]);
for (i = 0; i < (1 << (8 - vmpm->nlowbits)); i++)
arithmodel_encode_final(low_ams[i]);
for (i = 0; i < (1 << (8 - vmpm->nlowbits)); i++)
arithmodel_destroy(low_ams[i]);
arithcoder_encode_final(ac);
arithcoder_destroy(ac);
free(low_ams);
}
static void
decode(VMPM *vmpm)
{
Arithcoder *ac;
Arithmodel *am;
Arithmodel *bin_am;
unsigned int *index_to_symbol;
int i, n;
unsigned int j;
//debug_message_fn("()\n");
fatal("DECODING IS INVALID. NEED REIMPLEMENTATION.\n");
ac = arithcoder_arith_create();
arithcoder_decode_init(ac, vmpm->infile);
am = arithmodel_order_zero_create();
arithmodel_decode_init(am, ac);
bin_am = arithmodel_order_zero_create();
arithmodel_decode_init(bin_am, ac);
i = fgetc(vmpm->outfile);
vmpm->token_index[i] = vmpm->blocksize / ipow(vmpm->r, i);
for (; i >= 1; i--) {
unsigned int nsymbols = 0;
stat_message(vmpm, "D:Level %d (%d tokens, %d distinct): ", i, vmpm->token_index[i], vmpm->newtoken[i] - 1);
vmpm->token_index[i - 1] = 0;
if ((vmpm->tokens[i] = calloc(vmpm->token_index[i], sizeof(Token))) == NULL)
memory_error(NULL, MEMORY_ERROR);
/* Decode escape symbols at once. */
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
for (j = 0; j < vmpm->token_index[i]; j++) {
Index idx;
arithmodel_decode(bin_am, &idx);
if (idx) {
nsymbols++;
vmpm->token[i][j]->value = nsymbols;
} else {
vmpm->token[i][j]->value = 0;
}
}
/* Then decode non-escape symbols. */
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
for (j = 0; j < vmpm->token_index[i]; j++) {
if (vmpm->token[i][j]->value == 0) {
Token_value tv;
arithmodel_decode_cbt(bin_am, &tv, nsymbols - 1, 0, 1);
vmpm->token[i][j]->value = tv + 1;
stat_message(vmpm, "%d ", tv);
} else {
nsymbols++;
stat_message(vmpm, "e ");
}
}
stat_message(vmpm, "\n");
vmpm->token_index[i - 1] = nsymbols * vmpm->r;
}
if ((index_to_symbol = calloc(vmpm->alphabetsize, sizeof(unsigned int))) == NULL)
memory_error(NULL, MEMORY_ERROR);
n = 0;
arithmodel_order_zero_reset(am, 0, vmpm->alphabetsize - 1);
stat_message(vmpm, "D:Level 0 (%d tokens): ", vmpm->token_index[0]);
for (j = 0; j < vmpm->token_index[0]; j++) {
Index v;
if (arithmodel_decode(am, &v) == 2) {
arithmodel_decode_bits(bin_am, &vmpm->tokens[0][j].value, vmpm->bits_per_symbol, 0, 1);
index_to_symbol[n++] = vmpm->tokens[0][j].value;
stat_message(vmpm, "e ");
} else {
vmpm->tokens[0][j].value = index_to_symbol[v];
stat_message(vmpm, "%d ", v);
}
}
stat_message(vmpm, "\n");
free(index_to_symbol);
arithcoder_decode_final(ac);
arithmodel_destroy(am);
arithcoder_destroy(ac);
}
static void
encode(VMPM *vmpm)
{
Arithcoder *ac;
Arithmodel **ams;
Arithmodel *bin_am;
Arithmodel *level_am;
unsigned int j, pos, nsymbols, *s_to_i;
int i, match_found;
//debug_message("Recur3: %s()\n", __FUNCTION__);
if (!vmpm->outfile) {
debug_message_fnc("outfile is NULL.\n");
return;
}
ac = arithcoder_arith_create();
arithcoder_encode_init(ac, vmpm->outfile);
match_found = 0;
for (i = vmpm->I; i >= 1; i--) {
nsymbols = 0;
for (j = 0; j < vmpm->token_index[i]; j++) {
Token *t = vmpm->token[i][j];
Token_value tv = t->value - 1;
if (nsymbols == tv) {
nsymbols++;
} else {
match_found++;
break;
}
}
if (match_found) {
stat_message(vmpm, "Match found at Level %d\n", i);
break;
}
}
fputc(i, vmpm->outfile);
vmpm->I = i;
if ((ams = calloc(vmpm->I + 1, sizeof(Arithmodel **))) == NULL)
memory_error(NULL, MEMORY_ERROR);
for (i = 0; i <= (int)vmpm->I; i++) {
ams[i] = arithmodel_order_zero_create();
arithmodel_encode_init(ams[i], ac);
arithmodel_order_zero_reset(ams[i], 0, 0);
}
arithmodel_order_zero_reset(ams[0], 0, 1);
bin_am = arithmodel_order_zero_create();
arithmodel_encode_init(bin_am, ac);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
level_am = arithmodel_order_zero_create();
arithmodel_encode_init(level_am, ac);
arithmodel_order_zero_reset(level_am, 0, 0);
arithmodel_install_symbol(level_am, 1);
if ((s_to_i = malloc(vmpm->alphabetsize * sizeof(unsigned int))) == NULL)
memory_error(NULL, MEMORY_ERROR);
memset(s_to_i, 255, vmpm->alphabetsize * sizeof(unsigned int));
pos = 0;
for (j = 0; j < vmpm->token_index[vmpm->I]; j++)
pos += encode_recursively(vmpm, ams, bin_am, level_am, s_to_i, vmpm->I, j, pos);
for (i = vmpm->I - 1; i >= 0; i--) {
unsigned int n = (vmpm->newtoken[i + 1] - 1) * vmpm->r;
for (j = n; j < vmpm->token_index[i]; j++)
pos += encode_recursively(vmpm, ams, bin_am, level_am, s_to_i, i, j, pos);
}
stat_message(vmpm, "\n%d symbols encoded.\n", pos);
free(s_to_i);
for (i = 0; i <= (int)vmpm->I; i++)
arithmodel_encode_final(ams[i]);
arithmodel_encode_final(level_am);
arithcoder_encode_final(ac);
for (i = 0; i <= (int)vmpm->I; i++)
arithmodel_destroy(ams[i]);
arithmodel_destroy(level_am);
arithcoder_destroy(ac);
}
static void
decode(VMPM *vmpm)
{
Arithcoder *ac;
Arithmodel *am;
Arithmodel *bin_am;
unsigned int *index_to_symbol;
int i, n;
unsigned int j;
//debug_message_fn("()\n");
ac = arithcoder_arith_create();
arithcoder_decode_init(ac, vmpm->infile);
am = arithmodel_order_zero_create();
arithmodel_decode_init(am, ac);
bin_am = arithmodel_order_zero_create();
arithmodel_decode_init(bin_am, ac);
i = fgetc(vmpm->outfile);
vmpm->token_index[i] = vmpm->blocksize / ipow(vmpm->r, i);
for (; i >= 1; i--) {
unsigned int tmp;
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
arithmodel_decode_cbt(bin_am, &tmp, vmpm->token_index[i], 0, 1);
vmpm->newtoken[i] = tmp + 1;
stat_message(vmpm, "D:Level %d (%d tokens, %d distinct): ", i, vmpm->token_index[i], vmpm->newtoken[i] - 1);
vmpm->token_index[i - 1] = 0;
if ((vmpm->tokens[i] = calloc(vmpm->token_index[i], sizeof(Token))) == NULL)
memory_error(NULL, MEMORY_ERROR);
arithmodel_order_zero_reset(am, 0, vmpm->newtoken[i]);
vmpm->tokens[i][0].value = 1;
for (j = 1; j < vmpm->token_index[i]; j++) {
Index idx;
arithmodel_decode(am, &idx);
vmpm->tokens[i][j].value = idx + 1;
if (vmpm->tokens[i][j].value > vmpm->token_index[i - 1]) {
if (vmpm->tokens[i][j].value == vmpm->token_index[i - 1] + 1) {
stat_message(vmpm, "e ");
vmpm->token_index[i - 1]++;
} else
generic_error((char *)"Invalid token value.\n", INVALID_TOKEN_VALUE_ERROR);
} else {
stat_message(vmpm, "%d ", idx);
}
}
vmpm->token_index[i - 1] *= vmpm->r;
stat_message(vmpm, "\n");
}
if ((index_to_symbol = calloc(vmpm->alphabetsize, sizeof(unsigned int))) == NULL)
memory_error(NULL, MEMORY_ERROR);
n = 0;
arithmodel_order_zero_reset(am, 0, vmpm->alphabetsize - 1);
arithmodel_order_zero_reset(bin_am, 0, 0);
arithmodel_install_symbol(bin_am, 1);
arithmodel_install_symbol(bin_am, 1);
stat_message(vmpm, "D:Level 0 (%d tokens): ", vmpm->token_index[0]);
for (j = 0; j < vmpm->token_index[0]; j++) {
Index v;
if (arithmodel_decode(am, &v) == 2) {
stat_message(vmpm, "e ");
arithmodel_decode_bits(bin_am, &vmpm->tokens[0][j].value, vmpm->bits_per_symbol, 0, 1);
index_to_symbol[n++] = vmpm->tokens[0][j].value;
} else {
vmpm->tokens[0][j].value = index_to_symbol[v];
stat_message(vmpm, "%d ", v);
}
}
stat_message(vmpm, "\n");
free(index_to_symbol);
arithcoder_decode_final(ac);
arithmodel_destroy(bin_am);
arithmodel_destroy(am);
arithcoder_destroy(ac);
}