bool generate_decoder_tables(uint num_syms, const uint8* pCodesizes, decoder_tables* pTables, uint table_bits, const code_size_histogram &code_size_histo, bool sym_freq_all_ones) { uint min_codes[cMaxExpectedHuffCodeSize]; if ((!num_syms) || (table_bits > cMaxTableBits)) { LZHAM_LOG_ERROR(3004); return false; } pTables->m_num_syms = num_syms; uint sorted_positions[cMaxExpectedHuffCodeSize + 1]; uint next_code = 0; uint total_used_syms = 0; uint max_code_size = 0; uint min_code_size = UINT_MAX; for (uint i = 1; i <= cMaxExpectedHuffCodeSize; i++) { const uint n = code_size_histo.m_num_codes[i]; if (!n) pTables->m_max_codes[i - 1] = 0;//UINT_MAX; else { min_code_size = math::minimum(min_code_size, i); max_code_size = math::maximum(max_code_size, i); min_codes[i - 1] = next_code; pTables->m_max_codes[i - 1] = next_code + n - 1; pTables->m_max_codes[i - 1] = 1 + ((pTables->m_max_codes[i - 1] << (16 - i)) | ((1 << (16 - i)) - 1)); pTables->m_val_ptrs[i - 1] = total_used_syms; sorted_positions[i] = total_used_syms; next_code += n; total_used_syms += n; } next_code <<= 1; } pTables->m_total_used_syms = total_used_syms; if (total_used_syms > pTables->m_cur_sorted_symbol_order_size) { pTables->m_cur_sorted_symbol_order_size = total_used_syms; if (!math::is_power_of_2(total_used_syms)) pTables->m_cur_sorted_symbol_order_size = math::minimum<uint>(num_syms, math::next_pow2(total_used_syms)); if (pTables->m_sorted_symbol_order) { lzham_delete_array(pTables->m_malloc_context, pTables->m_sorted_symbol_order); pTables->m_sorted_symbol_order = NULL; } pTables->m_sorted_symbol_order = lzham_new_array<uint16>(pTables->m_malloc_context, pTables->m_cur_sorted_symbol_order_size); if (!pTables->m_sorted_symbol_order) { LZHAM_LOG_ERROR(3005); return false; } } pTables->m_min_code_size = static_cast<uint8>(min_code_size); pTables->m_max_code_size = static_cast<uint8>(max_code_size); if (sym_freq_all_ones) { if (min_code_size == max_code_size) { memcpy(pTables->m_sorted_symbol_order, g_uint16_sequence, num_syms * sizeof(uint16)); } else { LZHAM_ASSERT((min_code_size + 1) == max_code_size); LZHAM_ASSERT(pCodesizes[0] == max_code_size); LZHAM_ASSERT(pCodesizes[code_size_histo.m_num_codes[max_code_size]] == min_code_size); memcpy(pTables->m_sorted_symbol_order + sorted_positions[max_code_size], g_uint16_sequence, code_size_histo.m_num_codes[max_code_size] * sizeof(uint16)); memcpy(pTables->m_sorted_symbol_order + sorted_positions[min_code_size], g_uint16_sequence + code_size_histo.m_num_codes[max_code_size], code_size_histo.m_num_codes[min_code_size] * sizeof(uint16)); } #ifdef LZHAM_BUILD_DEBUG for (uint i = 0; i < num_syms; i++) { uint c = pCodesizes[i]; LZHAM_ASSERT(code_size_histo.m_num_codes[c]); uint sorted_pos = sorted_positions[c]++; LZHAM_ASSERT(sorted_pos < total_used_syms); LZHAM_ASSERT(pTables->m_sorted_symbol_order[sorted_pos] == i); } #endif } else { for (uint i = 0; i < num_syms; i++) { uint c = pCodesizes[i]; LZHAM_ASSERT(code_size_histo.m_num_codes[c]); uint sorted_pos = sorted_positions[c]++; LZHAM_ASSERT(sorted_pos < total_used_syms); pTables->m_sorted_symbol_order[sorted_pos] = static_cast<uint16>(i); } } if (table_bits <= pTables->m_min_code_size) table_bits = 0; pTables->m_table_bits = table_bits; if (table_bits) { uint table_size = 1 << table_bits; if (table_size > pTables->m_cur_lookup_size) { pTables->m_cur_lookup_size = table_size; if (pTables->m_lookup) { lzham_delete_array(pTables->m_malloc_context, pTables->m_lookup); pTables->m_lookup = NULL; } pTables->m_lookup = lzham_new_array<uint32>(pTables->m_malloc_context, table_size); if (!pTables->m_lookup) { LZHAM_LOG_ERROR(3006); return false; } } memset(pTables->m_lookup, 0xFF, static_cast<uint>(sizeof(pTables->m_lookup[0])) * (1UL << table_bits)); for (uint codesize = 1; codesize <= table_bits; codesize++) { if (!code_size_histo.m_num_codes[codesize]) continue; const uint fillsize = table_bits - codesize; const uint fillnum = 1 << fillsize; const uint min_code = min_codes[codesize - 1]; const uint max_code = pTables->get_unshifted_max_code(codesize); const uint val_ptr = pTables->m_val_ptrs[codesize - 1]; for (uint code = min_code; code <= max_code; code++) { const uint sym_index = pTables->m_sorted_symbol_order[ val_ptr + code - min_code ]; LZHAM_ASSERT( pCodesizes[sym_index] == codesize ); for (uint j = 0; j < fillnum; j++) { const uint t = j + (code << fillsize); LZHAM_ASSERT(t < (1U << table_bits)); LZHAM_ASSERT(pTables->m_lookup[t] == cUINT32_MAX); pTables->m_lookup[t] = sym_index | (codesize << 16U); } } } } for (uint i = 0; i < cMaxExpectedHuffCodeSize; i++) pTables->m_val_ptrs[i] -= min_codes[i]; pTables->m_table_max_code = 0; pTables->m_decode_start_code_size = pTables->m_min_code_size; if (table_bits) { uint i; for (i = table_bits; i >= 1; i--) { if (code_size_histo.m_num_codes[i]) { pTables->m_table_max_code = pTables->m_max_codes[i - 1]; break; } } if (i >= 1) { pTables->m_decode_start_code_size = table_bits + 1; for (i = table_bits + 1; i <= max_code_size; i++) { if (code_size_histo.m_num_codes[i]) { pTables->m_decode_start_code_size = i; break; } } } } // sentinels pTables->m_max_codes[cMaxExpectedHuffCodeSize] = UINT_MAX; pTables->m_val_ptrs[cMaxExpectedHuffCodeSize] = 0xFFFFF; pTables->m_table_shift = 32 - pTables->m_table_bits; return true; }
bool generate_decoder_tables(uint num_syms, const uint8* pCodesizes, decoder_tables* pTables, uint table_bits) { uint min_codes[cMaxExpectedCodeSize]; if ((!num_syms) || (table_bits > cMaxTableBits)) return false; pTables->m_num_syms = num_syms; uint num_codes[cMaxExpectedCodeSize + 1]; utils::zero_object(num_codes); for (uint i = 0; i < num_syms; i++) { uint c = pCodesizes[i]; num_codes[c]++; } uint sorted_positions[cMaxExpectedCodeSize + 1]; uint code = 0; uint total_used_syms = 0; uint max_code_size = 0; uint min_code_size = UINT_MAX; for (uint i = 1; i <= cMaxExpectedCodeSize; i++) { const uint n = num_codes[i]; if (!n) pTables->m_max_codes[i - 1] = 0;//UINT_MAX; else { min_code_size = math::minimum(min_code_size, i); max_code_size = math::maximum(max_code_size, i); min_codes[i - 1] = code; pTables->m_max_codes[i - 1] = code + n - 1; pTables->m_max_codes[i - 1] = 1 + ((pTables->m_max_codes[i - 1] << (16 - i)) | ((1 << (16 - i)) - 1)); pTables->m_val_ptrs[i - 1] = total_used_syms; sorted_positions[i] = total_used_syms; code += n; total_used_syms += n; } code <<= 1; } pTables->m_total_used_syms = total_used_syms; if (total_used_syms > pTables->m_cur_sorted_symbol_order_size) { pTables->m_cur_sorted_symbol_order_size = total_used_syms; if (!math::is_power_of_2(total_used_syms)) pTables->m_cur_sorted_symbol_order_size = math::minimum<uint>(num_syms, math::next_pow2(total_used_syms)); if (pTables->m_sorted_symbol_order) { lzham_delete_array(pTables->m_sorted_symbol_order); pTables->m_sorted_symbol_order = NULL; } pTables->m_sorted_symbol_order = lzham_new_array<uint16>(pTables->m_cur_sorted_symbol_order_size); if (!pTables->m_sorted_symbol_order) return false; } pTables->m_min_code_size = static_cast<uint8>(min_code_size); pTables->m_max_code_size = static_cast<uint8>(max_code_size); for (uint i = 0; i < num_syms; i++) { uint c = pCodesizes[i]; if (c) { LZHAM_ASSERT(num_codes[c]); uint sorted_pos = sorted_positions[c]++; LZHAM_ASSERT(sorted_pos < total_used_syms); pTables->m_sorted_symbol_order[sorted_pos] = static_cast<uint16>(i); } } if (table_bits <= pTables->m_min_code_size) table_bits = 0; pTables->m_table_bits = table_bits; if (table_bits) { uint table_size = 1 << table_bits; if (table_size > pTables->m_cur_lookup_size) { pTables->m_cur_lookup_size = table_size; if (pTables->m_lookup) { lzham_delete_array(pTables->m_lookup); pTables->m_lookup = NULL; } pTables->m_lookup = lzham_new_array<uint32>(table_size); if (!pTables->m_lookup) return false; } memset(pTables->m_lookup, 0xFF, static_cast<uint>(sizeof(pTables->m_lookup[0])) * (1UL << table_bits)); for (uint codesize = 1; codesize <= table_bits; codesize++) { if (!num_codes[codesize]) continue; const uint fillsize = table_bits - codesize; const uint fillnum = 1 << fillsize; const uint min_code = min_codes[codesize - 1]; const uint max_code = pTables->get_unshifted_max_code(codesize); const uint val_ptr = pTables->m_val_ptrs[codesize - 1]; for (uint code = min_code; code <= max_code; code++) { const uint sym_index = pTables->m_sorted_symbol_order[ val_ptr + code - min_code ]; LZHAM_ASSERT( pCodesizes[sym_index] == codesize ); for (uint j = 0; j < fillnum; j++) { const uint t = j + (code << fillsize); LZHAM_ASSERT(t < (1U << table_bits)); LZHAM_ASSERT(pTables->m_lookup[t] == UINT32_MAX); pTables->m_lookup[t] = sym_index | (codesize << 16U); } } } } for (uint i = 0; i < cMaxExpectedCodeSize; i++) pTables->m_val_ptrs[i] -= min_codes[i]; pTables->m_table_max_code = 0; pTables->m_decode_start_code_size = pTables->m_min_code_size; if (table_bits) { uint i; for (i = table_bits; i >= 1; i--) { if (num_codes[i]) { pTables->m_table_max_code = pTables->m_max_codes[i - 1]; break; } } if (i >= 1) { pTables->m_decode_start_code_size = table_bits + 1; for (uint i = table_bits + 1; i <= max_code_size; i++) { if (num_codes[i]) { pTables->m_decode_start_code_size = i; break; } } } } // sentinels pTables->m_max_codes[cMaxExpectedCodeSize] = UINT_MAX; pTables->m_val_ptrs[cMaxExpectedCodeSize] = 0xFFFFF; pTables->m_table_shift = 32 - pTables->m_table_bits; return true; }