TEST encoder_should_emit_series_of_same_byte_as_literal_then_backref() { heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7); ASSERT(hse); uint8_t input[5]; uint8_t output[1024]; uint16_t copied = 0; uint8_t expected[] = {0xb0, 0x80, 0x01, 0x80}; for (int i=0; i<5; i++) { input[i] = 'a'; } /* "aaaaa" */ memset(output, 0, 1024); ASSERT_EQ(HSER_SINK_OK, heatshrink_encoder_sink(hse, input, 5, &copied)); ASSERT_EQ(5, copied); /* Should get no output yet, since encoder doesn't know input is complete. */ copied = 0; HSE_poll_res pres = heatshrink_encoder_poll(hse, output, 1024, &copied); ASSERT_EQ(HSER_POLL_EMPTY, pres); ASSERT_EQ(0, copied); /* Mark input stream as done, to force small input to be processed. */ HSE_finish_res fres = heatshrink_encoder_finish(hse); ASSERT_EQ(HSER_FINISH_MORE, fres); pres = heatshrink_encoder_poll(hse, output, 1024, &copied); ASSERT_EQ(HSER_POLL_EMPTY, pres); ASSERT_EQ(4, copied); if (0) dump_buf("output", output, copied); for (int i=0; i<copied; i++) ASSERT_EQ(expected[i], output[i]); ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(hse)); heatshrink_encoder_free(hse); PASS(); }
TEST encoder_should_emit_data_without_repetitions_as_literal_sequence() { heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7); ASSERT(hse); uint8_t input[5]; uint8_t output[1024]; uint16_t copied = 0; uint8_t expected[] = { 0x80, 0x40, 0x60, 0x50, 0x38, 0x20 }; for (int i=0; i<5; i++) { input[i] = i; } memset(output, 0, 1024); ASSERT_EQ(HSER_SINK_OK, heatshrink_encoder_sink(hse, input, 5, &copied)); ASSERT_EQ(5, copied); /* Should get no output yet, since encoder doesn't know input is complete. */ copied = 0; HSE_poll_res pres = heatshrink_encoder_poll(hse, output, 1024, &copied); ASSERT_EQ(HSER_POLL_EMPTY, pres); ASSERT_EQ(0, copied); /* Mark input stream as done, to force small input to be processed. */ HSE_finish_res fres = heatshrink_encoder_finish(hse); ASSERT_EQ(HSER_FINISH_MORE, fres); pres = heatshrink_encoder_poll(hse, output, 1024, &copied); ASSERT_EQ(HSER_POLL_EMPTY, pres); for (int i=0; i<sizeof(expected); i++) { ASSERT_EQ(expected[i], output[i]); } ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(hse)); heatshrink_encoder_free(hse); PASS(); }
TEST encoder_poll_should_reject_nulls() { heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7); uint8_t output[256]; uint16_t output_size = 0; ASSERT_EQ(HSER_POLL_ERROR_NULL, heatshrink_encoder_poll(NULL, output, 256, &output_size)); ASSERT_EQ(HSER_POLL_ERROR_NULL, heatshrink_encoder_poll(hse, NULL, 256, &output_size)); ASSERT_EQ(HSER_POLL_ERROR_NULL, heatshrink_encoder_poll(hse, output, 256, NULL)); heatshrink_encoder_free(hse); PASS(); }
static int encoder_sink_read(config *cfg, heatshrink_encoder *hse, uint8_t *data, size_t data_sz) { size_t out_sz = 4096; uint8_t out_buf[out_sz]; memset(out_buf, 0, out_sz); size_t sink_sz = 0; size_t poll_sz = 0; HSE_sink_res sres; HSE_poll_res pres; HSE_finish_res fres; io_handle *out = cfg->out; size_t sunk = 0; do { if (data_sz > 0) { sres = heatshrink_encoder_sink(hse, &data[sunk], data_sz - sunk, &sink_sz); if (sres < 0) { die("sink"); } sunk += sink_sz; } do { pres = heatshrink_encoder_poll(hse, out_buf, out_sz, &poll_sz); if (pres < 0) { die("poll"); } if (handle_sink(out, poll_sz, out_buf) < 0) die("handle_sink"); } while (pres == HSER_POLL_MORE); if (poll_sz == 0 && data_sz == 0) { fres = heatshrink_encoder_finish(hse); if (fres < 0) { die("finish"); } if (fres == HSER_FINISH_DONE) { return 1; } } } while (sunk < data_sz); return 0; }
TEST gen() { heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7); uint8_t input[] = {'a', 'a', 'a', 'a', 'a'}; uint8_t output[1024]; uint16_t copied = 0; memset(output, 0, 1024); HSE_sink_res sres = heatshrink_encoder_sink(hse, input, sizeof(input), &copied); ASSERT_EQ(HSER_SINK_OK, sres); ASSERT_EQ(sizeof(input), copied); HSE_finish_res fres = heatshrink_encoder_finish(hse); ASSERT_EQ(HSER_FINISH_MORE, fres); ASSERT_EQ(HSER_POLL_EMPTY, heatshrink_encoder_poll(hse, output, 1024, &copied)); fres = heatshrink_encoder_finish(hse); ASSERT_EQ(HSER_FINISH_DONE, fres); if (0) { printf("{"); for (int i=0; i<copied; i++) printf("0x%02x, ", output[i]); printf("}\n"); } heatshrink_encoder_free(hse); PASS(); }
TEST encoder_poll_should_detect_repeated_substring_and_preserve_trailing_literal() { heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 3); uint8_t input[] = {'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd', 'e'}; uint8_t output[1024]; uint8_t expected[] = {0xb0, 0xd8, 0xac, 0x76, 0x40, 0x1b, 0xb2, 0x80 }; uint16_t copied = 0; memset(output, 0, 1024); HSE_sink_res sres = heatshrink_encoder_sink(hse, input, sizeof(input), &copied); ASSERT_EQ(HSER_SINK_OK, sres); ASSERT_EQ(sizeof(input), copied); HSE_finish_res fres = heatshrink_encoder_finish(hse); ASSERT_EQ(HSER_FINISH_MORE, fres); ASSERT_EQ(HSER_POLL_EMPTY, heatshrink_encoder_poll(hse, output, 1024, &copied)); fres = heatshrink_encoder_finish(hse); ASSERT_EQ(HSER_FINISH_DONE, fres); if (0) dump_buf("output", output, copied); ASSERT_EQ(sizeof(expected), copied); for (int i=0; i<sizeof(expected); i++) ASSERT_EQ(expected[i], output[i]); heatshrink_encoder_free(hse); PASS(); }
/** The core of compression. */ int buffer_compress(uint8_t *db, size_t length, uint8_t *comp) { size_t comp_sz = (db->item_cnt + (db->item_cnt / 2) + 4) * db->item_size; comp = malloc(comp_sz); memset(comp, 0, comp_sz); size_t count = 0; uint32_t sunk = 0; uint32_t polled = 0; while (sunk < input_size) { ASSERT(heatshrink_encoder_sink(&hse, &input[sunk], input_size - sunk, &count) >= 0); sunk += count; if (sunk == input_size) { ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(&hse)); } HSE_poll_res pres; do { /* "turn the crank" */ pres = heatshrink_encoder_poll(&hse, &comp[polled], comp_sz - polled, &count); ASSERT(pres >= 0); polled += count; } while (pres == HSER_POLL_MORE); ASSERT_EQ(HSER_POLL_EMPTY, pres); if (polled >= comp_sz) { printf("compression should never expand that muchr\r\n"); show_error(); return -1; } if (sunk == input_size) { ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(&hse)); } } return BUFFER_SUCCESS; }
/** gets data from array, writes to callback if nonzero. Returns total length. */ uint32_t heatshrink_encode(unsigned char *data, size_t dataLen, void (*callback)(unsigned char ch, uint32_t *cbdata), uint32_t *cbdata) { heatshrink_encoder hse; uint8_t outBuf[BUFFERSIZE]; heatshrink_encoder_reset(&hse); size_t i; size_t count = 0; size_t sunk = 0; size_t polled = 0; while (sunk < dataLen) { bool ok = heatshrink_encoder_sink(&hse, &data[sunk], dataLen - sunk, &count) >= 0; assert(ok);NOT_USED(ok); sunk += count; if (sunk == dataLen) { heatshrink_encoder_finish(&hse); } HSE_poll_res pres; do { pres = heatshrink_encoder_poll(&hse, outBuf, sizeof(outBuf), &count); assert(pres >= 0); if (callback) for (i=0;i<count;i++) callback(outBuf[i], cbdata); polled += count; } while (pres == HSER_POLL_MORE); assert(pres == HSER_POLL_EMPTY); if (sunk == dataLen) { heatshrink_encoder_finish(&hse); } } return (uint32_t)polled; }
TEST encoder_poll_should_indicate_when_no_input_is_provided() { heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7); uint8_t output[512]; uint16_t output_size = 0; HSE_poll_res res = heatshrink_encoder_poll(hse, output, 512, &output_size); ASSERT_EQ(HSER_POLL_EMPTY, res); heatshrink_encoder_free(hse); PASS(); }
static int compress_and_expand_and_check(uint8_t *input, uint32_t input_size) { // clear encoder&decoder state machines heatshrink_encoder_reset(&hse); heatshrink_decoder_reset(&hsd); // allocate memory for before and after size_t comp_sz = input_size + (input_size/2) + 4; uint8_t *comp = malloc(comp_sz); if (comp == NULL) printf("malloc fail\r\n"); memset(comp, 0, comp_sz); size_t count = 0; uint32_t sunk = 0; uint32_t polled = 0; while (sunk < input_size) { ASSERT(heatshrink_encoder_sink(&hse, &input[sunk], input_size - sunk, &count) >= 0); sunk += count; if (sunk == input_size) { ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(&hse)); } HSE_poll_res pres; do { /* "turn the crank" */ pres = heatshrink_encoder_poll(&hse, &comp[polled], comp_sz - polled, &count); ASSERT(pres >= 0); polled += count; } while (pres == HSER_POLL_MORE); ASSERT_EQ(HSER_POLL_EMPTY, pres); if (polled >= comp_sz) { printf("compression should never expand that muchr\r\n"); show_error(); return-1;} if (sunk == input_size) { ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(&hse)); } } for (int i = 0; i < polled; i++) { while(app_uart_put(comp[i]) != NRF_SUCCESS); // for(int time_ctr = 0; time_ctr <100000; time_ctr ++); } // while(app_uart_put(13) != NRF_SUCCESS); // printf("%d", polled); free(comp); }
size_t compressHeatshrink(char *in, int insize, char *out, int outsize, int level) { char *inp=in; char *outp=out; size_t len; int ws[]={5, 6, 8, 11, 13}; int ls[]={3, 3, 4, 4, 4}; HSE_poll_res pres; HSE_sink_res sres; size_t r; if (level==-1) level=8; level=(level-1)/2; //level is now 0, 1, 2, 3, 4 heatshrink_encoder *enc=heatshrink_encoder_alloc(ws[level], ls[level]); if (enc==NULL) { perror("allocating mem for heatshrink"); exit(1); } //Save encoder parms as first byte *outp=(ws[level]<<4)|ls[level]; outp++; outsize--; r=1; do { if (insize>0) { sres=heatshrink_encoder_sink(enc, inp, insize, &len); if (sres!=HSER_SINK_OK) break; inp+=len; insize-=len; if (insize==0) heatshrink_encoder_finish(enc); } do { pres=heatshrink_encoder_poll(enc, outp, outsize, &len); if (pres!=HSER_POLL_MORE && pres!=HSER_POLL_EMPTY) break; outp+=len; outsize-=len; r+=len; } while (pres==HSER_POLL_MORE); } while (insize!=0); if (insize!=0) { fprintf(stderr, "Heatshrink: Bug? insize is still %d. sres=%d pres=%d\n", insize, sres, pres); exit(1); } heatshrink_encoder_free(enc); return r; }
void compress(uint32_t buf, uint32_t size) { size_t count = 0; uint32_t sunk = 0; uint32_t polled = 0; // Compression in progress. //compression_in_progress = true; //printf("Buf %d compression start...\r\n", buf); // Clear encoder state machine heatshrink_encoder_reset(&hse); while (sunk < size) { ASSERTC(heatshrink_encoder_sink(&hse, &(data_buffers[buf][sunk]), size - sunk, &count) >= 0); sunk += count; if (sunk == size) { ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(&hse)); } HSE_poll_res pres; do { /* "turn the crank" */ pres = heatshrink_encoder_poll(&hse, &transmission_buffer[polled], TRANSMISSION_BUF_SIZE - polled, &count); ASSERTC(pres >= 0); polled += count; } while (pres == HSER_POLL_MORE); ASSERT_EQ(HSER_POLL_EMPTY, pres); if (polled >= TRANSMISSION_BUF_SIZE) { printf("compression should never expand that muchr\r\n"); } if (sunk == size) { ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(&hse)); } } // Finished compression. buffers_ready[buf] = false; printf("Buf %d compression done\r\n", buf); send_data(transmission_buffer, polled); }
TEST data_with_simple_repetition_should_match_with_absurdly_tiny_buffers() { heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 3); heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 3); uint8_t input[] = {'a', 'b', 'c', 'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd', 'e', 'a', 'b', 'c', 'd', 'e', 'f', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}; uint8_t comp[60]; uint8_t decomp[60]; uint16_t count = 0; int log = 0; if (log) dump_buf("input", input, sizeof(input)); for (int i=0; i<sizeof(input); i++) { ASSERT(heatshrink_encoder_sink(hse, &input[i], 1, &count) >= 0); } ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); uint16_t packed_count = 0; do { ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0); packed_count += count; } while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE); if (log) dump_buf("comp", comp, packed_count); for (int i=0; i<packed_count; i++) { HSD_sink_res sres = heatshrink_decoder_sink(hsd, &comp[i], 1, &count); //printf("sres is %d\n", sres); ASSERT(sres >= 0); } for (int i=0; i<sizeof(input); i++) { ASSERT(heatshrink_decoder_poll(hsd, &decomp[i], 1, &count) >= 0); } if (log) dump_buf("decomp", decomp, sizeof(input)); for (int i=0; i<sizeof(input); i++) ASSERT_EQ(input[i], decomp[i]); heatshrink_encoder_free(hse); heatshrink_decoder_free(hsd); PASS(); }
TEST data_without_duplication_should_match_with_absurdly_tiny_buffers(void) { heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 3); heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 3); uint8_t input[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; uint8_t comp[60]; uint8_t decomp[60]; size_t count = 0; int log = 0; if (log) dump_buf("input", input, sizeof(input)); for (uint32_t i=0; i<sizeof(input); i++) { ASSERT(heatshrink_encoder_sink(hse, &input[i], 1, &count) >= 0); } ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); size_t packed_count = 0; do { ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0); packed_count += count; } while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE); if (log) dump_buf("comp", comp, packed_count); for (uint32_t i=0; i<packed_count; i++) { HSD_sink_res sres = heatshrink_decoder_sink(hsd, &comp[i], 1, &count); //printf("sres is %d\n", sres); ASSERT(sres >= 0); } for (uint32_t i=0; i<sizeof(input); i++) { ASSERT(heatshrink_decoder_poll(hsd, &decomp[i], 1, &count) >= 0); } if (log) dump_buf("decomp", decomp, sizeof(input)); for (uint32_t i=0; i<sizeof(input); i++) ASSERT_EQ(input[i], decomp[i]); heatshrink_encoder_free(hse); heatshrink_decoder_free(hsd); PASS(); }
static int compress_and_expand_and_check(uint8_t *input, uint32_t input_size, cfg_info *cfg) { heatshrink_encoder *hse = heatshrink_encoder_alloc(cfg->window_sz2, cfg->lookahead_sz2); heatshrink_decoder *hsd = heatshrink_decoder_alloc(cfg->decoder_input_buffer_size, cfg->window_sz2, cfg->lookahead_sz2); size_t comp_sz = input_size + (input_size/2) + 4; size_t decomp_sz = input_size + (input_size/2) + 4; uint8_t *comp = malloc(comp_sz); uint8_t *decomp = malloc(decomp_sz); if (comp == NULL) FAILm("malloc fail"); if (decomp == NULL) FAILm("malloc fail"); memset(comp, 0, comp_sz); memset(decomp, 0, decomp_sz); uint16_t count = 0; if (cfg->log_lvl > 1) { printf("\n^^ COMPRESSING\n"); dump_buf("input", input, input_size); } uint32_t sunk = 0; uint32_t polled = 0; while (sunk < input_size) { ASSERT(heatshrink_encoder_sink(hse, &input[sunk], input_size - sunk, &count) >= 0); sunk += count; if (cfg->log_lvl > 1) printf("^^ sunk %d\n", count); if (sunk == input_size) { ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); } HSE_poll_res pres; do { /* "turn the crank" */ pres = heatshrink_encoder_poll(hse, &comp[polled], comp_sz - polled, &count); ASSERT(pres >= 0); polled += count; if (cfg->log_lvl > 1) printf("^^ polled %d\n", count); } while (pres == HSER_POLL_MORE); ASSERT_EQ(HSER_POLL_EMPTY, pres); if (polled >= comp_sz) FAILm("compression should never expand that much"); if (sunk == input_size) { ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(hse)); } } if (cfg->log_lvl > 0) printf("in: %u compressed: %u ", input_size, polled); uint32_t compressed_size = polled; sunk = 0; polled = 0; if (cfg->log_lvl > 1) { printf("\n^^ DECOMPRESSING\n"); dump_buf("comp", comp, compressed_size); } while (sunk < compressed_size) { ASSERT(heatshrink_decoder_sink(hsd, &comp[sunk], compressed_size - sunk, &count) >= 0); sunk += count; if (cfg->log_lvl > 1) printf("^^ sunk %d\n", count); if (sunk == compressed_size) { ASSERT_EQ(HSDR_FINISH_MORE, heatshrink_decoder_finish(hsd)); } HSD_poll_res pres; do { pres = heatshrink_decoder_poll(hsd, &decomp[polled], decomp_sz - polled, &count); ASSERT(pres >= 0); ASSERT(count > 0); polled += count; if (cfg->log_lvl > 1) printf("^^ polled %d\n", count); } while (pres == HSDR_POLL_MORE); ASSERT_EQ(HSDR_POLL_EMPTY, pres); if (sunk == compressed_size) { HSD_finish_res fres = heatshrink_decoder_finish(hsd); ASSERT_EQ(HSDR_FINISH_DONE, fres); } if (polled > input_size) { printf("\nExpected %d, got %d\n", input_size, polled); FAILm("Decompressed data is larger than original input"); } } if (cfg->log_lvl > 0) printf("decompressed: %u\n", polled); if (polled != input_size) { FAILm("Decompressed length does not match original input length"); } if (cfg->log_lvl > 1) dump_buf("decomp", decomp, polled); for (int i=0; i<input_size; i++) { if (input[i] != decomp[i]) { printf("*** mismatch at %d\n", i); if (0) { for (int j=0; j<=/*i*/ input_size; j++) { printf("in[%d] == 0x%02x ('%c') => out[%d] == 0x%02x ('%c') %c\n", j, input[j], isprint(input[j]) ? input[j] : '.', j, decomp[j], isprint(decomp[j]) ? decomp[j] : '.', input[j] == decomp[j] ? ' ' : 'X'); } } } ASSERT_EQ(input[i], decomp[i]); } free(comp); free(decomp); heatshrink_encoder_free(hse); heatshrink_decoder_free(hsd); PASS(); }
/* PHP */ static ZEND_FUNCTION(hs_compress) { zval *data; size_t data_len; size_t sink_sz = 0; size_t poll_sz = 0; HSE_sink_res sres; HSE_poll_res pres; HSE_finish_res fres; //Default window_sz2 and lookahead_sz2 values uint8_t window = DEF_WINDOW_SZ2; uint8_t lookahead = DEF_LOOKAHEAD_SZ2; if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "z|ll", &data, &window, &lookahead) == FAILURE) { RETURN_FALSE; } if (Z_TYPE_P(data) != IS_STRING) { zend_error(E_WARNING, "hs_compress : expects parameter to be string."); RETURN_FALSE; } //Allocate encoder heatshrink_encoder *hse = heatshrink_encoder_alloc(window, lookahead); if (hse == NULL) { zend_error(E_WARNING, "hs_compress : heatshrink_encoder_alloc error. Check window and lookahead values."); RETURN_FALSE; } data_len = Z_STRLEN_P(data); //TODO: think of better way to compute outbuff size size_t outbuff_len = data_len+1; unsigned char *outbuff = (unsigned char *) emalloc(outbuff_len); size_t sunk = 0; do { sres = heatshrink_encoder_sink(hse, (unsigned char *) &(Z_STRVAL_P(data)[sunk]), data_len - sunk, &sink_sz); if (sres < 0) { zend_error(E_WARNING, "hs_compress : heatshrink_encoder_sink error"); efree(outbuff); heatshrink_encoder_free(hse); RETURN_FALSE; } sunk += sink_sz; do { do { pres = heatshrink_encoder_poll(hse, &outbuff[poll_sz], outbuff_len - poll_sz, &sink_sz); if (pres < 0) { zend_error(E_WARNING, "hs_compress : heatshrink_encoder_poll error"); efree(outbuff); heatshrink_encoder_free(hse); RETURN_FALSE; } poll_sz += sink_sz; if (poll_sz == outbuff_len && pres == HSER_POLL_MORE) { //TODO: think of bettery way to compute buffer reallocation size outbuff_len += data_len/2; outbuff = erealloc(outbuff, outbuff_len); } } while (pres == HSER_POLL_MORE); if (sunk == data_len) { fres = heatshrink_encoder_finish(hse); if (fres < 0) { zend_error(E_WARNING, "hs_compress : heatshrink_encoder_finish error"); efree(outbuff); heatshrink_encoder_free(hse); RETURN_FALSE; } if (fres == HSER_FINISH_DONE) { RETVAL_STRINGL((char *) outbuff, poll_sz, 1); efree(outbuff); heatshrink_encoder_free(hse); return; } } } while (fres == HSER_FINISH_MORE); } while (sunk < data_len); zend_error(E_WARNING, "hs_compress : general error"); efree(outbuff); heatshrink_encoder_free(hse); RETURN_FALSE; }