static void send_gzip(h2o_ostream_t *_self, h2o_req_t *req, h2o_iovec_t *inbufs, size_t inbufcnt, int is_final) { gzip_encoder_t *self = (void *)_self; size_t outbufindex; /* initialize deflate (Z_BEST_SPEED for on-the-fly compression, memlevel set to 8 as suggested by the manual) */ if (self->bufs.size == 0) { deflateInit2(&self->zstream, Z_BEST_SPEED, Z_DEFLATED, WINDOW_BITS, 8, Z_DEFAULT_STRATEGY); expand_buf(&req->pool, &self->bufs); } /* compress */ outbufindex = 0; self->bufs.entries[0].len = 0; if (inbufcnt != 0) { size_t i; for (i = 0; i != inbufcnt - 1; ++i) if (inbufs[i].len != 0) outbufindex = compress_chunk(&req->pool, &self->zstream, inbufs[i].base, inbufs[i].len, Z_NO_FLUSH, &self->bufs, outbufindex); outbufindex = compress_chunk(&req->pool, &self->zstream, inbufs[i].base, inbufs[i].len, is_final ? Z_FINISH : Z_SYNC_FLUSH, &self->bufs, outbufindex); } else { outbufindex = compress_chunk(&req->pool, &self->zstream, "", 0, Z_FINISH, &self->bufs, outbufindex); } /* destroy deflate */ if (is_final) deflateEnd(&self->zstream); h2o_ostream_send_next(&self->super, req, self->bufs.entries, outbufindex + 1, is_final); }
static void do_compress(h2o_compress_context_t *_self, h2o_iovec_t *inbufs, size_t inbufcnt, int is_final, h2o_iovec_t **outbufs, size_t *outbufcnt) { struct st_gzip_context_t *self = (void *)_self; size_t outbufindex; h2o_iovec_t last_buf; outbufindex = 0; self->bufs.entries[0].len = 0; if (inbufcnt != 0) { size_t i; for (i = 0; i != inbufcnt - 1; ++i) outbufindex = compress_chunk(self, inbufs[i].base, inbufs[i].len, Z_NO_FLUSH, outbufindex); last_buf = inbufs[i]; } else { last_buf = h2o_iovec_init(NULL, 0); } outbufindex = compress_chunk(self, last_buf.base, last_buf.len, is_final ? Z_FINISH : Z_SYNC_FLUSH, outbufindex); *outbufs = self->bufs.entries; *outbufcnt = outbufindex + 1; if (is_final) { deflateEnd(&self->zs); self->zs_is_open = 0; } }
static int queue_envelope_dump_buffer(struct envelope *ep, char *evpbuf, size_t evpbufsize) { char *evp; size_t evplen; size_t complen; char compbuf[sizeof(struct envelope)]; size_t enclen; char encbuf[sizeof(struct envelope)]; evp = evpbuf; evplen = envelope_dump_buffer(ep, evpbuf, evpbufsize); if (evplen == 0) return (0); if (env->sc_queue_flags & QUEUE_COMPRESSION) { complen = compress_chunk(evp, evplen, compbuf, sizeof compbuf); if (complen == 0) return (0); evp = compbuf; evplen = complen; } if (env->sc_queue_flags & QUEUE_ENCRYPTION) { enclen = crypto_encrypt_buffer(evp, evplen, encbuf, sizeof encbuf); if (enclen == 0) return (0); evp = encbuf; evplen = enclen; } memmove(evpbuf, evp, evplen); return (evplen); }
static dav_error * _update_chunk_storage(const dav_resource *resource, const char *path, const struct data_treatments_s *dt, GHashTable *comp_opt) { GError *e = NULL; dav_error *de = NULL; const char *c = NULL; const request_rec *r = resource->info->request; c = g_hash_table_lookup(comp_opt, NS_COMPRESSION_OPTION); if(NULL != c && 0 == g_ascii_strcasecmp(c, NS_COMPRESSION_ON)) { DAV_DEBUG_REQ(r, 0, "In place chunk is compressed, uncompress it"); if(1 != uncompress_chunk(path, TRUE, &e)) { de = server_create_and_stat_error(request_get_server_config(r), r->pool, HTTP_INTERNAL_SERVER_ERROR, 0, apr_pstrcat(r->pool, "Failed to uncompress chunk : ", ((NULL != e)? e->message : "No error specified"), NULL)); if(NULL != e) g_clear_error(&e); return de; } DAV_DEBUG_REQ(r, 0, "Chunk uncompressed"); } if(COMPRESSION == data_treatments_get_type(dt)) { DAV_DEBUG_REQ(r, 0, "Re compressing chunk"); const char *algo = data_treatments_get_param(dt, DT_KEY_ALGO); const char *bs = data_treatments_get_param(dt, DT_KEY_BLOCKSIZE); if(!algo || !bs) { return server_create_and_stat_error(request_get_server_config(r), r->pool, HTTP_INTERNAL_SERVER_ERROR, 0, apr_pstrcat(r->pool, "Cannot compress chunk, missing info: ", algo, "|", bs, NULL)); } if(1 != compress_chunk(path, algo, g_ascii_strtoll(bs, NULL, 10), TRUE, &e)) { de = server_create_and_stat_error(request_get_server_config(r), r->pool, HTTP_INTERNAL_SERVER_ERROR, 0, apr_pstrcat(r->pool, "Failed to compress chunk : ", ((NULL != e)? e->message : "No error specified"), NULL)); if(NULL != e) g_clear_error(&e); return de; } } return NULL; }
int main(int argc, char** args) { int rc = -1; if (argc <= 1) { help(argc, args); return 1; } if (!parse_opt(argc, args)) { help(argc, args); return 1; } if (flag_help) { help(argc, args); return 0; } if (optind < argc) { GError *local_error = NULL; int i; for (i = optind; i < argc; i++) { PRINT_DEBUG("Going to work with chunk file [%s]\n", args[i]); /* Run compression */ if(compress_chunk(args[i], algo, blocksize, preserve, &local_error) != 1) { if(local_error) PRINT_ERROR("Failed to compress chunk [%s] :\n%s", args[i], local_error->message); else PRINT_ERROR("Failed to compress chunk [%s] : no error",args[i]); } else { PRINT_DEBUG("Chunk [%s] compressed\n",args[i]); } } PRINT_DEBUG("Process done\n"); } return rc; }
/* * Compresses a ztr (in memory). * Level is 0, 1, 2 or 3 (no compression, delta, delta + zlib, * chebyshev + zlib). */ int compress_ztr(ztr_t *ztr, int level) { int i; if (0 == level) return 0; for (i = 0; i < ztr->nchunks; i++) { /* { char str[5]; fprintf(stderr, "---- %.4s ----\n", ZTR_BE2STR(ztr->chunk[i].type,str)); } fprintf(stderr, "Uncomp length=%d\n", ztr->chunk[i].dlength); */ switch(ztr->chunk[i].type) { char *type; case ZTR_TYPE_SAMP: case ZTR_TYPE_SMP4: #ifdef ILLUMINA_GA compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_STHUFF, CODE_TRACES, 0); #else type = ztr_lookup_mdata_value(ztr, &ztr->chunk[i], "TYPE"); if (type && 0 == strcmp(type, "PYRW")) { /* Raw data is not really compressable */ } else if (type && 0 == strcmp(type, "PYNO")) { if (level > 1) { compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_16TO8, 0, 0); compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_ZLIB, Z_HUFFMAN_ONLY, 0); } } else { if (level <= 2) { /* * Experiments show that typically a double delta does * better than a single delta for 8-bit data, and the other * way around for 16-bit data */ compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_DELTA2, ztr->delta_level, 0); } else { compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_ICHEB, 0, 0); } compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_16TO8, 0, 0); if (level > 1) { compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_FOLLOW1,0, 0); /* compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_ZLIB, Z_HUFFMAN_ONLY); */ compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_RLE, 150, 0); compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_ZLIB, Z_HUFFMAN_ONLY, 0); } } #endif break; case ZTR_TYPE_BASE: #ifdef ILLUMINA_GA compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_STHUFF, CODE_DNA, 0); #else if (level > 1) { compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_ZLIB, Z_HUFFMAN_ONLY, 0); } #endif break; case ZTR_TYPE_CNF1: case ZTR_TYPE_CNF4: case ZTR_TYPE_CSID: #ifdef ILLUMINA_GA compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_RLE, 77, 0); compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_STHUFF, CODE_CONF_RLE, 0); #else compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_DELTA1, 1, 0); compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_RLE, 77, 0); if (level > 1) { compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_ZLIB, Z_HUFFMAN_ONLY, 0); } #endif break; case ZTR_TYPE_BPOS: compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_DELTA4, 1, 0); compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_32TO8, 0, 0); if (level > 1) { compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_ZLIB, Z_HUFFMAN_ONLY, 0); } break; case ZTR_TYPE_TEXT: #ifdef ILLUMINA_GA #else if (level > 1) { compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_ZLIB, Z_HUFFMAN_ONLY, 0); } #endif break; case ZTR_TYPE_FLWO: compress_chunk(ztr, &ztr->chunk[i], ZTR_FORM_XRLE, 0, 4); break; } /* fprintf(stderr, "Comp length=%d\n", ztr->chunk[i].dlength); */ } return 0; }