static int queue_envelope_load_buffer(struct envelope *ep, char *evpbuf, size_t evpbufsize) { char *evp; size_t evplen; char compbuf[sizeof(struct envelope)]; size_t complen; char encbuf[sizeof(struct envelope)]; size_t enclen; evp = evpbuf; evplen = evpbufsize; if (env->sc_queue_flags & QUEUE_ENCRYPTION) { enclen = crypto_decrypt_buffer(evp, evplen, encbuf, sizeof encbuf); if (enclen == 0) return (0); evp = encbuf; evplen = enclen; } if (env->sc_queue_flags & QUEUE_COMPRESSION) { complen = uncompress_chunk(evp, evplen, compbuf, sizeof compbuf); if (complen == 0) return (0); evp = compbuf; evplen = complen; } return (envelope_load_buffer(ep, evp, evplen)); }
/* * Parse the sequence * * Returns 0 on success. */ int parse_base(ztr_t *z, ztr_chunk_t *chunk, uint64_t *base_count) { int i; uncompress_chunk(z, chunk); for (i = 1; i < chunk->dlength; i++) { char base = chunk->data[i]; uint1 key; switch (base) { case 'A': case 'a': key = 0; break; case 'C': case 'c': key = 1; break; case 'G': case 'g': key = 2; break; case 'T': case 't': key = 3; break; default: key = 4; break; } base_count[key]++; } return 0; }
/* * Adds a key/value pair to a ztr TEXT chunk. * The 'ch' chunk may be explicitly specified in which case the text * is added to that chunk or it may be specified as NULL in which case * the key/value pair are added to the first available TEXT chunk, * possibly creating a new one if required. * * NOTE: If the key already exists in the text chunk this appends a new * copy; it does not overwrite the old one. * * Returns ztr text chunk ptr for success * NULL for failure */ ztr_chunk_t *ztr_add_text(ztr_t *z, ztr_chunk_t *ch, const char *key, const char *value) { ztr_chunk_t **text_chunks = NULL; int ntext_chunks; size_t key_len, value_len; char *cp; /* Find/create the appropriate chunk */ if (!ch) { text_chunks = ztr_find_chunks(z, ZTR_TYPE_TEXT, &ntext_chunks); if (!text_chunks) { ch = ztr_new_chunk(z, ZTR_TYPE_TEXT, NULL, 0, NULL, 0); } else { ch = text_chunks[0]; xfree(text_chunks); } } if (ch->type != ZTR_TYPE_TEXT) return NULL; /* Make sure it's not compressed */ uncompress_chunk(z, ch); /* Append key\0value\0 */ key_len = strlen(key); value_len = strlen(value); cp = ch->data; if (cp) { /* Set ch->dlength to the last non-nul byte of the previous value */ while (ch->dlength && ch->data[ch->dlength-1] == 0) ch->dlength--; } cp = realloc(ch->data, 1 + ch->dlength + key_len + value_len + 3); if (NULL == cp) return NULL; else ch->data = cp; cp = &ch->data[ch->dlength]; /* * Note this is a bit cryptic, but it works. * When appending to an existing text chunk we write a preceeding nul * to mark the end of the previous value (we rewound above specifically * for this case). * When creating a new chunk we still write a nul, but in this case it's * the RAW format byte. After the value we add an extra nul to * indicate the last entry. */ ch->dlength += 1+sprintf(cp, "%c%s%c%s%c", 0, key, 0, value, 0); return ch; }
/* * Uncompresses a ztr (in memory). */ int uncompress_ztr(ztr_t *ztr) { int i; for (i = 0; i < ztr->nchunks; i++) { /* { char str[5]; fprintf(stderr, "---- %.4s ----\n", ZTR_BE2STR(ztr->chunk[i].type,str)); } */ uncompress_chunk(ztr, &ztr->chunk[i]); } return 0; }
static dav_error * _update_chunk_storage(const dav_resource *resource, const char *path, const struct data_treatments_s *dt, GHashTable *comp_opt) { GError *e = NULL; dav_error *de = NULL; const char *c = NULL; const request_rec *r = resource->info->request; c = g_hash_table_lookup(comp_opt, NS_COMPRESSION_OPTION); if(NULL != c && 0 == g_ascii_strcasecmp(c, NS_COMPRESSION_ON)) { DAV_DEBUG_REQ(r, 0, "In place chunk is compressed, uncompress it"); if(1 != uncompress_chunk(path, TRUE, &e)) { de = server_create_and_stat_error(request_get_server_config(r), r->pool, HTTP_INTERNAL_SERVER_ERROR, 0, apr_pstrcat(r->pool, "Failed to uncompress chunk : ", ((NULL != e)? e->message : "No error specified"), NULL)); if(NULL != e) g_clear_error(&e); return de; } DAV_DEBUG_REQ(r, 0, "Chunk uncompressed"); } if(COMPRESSION == data_treatments_get_type(dt)) { DAV_DEBUG_REQ(r, 0, "Re compressing chunk"); const char *algo = data_treatments_get_param(dt, DT_KEY_ALGO); const char *bs = data_treatments_get_param(dt, DT_KEY_BLOCKSIZE); if(!algo || !bs) { return server_create_and_stat_error(request_get_server_config(r), r->pool, HTTP_INTERNAL_SERVER_ERROR, 0, apr_pstrcat(r->pool, "Cannot compress chunk, missing info: ", algo, "|", bs, NULL)); } if(1 != compress_chunk(path, algo, g_ascii_strtoll(bs, NULL, 10), TRUE, &e)) { de = server_create_and_stat_error(request_get_server_config(r), r->pool, HTTP_INTERNAL_SERVER_ERROR, 0, apr_pstrcat(r->pool, "Failed to compress chunk : ", ((NULL != e)? e->message : "No error specified"), NULL)); if(NULL != e) g_clear_error(&e); return de; } } return NULL; }
ztr_chunk_t *ztr_find_hcode_chunk(ztr_t *ztr, int code_set) { int i; if (code_set < CODE_USER) return NULL; /* computed on-the-fly or use a hard-coded set */ /* Check through chunks for undecoded HUFF chunks */ for (i = 0; i < ztr->nchunks; i++) { if (ztr->chunk[i].type == ZTR_TYPE_HUFF) { uncompress_chunk(ztr, &ztr->chunk[i]); if (ztr->chunk[i].dlength >= 2 && (unsigned char)ztr->chunk[i].data[1] == code_set) return &ztr->chunk[i]; } } return NULL; }
int decompress(struct strbuf *src, struct strbuf *dest) { z_stream stream; int ret, count = 0; struct strbuf temp; strbuf_init(&temp, 0); uncompress_setup(&stream, src, dest); ret = inflateInit(&stream); if (ret != Z_OK) zerr(ret); do { ret = uncompress_chunk(&stream, dest); if (ret == Z_BUF_ERROR) { (void)inflateEnd(&stream); return ret; } } while (ret != Z_STREAM_END); (void)inflateEnd(&stream); return ret; }
static void *ihy_filling_buffer(void *data) { /* contains uncompressed data, (ihy_buffer_content *) */ t_buffer buffer = ((struct ihy_streaming_data *)data)->buffer; ihy_data *ihy = ((struct ihy_streaming_data *)data)->ihy; ihy_chunk *chunk; struct ihy_buffer_content *to_add; unsigned int i = 0; while (i < ihy->NbChunk) { chunk = &ihy->DataChunks[i]; to_add = malloc(sizeof(struct ihy_buffer_content)); to_add->samples = calloc(ihy->ChunkSize * 2, 1); uncompress_chunk(chunk, to_add->samples, ihy->Channels); to_add->samplesSize = ihy->ChunkSize * 2; buffer_add(to_add, buffer); i++; } return NULL; }
/* * Searches through the cached huffman_codeset_t tables looking for a stored * huffman code of type 'code_set'. * NB: only code_sets >= CODE_USER will be stored here. * * Returns codes on success, * NULL on failure */ ztr_hcode_t *ztr_find_hcode(ztr_t *ztr, int code_set) { int i; if (code_set < CODE_USER) return NULL; /* computed on-the-fly or use a hard-coded set */ /* Check through chunks for undecoded HUFF chunks */ if (!ztr->hcodes_checked) { for (i = 0; i < ztr->nchunks; i++) { if (ztr->chunk[i].type == ZTR_TYPE_HUFF) { block_t *blk; huffman_codeset_t *cs; uncompress_chunk(ztr, &ztr->chunk[i]); blk = block_create((unsigned char *)(ztr->chunk[i].data+2), ztr->chunk[i].dlength-2); cs = restore_codes(blk, NULL); if (!cs) { block_destroy(blk, 1); return NULL; } cs->code_set = (unsigned char)(ztr->chunk[i].data[1]); ztr_add_hcode(ztr, cs, 1); block_destroy(blk, 1); } } ztr->hcodes_checked = 1; } /* Check cached copies */ for (i = 0; i < ztr->nhcodes; i++) { if (ztr->hcodes[i].codes->code_set == code_set) return &ztr->hcodes[i]; } return NULL; }
void ztr_process_text(ztr_t *ztr) { int i; ztr_chunk_t **text_chunks = NULL; int ntext_chunks = 0; ztr_text_t *zt = NULL; int nzt = 0; int nalloc = 0; if (ztr->text_segments) /* Already done */ return; text_chunks = ztr_find_chunks(ztr, ZTR_TYPE_TEXT, &ntext_chunks); if (!text_chunks) return; for (i = 0; i < ntext_chunks; i++) { char *data; uint4 length; char *ident, *value; /* Make sure it's not compressed */ uncompress_chunk(ztr, text_chunks[i]); data = text_chunks[i]->data; length = text_chunks[i]->dlength; if (!length) continue; /* Skip RAW header byte */ data++; length--; while (data - text_chunks[i]->data <= (ptrdiff_t)length && *(ident = data)) { data += strlen(ident)+1; value = data; if (value) data += strlen(value)+1; if (nzt + 1 > nalloc) { nalloc += 10; zt = (ztr_text_t *)xrealloc(zt, nalloc * sizeof(*zt)); } zt[nzt].ident = ident; zt[nzt].value = value; nzt++; } } ztr->text_segments = zt; ztr->ntext_segments = nzt; /* for (i = 0; i < ztr->ntext_segments; i++) { fprintf(stderr, "'%s' = '%s'\n", ztr->text_segments[i].ident, ztr->text_segments[i].value); } */ xfree(text_chunks); }
int check_trace_body(Settings *opts, Trace_reader *trc, srf_t *srf) { char name[512]; Spot_info spot = { NULL, 0, 0, 0, 0, 0, 0 }; ztr_t *ztr = NULL; ztr_chunk_t *chunk; int start_chunk = 0; int i; if (-1 == construct_trace_name(srf->th.id_prefix, (unsigned char *)srf->tb.read_id, srf->tb.read_id_length, name, sizeof(name))) { printf("Couldn't construct read name\n"); return -1; } if (0 != decode_name(name, &spot)) { printf("Couldn't decode read name.\n"); return -1; } if (opts->verbosity > 0) { printf("%s\n", name); } if (0 != read_trace_data(opts, trc, &spot)) { printf("Couldn't get trace data for %s\n", name); return -1; } mfseek(srf->mf, srf->mf_end, SEEK_SET); if (srf->tb.trace_size) { mfwrite(srf->tb.trace, 1, srf->tb.trace_size, srf->mf); free(srf->tb.trace); srf->tb.trace = NULL; } mftruncate(srf->mf, mftell(srf->mf)); mfseek(srf->mf, srf->mf_pos, SEEK_SET); if (srf->ztr) { start_chunk = srf->ztr->nchunks; ztr = ztr_dup(srf->ztr); } if (NULL == partial_decode_ztr(srf, srf->mf, ztr)) { printf("Couldn't decode ZTR data for %s\n", name); return -1; } for (i = start_chunk; i < ztr->nchunks; i++) { chunk = &ztr->chunk[i]; if (opts->verbosity > 1) { printf(" Chunk %d type %.4s\n", i, (char *) &chunk->type); } switch (chunk->type) { case ZTR_TYPE_BASE: if (0 != uncompress_chunk(ztr, chunk)) { printf("Couldn't uncompress BASE chunk\n"); return -1; } if (trc->seq_len + 1 != chunk->dlength || 0 != memcmp(trc->seq, chunk->data + 1, trc->seq_len)) { printf("Sequence differs for %s\n", name); return -1; } break; case ZTR_TYPE_CNF1: if (0 != uncompress_chunk(ztr, chunk)) { printf("Couldn't uncompress CNF1 chunk\n"); return -1; } if (trc->seq_len + 1 != chunk->dlength || 0 != memcmp(trc->qual, chunk->data + 1, trc->seq_len)) { printf("CNF1 confidence values differ for %s\n", name); return -1; } break; case ZTR_TYPE_CNF4: if (0 != uncompress_chunk(ztr, chunk)) { printf("Couldn't uncompress CNF4 chunk\n"); return -1; } if (0 != check_cnf4(ztr, chunk, trc)) { printf("CNF4 confidence values differ for %s\n", name); return -1; } break; case ZTR_TYPE_SMP4: if (0 != uncompress_chunk(ztr, chunk)) { printf("Couldn't uncompress SMP4 chunk\n"); return -1; } if (0 != check_trace(ztr, chunk, trc)) { printf("Trace doesn't match for %s\n", name); return -1; } break; default: printf("Found unexpected chunk type in trace body for %s\n", name); return -1; } } delete_ztr(ztr); return 0; }
int get_second_calls(ztr_t *ztr, size_t nbases, int **indexes) { ztr_chunk_t *smp4_chunk = NULL; char *type = NULL; size_t i; int max_base; int second_base; int base; uint16_t val; uint16_t max; uint16_t second; uint8_t *data[4]; for (i = 0; i < ztr->nchunks; i++) { if (ZTR_TYPE_SMP4 == ztr->chunk[i].type) { type = ztr_lookup_mdata_value(ztr, &ztr->chunk[i], "TYPE"); if (NULL == type || 0 == strcmp(type, "PROC") || 0 == strcmp(type, "SLXI")) { smp4_chunk = &ztr->chunk[i]; break; } } } if (NULL == smp4_chunk) return 1; if (0 != uncompress_chunk(ztr, smp4_chunk)) { printf("Couldn't uncompresss SMP4 chunk\n"); return -1; } if (smp4_chunk->dlength != nbases * 8 + 2) { printf("Trace and basecalls have different number of samples\n"); return -1; } *indexes = smalloc(nbases * sizeof(int)); if (NULL == type || type[0] == 'P') { data[0] = (uint8_t *) smp4_chunk->data + 2; data[1] = (uint8_t *) smp4_chunk->data + 2 + nbases * 2; data[2] = (uint8_t *) smp4_chunk->data + 2 + nbases * 4; data[3] = (uint8_t *) smp4_chunk->data + 2 + nbases * 6; } else { data[1] = (uint8_t *) smp4_chunk->data + 2; data[0] = (uint8_t *) smp4_chunk->data + 2 + nbases * 2; data[3] = (uint8_t *) smp4_chunk->data + 2 + nbases * 4; data[2] = (uint8_t *) smp4_chunk->data + 2 + nbases * 6; } for (i = 0; i < nbases; i++) { max = ((uint16_t) data[0][i * 2] << 8) | data[0][i * 2 + 1]; second = ((uint16_t) data[1][i * 2] << 8) | data[1][i * 2 + 1]; if (second > max) { val = max; max = second; second = val; max_base = 1; second_base = 0; } else { max_base = 0; second_base = 1; } for (base = 2; base < 4; base++) { val = ((uint16_t) data[base][i * 2] << 8) | data[base][i * 2 + 1]; if (val > max) { second = max; second_base = max_base; max = val; max_base = base; } else if (val > second) { second = val; second_base = base; } } (*indexes)[i] = second_base < max_base ? second_base : second_base - 1; } return 0; }
int check_trace_header(Settings *opts, srf_t *srf, Previous_data *seen) { int i; int j; int n; uint8_t *data; uint32_t val; ztr_chunk_t *chunk; if (NULL != srf->mf) { mfrecreate(srf->mf, NULL, 0); } else { srf->mf = mfcreate(NULL, 0); } if (NULL == srf->mf) die("%s\n", strerror(errno)); if (srf->th.trace_hdr_size) { if (1 != mfwrite(srf->th.trace_hdr, srf->th.trace_hdr_size, 1, srf->mf)) { die("mfwrite failed: %s\n", strerror(errno)); } } if (srf->ztr) delete_ztr(srf->ztr); mrewind(srf->mf); if (NULL != (srf->ztr = partial_decode_ztr(srf, srf->mf, NULL))) { srf->mf_pos = mftell(srf->mf); } else { /* We expect this to work for srfs made by illumina2srf */ printf("partial_decode_ztr failed for trace header\n"); srf->mf_pos = 0; return -1; } mfseek(srf->mf, 0, SEEK_END); srf->mf_end = mftell(srf->mf); /* Go through chunks */ for (i = 0; i < srf->ztr->nchunks; i++) { chunk = &srf->ztr->chunk[i]; if (opts->verbosity > 1) { printf(" Chunk %d type %.4s\n", i, (char *) &chunk->type); } switch (chunk->type) { case ZTR_TYPE_HUFF: break; case ZTR_TYPE_BPOS: if (0 != uncompress_chunk(srf->ztr, chunk)) { printf("Couldn't uncompress BPOS chunk\n"); return -1; } n = (chunk->dlength - 4) / 4; for (j = 0; j < n; j++) { data = (uint8_t *) &chunk->data[j * 4 + 4]; val = ( ((uint32_t) data[0]) << 24 | ((uint32_t) data[1]) << 16 | ((uint32_t) data[2]) << 8 | ((uint32_t) data[3])); if (val != j) { printf("BPOS data misses cycles\n"); return -1; } } break; case ZTR_TYPE_REGN: if (0 != uncompress_chunk(srf->ztr, chunk)) { printf("Couldn't uncompress REGN chunk\n"); return -1; } if (NULL == seen->regn) { /* Copy REGN chunk */ seen->regn_meta_sz = chunk->mdlength; seen->regn_meta = smalloc(seen->regn_meta_sz); memcpy(seen->regn_meta, chunk->mdata, seen->regn_meta_sz); seen->regn_sz = chunk->dlength; seen->regn = smalloc(seen->regn_sz); memcpy(seen->regn, chunk->data, seen->regn_sz); } else { /* Compare with last copy */ if (seen->regn_meta_sz != chunk->mdlength || seen->regn_sz != chunk->dlength || 0 != memcmp(seen->regn_meta, chunk->mdata, seen->regn_meta_sz) || 0 != memcmp(seen->regn, chunk->data, seen->regn_sz)) { printf("REGN chunk changed between header blocks\n"); return -1; } } break; case ZTR_TYPE_TEXT: if (0 != uncompress_chunk(srf->ztr, chunk)) { printf("Couldn't uncompress REGN chunk\n"); return -1; } if (NULL == seen->text) { seen->text_sz = chunk->dlength; seen->text = smalloc(seen->text_sz); memcpy(seen->text, chunk->data, seen->text_sz); if (0 != check_text(opts, seen)) return -1; } else { if (seen->text_sz != chunk->dlength || 0 != memcmp(seen->text, chunk->data, seen->text_sz)) { printf("New TEXT chunk found\n"); seen->text_sz = chunk->dlength; seen->text = srealloc(seen->text, seen->text_sz); memcpy(seen->text, chunk->data, seen->text_sz); if (0 != check_text(opts, seen)) return -1; } } break; default: printf("Found unexpected chunk type in header block\n"); return -1; } } return 0; }
/* * Parse the REGN chunk, add to regn HASH * * Returns corresponding HashItem * from regn Hash */ HashItem *parse_regn(ztr_t *z, ztr_chunk_t *chunk, HashTable *regn_hash) { char key[1024]; char *name; HashItem *hi; regn_t *regn; size_t l; uncompress_chunk(z, chunk); /* the hash key is a combination of the region names and boundaries */ name = ztr_lookup_mdata_value(z, chunk, "NAME"); l = snprintf(key, sizeof(key), "names=%s", name); if( chunk->dlength ){ int nbndy = (chunk->dlength-1)/4; uint4 *bndy = (uint4 *)(chunk->data+1); int ibndy; for (ibndy=0; ibndy<nbndy; ibndy++) { if( ibndy ) l += snprintf(key + l, sizeof(key) - l, ";%d", be_int4(bndy[ibndy])); else l += snprintf(key + l, sizeof(key) - l, " boundaries=%d", be_int4(bndy[ibndy])); } } if (NULL == (hi = (HashTableSearch(regn_hash, key, strlen(key))))) { int iregion, nregions = 0; char *coord; char *cp1; uint4 bndy[MAX_REGIONS]; int ibndy, nbndy = 0; HashData hd; if( NULL == (regn = (regn_t *)malloc(sizeof(regn_t)))) { return NULL; } coord = ztr_lookup_mdata_value(z, chunk, "COORD"); regn->coord = (NULL == coord ? 'B' : *coord ); regn->region_names = strdup(name); cp1 = strtok (regn->region_names,";"); while(cp1) { char *cp2; if(NULL == (cp2 = strchr(cp1,':'))) { fprintf(stderr, "Invalid region name/code pair %s\n", cp1); return NULL; } *cp2++ = '\0'; regn->name[nregions] = cp1; regn->code[nregions] = *cp2; nregions++; cp1 = strtok (NULL, ";"); } regn->nregions = nregions; if( chunk->dlength ) { nbndy = (chunk->dlength-1)/4; memcpy(bndy, chunk->data+1, chunk->dlength-1); } for( iregion=0, ibndy=0; iregion<nregions; iregion++) { /* start = (start + length of previous region) or 0 if no previous region */ /* length = (next boundary - start of region) or -1 if no next boundary */ if( regn->code[iregion] == 'E' ){ /* no sequence, length = 0 */ regn->start[iregion] = (iregion ? (regn->start[iregion-1] + regn->length[iregion-1]) : 0); regn->length[iregion] = 0; }else{ if( ibndy > nbndy ){ fprintf(stderr, "More name/code pairs than boundaries\n"); return NULL; } regn->start[iregion] = (iregion ? (regn->start[iregion-1] + regn->length[iregion-1]) : 0); regn->length[iregion] = (ibndy == nbndy ? -1 : (be_int4(bndy[ibndy])-regn->start[iregion])); ibndy++; } } regn->count = 1; hd.p = regn; if (NULL == (hi = HashTableAdd(regn_hash, key, strlen(key), hd, NULL))) { free(regn->region_names); free(regn); return NULL; } } else { regn = (regn_t *)(hi->data.p); regn->count++; } return hi; }