static int
queue_envelope_load_buffer(struct envelope *ep, char *evpbuf, size_t evpbufsize)
{
char *evp;
size_t evplen;
char compbuf[sizeof(struct envelope)];
size_t complen;
char encbuf[sizeof(struct envelope)];
size_t enclen;
evp = evpbuf;
evplen = evpbufsize;
if (env->sc_queue_flags & QUEUE_ENCRYPTION) {
enclen = crypto_decrypt_buffer(evp, evplen, encbuf, sizeof encbuf);
if (enclen == 0)
return (0);
evp = encbuf;
evplen = enclen;
}
if (env->sc_queue_flags & QUEUE_COMPRESSION) {
complen = uncompress_chunk(evp, evplen, compbuf, sizeof compbuf);
if (complen == 0)
return (0);
evp = compbuf;
evplen = complen;
}
return (envelope_load_buffer(ep, evp, evplen));
}
/*
* Parse the sequence
*
* Returns 0 on success.
*/
int parse_base(ztr_t *z, ztr_chunk_t *chunk, uint64_t *base_count) {
int i;
uncompress_chunk(z, chunk);
for (i = 1; i < chunk->dlength; i++) {
char base = chunk->data[i];
uint1 key;
switch (base) {
case 'A': case 'a':
key = 0;
break;
case 'C': case 'c':
key = 1;
break;
case 'G': case 'g':
key = 2;
break;
case 'T': case 't':
key = 3;
break;
default:
key = 4;
break;
}
base_count[key]++;
}
return 0;
}
/*
* Adds a key/value pair to a ztr TEXT chunk.
* The 'ch' chunk may be explicitly specified in which case the text
* is added to that chunk or it may be specified as NULL in which case
* the key/value pair are added to the first available TEXT chunk,
* possibly creating a new one if required.
*
* NOTE: If the key already exists in the text chunk this appends a new
* copy; it does not overwrite the old one.
*
* Returns ztr text chunk ptr for success
* NULL for failure
*/
ztr_chunk_t *ztr_add_text(ztr_t *z, ztr_chunk_t *ch,
const char *key, const char *value) {
ztr_chunk_t **text_chunks = NULL;
int ntext_chunks;
size_t key_len, value_len;
char *cp;
/* Find/create the appropriate chunk */
if (!ch) {
text_chunks = ztr_find_chunks(z, ZTR_TYPE_TEXT, &ntext_chunks);
if (!text_chunks) {
ch = ztr_new_chunk(z, ZTR_TYPE_TEXT, NULL, 0, NULL, 0);
} else {
ch = text_chunks[0];
xfree(text_chunks);
}
}
if (ch->type != ZTR_TYPE_TEXT)
return NULL;
/* Make sure it's not compressed */
uncompress_chunk(z, ch);
/* Append key\0value\0 */
key_len = strlen(key);
value_len = strlen(value);
cp = ch->data;
if (cp) {
/* Set ch->dlength to the last non-nul byte of the previous value */
while (ch->dlength && ch->data[ch->dlength-1] == 0)
ch->dlength--;
}
cp = realloc(ch->data, 1 + ch->dlength + key_len + value_len + 3);
if (NULL == cp)
return NULL;
else
ch->data = cp;
cp = &ch->data[ch->dlength];
/*
* Note this is a bit cryptic, but it works.
* When appending to an existing text chunk we write a preceeding nul
* to mark the end of the previous value (we rewound above specifically
* for this case).
* When creating a new chunk we still write a nul, but in this case it's
* the RAW format byte. After the value we add an extra nul to
* indicate the last entry.
*/
ch->dlength += 1+sprintf(cp, "%c%s%c%s%c", 0, key, 0, value, 0);
return ch;
}
/*
* Uncompresses a ztr (in memory).
*/
int uncompress_ztr(ztr_t *ztr) {
int i;
for (i = 0; i < ztr->nchunks; i++) {
/*
{
char str[5];
fprintf(stderr, "---- %.4s ----\n",
ZTR_BE2STR(ztr->chunk[i].type,str));
}
*/
uncompress_chunk(ztr, &ztr->chunk[i]);
}
return 0;
}
static dav_error *
_update_chunk_storage(const dav_resource *resource, const char *path, const struct data_treatments_s *dt, GHashTable *comp_opt)
{
GError *e = NULL;
dav_error *de = NULL;
const char *c = NULL;
const request_rec *r = resource->info->request;
c = g_hash_table_lookup(comp_opt, NS_COMPRESSION_OPTION);
if(NULL != c && 0 == g_ascii_strcasecmp(c, NS_COMPRESSION_ON)) {
DAV_DEBUG_REQ(r, 0, "In place chunk is compressed, uncompress it");
if(1 != uncompress_chunk(path, TRUE, &e)) {
de = server_create_and_stat_error(request_get_server_config(r),
r->pool, HTTP_INTERNAL_SERVER_ERROR, 0,
apr_pstrcat(r->pool, "Failed to uncompress chunk : ",
((NULL != e)? e->message : "No error specified"), NULL));
if(NULL != e)
g_clear_error(&e);
return de;
}
DAV_DEBUG_REQ(r, 0, "Chunk uncompressed");
}
if(COMPRESSION == data_treatments_get_type(dt)) {
DAV_DEBUG_REQ(r, 0, "Re compressing chunk");
const char *algo = data_treatments_get_param(dt, DT_KEY_ALGO);
const char *bs = data_treatments_get_param(dt, DT_KEY_BLOCKSIZE);
if(!algo || !bs) {
return server_create_and_stat_error(request_get_server_config(r),
r->pool, HTTP_INTERNAL_SERVER_ERROR, 0,
apr_pstrcat(r->pool, "Cannot compress chunk, missing info: ",
algo, "|", bs, NULL));
}
if(1 != compress_chunk(path, algo, g_ascii_strtoll(bs, NULL, 10), TRUE, &e)) {
de = server_create_and_stat_error(request_get_server_config(r),
r->pool, HTTP_INTERNAL_SERVER_ERROR, 0,
apr_pstrcat(r->pool, "Failed to compress chunk : ",
((NULL != e)? e->message : "No error specified"), NULL));
if(NULL != e)
g_clear_error(&e);
return de;
}
}
return NULL;
}
ztr_chunk_t *ztr_find_hcode_chunk(ztr_t *ztr, int code_set) {
int i;
if (code_set < CODE_USER)
return NULL; /* computed on-the-fly or use a hard-coded set */
/* Check through chunks for undecoded HUFF chunks */
for (i = 0; i < ztr->nchunks; i++) {
if (ztr->chunk[i].type == ZTR_TYPE_HUFF) {
uncompress_chunk(ztr, &ztr->chunk[i]);
if (ztr->chunk[i].dlength >= 2 &&
(unsigned char)ztr->chunk[i].data[1] == code_set)
return &ztr->chunk[i];
}
}
return NULL;
}
int decompress(struct strbuf *src, struct strbuf *dest)
{
z_stream stream;
int ret, count = 0;
struct strbuf temp;
strbuf_init(&temp, 0);
uncompress_setup(&stream, src, dest);
ret = inflateInit(&stream);
if (ret != Z_OK)
zerr(ret);
do {
ret = uncompress_chunk(&stream, dest);
if (ret == Z_BUF_ERROR) {
(void)inflateEnd(&stream);
return ret;
}
} while (ret != Z_STREAM_END);
(void)inflateEnd(&stream);
return ret;
}
static void *ihy_filling_buffer(void *data)
{
/* contains uncompressed data, (ihy_buffer_content *) */
t_buffer buffer = ((struct ihy_streaming_data *)data)->buffer;
ihy_data *ihy = ((struct ihy_streaming_data *)data)->ihy;
ihy_chunk *chunk;
struct ihy_buffer_content *to_add;
unsigned int i = 0;
while (i < ihy->NbChunk)
{
chunk = &ihy->DataChunks[i];
to_add = malloc(sizeof(struct ihy_buffer_content));
to_add->samples = calloc(ihy->ChunkSize * 2, 1);
uncompress_chunk(chunk, to_add->samples, ihy->Channels);
to_add->samplesSize = ihy->ChunkSize * 2;
buffer_add(to_add, buffer);
i++;
}
return NULL;
}
/*
* Searches through the cached huffman_codeset_t tables looking for a stored
* huffman code of type 'code_set'.
* NB: only code_sets >= CODE_USER will be stored here.
*
* Returns codes on success,
* NULL on failure
*/
ztr_hcode_t *ztr_find_hcode(ztr_t *ztr, int code_set) {
int i;
if (code_set < CODE_USER)
return NULL; /* computed on-the-fly or use a hard-coded set */
/* Check through chunks for undecoded HUFF chunks */
if (!ztr->hcodes_checked) {
for (i = 0; i < ztr->nchunks; i++) {
if (ztr->chunk[i].type == ZTR_TYPE_HUFF) {
block_t *blk;
huffman_codeset_t *cs;
uncompress_chunk(ztr, &ztr->chunk[i]);
blk = block_create((unsigned char *)(ztr->chunk[i].data+2),
ztr->chunk[i].dlength-2);
cs = restore_codes(blk, NULL);
if (!cs) {
block_destroy(blk, 1);
return NULL;
}
cs->code_set = (unsigned char)(ztr->chunk[i].data[1]);
ztr_add_hcode(ztr, cs, 1);
block_destroy(blk, 1);
}
}
ztr->hcodes_checked = 1;
}
/* Check cached copies */
for (i = 0; i < ztr->nhcodes; i++) {
if (ztr->hcodes[i].codes->code_set == code_set)
return &ztr->hcodes[i];
}
return NULL;
}
void ztr_process_text(ztr_t *ztr) {
int i;
ztr_chunk_t **text_chunks = NULL;
int ntext_chunks = 0;
ztr_text_t *zt = NULL;
int nzt = 0;
int nalloc = 0;
if (ztr->text_segments)
/* Already done */
return;
text_chunks = ztr_find_chunks(ztr, ZTR_TYPE_TEXT, &ntext_chunks);
if (!text_chunks)
return;
for (i = 0; i < ntext_chunks; i++) {
char *data;
uint4 length;
char *ident, *value;
/* Make sure it's not compressed */
uncompress_chunk(ztr, text_chunks[i]);
data = text_chunks[i]->data;
length = text_chunks[i]->dlength;
if (!length)
continue;
/* Skip RAW header byte */
data++;
length--;
while (data - text_chunks[i]->data <= (ptrdiff_t)length &&
*(ident = data)) {
data += strlen(ident)+1;
value = data;
if (value)
data += strlen(value)+1;
if (nzt + 1 > nalloc) {
nalloc += 10;
zt = (ztr_text_t *)xrealloc(zt, nalloc * sizeof(*zt));
}
zt[nzt].ident = ident;
zt[nzt].value = value;
nzt++;
}
}
ztr->text_segments = zt;
ztr->ntext_segments = nzt;
/*
for (i = 0; i < ztr->ntext_segments; i++) {
fprintf(stderr, "'%s' = '%s'\n",
ztr->text_segments[i].ident,
ztr->text_segments[i].value);
}
*/
xfree(text_chunks);
}
int check_trace_body(Settings *opts, Trace_reader *trc, srf_t *srf) {
char name[512];
Spot_info spot = { NULL, 0, 0, 0, 0, 0, 0 };
ztr_t *ztr = NULL;
ztr_chunk_t *chunk;
int start_chunk = 0;
int i;
if (-1 == construct_trace_name(srf->th.id_prefix,
(unsigned char *)srf->tb.read_id,
srf->tb.read_id_length,
name, sizeof(name))) {
printf("Couldn't construct read name\n");
return -1;
}
if (0 != decode_name(name, &spot)) {
printf("Couldn't decode read name.\n");
return -1;
}
if (opts->verbosity > 0) {
printf("%s\n", name);
}
if (0 != read_trace_data(opts, trc, &spot)) {
printf("Couldn't get trace data for %s\n", name);
return -1;
}
mfseek(srf->mf, srf->mf_end, SEEK_SET);
if (srf->tb.trace_size) {
mfwrite(srf->tb.trace, 1, srf->tb.trace_size, srf->mf);
free(srf->tb.trace);
srf->tb.trace = NULL;
}
mftruncate(srf->mf, mftell(srf->mf));
mfseek(srf->mf, srf->mf_pos, SEEK_SET);
if (srf->ztr) {
start_chunk = srf->ztr->nchunks;
ztr = ztr_dup(srf->ztr);
}
if (NULL == partial_decode_ztr(srf, srf->mf, ztr)) {
printf("Couldn't decode ZTR data for %s\n", name);
return -1;
}
for (i = start_chunk; i < ztr->nchunks; i++) {
chunk = &ztr->chunk[i];
if (opts->verbosity > 1) {
printf(" Chunk %d type %.4s\n", i,
(char *) &chunk->type);
}
switch (chunk->type) {
case ZTR_TYPE_BASE:
if (0 != uncompress_chunk(ztr, chunk)) {
printf("Couldn't uncompress BASE chunk\n");
return -1;
}
if (trc->seq_len + 1 != chunk->dlength
|| 0 != memcmp(trc->seq, chunk->data + 1, trc->seq_len)) {
printf("Sequence differs for %s\n", name);
return -1;
}
break;
case ZTR_TYPE_CNF1:
if (0 != uncompress_chunk(ztr, chunk)) {
printf("Couldn't uncompress CNF1 chunk\n");
return -1;
}
if (trc->seq_len + 1 != chunk->dlength
|| 0 != memcmp(trc->qual, chunk->data + 1, trc->seq_len)) {
printf("CNF1 confidence values differ for %s\n", name);
return -1;
}
break;
case ZTR_TYPE_CNF4:
if (0 != uncompress_chunk(ztr, chunk)) {
printf("Couldn't uncompress CNF4 chunk\n");
return -1;
}
if (0 != check_cnf4(ztr, chunk, trc)) {
printf("CNF4 confidence values differ for %s\n", name);
return -1;
}
break;
case ZTR_TYPE_SMP4:
if (0 != uncompress_chunk(ztr, chunk)) {
printf("Couldn't uncompress SMP4 chunk\n");
return -1;
}
if (0 != check_trace(ztr, chunk, trc)) {
printf("Trace doesn't match for %s\n", name);
return -1;
}
break;
default:
printf("Found unexpected chunk type in trace body for %s\n",
name);
return -1;
}
}
delete_ztr(ztr);
return 0;
}
int get_second_calls(ztr_t *ztr, size_t nbases, int **indexes) {
ztr_chunk_t *smp4_chunk = NULL;
char *type = NULL;
size_t i;
int max_base;
int second_base;
int base;
uint16_t val;
uint16_t max;
uint16_t second;
uint8_t *data[4];
for (i = 0; i < ztr->nchunks; i++) {
if (ZTR_TYPE_SMP4 == ztr->chunk[i].type) {
type = ztr_lookup_mdata_value(ztr, &ztr->chunk[i], "TYPE");
if (NULL == type
|| 0 == strcmp(type, "PROC")
|| 0 == strcmp(type, "SLXI")) {
smp4_chunk = &ztr->chunk[i];
break;
}
}
}
if (NULL == smp4_chunk) return 1;
if (0 != uncompress_chunk(ztr, smp4_chunk)) {
printf("Couldn't uncompresss SMP4 chunk\n");
return -1;
}
if (smp4_chunk->dlength != nbases * 8 + 2) {
printf("Trace and basecalls have different number of samples\n");
return -1;
}
*indexes = smalloc(nbases * sizeof(int));
if (NULL == type || type[0] == 'P') {
data[0] = (uint8_t *) smp4_chunk->data + 2;
data[1] = (uint8_t *) smp4_chunk->data + 2 + nbases * 2;
data[2] = (uint8_t *) smp4_chunk->data + 2 + nbases * 4;
data[3] = (uint8_t *) smp4_chunk->data + 2 + nbases * 6;
} else {
data[1] = (uint8_t *) smp4_chunk->data + 2;
data[0] = (uint8_t *) smp4_chunk->data + 2 + nbases * 2;
data[3] = (uint8_t *) smp4_chunk->data + 2 + nbases * 4;
data[2] = (uint8_t *) smp4_chunk->data + 2 + nbases * 6;
}
for (i = 0; i < nbases; i++) {
max = ((uint16_t) data[0][i * 2] << 8) | data[0][i * 2 + 1];
second = ((uint16_t) data[1][i * 2] << 8) | data[1][i * 2 + 1];
if (second > max) {
val = max; max = second; second = val;
max_base = 1;
second_base = 0;
} else {
max_base = 0;
second_base = 1;
}
for (base = 2; base < 4; base++) {
val = ((uint16_t) data[base][i * 2] << 8) | data[base][i * 2 + 1];
if (val > max) {
second = max;
second_base = max_base;
max = val;
max_base = base;
} else if (val > second) {
second = val;
second_base = base;
}
}
(*indexes)[i] = second_base < max_base ? second_base : second_base - 1;
}
return 0;
}
int check_trace_header(Settings *opts, srf_t *srf, Previous_data *seen) {
int i;
int j;
int n;
uint8_t *data;
uint32_t val;
ztr_chunk_t *chunk;
if (NULL != srf->mf) {
mfrecreate(srf->mf, NULL, 0);
} else {
srf->mf = mfcreate(NULL, 0);
}
if (NULL == srf->mf) die("%s\n", strerror(errno));
if (srf->th.trace_hdr_size) {
if (1 != mfwrite(srf->th.trace_hdr, srf->th.trace_hdr_size, 1, srf->mf)) {
die("mfwrite failed: %s\n", strerror(errno));
}
}
if (srf->ztr) delete_ztr(srf->ztr);
mrewind(srf->mf);
if (NULL != (srf->ztr = partial_decode_ztr(srf, srf->mf, NULL))) {
srf->mf_pos = mftell(srf->mf);
} else {
/* We expect this to work for srfs made by illumina2srf */
printf("partial_decode_ztr failed for trace header\n");
srf->mf_pos = 0;
return -1;
}
mfseek(srf->mf, 0, SEEK_END);
srf->mf_end = mftell(srf->mf);
/* Go through chunks */
for (i = 0; i < srf->ztr->nchunks; i++) {
chunk = &srf->ztr->chunk[i];
if (opts->verbosity > 1) {
printf(" Chunk %d type %.4s\n", i,
(char *) &chunk->type);
}
switch (chunk->type) {
case ZTR_TYPE_HUFF:
break;
case ZTR_TYPE_BPOS:
if (0 != uncompress_chunk(srf->ztr, chunk)) {
printf("Couldn't uncompress BPOS chunk\n");
return -1;
}
n = (chunk->dlength - 4) / 4;
for (j = 0; j < n; j++) {
data = (uint8_t *) &chunk->data[j * 4 + 4];
val = ( ((uint32_t) data[0]) << 24
| ((uint32_t) data[1]) << 16
| ((uint32_t) data[2]) << 8
| ((uint32_t) data[3]));
if (val != j) {
printf("BPOS data misses cycles\n");
return -1;
}
}
break;
case ZTR_TYPE_REGN:
if (0 != uncompress_chunk(srf->ztr, chunk)) {
printf("Couldn't uncompress REGN chunk\n");
return -1;
}
if (NULL == seen->regn) {
/* Copy REGN chunk */
seen->regn_meta_sz = chunk->mdlength;
seen->regn_meta = smalloc(seen->regn_meta_sz);
memcpy(seen->regn_meta, chunk->mdata, seen->regn_meta_sz);
seen->regn_sz = chunk->dlength;
seen->regn = smalloc(seen->regn_sz);
memcpy(seen->regn, chunk->data, seen->regn_sz);
} else {
/* Compare with last copy */
if (seen->regn_meta_sz != chunk->mdlength
|| seen->regn_sz != chunk->dlength
|| 0 != memcmp(seen->regn_meta, chunk->mdata, seen->regn_meta_sz)
|| 0 != memcmp(seen->regn, chunk->data, seen->regn_sz)) {
printf("REGN chunk changed between header blocks\n");
return -1;
}
}
break;
case ZTR_TYPE_TEXT:
if (0 != uncompress_chunk(srf->ztr, chunk)) {
printf("Couldn't uncompress REGN chunk\n");
return -1;
}
if (NULL == seen->text) {
seen->text_sz = chunk->dlength;
seen->text = smalloc(seen->text_sz);
memcpy(seen->text, chunk->data, seen->text_sz);
if (0 != check_text(opts, seen)) return -1;
} else {
if (seen->text_sz != chunk->dlength
|| 0 != memcmp(seen->text, chunk->data, seen->text_sz)) {
printf("New TEXT chunk found\n");
seen->text_sz = chunk->dlength;
seen->text = srealloc(seen->text, seen->text_sz);
memcpy(seen->text, chunk->data, seen->text_sz);
if (0 != check_text(opts, seen)) return -1;
}
}
break;
default:
printf("Found unexpected chunk type in header block\n");
return -1;
}
}
return 0;
}
/*
* Parse the REGN chunk, add to regn HASH
*
* Returns corresponding HashItem * from regn Hash
*/
HashItem *parse_regn(ztr_t *z, ztr_chunk_t *chunk, HashTable *regn_hash) {
char key[1024];
char *name;
HashItem *hi;
regn_t *regn;
size_t l;
uncompress_chunk(z, chunk);
/* the hash key is a combination of the region names and boundaries */
name = ztr_lookup_mdata_value(z, chunk, "NAME");
l = snprintf(key, sizeof(key), "names=%s", name);
if( chunk->dlength ){
int nbndy = (chunk->dlength-1)/4;
uint4 *bndy = (uint4 *)(chunk->data+1);
int ibndy;
for (ibndy=0; ibndy<nbndy; ibndy++) {
if( ibndy )
l += snprintf(key + l, sizeof(key) - l,
";%d", be_int4(bndy[ibndy]));
else
l += snprintf(key + l, sizeof(key) - l,
" boundaries=%d", be_int4(bndy[ibndy]));
}
}
if (NULL == (hi = (HashTableSearch(regn_hash, key, strlen(key))))) {
int iregion, nregions = 0;
char *coord;
char *cp1;
uint4 bndy[MAX_REGIONS];
int ibndy, nbndy = 0;
HashData hd;
if( NULL == (regn = (regn_t *)malloc(sizeof(regn_t)))) {
return NULL;
}
coord = ztr_lookup_mdata_value(z, chunk, "COORD");
regn->coord = (NULL == coord ? 'B' : *coord );
regn->region_names = strdup(name);
cp1 = strtok (regn->region_names,";");
while(cp1) {
char *cp2;
if(NULL == (cp2 = strchr(cp1,':'))) {
fprintf(stderr, "Invalid region name/code pair %s\n", cp1);
return NULL;
}
*cp2++ = '\0';
regn->name[nregions] = cp1;
regn->code[nregions] = *cp2;
nregions++;
cp1 = strtok (NULL, ";");
}
regn->nregions = nregions;
if( chunk->dlength ) {
nbndy = (chunk->dlength-1)/4;
memcpy(bndy, chunk->data+1, chunk->dlength-1);
}
for( iregion=0, ibndy=0; iregion<nregions; iregion++) {
/* start = (start + length of previous region) or 0 if no previous region */
/* length = (next boundary - start of region) or -1 if no next boundary */
if( regn->code[iregion] == 'E' ){
/* no sequence, length = 0 */
regn->start[iregion] = (iregion ? (regn->start[iregion-1] + regn->length[iregion-1]) : 0);
regn->length[iregion] = 0;
}else{
if( ibndy > nbndy ){
fprintf(stderr, "More name/code pairs than boundaries\n");
return NULL;
}
regn->start[iregion] = (iregion ? (regn->start[iregion-1] + regn->length[iregion-1]) : 0);
regn->length[iregion] = (ibndy == nbndy ? -1 : (be_int4(bndy[ibndy])-regn->start[iregion]));
ibndy++;
}
}
regn->count = 1;
hd.p = regn;
if (NULL == (hi = HashTableAdd(regn_hash, key, strlen(key), hd, NULL))) {
free(regn->region_names);
free(regn);
return NULL;
}
} else {
regn = (regn_t *)(hi->data.p);
regn->count++;
}
return hi;
}
