static
rc_t parse_read(SRF_context *ctx, ZTR_Context *ztr_ctx, const uint8_t *data, size_t size)
{
rc_t rc = 0;
size_t parsed;
uint8_t flags;
pstring readId;
ztr_raw_t ztr_raw;
ztr_t ztr;
enum ztr_chunk_type type;
fe_context_t* fe = (fe_context_t*)ctx;
*(void **)&fe->sequence =
*(void **)&fe->quality1 =
*(void **)&fe->quality4 =
*(void **)&fe->signal =
*(void **)&fe->intensity =
*(void **)&fe->noise = NULL;
rc = SRF_ParseReadChunk(data, size, &parsed, &flags, &readId);
if(rc) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rc);
return SRALoaderFile_LOG(ctx->file, klogErr, rc, "corrupt", NULL);
}
if(fe->defered != NULL)
ZTR_AddToBuffer(ztr_ctx, fe->defered, fe->defered_len);
ZTR_AddToBuffer(ztr_ctx, data + parsed, size - parsed);
if(fe->defered == NULL) {
rc = ZTR_ParseBlock(ztr_ctx, &ztr_raw);
if(rc == 0)
goto PARSE_BLOCK;
rc = ZTR_ParseHeader(ztr_ctx);
if(rc) {
return SRALoaderFile_LOG(ctx->file, klogErr, rc, "corrupt", NULL);
}
}
while (!ZTR_BufferIsEmpty(ztr_ctx)) {
rc = ZTR_ParseBlock(ztr_ctx, &ztr_raw);
PARSE_BLOCK:
if(rc != 0 || (rc = ZTR_ProcessBlock(ztr_ctx, &ztr_raw, &ztr, &type)) != 0 ) {
return SRALoaderFile_LOG(ctx->file, klogErr, rc, "corrupt", NULL);
}
switch (type) {
case READ:
if(ztr.sequence->datatype != i8) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnexpected);
return SRALoaderFile_LOG(ctx->file, klogErr, rc, "invalid data type for sequence data", NULL);
}
fe->sequence = ztr;
break;
case QUALITY1:
if(ztr.quality1->datatype != i8) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnexpected);
return SRALoaderFile_LOG(ctx->file, klogErr, rc, "invalid data type for quality1 data", NULL);
}
fe->quality1 = ztr;
break;
case QUALITY4:
if(ztr.quality4->datatype != i8) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnexpected);
return SRALoaderFile_LOG(ctx->file, klogErr, rc, "invalid data type for quality4 data", NULL);
}
fe->quality4 = ztr;
break;
case SIGNAL4:
if(ztr.signal4->Type != NULL && strncmp(ztr.signal4->Type, "SLXI", 4) == 0 ) {
if( !fe->skip_intensity ) {
fe->intensity = ztr;
} else if(ztr.signal4){
if(ztr.signal4->data) free(ztr.signal4->data);
free(ztr.signal4);
}
} else if(ztr.signal4->Type != NULL && strncmp(ztr.signal4->Type, "SLXN", 4) == 0 ) {
if( !fe->skip_noise ) {
fe->noise = ztr;
} else if(ztr.signal4){
if(ztr.signal4->data) free(ztr.signal4->data);
free(ztr.signal4);
}
} else if( !fe->skip_signal ) {
fe->signal = ztr;
} else if(ztr.signal4){
if(ztr.signal4->data) free(ztr.signal4->data);
free(ztr.signal4);
}
break;
default:
free(*(void **)&ztr);
case none:
case ignore:
if(ztr_raw.data) {
free(ztr_raw.data);
}
break;
}
if(ztr_raw.meta){
free(ztr_raw.meta);
ztr_raw.meta=NULL;
}
}
while(rc == 0) {
if(*(void **)&fe->sequence == NULL) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcConstraint, rcViolated);
SRALoaderFile_LOG(ctx->file, klogErr, rc, "missing sequence data", NULL);
break;
}
if(*(void **)&fe->quality4 == NULL && *(void **)&fe->quality1 == NULL) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcConstraint, rcViolated);
SRALoaderFile_LOG(ctx->file, klogErr, rc, "missing quality data", NULL);
break;
}
if( (rc = ILL_ZTR_Decompress(ztr_ctx, BASE, fe->sequence, fe->sequence)) != 0 ||
(rc = pstring_assign(&fe->read.seq, fe->sequence.sequence->data, fe->sequence.sequence->datasize)) != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "failed to decompress sequence data", NULL);
break;
}
if( *(void **)&fe->quality4 != NULL ) {
if( (rc = ILL_ZTR_Decompress(ztr_ctx, CNF4, fe->quality4, fe->sequence)) != 0 ||
(rc = pstring_assign(&fe->read.qual, fe->quality4.quality4->data, fe->quality4.quality4->datasize)) != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "failed to decompress quality4 data", NULL);
break;
}
fe->read.qual_type = ILLUMINAWRITER_COLMASK_QUALITY_LOGODDS4;
} else if( *(void **)&fe->quality1 != NULL ) {
if( (rc = ILL_ZTR_Decompress(ztr_ctx, CNF1, fe->quality1, fe->sequence)) != 0 ||
(rc = pstring_assign(&fe->read.qual, fe->quality1.quality1->data, fe->quality1.quality4->datasize)) != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "failed to decompress quality1 data", NULL);
break;
}
fe->read.qual_type = ILLUMINAWRITER_COLMASK_QUALITY_PHRED;
}
if( *(void **)&fe->signal != NULL ) {
if( (rc = ILL_ZTR_Decompress(ztr_ctx, SMP4, fe->signal, fe->sequence)) != 0 ||
(rc = pstring_assign(&fe->read.signal, fe->signal.signal4->data, fe->signal.signal4->datasize)) != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "failed to decompress signal data", NULL);
break;
}
}
if( *(void **)&fe->intensity != NULL ) {
if( (rc = ILL_ZTR_Decompress(ztr_ctx, SMP4, fe->intensity, fe->sequence)) != 0 ||
(rc = pstring_assign(&fe->read.intensity, fe->intensity.signal4->data, fe->intensity.signal4->datasize)) != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "failed to decompress intensity data", NULL);
break;
}
}
if( *(void **)&fe->noise != NULL ) {
if( (rc = ILL_ZTR_Decompress(ztr_ctx, SMP4, fe->noise, fe->sequence)) != 0 ||
(rc = pstring_assign(&fe->read.noise, fe->noise.signal4->data, fe->noise.signal4->datasize)) != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "failed to decompress noise data", NULL);
break;
}
}
rc = fe_new_read(fe, flags, &readId);
break;
}
if(fe->sequence.sequence) {
if(fe->sequence.sequence->data)
free(fe->sequence.sequence->data);
free(fe->sequence.sequence);
}
if(fe->quality1.quality1) {
if(fe->quality1.quality1->data)
free(fe->quality1.quality1->data);
free(fe->quality1.quality1);
}
if(fe->quality4.quality4) {
if(fe->quality4.quality4->data)
free(fe->quality4.quality4->data);
free(fe->quality4.quality4);
}
if(fe->signal.signal4) {
if(fe->signal.signal4->data)
free(fe->signal.signal4->data);
free(fe->signal.signal4);
}
if(fe->intensity.signal4) {
if(fe->intensity.signal4->data)
free(fe->intensity.signal4->data);
free(fe->intensity.signal4);
}
if(fe->noise.signal4) {
if(fe->noise.signal4->data)
free(fe->noise.signal4->data);
free(fe->noise.signal4);
}
return rc;
}
static
rc_t parse_v1_read(SRF_context *ctx, ZTR_Context *ztr_ctx, const uint8_t *data, size_t size)
{
rc_t rc = 0;
size_t i, parsed;
ztr_raw_t ztr_raw;
ztr_t ztr;
enum ztr_chunk_type type;
fe_context_t* fe = (fe_context_t*)ctx;
uint8_t flags;
pstring readId;
EAbisolidReadType read_type;
pstring label;
AbsolidRead read[ABSOLID_FMT_MAX_NUM_READS];
if( fe->region.nreads == 0 ) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcNotFound);
return SRALoaderFile_LOG(ctx->file, klogErr, rc, "missing region chunk before 1st read chunk", NULL);
}
if( (rc = SRF_ParseReadChunk(data, size, &parsed, &flags, &readId)) != 0 ) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rc);
return SRALoaderFile_LOG(ctx->file, klogErr, rc, "SRF parsing failure", NULL);
}
ABI_ZTR_AddToBuffer(ztr_ctx, data + parsed, size - parsed);
/* readId will have spotname */
if( (rc = fe_new_read(fe, &readId, &read_type, &label)) != 0 ) {
return SRALoaderFile_LOG(ctx->file, klogErr, rc, "parsing spot name suffix", NULL);
}
for(i = 0; i < sizeof(read) / sizeof(read[0]); i++) {
AbsolidRead_Init(&read[i]);
}
while(!ABI_ZTR_BufferIsEmpty(ztr_ctx)) {
if( (rc = ABI_ZTR_ParseBlock(ztr_ctx, &ztr_raw)) != 0 ||
(rc = ABI_ZTR_ProcessBlock(ztr_ctx, &ztr_raw, &ztr, &type)) != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "ZTR parsing failure", NULL);
break;
}
switch (type) {
case BASE:
if(ztr.sequence->datatype != i8) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnexpected);
SRALoaderFile_LOG(ctx->file, klogErr, rc, "read: expected 8-bit datatype", NULL);
} else if( read_type > eAbisolidReadType_SPOT ) {
int read_number = AbisolidReadType2ReadNumber[read_type];
if( (rc = pstring_assign(&read[read_number].seq, ztr.sequence->data, ztr.sequence->datasize)) == 0 ) {
/* grab 1st, may be the only cs_key */
read[read_number].cs_key = fe->region.cs_key[0];
for(i = 1; i < fe->region.nreads; i++) {
if( read_type == fe->region.type[i] ) {
read[read_number].cs_key = fe->region.cs_key[i];
break;
}
}
SRF_set_read_filter(&read[read_number].filter, flags);
rc = pstring_copy(&read[read_number].label, &label);
DEBUG_MSG(3, ("SRF READ: '%s'\n", read[read_number].seq.data));
}
if( rc != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "copying read", NULL);
}
} else {
for(i = 0; rc == 0 && i < fe->region.nreads; i++) {
int read_number = AbisolidReadType2ReadNumber[fe->region.type[i]];
size_t len = (i + 1 >= fe->region.nreads ? ztr.sequence->datasize : fe->region.start[i + 1]) - fe->region.start[i];
rc = pstring_assign(&read[read_number].seq, &ztr.sequence->data[fe->region.start[i]], len);
read[read_number].cs_key = fe->region.cs_key[i];
SRF_set_read_filter(&read[read_number].filter, flags);
if( fe->region.label[i].len != 0 ) {
rc = pstring_copy(&read[read_number].label, &fe->region.label[i]);
}
DEBUG_MSG(3, ("SRF READ[%u]: '%s'\n", i, read[read_number].seq.data));
}
if( rc != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "copying reads", NULL);
}
}
break;
case CNF1:
if(ztr.quality1->datatype != i8) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnexpected);
SRALoaderFile_LOG(ctx->file, klogErr, rc, "quality: expected 8-bit datatype", NULL);
} else if( read_type > eAbisolidReadType_SPOT ) {
int read_number = AbisolidReadType2ReadNumber[read_type];
if( (rc = pstring_assign(&read[read_number].qual, ztr.quality1->data, ztr.quality1->datasize)) == 0 ) {
DEBUG_MSG(3, ("SRF QUAL: %u bytes\n", read[read_number].qual.len));
}
if( rc != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "copying quality", NULL);
}
} else {
for(i = 0; rc == 0 && i < fe->region.nreads; i++) {
int read_number = AbisolidReadType2ReadNumber[fe->region.type[i]];
size_t len = (i + 1 >= fe->region.nreads ? ztr.quality1->datasize : fe->region.start[i + 1]) - fe->region.start[i];
rc = pstring_assign(&read[read_number].qual, &ztr.quality1->data[fe->region.start[i]], len);
DEBUG_MSG(3, ("SRF QUAL[%u]: %u bytes\n", i, read[read_number].qual.len));
}
if( rc != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "copying qualities", NULL);
}
}
break;
case SAMP:
if( !fe->skip_signal ) {
size_t i;
int stype = ABSOLID_FMT_COLMASK_NOTSET;
if(ztr.signal->datatype != f32) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnexpected);
SRALoaderFile_LOG(ctx->file, klogErr, rc, "signal: expected 32-bit float datatype", NULL);
} else if( (ztr.signal->datasize % sizeof(float)) != 0 ) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcInvalid);
SRALoaderFile_LOG(ctx->file, klogErr, rc, "signal: size not 32-bit float aligned", NULL);
} else if (ztr.signal->channel == NULL) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcIncomplete);
SRALoaderFile_LOG(ctx->file, klogErr, rc, "SIGNAL column: missing channel type", NULL);
} else if(strcmp(ztr.signal->channel, "0FAM") == 0) {
stype = ABSOLID_FMT_COLMASK_FAM;
} else if(strcmp(ztr.signal->channel, "1CY3") == 0) {
stype = ABSOLID_FMT_COLMASK_CY3;
} else if(strcmp(ztr.signal->channel, "2TXR") == 0) {
stype = ABSOLID_FMT_COLMASK_TXR;
} else if(strcmp(ztr.signal->channel, "3CY5") == 0) {
stype = ABSOLID_FMT_COLMASK_CY5;
} else {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnexpected);
SRALoaderFile_LOG(ctx->file, klogErr, rc, "SIGNAL column: unexpected channel type", NULL);
}
#if __BYTE_ORDER == __LITTLE_ENDIAN
for(i = 0; rc == 0 && i < ztr.signal->datasize; i += 4) {
uint32_t* r = (uint32_t*)&ztr.signal->data[i];
*r = bswap_32(*r);
}
#endif
if( rc == 0 ) {
if( read_type > eAbisolidReadType_SPOT ) {
int read_number = AbisolidReadType2ReadNumber[read_type];
pstring* d = NULL;
switch(stype) {
case ABSOLID_FMT_COLMASK_FAM:
read[read_number].fs_type = eAbisolidFSignalType_FAM;
d = &read[read_number].fxx;
break;
case ABSOLID_FMT_COLMASK_CY3:
d = &read[read_number].cy3;
break;
case ABSOLID_FMT_COLMASK_TXR:
d = &read[read_number].txr;
break;
case ABSOLID_FMT_COLMASK_CY5:
d = &read[read_number].cy5;
break;
}
if( d ) {
rc = pstring_assign(d, ztr.signal->data, ztr.signal->datasize);
DEBUG_MSG(3, ("SRF SIGNAL[%s]: %u bytes\n", ztr.signal->channel, d->len));
} else {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnrecognized);
}
if( rc != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "copying signal", NULL);
}
} else {
if( fe->region.nreads <= 0 || fe->region.nreads > ABSOLID_FMT_MAX_NUM_READS ) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnsupported);
SRALoaderFile_LOG(fe->ctx.file, klogErr, rc, "read count $(c)", PLOG_U8(c), fe->region.nreads);
}
for(i = 0; rc == 0 && i < fe->region.nreads; i++) {
pstring* d = NULL;
int read_number = AbisolidReadType2ReadNumber[fe->region.type[i]];
size_t len = (i + 1 >= fe->region.nreads) ? ztr.signal->datasize : (fe->region.start[i + 1] * sizeof(float));
len -= fe->region.start[i] * sizeof(float);
switch(stype) {
case ABSOLID_FMT_COLMASK_FAM:
read[read_number].fs_type = eAbisolidFSignalType_FAM;
d = &read[read_number].fxx;
break;
case ABSOLID_FMT_COLMASK_CY3:
d = &read[read_number].cy3;
break;
case ABSOLID_FMT_COLMASK_TXR:
d = &read[read_number].txr;
break;
case ABSOLID_FMT_COLMASK_CY5:
d = &read[read_number].cy5;
break;
}
if( d ) {
rc = pstring_assign(d, &ztr.signal->data[fe->region.start[i] * sizeof(float)], len);
DEBUG_MSG(3, ("SRF SIGNAL[%s]: %u bytes\n", ztr.signal->channel, d->len));
} else {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnrecognized);
}
}
if( rc != 0 ) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "copying signals", NULL);
}
}
}
}
break;
default:
break;
}
if(type != none && type != ignore) {
free(*(void **)&ztr);
}
}
if(rc == 0) {
if( read_type <= eAbisolidReadType_SPOT ) {
rc = SRAWriteAbsolid_Write(fe->writer, ctx->file, &readId, NULL, &read[0], &read[1]);
} else {
switch( AbisolidReadType2ReadNumber[read_type] ) {
case 0:
rc = SRAWriteAbsolid_Write(fe->writer, ctx->file, &readId, NULL, &read[0], NULL);
break;
case 1:
rc = SRAWriteAbsolid_Write(fe->writer, ctx->file, &readId, NULL, NULL, &read[1]);
break;
default:
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnsupported);
SRALoaderFile_LOG(ctx->file, klogErr, rc, "more than 2 reads", NULL);
break;
}
}
}
return rc;
}
