LIB_EXPORT rc_t CC KCurlRequestAddSFields( struct KCurlRequest *self, const String * fields )
{
rc_t rc = 0;
if ( self == NULL )
return RC( rcNS, rcFile, rcReading, rcSelf, rcNull );
if ( fields == NULL || fields->size == 0 )
return RC( rcNS, rcFile, rcReading, rcParam, rcNull );
if ( self->fields_chars == 0 )
{
rc = KDataBufferResize ( &self->fields, fields->size + 1 );
if ( rc == 0 )
{
memcpy( self->fields.base, fields->addr, fields->size );
( (uint8_t *)self->fields.base)[ fields->size ] = 0;
self->fields_chars = fields->size;
}
}
else
{
rc = KDataBufferResize ( &self->fields, self->fields_chars + fields->size + 1 );
if ( rc == 0 )
{
( (uint8_t *)self->fields.base)[ self->fields_chars ] = '&';
memcpy( &( ( (uint8_t *)self->fields.base)[ self->fields_chars + 1 ] ), fields->addr, fields->size );
self->fields_chars += ( fields->size + 1 );
( (uint8_t *)self->fields.base)[ self->fields_chars ] = 0;
}
}
return rc;
}
LIB_EXPORT rc_t CC KCurlRequestAddSField( struct KCurlRequest *self, const String * name, const String * value )
{
rc_t rc = 0;
size_t needed;
if ( self == NULL )
return RC( rcNS, rcFile, rcReading, rcSelf, rcNull );
if ( name == NULL || name->size == 0 || value == NULL || value->size == 0 )
return RC( rcNS, rcFile, rcReading, rcParam, rcNull );
needed = ( self->fields_chars + name->size + value->size + 3 );
rc = KDataBufferResize ( &self->fields, needed );
if ( rc == 0 )
{
size_t num_writ;
if ( self->fields_chars == 0 )
{
rc = string_printf ( ( char * )self->fields.base, needed, &num_writ,
"%S=%S", name, value );
}
else
{
uint8_t * dst = ( uint8_t * )self->fields.base;
rc = string_printf ( ( char * )( &dst[ self->fields_chars ] ), needed - self->fields_chars, &num_writ,
"&%S=%S", name, value );
}
if ( rc == 0 )
self->fields_chars += num_writ;
}
return rc;
}
static
rc_t CC build_scaffold_impl(void *const data, VXformInfo const *const info,
int64_t const row_id, VRowResult *const rslt,
uint32_t const argc, VRowData const argv[])
{
self_t const *const self = data;
INSDC_coord_one const *const start = argv[0].u.data.base;
INSDC_coord_len const *const length = argv[1].u.data.base;
NCBI_WGS_component_props const *const props = argv[2].u.data.base;
int64_t const *const join = argv[3].u.data.base;
unsigned const components = argv[0].u.data.elem_count;
rc_t rc;
assert(argv[1].u.data.elem_count == components);
assert(argv[2].u.data.elem_count == components);
assert(start != NULL);
assert(length != NULL);
assert(props != NULL);
assert(join != NULL);
assert(rslt->elem_bits == 8);
rslt->elem_count = total_length(components, &length[argv[1].u.data.first_elem]);
rc = KDataBufferResize(rslt->data, rslt->elem_count);
if (rc == 0)
rc = self->impl(self, rslt->data->base, components,
&start[argv[0].u.data.first_elem],
&length[argv[1].u.data.first_elem],
&props[argv[2].u.data.first_elem],
&join[argv[3].u.data.first_elem]);
return rc;
}
static rc_t read_bounded_quality_fix( struct cmn_iter * cmn,
uint32_t col_id,
KDataBuffer * qual_buffer,
char * q2a,
String * quality,
uint32_t fixed_len )
{
uint8_t * qual_values = NULL;
uint32_t num_qual;
rc_t rc = cmn_read_uint8_array( cmn, col_id, &qual_values, &num_qual );
num_qual = fixed_len;
if ( rc == 0 && num_qual > 0 && qual_values != NULL )
{
if ( num_qual > qual_buffer -> elem_count )
rc = KDataBufferResize ( qual_buffer, num_qual );
if ( rc == 0 )
{
uint32_t idx;
uint8_t * b = qual_buffer -> base;
for ( idx = 0; idx < num_qual; idx++ )
b[ idx ] = q2a[ qual_values[ idx ] ];
quality -> addr = qual_buffer -> base;
quality -> len = num_qual;
quality -> size = num_qual;
}
}
if ( rc != 0 )
{
quality -> addr = NULL;
quality -> len = 0;
quality -> size = 0;
rc = 0;
}
return rc;
}
static
rc_t CC seq_construct_read ( void *data, const VXformInfo *info, int64_t row_id,
VRowResult *rslt, uint32_t argc, const VRowData argv [] )
{
rc_t rc;
uint8_t *dst;
/* base pointers to input data */
const uint8_t *aligned = argv [ 0 ] . u . data . base;
const INSDC_coord_len *aligned_len = argv [ 1 ] . u . data . base;
const uint8_t *unaligned = argv [ 2 ] . u . data . base;
const INSDC_coord_len *unaligned_len = argv [ 3 ] . u . data . base;
/* the number of reads */
uint32_t i, seqlen, aligned_seqlen, unaligned_seqlen;
uint32_t nreads = argv [ 1 ] . u . data . elem_count;
/* require the same dimension for both inputs */
if ( argv [ 1 ] . u . data . elem_count != argv [ 3 ] . u . data . elem_count )
return RC ( rcXF, rcFunction, rcExecuting, rcData, rcInconsistent );
/* offset base pointers to start of row */
aligned += argv [ 0 ] . u . data . first_elem;
aligned_len += argv [ 1 ] . u . data . first_elem;
unaligned += argv [ 2 ] . u . data . first_elem;
unaligned_len += argv [ 3 ] . u . data . first_elem;
/* calculate new sequence length */
for ( i = seqlen = 0; i < nreads; ++ i )
seqlen += unaligned_len [ i ] != 0 ? unaligned_len [ i ] : aligned_len [ i ];
/* set output buffer size */
rslt -> data -> elem_bits = 8;
rc = KDataBufferResize ( rslt -> data, seqlen );
if ( rc != 0 )
return rc;
/* produce final sequence */
for ( dst = rslt -> data -> base, i = seqlen = aligned_seqlen = unaligned_seqlen = 0; i < nreads; ++ i )
{
/* give preference to unaligned */
if ( unaligned_len [ i ] != 0 )
{
memcpy ( & dst [ seqlen ], & unaligned [ unaligned_seqlen ], unaligned_len [ i ] );
seqlen += unaligned_len [ i ];
}
else
{
memcpy ( & dst [ seqlen ], & aligned [ aligned_seqlen ], aligned_len [ i ] );
seqlen += aligned_len [ i ];
}
aligned_seqlen += aligned_len [ i ];
unaligned_seqlen += unaligned_len [ i ];
}
rslt -> elem_count = seqlen;
return 0;
}
static
rc_t CC mismatch_restore_qual_impl ( void *data, const VXformInfo *info, int64_t row_id,
VRowResult *rslt, uint32_t argc, const VRowData argv [] )
{
rc_t rc;
unsigned i, j;
INSDC_quality_phred const *qual = argv[0].u.data.base;
unsigned const nqual = argv[0].u.data.elem_count;
bool const *mismatch = argv[1].u.data.base;
unsigned const nmis = argv[1].u.data.elem_count;
INSDC_quality_phred const rqual = (intptr_t)data;
INSDC_quality_phred *dst;
qual += argv[0].u.data.first_elem;
mismatch += argv[1].u.data.first_elem;
rslt->data->elem_bits = argv[0].u.data.elem_bits;
rc = KDataBufferResize( rslt -> data, nmis );
if ( rc != 0 ) return rc;
rslt -> elem_count = nmis;
dst = rslt -> data -> base;
for (j = i = 0; i != nmis; ++i) {
dst[i] = mismatch[i] ? qual[j++] : rqual;
}
assert(j == nqual);
return 0;
}
static
rc_t CC subtract_int64_t ( void *self, const VXformInfo *info,
int64_t row_id, VRowResult *rslt,
uint32_t argc, const VRowData argv [] )
{
rc_t rc;
uint32_t i;
int64_t *dst;
const int64_t *src = argv [ 0 ] . u . data . base;
uint64_t elem_count = argv [ 0 ] . u . data . elem_count;
assert ( rslt -> data -> elem_bits == 64 );
rc = KDataBufferResize ( rslt -> data, elem_count );
if ( rc != 0 )
return rc;
dst = rslt -> data -> base;
rslt -> elem_count = elem_count;
/* while this function is typically used on single-element rows,
we advertise taking rows of any length... */
for ( i = 0, src += argv [ 0 ] . u . data . first_elem; i < elem_count; ++ i )
dst [ i ] = src [ i ] - row_id;
return 0;
}
static rc_t CC perform_entrez_fetch ( void *self,
const VXformInfo *info, int64_t row_id, VRowResult *rslt,
uint32_t argc, const VRowData argv [] )
{
rc_t rc = 0;
entrez_context_t * ctx = ( entrez_context_t * )self;
if ( ctx == NULL )
rc = RC ( rcXF, rcFunction, rcReading, rcSelf, rcNull );
else
{
char * entrez_seq_id = "NC_000001";
uint32_t entrez_seq_len = 1024;
uint64_t entrez_row_id = 1;
uint32_t entrez_row_count = 1;
size_t buffsize;
if ( argc > 0 )
rc = vrowdata_get64( &( argv[0] ), &entrez_row_id );
if ( rc == 0 )
{
if ( argc > 1 )
rc = vrowdata_get32( &( argv[1] ), &entrez_seq_len );
if ( rc == 0 )
{
/* prepare the Fetcher with all the information it needs,
returns in buffsize the size of the buffer we have to provide */
rc = KEntrezFetcherSetup ( ctx->entrez_fetcher,
entrez_server, entrez_seq_id, entrez_seq_len,
entrez_row_id, entrez_row_count, &buffsize );
if ( rc == 0 )
{
rc = KDataBufferResize( rslt->data, buffsize );
if ( rc == 0 )
{
size_t num_read;
/* perform the fetch-operation directly into the KDatabuffer */
rc = KEntrezFetcherRead ( ctx->entrez_fetcher,
rslt->data->base,
buffsize, &num_read );
if ( rc == 0 )
{
rslt->elem_count = num_read;
rslt->elem_bits = 8;
}
}
}
}
}
}
return rc;
}
/*
function ascii NCBI:align:ref_seq_id ( I64 ref_id );
*/
static
rc_t CC align_ref_seq_id ( void *data, const VXformInfo *info,
int64_t row_id, VRowResult *rslt, uint32_t argc, const VRowData argv[] )
{
rc_t rc;
RefSeqID const *self = ( void const * )data;
char const *name = NULL;
uint32_t name_len;
/* get start and length of reference segment */
int64_t const *ref_id = argv[REF_ID].u.data.base;
assert( argv[ REF_ID ].u.data.elem_bits == sizeof( *ref_id ) * 8) ;
if ( self->curs == NULL || argv[ REF_ID ].u.data.elem_count == 0 )
{
rslt->elem_count = 0;
return 0;
}
ref_id += argv[ REF_ID] .u.data.first_elem;
SUB_DEBUG( ( "SUB.Rd in 'align_ref_seq_id.c' at #%lu\n", ref_id[ 0 ] ) );
rc = VCursorCellDataDirect( self->curs, ref_id[ 0 ], self->seqID_idx, NULL, (void const **)&name, NULL, &name_len );
if ( GetRCState( rc ) == rcNotFound && GetRCObject( rc ) == rcRow )
{
name = "";
name_len = 0;
}
else if ( rc != 0 )
{
return rc;
}
rc = KDataBufferCast( rslt->data, rslt->data, sizeof( name[ 0 ] ) * 8, true );
if ( rc != 0 )
return rc;
rc = KDataBufferResize( rslt->data, name_len );
if ( rc != 0 )
return rc;
memmove( rslt->data->base, name, sizeof( name[ 0 ] ) * name_len );
rslt->elem_count = name_len;
rslt->elem_bits = sizeof( name[ 0 ] ) * 8;
return rc;
}
static size_t CC KCurlFileCallback( void *ptr, size_t size, size_t nmemb, void *data )
{
rc_t rc = 0;
size_t given_bytes = size * nmemb; /* calculate the size given in ptr */
ReadContext *ctx = ( ReadContext * )data;
if ( ctx != NULL && ctx->buffer != NULL )
{
rc = KDataBufferResize ( ctx->buffer, ctx->num_read + given_bytes + 2 );
if ( rc == 0 )
{
memcpy( &( ( (uint8_t *)ctx->buffer->base )[ ctx->num_read ] ), ptr, given_bytes );
ctx->num_read += given_bytes;
}
}
return given_bytes;
}
/* Open
*/
rc_t KColumnIdx2OpenRead ( KColumnIdx2 *self,
const KDirectory *dir, uint64_t eof )
{
rc_t rc;
rc = KDataBufferMake ( & self -> cstorage, sizeof ( KColumnIdx2BlockCache ) * 8, 0 );
if ( rc != 0 )
{
memset ( self, 0, sizeof * self );
return rc;
}
self -> last = 0;
if ( eof == 0 )
{
#if 0
KDataBufferResize(&self->cstorage,0);
self -> last = 0;
#endif
self -> eof = 0;
self -> f = NULL;
return 0;
}
rc = KDirectoryVOpenFileRead ( dir,
& self -> f, "idx2", NULL );
#if IDX2_READ_FILE_BUFFER
if ( rc == 0 )
{
const KFile * orig = self -> f;
rc = KBufFileMakeRead ( & self -> f, self -> f, IDX2_READ_FILE_BUFFER );
if ( rc == 0 )
{
KFileRelease ( orig );
}
else
{
self -> f = orig;
rc = 0;
}
}
#endif
if ( rc == 0 )
rc = KColumnIdx2Init ( self, eof );
return rc;
}
rc_t BlobHeadersSerialize ( BlobHeaders *self, KDataBuffer *buffer,
uint64_t offset, uint64_t *psz )
{
rc_t rc;
uint64_t sz = 0;
assert(buffer->elem_bits == 8);
assert(buffer->bit_offset == 0);
rc = KDataBufferResize(buffer, 1);
if (rc)
return rc;
((uint8_t *)(buffer->base))[offset] = 0;
rc = serialize(self, buffer, offset + 1, &sz);
*psz = sz - offset;
return rc;
}
static
rc_t CC row_id_func ( void *self, const VXformInfo *info,
int64_t row_id, VRowResult *rslt, uint32_t argc, const VRowData in [] )
{
assert ( rslt -> data -> elem_bits == 64 );
if ( rslt -> data -> elem_count == 0 )
{
rc_t rc = KDataBufferResize ( rslt -> data, 1 );
if ( rc != 0 )
return rc;
}
* ( int64_t* ) rslt -> data -> base = row_id;
rslt -> elem_count = 1;
return 0;
}
static int string_buffer_add(KDataBuffer *const self,
unsigned const len, char const name[/* len */])
{
size_t const rslt = self->elem_count;
size_t const newsize = rslt + len;
if (KDataBufferResize(self, newsize) != 0)
return -1;
#if 1
PLOGMSG(klogInfo, (klogInfo, "New spot group '$(NAME)'", "NAME=%.*s", (int)len, name));
#endif
memcpy(((char *)self->base) + rslt, name, len);
return (int)rslt;
}
static
rc_t CC dynamic_read_desc_static ( void *self, const VXformInfo *info, int64_t row_id,
VRowResult *rslt, uint32_t argc, const VRowData argv [] )
{
rc_t rc;
dynamic_read_desc *p;
KDataBuffer *dst = rslt -> data;
/* severe error if adapter is longer than spot */
if ( argv [ 0 ] . u . data . elem_count < argv [ 1 ] . u . data . elem_count )
return RC ( rcSRA, rcFunction, rcExecuting, rcData, rcCorrupt );
/* the buffer should have already been given the correct element size */
if ( dst -> elem_bits != 32 * 3 )
{
rc = KDataBufferCast ( dst, dst, 32 * 3, true );
if ( rc != 0 )
return rc;
}
/* we always produce 2 reads */
if ( dst -> elem_count != 2 )
{
rc = KDataBufferResize ( dst, 2 );
if ( rc != 0 )
return rc;
}
p = dst -> base;
/* adapter */
p [ 0 ] [ dyn_read_type ] = SRA_READ_TYPE_TECHNICAL;
p [ 0 ] [ dyn_read_start ] = 0;
assert(argv [ 1 ] . u . data . elem_count >> 32 == 0);
p [ 0 ] [ dyn_read_len ] = (uint32_t)argv [ 1 ] . u . data . elem_count;
/* fragment */
p [ 1 ] [ dyn_read_type ] = SRA_READ_TYPE_BIOLOGICAL;
p [ 1 ] [ dyn_read_start ] = p [ 0 ] [ dyn_read_len ];
assert(argv [ 0 ] . u . data . elem_count >> 32 == 0);
p [ 1 ] [ dyn_read_len ] = (uint32_t)(argv [ 0 ] . u . data . elem_count) - p [ 0 ] [ dyn_read_len ];
rslt -> elem_count = 2;
return 0;
}
static
rc_t CC row_len_func ( void *self, const VXformInfo *info,
int64_t row_id, VRowResult *rslt, uint32_t argc, const VRowData in[] )
{
assert ( rslt -> data -> elem_bits == 32 );
if ( rslt -> data -> elem_count == 0 )
{
rc_t rc = KDataBufferResize ( rslt -> data, 1 );
if ( rc != 0 )
return rc;
}
/* for the moment row length are 32-bit */
* ( uint32_t* ) rslt -> data -> base = (uint32_t)in [ 0 ] . u . data . elem_count;
rslt -> elem_count = 1;
return 0;
}
static group_stats_t *stats_data_get_group(stats_data_t *const self,
unsigned const len,
char const name[])
{
unsigned i;
unsigned which;
group_stats_t *fnd;
for (i = 0; i < STATS_DATA_MRU_COUNT && i < self->count; ++i) {
group_stats_t *const k = self->mru[i];
if (k && stats_data_name_cmp(self, k, len, name) == 0) {
fnd = k;
goto UPDATE_MRU;
}
}
which = stats_data_search(self, len, name);
if (which < self->count && stats_data_name_cmp_idx(self, which, len, name) == 0)
fnd = get_group(self, which);
else {
int const offset = string_buffer_add(&self->names, len, name);
unsigned const move = self->count - which;
if (offset < 0)
return NULL;
if (++self->count > MAX_GROUP_COUNT)
return get_group(self, MAX_GROUP_COUNT);
if (KDataBufferResize(&self->group, self->count) != 0)
return NULL;
fnd = get_group(self, which);
assert(fnd + move + 1 <= get_group(self, self->count) || move == 0);
memmove(fnd + 1, fnd, move * sizeof(*fnd));
group_stats_init(fnd, offset, len, self->count);
stats_data_invald_mru(self);
}
UPDATE_MRU:
stats_data_update_mru(self, fnd);
return fnd;
}
/* MARK: SequenceRecord Object */
static
rc_t SequenceRecordResize(SequenceRecord *self,
KDataBuffer *storage,
unsigned numreads,
unsigned seqLen)
{
size_t sz;
rc_t rc;
sz = seqLen * (sizeof(self->seq[0]) + sizeof(self->qual[0])) +
numreads * (sizeof(self->ti) +
sizeof(self->readStart[0]) +
sizeof(self->readLen[0]) +
sizeof(self->aligned[0]) +
sizeof(self->orientation[0]) +
sizeof(self->alignmentCount[0]) +
sizeof(self->cskey[0])
);
storage->elem_bits = 8;
rc = KDataBufferResize(storage, sz);
if (rc)
return rc;
self->numreads = numreads;
self->ti = (uint64_t *)storage->base;
self->readStart = (uint32_t *)&self->ti[numreads];
self->readLen = (uint32_t *)&self->readStart[numreads];
self->aligned = (bool *)&self->readLen[numreads];
self->orientation = (uint8_t *)&self->aligned[numreads];
self->is_bad = (uint8_t *)&self->orientation[numreads];
self->alignmentCount = (uint8_t *)&self->is_bad[numreads];
self->cskey = (char *)&self->alignmentCount[numreads];
self->seq = (char *)&self->cskey[numreads];
self->qual = (uint8_t *)&self->seq[seqLen];
self->spotGroup = NULL;
self->spotGroupLen = 0;
return 0;
}
static
rc_t CC vec_sum_row_func(void *const Self,
VXformInfo const *info,
int64_t const row_id,
VRowResult *const rslt,
uint32_t const argc,
VRowData const argv[])
{
self_t const *self = Self;
rc_t rc;
assert(argc == 1);
rslt->data->elem_bits = rslt->elem_bits;
rc = KDataBufferResize(rslt->data, rslt->elem_count = 1);
if (rc == 0) {
self->f(rslt->data->base,
argv[0].u.data.base, argv[0].u.data.first_elem,
argv[0].u.data.elem_count, 1);
}
return rc;
}
/*
function ascii NCBI:align:ref_name ( I64 ref_id );
*/
static
rc_t CC align_ref_name ( void *data, const VXformInfo *info,
int64_t row_id, VRowResult *rslt, uint32_t argc, const VRowData argv[] )
{
rc_t rc;
RefName const *self = (void const *)data;
char const *name = NULL;
uint32_t name_len;
/* get start and length of reference segment */
int64_t const *ref_id = argv[REF_ID].u.data.base;
if (argv[REF_ID].u.data.elem_count == 0)
rc = RC(rcAlign, rcFunction, rcExecuting, rcRow, rcNotFound);
else {
assert(argv[REF_ID].u.data.elem_bits == sizeof(*ref_id) * 8);
ref_id += argv[REF_ID].u.data.first_elem;
rc = VCursorCellDataDirect(self->curs, ref_id[0], self->name_idx, NULL, (void const **)&name, NULL, &name_len);
}
if (GetRCState(rc) == rcNotFound && GetRCObject(rc) == rcRow) {
name = "";
name_len = 0;
}
else if (rc) return rc;
rc = KDataBufferCast(rslt->data, rslt->data, sizeof(name[0]) * 8, true);
if (rc) return rc;
rc = KDataBufferResize(rslt->data, name_len);
if (rc) return rc;
memcpy(rslt->data->base, name, sizeof(name[0]) * name_len);
rslt->elem_count = name_len;
rslt->elem_bits = sizeof(name[0]) * 8;
return rc;
}
static
rc_t CC generate_mismatch_qual ( void *data, const VXformInfo *info,
int64_t row_id, VRowResult *rslt, uint32_t argc, const VRowData argv[] )
{
INSDC_quality_phred const *qual = argv[QUAL].u.data.base;
bool const *has_mismatch = argv[HAS_MISMATCH].u.data.base;
INSDC_coord_len const *left = argv[LEFT_CLIP].u.data.base;
INSDC_coord_len const *right = argv[RIGHT_CLIP].u.data.base;
unsigned const readlen = argv[QUAL].u.data.elem_count;
INSDC_quality_phred *dst;
rc_t rc;
unsigned i;
unsigned j;
assert(argv[QUAL].u.data.elem_bits == sizeof(qual[0]) * 8);
assert(argv[HAS_MISMATCH].u.data.elem_bits == sizeof(has_mismatch[0]) * 8);
assert(argv[LEFT_CLIP].u.data.elem_bits == sizeof(left[0]) * 8);
assert(argv[RIGHT_CLIP].u.data.elem_bits == sizeof(right[0]) * 8);
assert(argv[HAS_MISMATCH].u.data.elem_count == readlen);
qual += argv[QUAL].u.data.first_elem;
has_mismatch += argv[HAS_MISMATCH].u.data.first_elem;
left += argv[LEFT_CLIP ].u.data.first_elem;
right += argv[RIGHT_CLIP].u.data.first_elem;
rslt->data->elem_bits = argv[QUAL].u.data.elem_bits;
rc = KDataBufferResize(rslt->data, argv[QUAL].u.data.elem_count);
if (rc)
return rc;
dst = rslt->data->base;
for (j = 0, i = left[0]; i != readlen - right[0]; ++i) {
if (has_mismatch[i])
dst[j++] = qual[i];
}
rslt->elem_count = j;
return 0;
}
static
rc_t CC environment_read_func(
void *Self,
const VXformInfo *info,
int64_t row_id,
VRowResult *rslt,
uint32_t argc,
const VRowData argv[]
) {
const KDataBuffer *value = Self;
rc_t rc = 0;
rslt->data->elem_bits = value->elem_bits;
rslt->data->elem_count = 0;
rc = KDataBufferResize(rslt->data, value->elem_count);
if (rc == 0) {
memcpy(rslt->data->base, value->base, KDataBufferBytes(value));
rc = KDataBufferCast(rslt->data, rslt->data, rslt->elem_bits, true);
if (rc == 0)
rslt->elem_count = rslt->data->elem_count;
}
return rc;
}
static
rc_t CC raw_restore_read_impl ( void *data, const VXformInfo *info, int64_t row_id,
VRowResult *rslt, uint32_t argc, const VRowData argv [] )
{
rc_t rc;
int i, j;
const INSDC_4na_bin *qual = argv[ 0 ].u.data.base;
const uint32_t read_len = (uint32_t)argv[ 0 ].u.data.elem_count;
const uint8_t *strand = argv[ 1 ].u.data.base;
const uint32_t strand_len = (uint32_t)argv[ 1 ].u.data.elem_count;
INSDC_4na_bin *dst;
assert( argv[ 0 ].u.data.elem_bits == 8 );
assert( argv[ 1 ].u.data.elem_bits == 8 );
assert( strand_len == 1 );
qual += argv[ 0 ].u.data.first_elem;
strand += argv[ 1 ].u.data.first_elem;
/* resize output row for the total number of reads */
rslt -> data -> elem_bits = 8;
rc = KDataBufferResize ( rslt -> data, read_len );
if ( rc != 0 )
return rc;
rslt -> elem_count = read_len;
dst = rslt -> data -> base;
/**** MAIN RESTORATION LOOP ***/
if ( strand[ 0 ] == false ) /*** nothing to do **/
{
memmove( dst, qual, read_len );
} else for ( i = 0, j = read_len - 1; i < (int)read_len; i++, j-- )
{
dst[ i ] = qual[ j ];
}
return 0;
}
static
rc_t CC get_sam_flags_impl(void *data, const VXformInfo *info,
int64_t row_id, VRowResult *rslt,
uint32_t nreads,
uint32_t argc, const VRowData argv [] )
{
rc_t rc;
int32_t *dst;
const INSDC_coord_one *rid = argv[ 1 ].u.data.base; /* SEQ_READ_ID */
const int32_t *tlen = argv[ 2 ].u.data.base; /* TEMPLATE_LEN */
const bool *ro1 = argv[ 3 ].u.data.base; /* REF_ORIENTATION */
const bool *ro2 = argv[ 4 ].u.data.base; /* MATE_REF_ORIENTATION */
const bool *sec = argv[ 5 ].u.data.base;
const bool mate_present = ( argv[ 4 ].u.data.elem_count > 0 );
const SRAReadFilter *flt = argc > 6 ? argv[ 6 ].u.data.base : NULL;
assert( argv[ 1 ].u.data.elem_count == 1 );
assert( argv[ 2 ].u.data.elem_count == 1 );
assert( argv[ 3 ].u.data.elem_count == 1 );
assert( argv[ 5 ].u.data.elem_count == 1 );
rc = KDataBufferResize( rslt->data, 1 );
if( rc != 0 )
return rc;
rslt->elem_count=1;
dst = rslt->data->base;
dst[ 0 ] = 0;
if( nreads == 0 )
return 0;
rid += argv[ 1 ].u.data.first_elem;
tlen += argv[ 2 ].u.data.first_elem;
ro1 += argv[ 3 ].u.data.first_elem;
ro2 += argv[ 4 ].u.data.first_elem;
sec += argv[ 5 ].u.data.first_elem;
if ( flt != NULL )
flt += argv[ 6 ].u.data.first_elem;
/***************** SAM FLAGS************
Bit Description
0x001 template having multiple fragments in sequencing
0x002 each fragment properly aligned according to the aligner
0x004 fragment unmapped
0x008 next fragment in the template unmapped
0x010 SEQ being reverse complemented
0x020 SEQ of the next fragment in the template being reversed
0x040 the first fragment in the template
0x080 the last fragment in the template
0x100 secondary alignment
0x200 not passing quality controls
0x400 PCR or optical duplicate
****************************/
if ( ro1[ 0 ] )
dst[ 0 ] |= 0x10;
if ( sec[ 0 ] )
dst[ 0 ] |= 0x100;
if ( nreads > 1 )
{
if ( rid[ 0 ] == 1 )
dst[ 0 ] |= 0x40;
if ( rid[ 0 ] == nreads )
dst[ 0 ] |= 0x80;
dst[ 0 ] |= 0x1;
if( mate_present )
{
#if PROPER_ALIGNED_MEANS_SAME_CHROMOSOME
if ( tlen[ 0 ] != 0 )
#endif
dst[ 0 ] |= 0x2;
if ( ro2 [ 0 ] )
dst[ 0 ] |= 0x20;
}
else
{
dst[ 0 ] |= 0x8;
}
}
if ( flt != NULL )
{
if ( flt[ 0 ] == SRA_READ_FILTER_REJECT )
{
dst[ 0 ] |= 0x200;
}
else if ( flt[ 0 ] == SRA_READ_FILTER_CRITERIA )
{
dst[ 0 ] |= 0x400;
}
}
return rc;
}
/*
function INSDC:coord:zero NCBI:align:ref_pos ( I64 ref_id, INSDC:coord:zero ref_start );
*/
static
rc_t CC align_ref_pos ( void *data, const VXformInfo *info,
int64_t row_id, VRowResult *rslt, uint32_t argc, const VRowData argv[] )
{
rc_t rc = 0;
RefPos const *self = ( void const * )data;
int64_t ref_row_id = 0;
INSDC_coord_zero *ref_pos;
unsigned const ploidy = ( unsigned const )argv[ REF_START ].u.data.elem_count;
unsigned i;
/* get start and length of reference segment */
int64_t const *ref_id = 0;
INSDC_coord_zero const *ref_start;
assert( argv[ REF_ID ].u.data.elem_bits == sizeof( *ref_id ) * 8 );
assert( argv[ REF_START ].u.data.elem_bits == sizeof( *ref_start ) * 8 );
ref_start = argv[ REF_START ].u.data.base;
ref_start += argv[ REF_START ].u.data.first_elem;
if ( self->curs != NULL )
{
char const *name = NULL;
uint32_t name_len;
BSTRowRange *brr;
ref_id = argv[ REF_ID ].u.data.base;
ref_id += argv[ REF_ID ].u.data.first_elem;
brr = ( BSTRowRange * )BSTreeFind( &self->tr_range, &ref_id[ 0 ], row_range_cmp );
if ( brr == NULL )
{
RowRange *new_rr;
SUB_DEBUG( ( "SUB.Rd in 'align-ref-pos.c' at #%lu\n", ref_id[ 0 ] ) );
rc = VCursorCellDataDirect( self->curs, ref_id[ 0 ], self->name_idx, NULL, (void const **)&name, NULL, &name_len );
if ( rc != 0 )
return rc;
rc = VCursorParamsSet( ( struct VCursorParams const * )self->curs, "QUERY_SEQ_NAME", "%.*s", name_len, name );
if ( rc != 0 )
return rc;
rc = VCursorCellDataDirect( self->curs, ref_id[ 0 ], self->name_range_idx, NULL, (void const **)&new_rr, NULL, NULL );
if ( rc != 0 )
return rc;
brr = malloc( sizeof( *brr ) );
if ( brr == NULL )
{
return RC( rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted );
}
else
{
memcpy( &brr->rr, new_rr, sizeof( *new_rr ) );
BSTreeInsert( ( BSTree* )&self->tr_range, ( BSTNode* )brr, row_range_sort );
}
}
ref_row_id = brr->rr.start_id;
}
rc = KDataBufferResize( rslt->data, ploidy );
if ( rc != 0 )
return rc;
ref_pos = rslt->data->base;
for ( i = 0; i != ploidy; ++i )
{
ref_pos[ i ] = ref_start[ i ];
if ( self->curs != NULL )
{
ref_pos[ i ] += ( INSDC_coord_zero )( ( ref_id[ 0 ] - ref_row_id ) * self->max_seq_len );
}
}
rslt->elem_count = ploidy;
rslt->elem_bits = sizeof( ref_pos[ 0 ] ) * 8;
return rc;
}
static
rc_t CC seq_restore_read_impl1 ( void *data, const VXformInfo *info, int64_t row_id,
VRowResult *rslt, uint32_t argc, const VRowData argv [] )
{
rc_t rc;
int i;
Read_Restorer *self = data;
INSDC_4na_bin *dst;
INSDC_coord_len len;
uint32_t src_len = (uint32_t)argv[ 0 ].u.data.elem_count;
const INSDC_4na_bin *src = argv[ 0 ].u.data.base;
const uint32_t num_reads = (uint32_t)argv[ 1 ].u.data.elem_count;
const int64_t *align_id = argv[ 1 ].u.data.base;
const INSDC_coord_len *read_len = argv[ 2 ].u.data.base;
const uint8_t *read_type = argv[ 3 ].u.data.base;
bool is_sequential = false;
assert( argv[ 0 ].u.data.elem_bits == 8 );
assert( argv[ 1 ].u.data.elem_bits == 64 );
assert( argv[ 2 ].u.data.elem_bits == sizeof( INSDC_coord_len ) * 8 );
assert( argv[ 2 ].u.data.elem_count == num_reads );
assert( argv[ 3 ].u.data.elem_count == num_reads );
src += argv [ 0 ] . u . data . first_elem;
align_id += argv [ 1 ] . u . data . first_elem;
read_len += argv [ 2 ] . u . data . first_elem;
read_type += argv [ 3 ] . u . data . first_elem;
if ( row_id != self->last_row_id && row_id != self->last_row_id + 1 )
{
self->first_sequential_row_id = row_id;
is_sequential = false;
}
else if ( row_id > self->first_sequential_row_id + 100 )
{
is_sequential = true;
}
self->last_row_id = row_id;
/* is_sequential = false; forcing it to false ... Sept. 16th 2015 to analyze prefetching */
for ( i = 0, len = 0; i < (int)num_reads; i++ )
{
len += read_len[ i ];
}
/* resize output row */
rslt->data->elem_bits = 8;
rc = KDataBufferResize( rslt->data, len );
rslt->elem_count = len;
dst = rslt->data->base;
if ( rc == 0 && len > 0 )
{
if ( len == src_len ) /*** shortcut - all data is local ***/
{
memmove( dst, src, len );
}
else
{
if ( is_sequential &&
( row_id < self->prefetch_start_id || row_id > self->prefetch_stop_id ) )
{ /* do prefetch */
uint32_t num_rows = ( argv[ 1 ].u.data.base_elem_count - argv[ 1 ].u.data.first_elem );
/*
KTimeMs_t ts = KTimeMsStamp();
fprintf( stderr, "\nprefetch row_id #%lu ( start_id #%lu, stop_id #%lu ) num_rows = %d\n", row_id, self->prefetch_start_id, self->prefetch_stop_id, num_rows );
*/
VCursorDataPrefetch( self->curs,
align_id,
self->read_idx,
num_rows,
1,
INT64_MAX,
true );
/*
ts = KTimeMsStamp() - ts;
fprintf( stderr, "prefetch done in %lu ms\n", ts );
*/
self->prefetch_start_id=row_id;
self->prefetch_stop_id =argv[1].blob_stop_id;
}
for( i = 0; i < (int)num_reads && rc == 0; i++ ) /*** checking read by read ***/
{
if ( align_id[ i ] > 0 )
{
const INSDC_4na_bin *r_src;
uint32_t r_src_len;
SUB_DEBUG( ( "SUB.Rd in 'seq-restore-read.c' at #%lu\n", align_id[ i ] ) );
rc = VCursorCellDataDirect( self -> curs, align_id[ i ], self -> read_idx,
NULL, ( const void** ) &r_src, NULL, &r_src_len );
if ( rc == 0 )
{
if ( r_src_len == read_len[ i ] )
{
if ( read_type[ i ] & SRA_READ_TYPE_FORWARD )
{
memmove( dst, r_src, read_len[ i ] );
}
else if ( read_type[ i ] & SRA_READ_TYPE_REVERSE )
{
int j, k;
for( j = 0, k = read_len[ i ] - 1; j < (int)read_len[ i ]; j++, k-- )
{
dst[ j ] = map [ r_src[ k ] & 15 ];
}
}
else
{
rc = RC( rcXF, rcFunction, rcExecuting, rcData, rcInconsistent );
}
}
else
{
rc = RC( rcXF, rcFunction, rcExecuting, rcData, rcInconsistent );
}
}
}
else /*** data is in READ column **/
{
if ( src_len >= read_len[ i ] )
{
memmove( dst, src, read_len[ i ] );
src_len -= read_len[ i ];
src += read_len[ i ];
}
else
{
return RC( rcXF, rcFunction, rcExecuting, rcData, rcInconsistent );
}
}
dst += read_len[ i ];
}
}
}
return rc;
}
/* --------------------------------------------------------------------------------------
argv[ 0 ] ... CMP_READ
argv[ 1 ] ... PRIM_ALIG_ID
argv[ 2 ] ... READ_LEN
argv[ 3 ] ... READ_TYPE
-------------------------------------------------------------------------------------- */
static rc_t CC seq_restore_read_impl2 ( void *data, const VXformInfo *info, int64_t row_id,
VRowResult *rslt, uint32_t argc, const VRowData argv [] )
{
rc_t rc;
Read_Restorer *self = data;
INSDC_4na_bin *dst;
INSDC_coord_len len;
id_list align_ids;
uint32_t i;
uint32_t src_len = (uint32_t)argv[ 0 ] . u . data . elem_count;
const INSDC_4na_bin * src = argv[ 0 ] . u . data.base;
const uint32_t num_reads = (uint32_t)argv[ 1 ]. u . data . elem_count;
const INSDC_coord_len * read_len = argv[ 2 ] . u . data.base;
const uint8_t *read_type = argv[ 3 ] . u . data.base;
int64_t last_row_id = argv[ 1 ] . blob_stop_id;
align_ids.list = ( int64_t * )argv[ 1 ].u.data.base;
align_ids.count = ( uint32_t )( argv[ 1 ].u.data.base_elem_count - argv[ 1 ].u.data.first_elem );
assert( argv[ 0 ].u.data.elem_bits == 8 );
assert( argv[ 1 ].u.data.elem_bits == 64 );
assert( argv[ 2 ].u.data.elem_bits == sizeof( INSDC_coord_len ) * 8 );
assert( argv[ 2 ].u.data.elem_count == num_reads );
assert( argv[ 3 ].u.data.elem_count == num_reads );
src += argv [ 0 ] . u . data . first_elem;
align_ids.list += argv [ 1 ] . u . data . first_elem;
read_len += argv [ 2 ] . u . data . first_elem;
read_type += argv [ 3 ] . u . data . first_elem;
handle_caching( self, &align_ids, row_id, last_row_id );
for ( i = 0, len = 0; i < num_reads; i++ )
len += read_len[ i ];
/* resize output row */
rslt->data->elem_bits = 8;
rc = KDataBufferResize( rslt->data, len );
rslt->elem_count = len;
dst = rslt->data->base;
if ( rc == 0 && len > 0 )
{
if ( len == src_len ) /*** shortcut - all data is local ***/
memmove( dst, src, len );
else
{
rr_entry * ep;
const INSDC_4na_bin * rd;
uint32_t rd_len;
bool found_in_cache;
for ( i = 0; i < num_reads && rc == 0; i++ ) /*** checking read by read ***/
{
int64_t align_id = align_ids.list[ i ];
if ( align_id > 0 )
{
found_in_cache = false;
if ( self -> read_store != NULL )
found_in_cache = rr_get_read ( self -> read_store, align_id, &ep );
if ( found_in_cache )
{
/* we found it in the cache... */
rd = &( ep->read[ 0 ] );
rd_len = ep->read_len;
}
else
{
/* we did not find it in the cache, get it from the alignment-table... */
rc = VCursorCellDataDirect( self -> curs, align_id, self -> read_idx,
NULL, ( const void** ) &rd, NULL, &rd_len );
}
if ( rc == 0 )
{
if ( rd_len == read_len[ i ] )
{
if ( read_type[ i ] & SRA_READ_TYPE_FORWARD )
{
memmove( dst, rd, read_len[ i ] );
}
else if ( read_type[ i ] & SRA_READ_TYPE_REVERSE )
{
int j, k;
for( j = 0, k = read_len[ i ] - 1; j < (int)read_len[ i ]; j++, k-- )
{
dst[ j ] = map [ rd[ k ] & 15 ];
}
}
else
{
rc = RC( rcXF, rcFunction, rcExecuting, rcData, rcInconsistent );
}
}
else
{
rc = RC( rcXF, rcFunction, rcExecuting, rcData, rcInconsistent );
}
}
}
else /*** data is in READ column **/
{
if ( src_len >= read_len[ i ] )
{
memmove( dst, src, read_len[ i ] );
src_len -= read_len[ i ];
src += read_len[ i ];
}
else
{
return RC( rcXF, rcFunction, rcExecuting, rcData, rcInconsistent );
}
}
dst += read_len[ i ];
}
}
}
return rc;
}
static rc_t CC KNSManagerNewReleaseVersionImpl(const struct KNSManager *self,
SraReleaseVersion *newVersion)
{
rc_t rc = 0;
KDataBuffer result;
KHttpRequest *req = NULL;
KHttpResult *rslt = NULL;
if (newVersion == NULL) {
return RC(rcNS, rcArgv, rcAccessing, rcParam, rcNull);
}
memset(newVersion, 0, sizeof *newVersion);
if (self == NULL) {
return RC(rcNS, rcArgv, rcAccessing, rcSelf, rcNull);
}
memset(&result, 0, sizeof result);
if (rc == 0) {
rc = KNSManagerMakeRequest(self, &req, 0x01010000, NULL,
"https://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/sratoolkit.current.version"
);
}
if (rc == 0) {
rc = KHttpRequestGET(req, &rslt);
}
if (rc == 0) {
uint32_t code = 0;
rc = KHttpResultStatus(rslt, &code, NULL, 0, NULL);
if (rc == 0) {
if (code != 200) {
rc = RC(rcNS, rcFile, rcReading, rcFile, rcInvalid);
}
}
}
if (rc == 0) {
size_t total = 0;
KStream *response = NULL;
rc = KHttpResultGetInputStream(rslt, &response);
if (rc == 0) {
rc = KDataBufferMakeBytes(&result, 1024);
}
while (rc == 0) {
size_t num_read = 0;
uint8_t *base = NULL;
uint64_t avail = result.elem_count - total;
if (avail < 256) {
rc = KDataBufferResize(&result, result.elem_count + 1024);
if (rc != 0) {
break;
}
}
base = result.base;
rc = KStreamRead(response, &base[total], result.elem_count - total,
&num_read);
if (num_read > 0 || rc != 0) {
DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS), ("KStreamRead"
"(sratoolkit.current.version, %zu) = %R\n", num_read, rc));
}
if (rc != 0) {
/* TBD - look more closely at rc */
if (num_read > 0) {
rc = 0;
}
else {
break;
}
}
if (num_read == 0) {
break;
}
total += num_read;
}
RELEASE(KStream, response);
if (rc == 0) {
DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
("sratoolkit.current.version (%zu)\n", total));
result.elem_count = total;
}
}
if (rc == 0) {
const char *start = (const void*)(result.base);
size_t size = KDataBufferBytes(&result);
DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
("sratoolkit.current.version = '%.*s'\n", (uint32_t)size, start));
rc = SraReleaseVersionInit(newVersion, start, size);
}
KDataBufferWhack(&result);
RELEASE(KHttpResult, rslt);
RELEASE(KHttpRequest, req);
return rc;
}
rc_t KColumnIdx2LocateBlob ( const KColumnIdx2 *self,
KColBlobLoc *loc, const KColBlockLoc *bloc,
int64_t first, int64_t upper, bool bswap )
{
rc_t rc;
uint32_t count;
size_t block_size, orig;
uint32_t slot=0;
void *block;
/* check within page cache */
if(self -> cstorage.elem_count > 0){
KColumnIdx2BlockCache * cache=(KColumnIdx2BlockCache *)self -> cstorage.base;
assert(self->last < self->cstorage.elem_count);
if(bloc -> start_id == cache[self->last].start_id){
rc = KColIdxBlockLocateBlob ( & cache[self->last].iblk, loc, bloc, ( uint32_t ) cache[self->last].count, first, upper );
if ( rc == 0) return 0;
} else {
uint32_t lower,upper,pivot;
if(bloc -> start_id > cache[self->last].start_id){
lower=self->last+1;
upper=self->cstorage.elem_count;
} else {
lower=0;
upper = self->last;
}
while(lower < upper){
pivot = (lower + upper) / 2;
if(bloc -> start_id == cache[pivot].start_id){
KColumnIdx2 * nc_self=(KColumnIdx2 *)self;
nc_self->last = pivot;
rc = KColIdxBlockLocateBlob ( & cache[self->last].iblk, loc, bloc, ( uint32_t ) cache[self->last].count, first, upper );
if ( rc == 0) return 0;
goto BSEARCH_DONE;
} else if(bloc -> start_id < cache[pivot].start_id){
upper=pivot;
} else {
lower =pivot+1;
}
}
assert(lower == upper );
slot = upper;
}
}
BSEARCH_DONE:
/* file may be empty or non-existent */
if ( self -> eof == 0 )
return RC ( rcDB, rcColumn, rcSelecting, rcBlob, rcNotFound );
/* TBD - compression not supported */
if ( bloc -> u . blk . compressed )
return RC ( rcDB, rcColumn, rcSelecting, rcData, rcUnsupported );
/* determine the number of entries in block */
orig = bloc -> u . blk . size;
count = KColBlockLocEntryCount ( bloc, & orig );
/* determine the size to allocate */
block_size = KColBlockLocAllocSize ( bloc, orig, count );
/* allocate a block */
block = malloc ( block_size );
if ( block == NULL )
rc = RC ( rcDB, rcColumn, rcSelecting, rcMemory, rcExhausted );
else
{
size_t num_read;
rc = KFileReadAll ( self -> f, bloc -> pg, block, orig, & num_read );
if ( rc == 0 )
{
if ( num_read != orig )
rc = RC ( rcDB, rcIndex, rcReading, rcTransfer, rcIncomplete );
else
{
KColIdxBlock iblk;
rc = KColIdxBlockInit ( & iblk, bloc, orig, block, block_size, bswap );
if ( rc == 0 )
{
rc = KColIdxBlockLocateBlob ( & iblk,
loc, bloc, count, first, upper );
if ( rc == 0 )
{
KColumnIdx2BlockCache * cache;
KDataBufferResize(&((KColumnIdx2 *)self)->cstorage,self->cstorage.elem_count+1);
cache=(KColumnIdx2BlockCache *)self -> cstorage.base;
if(slot < self->cstorage.elem_count -1){ /** not adding to the end **/
memmove(cache+slot+1,cache+slot,sizeof(*cache)*(self->cstorage.elem_count - slot - 1));
}
cache += slot;
cache -> block = block;
cache -> start_id = bloc -> start_id;
cache -> count = count;
cache -> iblk = iblk;
return 0;
}
}
}
}
free ( block );
}
return rc;
}
static rc_t serialize1( const BlobHeaders *self, KDataBuffer *buffer, uint64_t offset, uint64_t *size ) {
uint8_t flags = 0;
uint8_t version = 0;
uint32_t fmt = 0;
uint64_t osize = 0;
uint32_t op_count = 0;
uint32_t arg_count = 0;
rc_t rc;
uint64_t sz;
uint8_t *dst;
uint64_t dsize;
if (self->data) {
version = self->data->version;
flags = self->data->flags;
fmt = self->data->fmt;
osize = self->data->osize;
op_count = self->data->op_count;
arg_count = self->data->arg_count;
}
sz = 2 + 4 * 5 + op_count + arg_count * 10;
rc = KDataBufferResize(buffer, sz + offset);
if (rc)
return rc;
dst = buffer->base;
dst += offset;
dsize = buffer->elem_count - offset;
*dst++ = flags; --dsize;
*dst++ = version; --dsize;
rc = vlen_encode1(dst, dsize, &sz, fmt);
if (rc)
return rc;
dst += sz; dsize -= sz;
rc = vlen_encode1(dst, dsize, &sz, osize);
if (rc)
return rc;
dst += sz; dsize -= sz;
rc = vlen_encode1(dst, dsize, &sz, op_count);
if (rc)
return rc;
dst += sz; dsize -= sz;
rc = vlen_encode1(dst, dsize, &sz, arg_count);
if (rc)
return rc;
dst += sz; dsize -= sz;
if (op_count > 0) {
memmove(dst, self->data->ops, op_count);
dst += op_count;
dsize -= op_count;
}
if (arg_count > 0) {
rc = vlen_encode(dst, dsize, &sz, self->data->args, arg_count);
if (rc)
return rc;
dst += sz; /* dsize -= sz */;
}
*size = (uint64_t)( dst - (uint8_t *)buffer->base );
return 0;
}
