static
rc_t SFFLoaderFmtWriteDataFile(SFFLoaderFmt* self, const SRALoaderFile* file)
{
rc_t rc = 0;
while( rc == 0 ) {
if( self->curr_read_number == 0 ) {
if( (rc = SFFLoaderFmtReadCommonHeader(self, file)) == 0 ) {
DEBUG_MSG (5, ("%s: Common header ok: %u reads\n", self->file_name, self->header.number_of_reads));
DEBUG_MSG (8, ("%s: flow_chars: [%hu] %s\n", self->file_name, self->header.num_flows_per_read, self->flow_chars.data));
DEBUG_MSG (8, ("%s: key_seq: [%hu] %s\n", self->file_name, self->header.key_length, self->key_seq.data));
} else if( GetRCObject(rc) == (enum RCObject)rcData && GetRCState(rc) == rcIgnored ) {
rc = 0;
break;
}
}
if( rc == 0 && self->header.number_of_reads != 0 &&
(rc = SFFLoaderFmtSkipIndex(self, file)) == 0 &&
(rc = SFFLoaderFmtReadDataHeader(self, file)) == 0 &&
(rc = SFFLoaderFmtReadData(self, file)) == 0 ) {
if( self->w454 ) {
rc = SRAWriter454_WriteRead(self->w454, file, &self->name, &self->read, &self->quality,
self->skip_signal ? NULL : &self->signal,
self->skip_signal ? NULL : &self->position,
self->read_header.clip_quality_left, self->read_header.clip_quality_right,
self->read_header.clip_adapter_left, self->read_header.clip_adapter_right);
} else {
rc = SRAWriterIonTorrent_WriteRead(self->wIonTorrent, file, &self->name, &self->read, &self->quality,
self->skip_signal ? NULL : &self->signal,
self->skip_signal ? NULL : &self->position,
self->read_header.clip_quality_left, self->read_header.clip_quality_right,
self->read_header.clip_adapter_left, self->read_header.clip_adapter_right);
}
if( rc == 0 ) {
++self->curr_read_number;
}
}
if( rc != 0 && (GetRCObject(rc) != rcTransfer && GetRCState(rc) != rcDone) ) {
SRALoaderFile_LOG(file, klogErr, rc, "on or about read #$(i)", PLOG_U32(i), self->curr_read_number + 1);
} else if( self->curr_read_number == self->header.number_of_reads ) {
DEBUG_MSG(5, ("%s: done loading declared %u reads\n", self->file_name, self->curr_read_number));
self->curr_read_number = 0;
/* will skip indexes if they are at eof */
if( (rc = SFFLoaderFmtSkipIndex(self, file)) == 0 ) {
/* This should be the end of file and/or beginning of next */
if( (rc = SFFLoaderFmt_ReadBlock(self, file, 0, "EOF", false)) == 0 ) {
if( self->file_buf == NULL ) {
DEBUG_MSG(5, ("%s: EOF detected\n", self->file_name));
self->index_correction = 0;
break;
}
}
}
}
}
return rc;
}
LIB_EXPORT rc_t CC AlignAccessAlignmentEnumeratorNext(const AlignAccessAlignmentEnumerator *cself) {
rc_t rc;
AlignAccessAlignmentEnumerator *self = (AlignAccessAlignmentEnumerator *)cself;
int32_t refSeqID;
AGAIN:
if (self->innerSelf != NULL) {
BAMAlignmentRelease(self->innerSelf);
self->innerSelf = NULL;
}
if (self->atend != 0)
return AlignAccessAlignmentEnumeratorEOFCode;
if (self->slice == NULL) {
rc = BAMFileRead2(self->parent->innerSelf, &self->innerSelf);
if (rc) {
if (GetRCState(rc) == rcNotFound && GetRCObject(rc) == rcRow) {
self->atend = 1;
rc = AlignAccessAlignmentEnumeratorEOFCode;
}
}
return rc;
}
rc = BAMFileReadSlice(self->parent->innerSelf, &self->innerSelf, self->slice);
if (rc) {
if (GetRCState(rc) == rcNotFound && GetRCObject(rc) == rcRow) {
self->atend = 1;
rc = AlignAccessAlignmentEnumeratorEOFCode;
}
return rc;
}
if (!BAMAlignmentIsMapped(self->innerSelf))
goto AGAIN;
BAMAlignmentGetRefSeqId(self->innerSelf, &refSeqID);
if (self->refSeqID != refSeqID) {
self->atend = 1;
rc = AlignAccessAlignmentEnumeratorEOFCode;
}
else if (self->endpos != 0) {
int64_t pos;
uint32_t length;
BAMAlignmentGetPosition2(self->innerSelf, &pos, &length);
if (pos >= (int64_t)self->endpos) {
self->atend = 1;
rc = AlignAccessAlignmentEnumeratorEOFCode;
}
else {
int64_t const endpos = pos + length;
if (endpos <= self->startpos)
goto AGAIN;
}
}
return rc;
}
static
rc_t RefNameMake ( RefName **objp, const VTable *tbl, const VCursor *native_curs )
{
rc_t rc;
/* create the object */
RefName *obj = malloc ( sizeof * obj );
if ( obj == NULL ) {
rc = RC ( rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted );
} else {
obj->curs=NULL;
/* open the reference cursor */
rc = AlignRefTableCursor(tbl, native_curs, &obj->curs, NULL);
if(rc == 0){
/* add columns to cursor */
rc = VCursorAddColumn(obj->curs, &obj->name_idx, "(utf8)REF_NAME");
if( GetRCObject(rc) == rcColumn && GetRCState(rc) == rcNotFound ) {
rc = VCursorAddColumn(obj->curs, &obj->name_idx, "(utf8)NAME");
}
if(GetRCState(rc) == rcExists){
rc = 0;
}
if( rc == 0 ) {
*objp = obj;
return 0;
}
VCursorRelease ( obj -> curs );
}
free ( obj );
}
return rc;
}
static rc_t TableWriterSeq_CollectStatistics(TableWriterSeq *self, const TableWriterData* primary_alignment_id)
{
rc_t rc;
int64_t mate[2] = {0, 0};
uint64_t distance = 0;
uint64_t count = 0;
memcpy(mate, primary_alignment_id->buffer, sizeof(mate[0]) * primary_alignment_id->elements);
if (mate[0] && mate[1]) {
distance = (mate[0] < mate[1]) ? (mate[1] - mate[0]) : (mate[0] - mate[1]);
}
else
return 0;
if (self->stats == NULL) {
rc = KVectorMake(&self->stats); if (rc) return rc;
}
rc = KVectorGetU64(self->stats, distance, &count);
if (GetRCObject(rc) == rcItem && GetRCState(rc) == rcNotFound) {
rc = KVectorSetU64(self->stats, distance, 1);
if (rc == 0) ++self->statsCount;
}
else if (rc == 0) {
++count;
rc = KVectorSetU64(self->stats, distance, count);
}
return rc;
}
CVDBTable::CVDBTable(const CVDB& db,
const char* table_name,
EMissing missing)
: m_Db(db),
m_Name(table_name)
{
DECLARE_SDK_GUARD();
if ( rc_t rc = VDatabaseOpenTableRead(db, x_InitPtr(), table_name) ) {
*x_InitPtr() = 0;
RCState rc_state = GetRCState(rc);
int rc_object = GetRCObject(rc);
if ( rc_state == rcNotFound &&
(rc_object == rcParam ||
rc_object == rcPath) ) {
// missing table in the DB
if ( missing != eMissing_Throw ) {
return;
}
NCBI_THROW2_FMT(CSraException, eNotFoundTable,
"Cannot open VDB table: "<<*this, rc);
}
else {
// other errors
NCBI_THROW2_FMT(CSraException, eOtherError,
"Cannot open VDB table: "<<*this, rc);
}
}
}
rc_t BufferQRelease (const BufferQ *cself)
{
BufferQ * self = (BufferQ*)cself;
rc_t rc = 0;
if ( self != NULL )
{
if ( atomic32_dec_and_test (&self->refcount))
{
const Buffer * b;
while (rc == 0)
{
rc = BufferQPopBuffer (self, &b, NULL);
BufferRelease (b);
}
/* this might need rework especially if KQueue changes */
if ((GetRCState(rc) == rcExhausted) && (GetRCObject(rc) == rcTimeout))
rc = 0;
if (rc == 0)
{
rc = KQueueRelease (self->q);
if (rc == 0)
{
free (self);
return 0;
}
}
atomic32_inc (&((BufferQ*)self)->refcount);
}
}
return rc;
}
static
rc_t SRAFastqFile_Read(const SRAFastqFile* self, uint64_t pos, void *buffer, size_t size, size_t *num_read)
{
rc_t rc = 0;
if( pos >= self->file_sz ) {
*num_read = 0;
} else if( (rc = KLockAcquire(self->lock)) == 0 ) {
do {
if( pos < self->from || pos >= (self->from + self->size) ) {
int64_t id = 0;
uint64_t id_qty = 0;
DEBUG_MSG(10, ("Caching for pos %lu %lu bytes\n", pos, size - *num_read));
if( (rc = KIndexFindU64(self->kidx, pos, &((SRAFastqFile*)self)->from, &((SRAFastqFile*)self)->size, &id, &id_qty)) == 0 ) {
DEBUG_MSG(10, ("Caching from %lu:%lu, %lu bytes\n", self->from, self->from + self->size - 1, self->size));
DEBUG_MSG(10, ("Caching spot %ld, %lu spots\n", id, id_qty));
if( (rc = FastqReaderSeekSpot(self->reader, id)) == 0 ) {
size_t inbuf = 0, w = 0;
char* b = self->buf;
uint64_t left = self->buffer_sz;
do {
if( (rc = FastqReader_GetCurrentSpotSplitData(self->reader, b, left, &w)) != 0 ) {
break;
}
b += w; left -= w; inbuf += w; --id_qty;
} while( id_qty > 0 && (rc = FastqReaderNextSpot(self->reader)) == 0);
if( GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted ) {
DEBUG_MSG(10, ("No more rows\n"));
rc = 0;
}
DEBUG_MSG(8, ("Cached %u bytes\n", inbuf));
if( self->gzipped != NULL ) {
size_t compressed = 0;
if( (rc = ZLib_DeflateBlock(self->buf, inbuf, self->gzipped, self->buffer_sz, &compressed)) == 0 ) {
char* b = self->buf;
((SRAFastqFile*)self)->buf = self->gzipped;
((SRAFastqFile*)self)->gzipped = b;
((SRAFastqFile*)self)->size = compressed;
DEBUG_MSG(10, ("gzipped %lu bytes\n", self->size));
}
}
}
}
}
if( rc == 0 ) {
off_t from = pos - self->from;
size_t q = (self->size - from) > (size - *num_read) ? (size - *num_read) : (self->size - from);
DEBUG_MSG(10, ("Copying from %lu %u bytes\n", from, q));
memcpy(&((char*)buffer)[*num_read], &self->buf[from], q);
*num_read = *num_read + q;
pos += q;
}
} while( rc == 0 && *num_read < size && pos < self->file_sz );
ReleaseComplain(KLockUnlock, self->lock);
}
return rc;
}
rc_t CopierDoOne (Copier * self)
{
rc_t rc = 0;
const Buffer * b;
LOGMSG (klogDebug10, "CopierDoOne");
rc = Quitting();
if (rc == 0)
{
LOGMSG (klogDebug10, "call BufferQPopBuffer");
rc = BufferQPopBuffer (self->q, &b, NULL);
if (rc == 0)
{
size_t w;
size_t z;
LOGMSG (klogDebug10, "call BufferContentGetSize");
z = BufferContentGetSize (b);
rc = KFileWrite (self->f, self->o, b, z, &w);
self->o += w;
if (w != z)
rc = RC (rcExe, rcFile, rcWriting, rcTransfer, rcIncomplete);
else
rc = BufferRelease (b);
}
/* ow this is ugly! */
/* is the rc a "exhausted" on a timeout? */
else if ((GetRCObject(rc) == rcTimeout) && (GetRCState(rc) == rcExhausted))
{
rc = 0;
LOGMSG (klogDebug10, "CopierDoOne timeout");
/* if so is the queue also sealed? */
if (BufferQSealed (self->q) == true)
{
LOGMSG (klogDebug10, "CopierDoOne sealed");
/* if both then we are done and so signal */
rc = KFileRelease (self->f);
PLOGMSG (klogDebug10, "CopierDoOne back from KFileRelease $(rc)",PLOG_U32(rc),rc);
if (rc == 0)
{
self->f = NULL;
rc = BufferQRelease (self->q);
if (rc == 0)
{
self->q = NULL;
rc = RC (rcExe, rcNoTarg, rcCopying, rcNoTarg, rcDone );
}
}
}
}
else
LOGMSG (klogDebug10, "CopierDoOne pop failure");
}
else
LOGMSG (klogDebug10, "CopierDoOne: quitting");
return rc;
}
static
rc_t AbsolidLabelerFilter_GetKeySet(const SRASplitter* cself, const SRASplitter_Keys** key, uint32_t* keys, spotid_t spot, const readmask_t* readmask)
{
rc_t rc = 0;
AbsolidLabelerFilter* self = (AbsolidLabelerFilter*)cself;
if( self == NULL || key == NULL ) {
rc = RC(rcExe, rcNode, rcExecuting, rcParam, rcInvalid);
} else {
*keys = 0;
if( rc == 0 && (rc = AbsolidReaderSeekSpot(self->reader, spot)) == 0 ) {
uint32_t num_reads = 0;
if( (rc = AbsolidReader_SpotInfo(self->reader, NULL, NULL, NULL, &num_reads)) == 0 ) {
uint32_t readId, q;
const char* read_label;
INSDC_coord_len read_label_sz = 0;
INSDC_coord_len read_len = 0;
*key = self->keys;
*keys = sizeof(self->keys) / sizeof(self->keys[0]);
for(q = 0; q < *keys; q++) {
clear_readmask(self->keys[q].readmask);
}
for(q = 0, readId = 0; rc == 0 && readId < num_reads; readId++) {
if( !isset_readmask(readmask, readId) ) {
continue;
}
rc = AbsolidReader_SpotReadInfo(self->reader, readId + 1, NULL, &read_label, &read_label_sz, NULL, &read_len);
if( read_len == 0 ) {
continue;
}
if( self->is_platform_cs_native ) {
uint32_t i;
for(i = 0; i < *keys; i++) {
if( read_label_sz == self->key_sz[i] && strncmp(read_label, self->keys[i].key, read_label_sz) == 0 ) {
set_readmask(self->keys[i].readmask, readId);
break;
}
}
if( i >= *keys ) {
rc = RC(rcExe, rcNode, rcExecuting, rcTag, rcUnexpected);
}
} else {
/* since prev filter leaves only 2 first bio reads labels F3 and R3 wil be used */
set_readmask(self->keys[q].readmask, readId);
}
q++;
}
}
} else if( GetRCObject(rc) == rcRow && GetRCState(rc) == rcNotFound ) {
SRA_DUMP_DBG (3, ("%s skipped %u row\n", __func__, spot));
rc = 0;
}
}
return rc;
}
CVDBTable::CVDBTable(const CVDBMgr& mgr,
const string& acc_or_path,
EMissing missing)
: m_Name(acc_or_path)
{
*x_InitPtr() = 0;
VSchema *schema;
DECLARE_SDK_GUARD();
if ( rc_t rc = SRASchemaMake(&schema, mgr) ) {
NCBI_THROW2(CSraException, eInitFailed,
"Cannot make default SRA schema", rc);
}
string path = CVPath::ConvertAccOrSysPathToPOSIX(acc_or_path);
if ( rc_t rc = VDBManagerOpenTableRead(mgr, x_InitPtr(), schema, "%.*s",
int(path.size()), path.data()) ) {
*x_InitPtr() = 0;
VSchemaRelease(schema);
if ( (GetRCObject(rc) == RCObject(rcDirectory) ||
GetRCObject(rc) == RCObject(rcPath)) &&
GetRCState(rc) == rcNotFound ) {
// no SRA accession
if ( missing != eMissing_Throw ) {
return;
}
NCBI_THROW2_FMT(CSraException, eNotFoundTable,
"Cannot open SRA table: "<<acc_or_path, rc);
}
else if ( GetRCObject(rc) == RCObject(rcDatabase) &&
GetRCState(rc) == rcIncorrect ) {
// invalid SRA database
NCBI_THROW2_FMT(CSraException, eDataError,
"Cannot open SRA table: "<<acc_or_path, rc);
}
else {
// other errors
NCBI_THROW2_FMT(CSraException, eOtherError,
"Cannot open SRA table: "<<acc_or_path, rc);
}
}
VSchemaRelease(schema);
}
static
rc_t parse_header(SRF_context *ctx, ZTR_Context *ztr_ctx, const uint8_t *data, size_t size)
{
rc_t rc = 0;
size_t parsed;
char prefixType;
uint32_t counter;
ztr_raw_t ztr_raw;
ztr_t ztr;
enum ztr_chunk_type type;
fe_context_t* fe = (fe_context_t*)ctx;
rc = SRF_ParseDataChunk(data, size, &parsed, &fe->name_prefix, &prefixType, &counter);
if(rc) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcInvalid);
return SRALoaderFile_LOG(ctx->file, klogErr, rc, "corrupt", NULL);
}
if(fe->defered != NULL) {
free((void *)fe->defered);
fe->defered = NULL;
}
if(parsed == size)
return 0;
rc = ZTR_AddToBuffer(ztr_ctx, data + parsed, size - parsed);
if(rc)
return rc;
if((rc = ZTR_ParseHeader(ztr_ctx)) != 0) {
return SRALoaderFile_LOG(ctx->file, klogErr, rc, "corrupt", NULL);
}
while (rc == 0 && !ZTR_BufferIsEmpty(ztr_ctx)) {
if((rc = ZTR_ParseBlock(ztr_ctx, &ztr_raw)) != 0) {
if(GetRCState(rc) == rcInsufficient && GetRCObject(rc) == (enum RCObject)rcData)
rc = ZTR_BufferGetRemainder(ztr_ctx, &fe->defered, &fe->defered_len);
break;
}
if((rc = ZTR_ProcessBlock(ztr_ctx, &ztr_raw, &ztr, &type)) != 0) {
SRALoaderFile_LOG(ctx->file, klogErr, rc, "corrupt", NULL);
break;
}
if(*(void **)&ztr != NULL)
free(*(void **)&ztr);
if(ztr_raw.meta != NULL)
free(ztr_raw.meta);
if(ztr_raw.data != NULL)
free(ztr_raw.data);
}
return rc;
}
static rc_t bam_header(const VDatabase* db) {
rc_t rc = 0;
const char path[] = "BAM_HEADER";
const KMetadata* meta = NULL;
const KMDataNode* node = NULL;
char* buffer = NULL;
assert(db);
if (rc == 0) {
rc = VDatabaseOpenMetadataRead(db, &meta);
DISP_RC(rc, "while calling VDatabaseOpenMetadataRead");
}
if (rc == 0) {
rc = KMetadataOpenNodeRead(meta, &node, "%s", path);
if (GetRCState(rc) == rcNotFound)
{ rc = 0; }
else {
DISP_RC2(rc, path, "while calling KMetadataOpenNodeRead");
if (rc == 0) {
int i = 0;
size_t bsize = 0;
size_t size = 1024;
for (i = 0; i < 2; ++i) {
free(buffer);
bsize = size + 1;
buffer = malloc(bsize);
if (buffer == NULL) {
rc = RC(rcExe, rcStorage, rcAllocating,
rcMemory, rcExhausted);
}
else {
rc = KMDataNodeReadCString(node, buffer, bsize, &size);
if (rc == 0) {
break;
}
else if (i == 0
&& GetRCObject(rc) == (enum RCObject)rcBuffer
&& GetRCState (rc) == rcInsufficient)
{
rc = 0;
}
}
DISP_RC2(rc, path, "while calling KMDataNodeReadCString");
}
}
}
}
if (rc == 0 && buffer)
{ OUTMSG(("BAM_HEADER: {\n%s}\n\n", buffer)); }
DESTRUCT(KMDataNode, node);
DESTRUCT(KMetadata, meta);
free(buffer);
return rc;
}
/* you cannot addref to this dir object cause it's created on stack silently */
static
rc_t CC DirVisitor(const KDirectory *dir, uint32_t type, const char *name, void *data)
{
rc_t rc = 0;
DirVisit_Data* d = (DirVisit_Data*)data;
if( (type & ~kptAlias) == kptFile ) {
if (strcmp(&name[strlen(name) - 4], ".tsv") == 0 ||
strcmp(&name[strlen(name) - 8], ".tsv.bz2") == 0 ||
strcmp(&name[strlen(name) - 7], ".tsv.gz") == 0)
{
char buf[4096];
const CGLoaderFile* file;
FGroupKey key;
if( (rc = KDirectoryResolvePath(dir, true, buf, sizeof(buf), name)) == 0 &&
(rc = CGLoaderFile_Make(&file, d->dir, buf, NULL, !d->param->no_read_ahead)) == 0 &&
(rc = FGroupKey_Make(&key, file, d->param)) == 0 ) {
FGroupMAP* found = (FGroupMAP*)BSTreeFind(d->tree, &key, FGroupMAP_Cmp);
DEBUG_MSG(5, ("file %s recognized\n", name));
if( found != NULL ) {
rc = FGroupMAP_Set(found, file);
} else {
FGroupMAP* x = calloc(1, sizeof(*x));
if( x == NULL ) {
rc = RC(rcExe, rcFile, rcInserting, rcMemory, rcExhausted);
} else {
memcpy(&x->key, &key, sizeof(key));
if( (rc = FGroupMAP_Set(x, file)) == 0 ) {
rc = BSTreeInsertUnique(d->tree, &x->dad, NULL, FGroupMAP_Sort);
}
}
}
} else if( GetRCObject(rc) == rcItem && GetRCState(rc) == rcIgnored ) {
DEBUG_MSG(5, ("file %s ignored\n", name));
rc = CGLoaderFile_Release(file, true);
file = NULL;
}
if( rc != 0 && file != NULL ) {
CGLoaderFile_LOG(file, klogErr, rc, NULL, NULL);
CGLoaderFile_Release(file, true);
}
} else if( strcmp(&name[strlen(name) - 4], ".tar") == 0 ) {
const KDirectory* tmp = d->dir;
if( (rc = KDirectoryOpenArcDirRead(dir, &d->dir, true, name, tocKFile, KArcParseTAR, NULL, NULL)) == 0 ) {
rc = KDirectoryVisit(d->dir, true, DirVisitor, d, ".");
KDirectoryRelease(d->dir);
}
d->dir = tmp;
}
}
return rc;
}
/* OpenRead
* open an existing table
*
* "tbl" [ OUT ] - return parameter for table
*
* "spec" [ IN ] - NUL terminated UTF-8 string giving path
* to table.
*/
LIB_EXPORT rc_t CC SRAMgrVOpenTableRead ( const SRAMgr *self,
const SRATable **crslt, const char *spec, va_list args )
{
rc_t rc;
char tblpath [ 4096 ];
int num_writ = vsnprintf ( tblpath, sizeof tblpath, spec, args );
if ( num_writ < 0 || ( size_t ) num_writ >= sizeof tblpath )
rc = RC ( rcSRA, rcMgr, rcOpening, rcPath, rcExcessive );
else
{
SRATable **rslt = (SRATable **)crslt; /* to avoid "const_casts" below */
rc = SRACacheGetTable( self->cache, tblpath, crslt );
if (rc == 0 )
{
if ( *crslt == NULL )
{
rc = SRAMgrOpenAltTableRead ( self, crslt, "SEQUENCE", tblpath );
if ( rc == 0 )
{
rc = SRACacheAddTable( self->cache, tblpath, *rslt);
if ( GetRCObject(rc) == rcParam && GetRCState(rc) == rcExists )
{ /* the same object has appeared in the cache since our call to SRACacheGetTable above; */
rc = 0; /* return the new object, never mind the cache */
}
else if ( ! SRACacheMetricsLessThan(&self->cache->current, &self->cache->hardThreshold) )
rc = SRACacheFlush(self->cache);
}
}
}
else if ( (GetRCObject(rc) == rcName && GetRCState(rc) == rcInvalid) || /* accessions with irregular names are not cached */
(GetRCObject(rc) == rcParam && GetRCState(rc) == rcBusy) ) /* in cache but in use */
{
rc = SRAMgrOpenAltTableRead ( self, crslt, "SEQUENCE", tblpath );
if (rc == 0)
(*rslt)->mgr = SRAMgrAttach(self);
}
}
return rc;
}
string CVDBMgr::GetCacheRoot() const
{
const VPath* ret;
if ( rc_t rc = VDBManagerGetCacheRoot(*this, &ret) ) {
if ( GetRCObject(rc) == RCObject(rcPath) &&
GetRCState(rc) == rcNotFound ) {
return kEmptyStr;
}
NCBI_THROW2(CSraException, eOtherError,
"CVDBMgr: Cannot get cache root", rc);
}
return CVPath(ret).ToString(CVPath::eSys);
}
static
rc_t RefPosMake ( RefPos **objp, const VTable *tbl, const VCursor *native_curs )
{
rc_t rc;
/* create the object */
RefPos *obj = malloc ( sizeof * obj );
if ( obj == NULL ) {
rc = RC ( rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted );
} else {
obj->curs=NULL;
BSTreeInit(&obj->tr_range);
/* open the reference table cursor*/
if( (rc = AlignRefTableCursor(tbl, native_curs, &obj->curs, NULL)) == 0 ) {
uint32_t itmp;
if( (rc = VCursorAddColumn(obj->curs, &itmp, "(U32)MAX_SEQ_LEN")) == 0 || GetRCState(rc) == rcExists) {
const void *base;
uint32_t row_len;
rc = VCursorCellDataDirect(obj->curs, 1, itmp, NULL, &base, NULL, &row_len);
if(rc == 0) {
assert(row_len == 1);
memcpy(&obj->max_seq_len, base, 4);
}
}
if( GetRCObject(rc) == rcColumn && GetRCState(rc) == rcNotFound ) {
/*** no MAX_SEQ_LEN means that REF_POS==REF_START **/
VCursorRelease(obj->curs);
obj->curs = NULL;
obj->max_seq_len = 0;
obj->name_range_idx = 0;
obj->name_idx = 0;
rc = 0;
} else if( rc == 0 ) {
/* add columns to cursor */
rc = VCursorAddColumn(obj->curs, &obj->name_idx, "(utf8)NAME");
if(rc == 0 || GetRCState(rc) == rcExists)
rc = VCursorAddColumn(obj->curs, &obj->name_range_idx, "NAME_RANGE");
if(GetRCState(rc) == rcExists)
rc = 0;
}
}
if( rc == 0 ) {
*objp = obj;
} else {
VCursorRelease(obj->curs);
free(obj);
}
}
return rc;
}
static
rc_t sra_meta_stats_node_read(KMDataNode* node, void* value)
{
rc_t rc = KMDataNodeReadAsU64(node, value);
if ( rc != 0 )
{
if ( GetRCState ( rc ) == rcIncomplete && GetRCObject ( rc ) == rcTransfer )
{
* ( uint64_t* ) value = 0;
rc = 0;
}
}
return rc;
}
/* Open
* open cursor, resolving schema
* for the set of opened columns
*
* NB - there is no corresponding "Close"
* use "Release" instead.
*/
rc_t VTableReadCursorOpen ( const VTableCursor *cself )
{
rc_t rc;
VTableCursor *self = ( VTableCursor* ) cself;
if ( self == NULL )
rc = RC ( rcVDB, rcCursor, rcOpening, rcSelf, rcNull );
else
{
VLinker *ld = self -> tbl -> linker;
KDlset *libs;
rc = VLinkerOpen ( ld, & libs );
if ( rc == 0 )
{
rc = VCursorOpenRead ( self, libs );
if ( rc == 0 )
{
int64_t first;
uint64_t count;
rc = VCursorIdRange ( & self -> dad, 0, & first, & count );
if ( rc != 0 )
{
/* permit empty open when run from sradb */
if ( GetRCState ( rc ) == rcEmpty && GetRCObject ( rc ) == rcRange &&
self -> permit_add_column && VectorLength ( & self -> dad . row ) == 0 )
{
rc = 0;
}
}
else if ( count != 0 )
{
/* set initial row id to starting row */
self -> dad . start_id =
self -> dad . end_id =
self -> dad . row_id = first;
}
if ( rc != 0 )
self -> dad . state = vcFailed;
}
KDlsetRelease ( libs );
}
}
return rc;
}
LIB_EXPORT
rc_t CC AlignAccessDBWindowedAlignments(const AlignAccessDB *self,
AlignAccessAlignmentEnumerator **align_enum,
const char *refSeqName, uint64_t pos, uint64_t wsize
)
{
AlignAccessAlignmentEnumerator *lhs;
unsigned i, n;
const BAMRefSeq *rs;
uint64_t endpos = pos + wsize;
rc_t rc;
BAMFileSlice *slice;
*align_enum = NULL;
BAMFileGetRefSeqCount(self->innerSelf, &n);
for (i = 0; i != n; ++i) {
BAMFileGetRefSeq(self->innerSelf, i, &rs);
if (strcmp(refSeqName, rs->name) == 0)
break;
}
if (i == n || pos >= rs->length) {
return RC(rcAlign, rcTable, rcConstructing, rcParam, rcInvalid);
}
if (endpos > rs->length || wsize == 0)
endpos = rs->length;
rc = BAMFileMakeSlice(self->innerSelf, &slice, i, pos, endpos);
if (rc == 0 && slice == NULL)
return RC(rcAlign, rcTable, rcConstructing, rcMemory, rcExhausted);
if ( rc != 0 )
{
if ( GetRCState( rc ) == rcNotFound && GetRCObject( rc ) == (enum RCObject)rcData )
rc = AlignAccessAlignmentEnumeratorEOFCode;
return rc;
}
rc = AlignAccessDBMakeEnumerator(self, &lhs);
if (rc)
return rc;
lhs->refSeqID = i;
lhs->endpos = endpos;
lhs->startpos = pos;
lhs->slice = slice;
*align_enum = lhs;
return AlignAccessAlignmentEnumeratorNext(*align_enum);
}
/*
function ascii NCBI:align:ref_seq_id ( I64 ref_id );
*/
static
rc_t CC align_ref_seq_id ( void *data, const VXformInfo *info,
int64_t row_id, VRowResult *rslt, uint32_t argc, const VRowData argv[] )
{
rc_t rc;
RefSeqID const *self = ( void const * )data;
char const *name = NULL;
uint32_t name_len;
/* get start and length of reference segment */
int64_t const *ref_id = argv[REF_ID].u.data.base;
assert( argv[ REF_ID ].u.data.elem_bits == sizeof( *ref_id ) * 8) ;
if ( self->curs == NULL || argv[ REF_ID ].u.data.elem_count == 0 )
{
rslt->elem_count = 0;
return 0;
}
ref_id += argv[ REF_ID] .u.data.first_elem;
SUB_DEBUG( ( "SUB.Rd in 'align_ref_seq_id.c' at #%lu\n", ref_id[ 0 ] ) );
rc = VCursorCellDataDirect( self->curs, ref_id[ 0 ], self->seqID_idx, NULL, (void const **)&name, NULL, &name_len );
if ( GetRCState( rc ) == rcNotFound && GetRCObject( rc ) == rcRow )
{
name = "";
name_len = 0;
}
else if ( rc != 0 )
{
return rc;
}
rc = KDataBufferCast( rslt->data, rslt->data, sizeof( name[ 0 ] ) * 8, true );
if ( rc != 0 )
return rc;
rc = KDataBufferResize( rslt->data, name_len );
if ( rc != 0 )
return rc;
memmove( rslt->data->base, name, sizeof( name[ 0 ] ) * name_len );
rslt->elem_count = name_len;
rslt->elem_bits = sizeof( name[ 0 ] ) * 8;
return rc;
}
static
rc_t LocalRefIDMake ( LocalRefID **objp, const VTable *tbl, const VCursor *native_curs)
{
rc_t rc;
/* create the object */
LocalRefID *obj = malloc ( sizeof * obj );
if ( obj == NULL )
{
rc = RC( rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted );
}
else
{
const VCursor *curs=NULL;
/* open the reference table cursor*/
rc = AlignRefTableCursor( tbl, native_curs, &curs, NULL );
if ( rc == 0 )
{
uint32_t itmp;
rc = VCursorAddColumn( curs, &itmp, "(U32)MAX_SEQ_LEN" );
if ( rc == 0 || GetRCState( rc ) == rcExists )
{
const void *base;
uint32_t row_len;
rc = VCursorCellDataDirect( curs, 1, itmp, NULL, &base, NULL, &row_len );
if ( rc == 0 )
{
assert( row_len == 1 );
memmove( &obj->max_seq_len, base, 4 );
}
}
if ( ( GetRCObject( rc ) == ( enum RCObject )rcColumn ) && ( GetRCState( rc ) == rcNotFound ) )
{
obj->max_seq_len = 0;
rc = 0;
}
VCursorRelease( curs );
if ( rc == 0 )
{
*objp = obj;
return 0;
}
}
free ( obj );
}
return rc;
}
static
rc_t init_self( self_t *self, VTable const * const srctbl, char const column[] )
{
VDatabase const * db;
rc_t rc = VTableOpenParentRead( srctbl, &db );
if ( rc == 0 )
{
VTable const * tbl;
rc = VDatabaseOpenTableRead( db, &tbl, "SEQUENCE" );
VDatabaseRelease( db );
if ( rc == 0 )
{
bool has_column = does_table_have_column( tbl, column );
if ( !has_column )
VTableRelease( tbl );
else
{
VCursor const * curs;
#if CURSOR_CACHE_SIZE
rc = VTableCreateCachedCursorRead( tbl, &curs, CURSOR_CACHE_SIZE );
#else
rc = VTableCreateCursorRead( tbl, &curs );
#endif
VTableRelease( tbl );
if ( rc == 0 )
{
uint32_t col_idx;
rc = VCursorAddColumn( curs, &col_idx, "%s", column );
if ( rc == 0 )
{
rc = VCursorOpen( curs );
if ( rc == 0 )
{
self->curs = curs;
self->col_idx = col_idx;
return 0;
}
if ( GetRCObject( rc ) == (enum RCObject)rcColumn && GetRCState( rc ) == rcUndefined )
rc = 0;
}
VCursorRelease( curs );
}
}
}
}
return rc;
}
static
rc_t RefSeqIDMake ( RefSeqID **objp, const VTable *tbl, const VCursor *native_curs )
{
rc_t rc;
/* create the object */
RefSeqID *obj = malloc ( sizeof * obj );
if ( obj == NULL )
{
rc = RC( rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted );
}
else
{
obj->curs=NULL;
SUB_DEBUG( ( "SUB.Make in 'align_ref_seq_id.c'\n" ) );
/* open the reference cursor */
rc = AlignRefTableCursor( tbl, native_curs, &obj->curs, NULL );
if ( rc == 0 )
{
/* add columns to cursor */
rc = VCursorAddColumn( obj->curs, &obj->seqID_idx, "SEQ_ID" );
if ( GetRCObject( rc ) == ( enum RCObject ) rcColumn && GetRCState( rc ) == rcNotFound )
{
rc = VCursorAddColumn( obj->curs, &obj->seqID_idx, "REF_SEQ_ID" );
}
if ( GetRCState( rc ) == rcExists )
{
rc = 0;
}
if ( rc == 0 )
{
*objp = obj;
return 0;
}
VCursorRelease ( obj -> curs );
}
free ( obj );
}
return rc;
}
/* ReadExactly
* TimedReadExactly
* read from stream until "bytes" have been retrieved
* or return incomplete transfer
*
* "buffer" [ OUT ] and "bytes" [ IN ] - return buffer for read
*
* "tm" [ IN/OUT, NULL OKAY ] - an optional indicator of
* blocking behavior. not all implementations will support
* timed reads. a NULL timeout will block indefinitely,
* a value of "tm->mS == 0" will have non-blocking behavior
* if supported by implementation, and "tm->mS > 0" will indicate
* a maximum wait timeout.
*/
LIB_EXPORT rc_t CC KStreamReadExactly ( const KStream *self,
void *buffer, size_t bytes )
{
rc_t rc;
uint8_t *b;
size_t total, count;
if ( self == NULL )
return RC ( rcNS, rcStream, rcReading, rcSelf, rcNull );
if ( ! self -> read_enabled )
return RC ( rcNS, rcStream, rcReading, rcStream, rcNoPerm );
if ( bytes == 0 )
return 0;
if ( buffer == NULL )
return RC ( rcNS, rcStream, rcReading, rcBuffer, rcNull );
switch ( self -> vt -> v1 . maj )
{
case 1:
assert ( bytes != 0 );
for ( b = buffer, total = 0; total < bytes; total += count )
{
count = 0;
rc = ( * self -> vt -> v1 . read ) ( self, b + total, bytes - total, & count );
if ( rc != 0 )
{
if ( GetRCObject ( rc ) != rcTimeout || GetRCState ( rc ) != rcExhausted )
break;
}
else if ( count == 0 )
{
rc = RC ( rcNS, rcStream, rcReading, rcTransfer, rcIncomplete );
break;
}
}
break;
default:
return RC ( rcNS, rcStream, rcReading, rcInterface, rcBadVersion );
}
return rc;
}
TVDBRowIdRange CVDBTableIndex::Find(const string& value) const
{
TVDBRowIdRange range;
if ( rc_t rc = KIndexFindText(*this, value.c_str(),
&range.first, &range.second, 0, 0) ) {
if ( GetRCObject(rc) == RCObject(rcString) &&
GetRCState(rc) == rcNotFound ) {
// no such value
range.first = range.second = 0;
}
else {
NCBI_THROW2_FMT(CSraException, eOtherError,
"Cannot find value in index: "<<*this<<": "<<value,
rc);
}
}
return range;
}
rc_t XML_ParseBool(const KXMLNode* xml_node, const char* attr, bool* val, bool optional)
{
rc_t rc;
char b[16];
size_t sz;
if( (rc = KXMLNodeReadAttrCString(xml_node, attr, b, sizeof(b), &sz)) == 0 ) {
if( strcasecmp(b, "true") == 0 || strcasecmp(b, "on") == 0 || strcasecmp(b, "yes") == 0 ) {
*val = true;
} else if( strcasecmp(b, "false") == 0 || strcasecmp(b, "off") == 0 || strcasecmp(b, "no") == 0 ) {
*val = false;
} else {
rc = RC(rcExe, rcDoc, rcReading, rcAttr, rcInvalid);
}
} else if( optional && GetRCObject(rc) == rcAttr && GetRCState(rc) == rcNotFound ) {
rc = 0;
}
return rc;
}
bool CC FGroupMAP_LoadReads( BSTNode *node, void *data )
{
FGroupMAP* n = (FGroupMAP*)node;
FGroupMAP_LoadData* d = (FGroupMAP_LoadData*)data;
DEBUG_MSG(5, (" started\n", FGroupKey_Validate(&n->key)));
while( d->rc == 0 )
{
if( (d->rc = CGLoaderFile_GetRead(n->seq, d->db.reads)) == 0 ) {
if( (d->db.reads->flags & (cg_eLeftHalfDnbNoMatches | cg_eLeftHalfDnbMapOverflow)) &&
(d->db.reads->flags & (cg_eRightHalfDnbNoMatches | cg_eRightHalfDnbMapOverflow)) ) {
d->db.mappings->map_qty = 0;
} else {
d->rc = CGLoaderFile_GetMapping(n->align, d->db.mappings);
}
/* alignment written 1st than sequence -> primary_alignment_id must be set!! */
if( d->rc == 0 && (d->rc = CGWriterAlgn_Write(d->db.walgn, d->db.reads)) == 0 ) {
d->rc = CGWriterSeq_Write(d->db.wseq);
}
}
if( GetRCState(d->rc) == rcDone && GetRCObject(d->rc) == rcData ) {
bool eof = false;
d->rc = 0;
if( n->align == NULL || ((d->rc = CGLoaderFile_IsEof(n->align, &eof)) == 0 && eof) ) {
/* mappings file EOF detected ok */
DEBUG_MSG(5, (" done\n", FGroupKey_Validate(&n->key)));
break;
} else if( d->rc == 0 ) {
/* not EOF */
d->rc = RC(rcExe, rcFile, rcReading, rcData, rcUnexpected);
CGLoaderFile_LOG(n->align, klogErr, d->rc,
"extra mappings, possible that corresponding reads file is truncated", NULL);
}
}
d->rc = d->rc ? d->rc : Quitting();
}
if( d->rc != 0 ) {
CGLoaderFile_LOG(n->seq, klogErr, d->rc, NULL, NULL);
CGLoaderFile_LOG(n->align, klogErr, d->rc, NULL, NULL);
}
FGroupMAP_CloseFiles(n);
return d->rc != 0;
}
rc_t XML_ParseTimestamp(const KXMLNode* xml_node, const char* attr, KTime_t* timestamp, bool optional)
{
rc_t rc;
char ts[128];
size_t sz;
if( (rc = KXMLNodeReadAttrCString(xml_node, attr, ts, sizeof(ts), &sz)) == 0 ) {
struct tm tm;
memset(&tm, 0, sizeof(tm));
if( strptime(ts, "%Y-%m-%dT%H:%M:%S", &tm) != NULL ) {
*timestamp = mktime(&tm);
} else {
rc = RC(rcExe, rcDoc, rcReading, rcAttr, rcInvalid);
}
} else if( optional && GetRCObject(rc) == rcAttr && GetRCState(rc) == rcNotFound ) {
rc = 0;
}
return rc;
}
static
rc_t VCursor_ReadPseudoMeta(rc_t rc, const VCursor *self,
const char *name, void *buffer, uint32_t blen, State *state)
{
uint32_t idx = ~0;
uint32_t row_len = ~0;
assert(state);
state->rc = 0;
if (rc != 0)
{ return rc; }
state->state = eNotRead;
rc = VCursorAddColumn(self, &idx, name);
if (rc != 0) {
state->rc = rc;
if (GetRCObject(rc) == rcColumn && GetRCState(rc) == rcNotFound) {
rc = 0;
state->state = eNotFound;
}
else {
state->state = eFailed;
}
}
if (state->rc == 0) {
rc = VCursorReadDirect(self, 1, idx, blen * 8, buffer, blen, &row_len);
state->rc = rc;
if (rc != 0)
{ state->state = eFailed; }
else
{ state->state = eRead; }
}
return rc;
}
/*
function ascii NCBI:align:ref_name ( I64 ref_id );
*/
static
rc_t CC align_ref_name ( void *data, const VXformInfo *info,
int64_t row_id, VRowResult *rslt, uint32_t argc, const VRowData argv[] )
{
rc_t rc;
RefName const *self = (void const *)data;
char const *name = NULL;
uint32_t name_len;
/* get start and length of reference segment */
int64_t const *ref_id = argv[REF_ID].u.data.base;
if (argv[REF_ID].u.data.elem_count == 0)
rc = RC(rcAlign, rcFunction, rcExecuting, rcRow, rcNotFound);
else {
assert(argv[REF_ID].u.data.elem_bits == sizeof(*ref_id) * 8);
ref_id += argv[REF_ID].u.data.first_elem;
rc = VCursorCellDataDirect(self->curs, ref_id[0], self->name_idx, NULL, (void const **)&name, NULL, &name_len);
}
if (GetRCState(rc) == rcNotFound && GetRCObject(rc) == rcRow) {
name = "";
name_len = 0;
}
else if (rc) return rc;
rc = KDataBufferCast(rslt->data, rslt->data, sizeof(name[0]) * 8, true);
if (rc) return rc;
rc = KDataBufferResize(rslt->data, name_len);
if (rc) return rc;
memcpy(rslt->data->base, name, sizeof(name[0]) * name_len);
rslt->elem_count = name_len;
rslt->elem_bits = sizeof(name[0]) * 8;
return rc;
}
