/* Make
* make a memory bank
*
* "block_size" [ IN ] - ignored
*
* "limit" [ IN, DFLT ZERO ] - ignored
*
* "backing" [ IN ] - required.
*/
LIB_EXPORT rc_t CC KMemBankMake ( KMemBank **bankp,
size_t block_size, uint64_t limit, KFile *backing )
{
assert(backing != NULL);
assert(bankp != NULL);
if (bankp == NULL || backing == NULL) {
PLOGMSG(klogFatal, (klogFatal, "PROGRAMMER ERROR - " __FILE__ ":$(line)", "line=%i", __LINE__));
abort();
return RC(rcFS, rcMemory, rcConstructing, rcParam, rcNull);
}
else {
KMemBank *const self = newMemBank();
if (self == NULL) {
PLOGMSG(klogFatal, (klogFatal, "OUT OF MEMORY - " __FILE__ ":$(line)", "line=%i", __LINE__));
abort();
return RC(rcFS, rcMemory, rcConstructing, rcMemory, rcExhausted);
}
if (backing) {
self->pf = backing;
KFileAddRef(self->pf);
}
*bankp = self;
return 0;
}
}
static
rc_t XML_Open(const char* path, const FSNode** tree)
{
rc_t rc = 0;
char errmsg[4096] = "";
KDirectory *dir = NULL;
PLOGMSG(klogInfo, (klogInfo, "Reading XML file '$(x)'", PLOG_S(x), path));
if( (rc = KDirectoryNativeDir(&dir)) == 0 ) {
const KFile* file = NULL;
if( (rc = KDirectoryOpenFileRead(dir, &file, "%s", path)) == 0 ) {
if( (rc = FSNode_Make((FSNode**)tree, "ROOT", &RootNode_vtbl)) == 0 ) {
const KXMLDoc* xmldoc = NULL;
if( (rc = KXMLMgrMakeDocRead(g_xmlmgr, &xmldoc, file)) == 0 ) {
const KXMLNodeset* ns = NULL;
if( (rc = KXMLDocOpenNodesetRead(xmldoc, &ns, "/FUSE/*")) == 0 ) {
uint32_t count = 0;
if( (rc = KXMLNodesetCount(ns, &count)) == 0 ) {
if( count == 0 ) {
rc = RC(rcExe, rcDoc, rcValidating, rcData, rcEmpty);
} else {
uint32_t i = 0;
while(rc == 0 && i < count) {
const KXMLNode* n = NULL;
if( (rc = KXMLNodesetGetNodeRead(ns, &n, i++)) == 0 ) {
SRAConfigFlags flags = ~0;
errmsg[0] = '\0';
rc = XML_ValidateNode((FSNode*)*tree, n, flags, errmsg);
ReleaseComplain(KXMLNodeRelease, n);
}
}
if( rc == 0 ) {
rc = SRAList_NextVersion();
}
}
}
ReleaseComplain(KXMLNodesetRelease, ns);
}
ReleaseComplain(KXMLDocRelease, xmldoc);
}
if( rc != 0 ) {
FSNode_Release(*tree);
*tree = NULL;
}
}
ReleaseComplain(KFileRelease, file);
}
ReleaseComplain(KDirectoryRelease, dir);
}
if( rc == 0 ) {
PLOGMSG(klogInfo, (klogInfo, "XML file '$(x)' ok", PLOG_S(x), path));
} else {
if( strlen(errmsg) < 1 ) {
strcpy(errmsg, path);
}
LOGERR(klogErr, rc, errmsg);
}
return rc;
}
static rc_t MaxNReadsValidator_GetKey( const SRASplitter* cself,
const char** key, spotid_t spot, readmask_t* readmask )
{
rc_t rc = 0;
MaxNReadsValidator* self = ( MaxNReadsValidator* )cself;
if ( self == NULL || key == NULL )
{
rc = RC( rcSRA, rcNode, rcExecuting, rcParam, rcNull );
}
else
{
const void* nreads = NULL;
bitsz_t o = 0, sz = 0;
uint64_t nn = 0;
*key = "";
if ( self->col != NULL )
{
rc = SRAColumnRead( self->col, spot, &nreads, &o, &sz );
if ( rc == 0 )
{
switch( sz )
{
case 8:
nn = *((const uint8_t*)nreads);
break;
case 16:
nn = *((const uint16_t*)nreads);
break;
case 32:
nn = *((const uint32_t*)nreads);
break;
case 64:
nn = *((const uint64_t*)nreads);
break;
default:
rc = RC( rcSRA, rcNode, rcExecuting, rcData, rcUnexpected );
break;
}
if ( nn > nreads_max )
{
clear_readmask( readmask );
PLOGMSG(klogWarn, (klogWarn, "too many reads $(nreads) at spot id $(row), maximum $(max) supported, skipped",
PLOG_3(PLOG_U64(nreads),PLOG_I64(row),PLOG_U32(max)), nn, spot, nreads_max));
}
else if ( nn == nreads_max - 1 )
{
PLOGMSG(klogWarn, (klogWarn, "too many reads $(nreads) at spot id $(row), truncated to $(max)",
PLOG_3(PLOG_U64(nreads),PLOG_I64(row),PLOG_U32(max)), nn + 1, spot, nreads_max));
}
}
}
}
return rc;
}
rc_t runChecks(const TestCase& test_case, const VCursor * cursor, uint32_t name_idx, uint32_t name_range_idx)
{
rc_t rc;
int64_t first_id;
uint64_t count_id;
rc = VCursorIdRange( cursor, name_idx, &first_id, &count_id );
if (rc != 0)
{
LOGERR( klogInt, rc, "VCursorIdRange() failed" );
return rc;
}
for (uint64_t row_id = first_id; row_id < first_id + count_id; ++row_id)
{
const char * name = NULL;
uint32_t name_len;
RowRange *row_range;
rc = VCursorCellDataDirect( cursor, row_id, name_idx, NULL, (void const **)&name, NULL, &name_len );
if ( rc != 0 )
return rc;
rc = VCursorParamsSet( ( struct VCursorParams const * )cursor, "QUERY_NAME", "%.*s", name_len, name );
if ( rc != 0 )
return rc;
rc = VCursorCellDataDirect( cursor, row_id, name_range_idx, NULL, (void const **)&row_range, NULL, NULL );
if ( rc != 0 )
return rc;
std::string name_str(name, name_len);
if (test_case.key_ranges.find(name_str) == test_case.key_ranges.end())
{
PLOGMSG( klogInt, (klogErr, "Unexpected name '$(NAME)' in test case '$(TC_NAME)'", "TC_NAME=%s,NAME=%s", test_case_name, name_str.c_str()) );
return 1;
}
RowRange row_range_exp = test_case.key_ranges.find(name_str)->second;
if (row_range->start_id != row_range_exp.start_id || row_range->stop_id != row_range_exp.stop_id)
{
PLOGMSG( klogInt, (klogErr, "Row range for name '$(NAME)' in test case '$(TC_NAME)' does not match. Expected: $(EXP_S)-$(EXP_F), actual: $(ACT_S)-$(ACT_F)",
"TC_NAME=%s,NAME=%s,EXP_S=%ld,EXP_F=%ld,ACT_S=%ld,ACT_F=%ld",
test_case_name, name_str.c_str(), row_range_exp.start_id, row_range_exp.stop_id, row_range->start_id, row_range->stop_id) );
return 1;
}
}
return rc;
}
static
rc_t stats_data_update_group(stats_data_t *self,
int64_t spot_id,
uint32_t spot_len,
uint32_t bio_spot_len,
uint32_t cmp_spot_len,
char const grp[],
uint64_t grp_len)
{
if (grp_len == 0 || grp == NULL || grp[0] == '\0' ||
(grp_len == 7 && strncasecmp("default", grp, 7) == 0))
{
group_stats_update(&self->deflt, spot_id, spot_len, bio_spot_len, cmp_spot_len);
}
else
{
group_stats_t *const fnd = stats_data_get_group(self, (unsigned)grp_len, grp);
if (fnd == NULL)
return RC(rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted);
if (fnd - get_group(self, 0) < MAX_GROUP_COUNT)
group_stats_update(fnd, spot_id, spot_len, bio_spot_len, cmp_spot_len);
else {
KDataBufferWhack(&self->group);
KDataBufferWhack(&self->names);
stats_data_init_funcs(self, false);
(void)PLOGMSG(klogWarn, (klogWarn, "Too many spot groups ($(count)); dropping group stats", "count=%u", (unsigned)(self->count)));
}
}
return 0;
}
rc_t ProcessOneDo (ProcessOne * self)
{
static const char F[] = PLOG_2(PLOG_S(p),PLOG_S(t));
enum KPathType type;
rc_t rc = 0;
type = KDirectoryPathType (self->dir, self->path);
switch (type)
{
case kptFile:
rc = ProcessOneDoFile (self);
break;
/* case kptDir: */
/* break; */
/* case kptAlias|kptFile: */
/* break; */
/* case kptAlias|kptDir: */
/* break; */
default:
PLOGMSG (klogInfo, "+ Skipping $(p) of type $(t)", F, self->path, typeToString(type));
break;
}
return rc;
}
rc_t ProcessOneDoFile (ProcessOne * self)
{
rc_t rc = 0;
KFile * mfile;
PLOGMSG (klogInfo, "ProcessOneDoFile: $(F)", PLOG_S(F), self->path);
rc = KFileMakeNewMD5Read (&mfile, self->file, self->md5, self->path);
if (rc == 0)
{
const KFile * bfile;
rc = KFileMakeBuf (&bfile, mfile, 64*1024);
if (rc == 0)
{
/* add more here */
KFileRelease (bfile);
}
else
{
pLOGERR (klogErr, rc, "Failure to initiate buffer $(F)", PLOG_S(F), self->path);
KFileRelease (mfile);
}
}
else
pLOGERR (klogErr, rc, "Failure to initiate MD5 summing $(F)", PLOG_S(F), self->path);
return rc;
}
rc_t BufferQPopBuffer (BufferQ * self, const Buffer ** buff, timeout_t * tm)
{
rc_t rc = 0;
timeout_t t;
void * p;
LOGMSG (klogDebug10, "BufferQPopBuffer");
assert (self != NULL);
assert (buff != NULL);
if (tm == NULL)
{
LOGMSG (klogDebug10, "BufferQPopBuffer tm was NULL");
tm = &t;
rc = TimeoutInit (tm, self->timeout);
}
if (rc == 0)
{
LOGMSG (klogDebug10, "BufferQPopBuffer call KQueuePop");
rc = KQueuePop (self->q, &p, tm);
PLOGMSG (klogDebug10, "BufferQPopBuffer back from KQueuePop $(rc)", PLOG_U32(rc), rc);
if (rc == 0)
*buff = p;
else
{
*buff = NULL;
}
}
LOGMSG (klogDebug10, "leave BufferQPopBuffer");
return rc;
}
rc_t load_consensus_src( con_ctx * sctx, KDirectory * hdf5_src )
{
BaseCalls_cmn ConsensusTab;
rc_t rc = 0;
if ( sctx->lctx->check_src_obj )
rc = check_src_objects( hdf5_src, consensus_groups_to_check,
consensus_tables_to_check, false );
if ( rc == 0 )
rc = open_BaseCalls_cmn( hdf5_src, &ConsensusTab, true,
"PulseData/ConsensusBaseCalls", sctx->lctx->cache_content, true );
if ( rc == 0 )
{
uint64_t total_bases = zmw_total( &ConsensusTab.zmw );
uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ];
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading consensus-table ( $(bases) bases / $(spots) spots ):",
"bases=%lu,spots=%lu", total_bases, total_spots ));
KLogLevelSet( tmp_lvl );
if ( !check_Consensus_totalcount( &ConsensusTab, total_bases ) )
rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
else
rc = zmw_for_each( &ConsensusTab.zmw, &sctx->lctx->xml_progress, sctx->cursor,
sctx->lctx->with_progress, sctx->col_idx, NULL,
true, consensus_load_spot, &ConsensusTab );
close_BaseCalls_cmn( &ConsensusTab );
}
return rc;
}
static
rc_t VProdResolveColExpr ( const VProdResolve *self, VProduction **out,
VFormatdecl *fd, const SSymExpr *x, bool casting )
{
rc_t rc;
const SNameOverload *sname;
const KSymbol *sym = x -> _sym;
BSTree ordered;
uint32_t i, count;
SColumnBestFit buff [ 16 ], * nodes = buff;
/* fail if "fd" has a format */
if ( fd -> fmt != 0 )
{
PLOGMSG ( klogWarn, ( klogWarn, "illegal cast of column '$(name)'"
, "name=%.*s"
, ( int ) sym -> name . size
, sym -> name . addr ));
return 0;
}
/* allocate nodes for indexing columns */
sname = sym -> u . obj;
count = VectorLength ( & sname -> items );
if ( count > sizeof buff / sizeof buff [ 0 ] )
{
nodes = malloc ( sizeof * nodes * count );
if ( nodes == NULL )
return RC ( rcVDB, rcProduction, rcResolving, rcMemory, rcExhausted );
}
/* insert columns into ordered tree */
BSTreeInit ( & ordered );
for ( i = VectorStart ( & sname -> items ), count += i; i < count; ++ i )
{
/* get SColumn */
nodes [ i ] . scol = ( const void* ) VectorGet ( & sname -> items, i );
/* perform type cast and measure distance */
if ( casting ?
VTypedeclCommonAncestor ( & nodes [ i ] . scol -> td, self -> schema,
& fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) :
VTypedeclToTypedecl ( & nodes [ i ] . scol -> td, self -> schema,
& fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) )
{
BSTreeInsert ( & ordered, & nodes [ i ] . n, order_column );
}
}
/* try to resolve each in order */
rc = VProdResolveBestColumn ( self, out, & ordered, x -> alt );
if ( nodes != buff )
free ( nodes );
return rc;
}
static
rc_t CCCopyDoDirectory (CCCopy * self)
{
rc_t rc = 0;
PLOGMSG (klogInfo, "CCCopyDoDirectory $(d)", PLOG_S(d), self->path);
rc = RC (rcExe, rcDirectory, rcCopying, rcParam, rcUnsupported);
return rc;
}
static rc_t consensus_loader( ld_context *lctx, const char * table_name,
bool cache_content )
{
uint32_t col_idx[ consensus_tab_count ];
rc_t rc = add_columns( lctx->cursor, consensus_tab_count, -1,
col_idx, consensus_tab_names );
if ( rc == 0 )
{
rc = VCursorOpen( lctx->cursor );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot open cursor on consensus-table" );
else
{
BaseCalls_cmn ConsensusTab;
const INSDC_SRA_platform_id platform = SRA_PLATFORM_PACBIO_SMRT;
rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_PLATFORM ],
sizeof platform * 8, &platform, 0, 1 );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for platform-column" );
else
{
const INSDC_SRA_read_filter filter = SRA_READ_FILTER_PASS;
rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_READ_FILTER ],
sizeof filter * 8, &filter, 0, 1 );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for read-filter-column" );
}
if ( rc == 0 )
rc = open_BaseCalls_cmn( lctx->hdf5_dir, &ConsensusTab, true,
"PulseData/ConsensusBaseCalls", cache_content, true );
if ( rc == 0 )
{
uint64_t total_bases = zmw_total( &ConsensusTab.zmw );
uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ];
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading consensus-table ( $(bases) bases / $(spots) spots ):",
"bases=%lu,spots=%lu", total_bases, total_spots ));
KLogLevelSet( tmp_lvl );
if ( check_Consensus_totalcount( &ConsensusTab, total_bases ) )
rc = zmw_for_each( &ConsensusTab.zmw, lctx, col_idx, NULL,
true, consensus_load_spot, &ConsensusTab );
else
rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
close_BaseCalls_cmn( &ConsensusTab );
}
}
}
return rc;
}
rc_t CopierDoOne (Copier * self)
{
rc_t rc = 0;
const Buffer * b;
LOGMSG (klogDebug10, "CopierDoOne");
rc = Quitting();
if (rc == 0)
{
LOGMSG (klogDebug10, "call BufferQPopBuffer");
rc = BufferQPopBuffer (self->q, &b, NULL);
if (rc == 0)
{
size_t w;
size_t z;
LOGMSG (klogDebug10, "call BufferContentGetSize");
z = BufferContentGetSize (b);
rc = KFileWrite (self->f, self->o, b, z, &w);
self->o += w;
if (w != z)
rc = RC (rcExe, rcFile, rcWriting, rcTransfer, rcIncomplete);
else
rc = BufferRelease (b);
}
/* ow this is ugly! */
/* is the rc a "exhausted" on a timeout? */
else if ((GetRCObject(rc) == rcTimeout) && (GetRCState(rc) == rcExhausted))
{
rc = 0;
LOGMSG (klogDebug10, "CopierDoOne timeout");
/* if so is the queue also sealed? */
if (BufferQSealed (self->q) == true)
{
LOGMSG (klogDebug10, "CopierDoOne sealed");
/* if both then we are done and so signal */
rc = KFileRelease (self->f);
PLOGMSG (klogDebug10, "CopierDoOne back from KFileRelease $(rc)",PLOG_U32(rc),rc);
if (rc == 0)
{
self->f = NULL;
rc = BufferQRelease (self->q);
if (rc == 0)
{
self->q = NULL;
rc = RC (rcExe, rcNoTarg, rcCopying, rcNoTarg, rcDone );
}
}
}
}
else
LOGMSG (klogDebug10, "CopierDoOne pop failure");
}
else
LOGMSG (klogDebug10, "CopierDoOne: quitting");
return rc;
}
rc_t CGWriterAlgn_Whack(const CGWriterAlgn* cself, bool commit, uint64_t* rows_1st, uint64_t* rows_2nd)
{
rc_t rc = 0;
if( cself != NULL ) {
CGWriterAlgn* self = (CGWriterAlgn*)cself;
rc_t rc1 = TableWriterAlgn_Whack(cself->primary, commit, rows_1st);
rc_t rc2 = TableWriterAlgn_Whack(cself->secondary, commit, rows_2nd);
if( self->forced_pairs_cnt > 0 ) {
PLOGMSG(klogInfo, (klogInfo, "$(forced_pairs_cnt) forced pairs to PRIMARY", "forced_pairs_cnt=%lu", self->forced_pairs_cnt));
}
if( self->dropped_mates_cnt > 0 ) {
PLOGMSG(klogInfo, (klogInfo, "$(dropped_mates_cnt) dropped duplicate mates in SECONDARY", "dropped_mates_cnt=%lu", self->dropped_mates_cnt));
}
rc = rc1 ? rc1 : rc2;
free(self);
}
return rc;
}
static rc_t passes_load_loop( ld_context *lctx, VCursor * cursor, Passes_src *tab,
uint32_t *col_idx )
{
rc_t rc = 0;
KLogLevel tmp_lvl;
pass_block block;
pl_progress *progress;
uint64_t pos = 0;
uint64_t total_passes = tab->AdapterHitBefore.extents[0];
pl_progress_make( &progress, total_passes );
rc = progress_chunk( &lctx->xml_progress, total_passes );
tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading passes-table ( $(rows) rows ) :",
"rows=%lu", total_passes ));
KLogLevelSet( tmp_lvl );
while( pos < total_passes && rc == 0 )
{
rc = pass_block_read_from_src( tab, total_passes, pos, &block );
if ( rc == 0 )
{
uint32_t i;
for ( i = 0; i < block.n_read && rc == 0; ++i )
{
rc = Quitting();
if ( rc == 0 )
{
rc = passes_load_pass( cursor, &block, i, col_idx );
if ( rc == 0 )
{
rc = progress_step( lctx->xml_progress );
if ( lctx->with_progress )
pl_progress_increment( progress, 1 );
}
}
else
LOGERR( klogErr, rc, "...loading passes interrupted" );
}
pos += block.n_read;
}
}
pl_progress_destroy( progress );
if ( rc == 0 )
{
rc = VCursorCommit( cursor );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot commit cursor on PASSES-tab" );
}
return rc;
}
rc_t WriteFileMeta(SIndexObj* obj)
{
rc_t rc = 0;
KMDataNode* nd = NULL;
PLOGMSG(klogInfo, (klogInfo, "Meta $(f) on index $(i): file size $(s), buffer $(b)",
PLOG_4(PLOG_S(f),PLOG_S(i),PLOG_U64(s),PLOG_U32(b)), obj->file, obj->index, obj->file_size, obj->buffer_sz));
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Format")) == 0 ) {
KMDataNode* opt = NULL;
rc = KMDataNodeWriteCString(nd, obj->format);
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(nd, &opt, "Options")) == 0 ) {
KMDataNode* ond = NULL;
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "accession")) == 0 ) {
rc = KMDataNodeWriteCString(ond, g_accession);
KMDataNodeRelease(ond);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "minSpotId")) == 0 ) {
rc = KMDataNodeWriteB64(ond, &obj->minSpotId);
KMDataNodeRelease(ond);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "maxSpotId")) == 0 ) {
rc = KMDataNodeWriteB64(ond, &obj->maxSpotId);
KMDataNodeRelease(ond);
}
KMDataNodeRelease(opt);
}
KMDataNodeRelease(nd);
}
if( rc == 0 && obj->file_size > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Size")) == 0 ) {
rc = KMDataNodeWriteB64(nd, &obj->file_size);
KMDataNodeRelease(nd);
}
if( rc == 0 && obj->buffer_sz > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Buffer")) == 0 ) {
rc = KMDataNodeWriteB32(nd, &obj->buffer_sz);
KMDataNodeRelease(nd);
}
if( rc == 0 && strlen(obj->index) > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Index")) == 0 ) {
rc = KMDataNodeWriteCString(nd, obj->index);
KMDataNodeRelease(nd);
}
if( rc == 0 && obj->file_size > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "md5")) == 0 ) {
char x[5];
int i;
for( i = 0; rc == 0 && i < sizeof(obj->md5_digest); i++ ) {
int l = snprintf(x, 4, "%02x", obj->md5_digest[i]);
rc = KMDataNodeAppend(nd, x, l);
}
KMDataNodeRelease(nd);
}
return rc;
}
static
rc_t XMLThread( const KThread *self, void *data )
{
KDirectory *dir = NULL;
PLOGMSG(klogInfo, (klogInfo, "XML sync thread started with $(s) sec", PLOG_U32(s), g_xml_sync));
do {
rc_t rc = 0;
KTime_t dt = 0;
DEBUG_MSG(8, ("XML sync thread checking %s\n", g_xml_path));
if( (rc = KDirectoryNativeDir(&dir)) == 0 ) {
rc = KDirectoryDate(dir, &dt, "%s", g_xml_path);
ReleaseComplain(KDirectoryRelease, dir);
}
if( rc == 0 ) {
if( dt != g_xml_mtime ) {
const FSNode* new_root = NULL;
PLOGMSG(klogInfo, (klogInfo, "File $(f) changed ($(m) <> $(d)), updating...",
PLOG_3(PLOG_S(f),PLOG_I64(m),PLOG_I64(d)), g_xml_path, g_xml_mtime, dt));
if( XML_Open(g_xml_path, &new_root) == 0 ) {
if( (rc = XMLLock(true)) == 0 ) {
const FSNode* old_root = g_root;
g_root = new_root;
g_xml_mtime = dt;
XMLUnlock();
FSNode_Release(old_root);
PLOGMSG(klogInfo, (klogInfo, "Data from $(f) updated successfully", PLOG_S(f), g_xml_path));
}
}
} else {
DEBUG_MSG(8, ("XML sync thread up-to-date %s\n", g_xml_path));
}
} else {
LOGERR(klogErr, rc, g_xml_path);
}
SRAList_PostRefresh();
sleep(g_xml_sync);
} while( g_xml_sync > 0 );
LOGMSG(klogInfo, "XML sync thread ended");
return 0;
}
void get_ref_var_object (KSearch::CVRefVariation& obj, size_t ref_pos, size_t del_len,
char const* allele, size_t allele_len, ngs::ReferenceSequence const& ref_seq,
std::string & ref_allele)
{
size_t var_len = allele_len;
size_t chunk_size = 5000; // TODO: add the method Reference[Sequence].getChunkSize() to the API
size_t chunk_no = ref_pos / chunk_size;
size_t ref_pos_in_slice = ref_pos % chunk_size;
size_t bases_start = chunk_no * chunk_size;
size_t chunk_no_last = ref_seq.getLength() / chunk_size;
bool cont = false;
size_t chunk_no_start = chunk_no, chunk_no_end = chunk_no;
// optimization: first look into the current chunk only (using ngs::StringRef)
{
ngs::StringRef ref_chunk = ref_seq.getReferenceChunk ( bases_start );
if ( ! check_ref_slice (ref_chunk.data() + ref_pos_in_slice, del_len) )
{
PLOGMSG ( klogWarn,
( klogWarn,
"The selected reference region [$(REFSLICE)] does not contain valid bases, skipping...",
"REFSLICE=%.*s",
(int)del_len, ref_chunk.data() + ref_pos_in_slice ));
}
cont = Common::find_variation_core_step ( obj, g_Params.alg,
ref_chunk.data(), ref_chunk.size(), ref_pos_in_slice,
allele, var_len, del_len,
chunk_size, chunk_no_last, bases_start, chunk_no_start, chunk_no_end );
if ( !cont )
ref_allele.assign (ref_chunk.data() + obj.GetAlleleStartRelative(), obj.GetAlleleLenOnRef());
}
// general case - expanding ref_slice to multiple chunks
if ( cont )
{
ngs::String ref_slice;
while ( cont )
{
ref_slice = ref_seq.getReferenceBases (
bases_start, (chunk_no_end - chunk_no_start + 1)*chunk_size );
cont = Common::find_variation_core_step ( obj, g_Params.alg,
ref_slice.c_str(), ref_slice.size(), ref_pos_in_slice,
allele, var_len, del_len,
chunk_size, chunk_no_last, bases_start, chunk_no_start, chunk_no_end );
}
ref_allele.assign (ref_slice.c_str() + obj.GetAlleleStartRelative(), obj.GetAlleleLenOnRef());
}
}
static rc_t Run(const CmdLine* args)
{
rc_t rc = 0;
Db db;
SpotIterator it;
assert(args);
if (!SpotIteratorFileExists(args->file)) {
rc = RC(rcExe, rcFile, rcOpening, rcFile, rcNotFound);
PLOGERR(klogErr,
(klogErr, rc, "Cannot find '$(path)'", "path=%s", args->file));
}
else if (!SpotIteratorFileExists(args->table)) {
rc = RC(rcExe, rcTable, rcOpening, rcTable, rcNotFound);
PLOGERR(klogErr,
(klogErr, rc, "Cannot find '$(path)'", "path=%s", args->table));
}
{
rc_t rc2 = DbInit(rc, args, &db);
if (rc == 0)
{ rc = rc2; }
}
if (rc == 0) {
rc = SpotIteratorInit(args->file, &db, &it);
}
if (rc == 0) {
rc = Work(&db, &it);
}
if (rc == 0) {
PLOGMSG(klogInfo, (klogInfo,
"Success: redacted $(redacted) spots out of $(all)",
"redacted=%d,all=%d", db.redactedSpots, db.nSpots));
}
{
rc_t rc2 = SpotIteratorDestroy(&it);
if (rc == 0)
{ rc = rc2; }
}
{
rc_t rc2 = DbDestroy(&db);
if (rc == 0)
{ rc = rc2; }
}
return rc;
}
static void seq_load_info( regions_stat * stat )
{
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
if ( stat->expands_a > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"adapter expanded: $(times) ( in $(spots) spots )",
"times=%u,spots=%u",
stat->expands_a, stat->expands_spots ));
if ( stat->expands_i > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"insert expanded : $(times) ( in $(spots) spots )",
"times=%u,spots=%u",
stat->expands_i, stat->expands_spots ));
if ( stat->inserts > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"regions inserted: $(times) ( in $(spots) spots )",
"times=%u,spots=%u",
stat->inserts, stat->inserts_spots ));
if ( stat->end_gap > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"end-gap inserted: $(times)",
"times=%u", stat->end_gap ));
if ( stat->overlapps > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"overlapping rngs: $(times)",
"times=%u", stat->overlapps ));
if ( stat->removed > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"removed rgns : $(times)",
"times=%u", stat->removed ));
KLogLevelSet( tmp_lvl );
}
int expand_variations (int argc, char** argv)
{
int ret;
try
{
KApp::CArgs args;
args.MakeAndHandle (argc, argv, Options, countof (Options));
if (args.GetOptionCount (OPTION_ALG) == 1)
{
char const* alg = args.GetOptionValue ( OPTION_ALG, 0 );
if (!strcmp(alg, PARAM_ALG_SW))
g_Params.alg = ::refvarAlgSW;
else if (!strcmp(alg, PARAM_ALG_RA))
g_Params.alg = ::refvarAlgRA;
else
{
PLOGMSG ( klogErr, ( klogErr,
"Error: Unknown algorithm specified: \"$(ALG)\"", "ALG=%s", alg ));
return 3;
}
}
ret = expand_variations_impl ();
}
catch ( ngs::ErrorMsg const& e )
{
PLOGMSG ( klogErr,
( klogErr, "ngs::ErrorMsg: $(WHAT)", "WHAT=%s", e.what()
));
ret = 3;
}
catch (...)
{
Utils::HandleException ();
ret = 3;
}
return ret;
}
rc_t CCCopyDo (CCCopy * self)
{
rc_t rc = 0;
enum KPathType type;
assert (self != NULL);
assert (self->path != NULL);
type = KDirectoryVPathType (self->in, self->path, NULL);
switch (type & ~kptAlias)
{
case kptNotFound:
rc = RC (rcExe, rcPath, rcAccessing, rcPath, rcNotFound);
break;
default:
case kptBadPath:
rc = RC (rcExe, rcPath, rcAccessing, rcPath, rcInvalid);
break;
case kptFile:
rc = CCCopyDoFile (self);
break;
case kptDir:
#if 0
rc = CCCopyDoDirectory (self);
#else
rc = PLOGMSG (klogInfo, "Ignoring directory $(p)",PLOG_S(p),self->path);
#endif
break;
case kptCharDev:
rc = PLOGMSG (klogInfo, "Ignoring kptCharDev $(p)",PLOG_S(p),self->path);
break;
case kptBlockDev:
rc = PLOGMSG (klogInfo, "Ignoring kptBlockDev $(p)",PLOG_S(p),self->path);
break;
case kptFIFO:
rc = PLOGMSG (klogInfo, "Ignoring kptFIFO $(p)",PLOG_S(p),self->path);
break;
}
return rc;
}
/* OpenUpdate
* finish create operation
*/
static
rc_t VDatabaseOpenUpdate ( VDatabase *self, const char *decl )
{
/* open metadata */
rc_t rc = KDatabaseOpenMetadataUpdate ( self -> kdb, & self -> meta );
if ( rc == 0 )
{
/* fetch stored schema */
rc = VDatabaseLoadSchema ( self );
if ( rc == 0 )
{
/* fetch requested schema */
const SDatabase *sdb = self -> sdb;
if ( decl != NULL && decl [ 0 ] != 0 )
{
uint32_t type;
const SNameOverload *name;
sdb = ( self -> dad != NULL ) ?
SDatabaseFind ( self -> dad -> sdb, self -> schema,
& name, & type, decl, "VDatabaseOpenUpdate" ):
VSchemaFind ( self -> schema,
& name, & type, decl, "VDatabaseOpenUpdate", true );
if ( sdb != NULL && type != eDatabase )
{
PLOGMSG ( klogWarn, ( klogWarn, "expression '$(expr)' is not a database",
"expr=%s", decl ));
sdb = NULL;
}
}
/* error if the two definitions differ */
if ( sdb != NULL && self -> sdb != NULL && sdb != self -> sdb )
rc = RC ( rcVDB, rcDatabase, rcOpening, rcSchema, rcIncorrect );
else if ( sdb == NULL && self -> sdb == NULL )
rc = RC ( rcVDB, rcDatabase, rcOpening, rcSchema, rcNotFound );
else
{
if ( sdb != NULL )
self -> sdb = sdb;
/* write schema to metadata */
rc = VDatabaseStoreSchema ( self );
if ( rc == 0 )
return 0;
}
}
}
DBGMSG(DBG_VDB, DBG_FLAG(DBG_VDB_VDB), ("VDatabaseOpenUpdate = %d\n", rc));
return rc;
}
/* AddRef
* Release
*/
LIB_EXPORT rc_t CC KMemBankAddRef(const KMemBank *cself)
{
KMemBank *const self = (KMemBank *)cself;
if (self != NULL) {
int const rslt = KRefcountAdd(&self->refcount, "KMemBank");
assert(rslt == krefOkay);
if (rslt != krefOkay) {
PLOGMSG(klogFatal, (klogFatal, "LOGIC ERROR - " __FILE__ ":$(line)", "line=%i", __LINE__));
abort();
return RC(rcFS, rcMemory, rcAttaching, rcConstraint, rcViolated);
}
}
return 0;
}
static
rc_t VProdResolveProdExpr ( const VProdResolve *self, VProduction **out, const KSymbol *sym )
{
const SProduction *sprod = sym -> u . obj;
if ( ! sprod -> trigger )
return VProdResolveSProduction ( self, out, sprod );
PLOGMSG ( klogWarn, ( klogWarn, "trigger production '$(trig)' used in expression"
, "trig=%.*s"
, ( int ) sym -> name . size
, sym -> name . addr ));
return 0;
}
static rc_t remove_path( KDirectory * dir, const char * path, bool quiet )
{
rc_t rc;
if ( !quiet )
{
PLOGMSG( klogInfo, ( klogInfo, "removing '$(path)'", "path=%s", path ));
}
rc = KDirectoryRemove ( dir, true, path );
if ( rc != 0 )
{
LOGERR( klogErr, rc, "remove_path:KDirectoryRemove() failed" );
}
return rc;
}
/* ParamExpr
* resolve a simple parameter by name
*/
LIB_EXPORT rc_t CC VProdResolveParamExpr ( const VProdResolve *self, VProduction **out, const KSymbol *sym )
{
const SProduction *sprod = sym -> u . obj;
VProduction *vprod = VCursorCacheGet ( self -> cache, & sprod -> cid );
if ( vprod != NULL )
{
* out = vprod;
return 0;
}
PLOGMSG ( klogWarn, ( klogWarn, "unknown parameter '$(param)' used in expression"
, "param=%.*s"
, ( int ) sprod -> name -> name . size
, sprod -> name -> name . addr ));
return 0;
}
static
rc_t VProdResolvePhysExpr ( const VProdResolve *self,
VProduction **out, const KSymbol *sym )
{
if ( self -> chain == chainEncoding )
{
assert ( self -> curs -> read_only == false );
PLOGMSG ( klogWarn, ( klogWarn, "illegal access of physical column '$(name)'"
, "name=%.*s"
, ( int ) sym -> name . size
, sym -> name . addr ));
return 0;
}
return VProdResolveSPhysMember ( self, out, sym -> u . obj );
}
static int string_buffer_add(KDataBuffer *const self,
unsigned const len, char const name[/* len */])
{
size_t const rslt = self->elem_count;
size_t const newsize = rslt + len;
if (KDataBufferResize(self, newsize) != 0)
return -1;
#if 1
PLOGMSG(klogInfo, (klogInfo, "New spot group '$(NAME)'", "NAME=%.*s", (int)len, name));
#endif
memcpy(((char *)self->base) + rslt, name, len);
return (int)rslt;
}
void seq_report_totals( ld_context *lctx )
{
const char* accession;
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
accession = strrchr( lctx->dst_path, '/' );
if( accession == NULL )
accession = lctx->dst_path;
else
accession++;
PLOGMSG( klogInfo, ( klogInfo, "loaded",
"severity=total,status=success,accession=%s,spot_count=%lu,base_count=%lu,bad_spots=0",
accession, lctx->total_seq_spots, lctx->total_seq_bases ));
KLogLevelSet( tmp_lvl );
}