/* Make * make a memory bank * * "block_size" [ IN ] - ignored * * "limit" [ IN, DFLT ZERO ] - ignored * * "backing" [ IN ] - required. */ LIB_EXPORT rc_t CC KMemBankMake ( KMemBank **bankp, size_t block_size, uint64_t limit, KFile *backing ) { assert(backing != NULL); assert(bankp != NULL); if (bankp == NULL || backing == NULL) { PLOGMSG(klogFatal, (klogFatal, "PROGRAMMER ERROR - " __FILE__ ":$(line)", "line=%i", __LINE__)); abort(); return RC(rcFS, rcMemory, rcConstructing, rcParam, rcNull); } else { KMemBank *const self = newMemBank(); if (self == NULL) { PLOGMSG(klogFatal, (klogFatal, "OUT OF MEMORY - " __FILE__ ":$(line)", "line=%i", __LINE__)); abort(); return RC(rcFS, rcMemory, rcConstructing, rcMemory, rcExhausted); } if (backing) { self->pf = backing; KFileAddRef(self->pf); } *bankp = self; return 0; } }
static rc_t XML_Open(const char* path, const FSNode** tree) { rc_t rc = 0; char errmsg[4096] = ""; KDirectory *dir = NULL; PLOGMSG(klogInfo, (klogInfo, "Reading XML file '$(x)'", PLOG_S(x), path)); if( (rc = KDirectoryNativeDir(&dir)) == 0 ) { const KFile* file = NULL; if( (rc = KDirectoryOpenFileRead(dir, &file, "%s", path)) == 0 ) { if( (rc = FSNode_Make((FSNode**)tree, "ROOT", &RootNode_vtbl)) == 0 ) { const KXMLDoc* xmldoc = NULL; if( (rc = KXMLMgrMakeDocRead(g_xmlmgr, &xmldoc, file)) == 0 ) { const KXMLNodeset* ns = NULL; if( (rc = KXMLDocOpenNodesetRead(xmldoc, &ns, "/FUSE/*")) == 0 ) { uint32_t count = 0; if( (rc = KXMLNodesetCount(ns, &count)) == 0 ) { if( count == 0 ) { rc = RC(rcExe, rcDoc, rcValidating, rcData, rcEmpty); } else { uint32_t i = 0; while(rc == 0 && i < count) { const KXMLNode* n = NULL; if( (rc = KXMLNodesetGetNodeRead(ns, &n, i++)) == 0 ) { SRAConfigFlags flags = ~0; errmsg[0] = '\0'; rc = XML_ValidateNode((FSNode*)*tree, n, flags, errmsg); ReleaseComplain(KXMLNodeRelease, n); } } if( rc == 0 ) { rc = SRAList_NextVersion(); } } } ReleaseComplain(KXMLNodesetRelease, ns); } ReleaseComplain(KXMLDocRelease, xmldoc); } if( rc != 0 ) { FSNode_Release(*tree); *tree = NULL; } } ReleaseComplain(KFileRelease, file); } ReleaseComplain(KDirectoryRelease, dir); } if( rc == 0 ) { PLOGMSG(klogInfo, (klogInfo, "XML file '$(x)' ok", PLOG_S(x), path)); } else { if( strlen(errmsg) < 1 ) { strcpy(errmsg, path); } LOGERR(klogErr, rc, errmsg); } return rc; }
static rc_t MaxNReadsValidator_GetKey( const SRASplitter* cself, const char** key, spotid_t spot, readmask_t* readmask ) { rc_t rc = 0; MaxNReadsValidator* self = ( MaxNReadsValidator* )cself; if ( self == NULL || key == NULL ) { rc = RC( rcSRA, rcNode, rcExecuting, rcParam, rcNull ); } else { const void* nreads = NULL; bitsz_t o = 0, sz = 0; uint64_t nn = 0; *key = ""; if ( self->col != NULL ) { rc = SRAColumnRead( self->col, spot, &nreads, &o, &sz ); if ( rc == 0 ) { switch( sz ) { case 8: nn = *((const uint8_t*)nreads); break; case 16: nn = *((const uint16_t*)nreads); break; case 32: nn = *((const uint32_t*)nreads); break; case 64: nn = *((const uint64_t*)nreads); break; default: rc = RC( rcSRA, rcNode, rcExecuting, rcData, rcUnexpected ); break; } if ( nn > nreads_max ) { clear_readmask( readmask ); PLOGMSG(klogWarn, (klogWarn, "too many reads $(nreads) at spot id $(row), maximum $(max) supported, skipped", PLOG_3(PLOG_U64(nreads),PLOG_I64(row),PLOG_U32(max)), nn, spot, nreads_max)); } else if ( nn == nreads_max - 1 ) { PLOGMSG(klogWarn, (klogWarn, "too many reads $(nreads) at spot id $(row), truncated to $(max)", PLOG_3(PLOG_U64(nreads),PLOG_I64(row),PLOG_U32(max)), nn + 1, spot, nreads_max)); } } } } return rc; }
rc_t runChecks(const TestCase& test_case, const VCursor * cursor, uint32_t name_idx, uint32_t name_range_idx) { rc_t rc; int64_t first_id; uint64_t count_id; rc = VCursorIdRange( cursor, name_idx, &first_id, &count_id ); if (rc != 0) { LOGERR( klogInt, rc, "VCursorIdRange() failed" ); return rc; } for (uint64_t row_id = first_id; row_id < first_id + count_id; ++row_id) { const char * name = NULL; uint32_t name_len; RowRange *row_range; rc = VCursorCellDataDirect( cursor, row_id, name_idx, NULL, (void const **)&name, NULL, &name_len ); if ( rc != 0 ) return rc; rc = VCursorParamsSet( ( struct VCursorParams const * )cursor, "QUERY_NAME", "%.*s", name_len, name ); if ( rc != 0 ) return rc; rc = VCursorCellDataDirect( cursor, row_id, name_range_idx, NULL, (void const **)&row_range, NULL, NULL ); if ( rc != 0 ) return rc; std::string name_str(name, name_len); if (test_case.key_ranges.find(name_str) == test_case.key_ranges.end()) { PLOGMSG( klogInt, (klogErr, "Unexpected name '$(NAME)' in test case '$(TC_NAME)'", "TC_NAME=%s,NAME=%s", test_case_name, name_str.c_str()) ); return 1; } RowRange row_range_exp = test_case.key_ranges.find(name_str)->second; if (row_range->start_id != row_range_exp.start_id || row_range->stop_id != row_range_exp.stop_id) { PLOGMSG( klogInt, (klogErr, "Row range for name '$(NAME)' in test case '$(TC_NAME)' does not match. Expected: $(EXP_S)-$(EXP_F), actual: $(ACT_S)-$(ACT_F)", "TC_NAME=%s,NAME=%s,EXP_S=%ld,EXP_F=%ld,ACT_S=%ld,ACT_F=%ld", test_case_name, name_str.c_str(), row_range_exp.start_id, row_range_exp.stop_id, row_range->start_id, row_range->stop_id) ); return 1; } } return rc; }
static rc_t stats_data_update_group(stats_data_t *self, int64_t spot_id, uint32_t spot_len, uint32_t bio_spot_len, uint32_t cmp_spot_len, char const grp[], uint64_t grp_len) { if (grp_len == 0 || grp == NULL || grp[0] == '\0' || (grp_len == 7 && strncasecmp("default", grp, 7) == 0)) { group_stats_update(&self->deflt, spot_id, spot_len, bio_spot_len, cmp_spot_len); } else { group_stats_t *const fnd = stats_data_get_group(self, (unsigned)grp_len, grp); if (fnd == NULL) return RC(rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted); if (fnd - get_group(self, 0) < MAX_GROUP_COUNT) group_stats_update(fnd, spot_id, spot_len, bio_spot_len, cmp_spot_len); else { KDataBufferWhack(&self->group); KDataBufferWhack(&self->names); stats_data_init_funcs(self, false); (void)PLOGMSG(klogWarn, (klogWarn, "Too many spot groups ($(count)); dropping group stats", "count=%u", (unsigned)(self->count))); } } return 0; }
rc_t ProcessOneDo (ProcessOne * self) { static const char F[] = PLOG_2(PLOG_S(p),PLOG_S(t)); enum KPathType type; rc_t rc = 0; type = KDirectoryPathType (self->dir, self->path); switch (type) { case kptFile: rc = ProcessOneDoFile (self); break; /* case kptDir: */ /* break; */ /* case kptAlias|kptFile: */ /* break; */ /* case kptAlias|kptDir: */ /* break; */ default: PLOGMSG (klogInfo, "+ Skipping $(p) of type $(t)", F, self->path, typeToString(type)); break; } return rc; }
rc_t ProcessOneDoFile (ProcessOne * self) { rc_t rc = 0; KFile * mfile; PLOGMSG (klogInfo, "ProcessOneDoFile: $(F)", PLOG_S(F), self->path); rc = KFileMakeNewMD5Read (&mfile, self->file, self->md5, self->path); if (rc == 0) { const KFile * bfile; rc = KFileMakeBuf (&bfile, mfile, 64*1024); if (rc == 0) { /* add more here */ KFileRelease (bfile); } else { pLOGERR (klogErr, rc, "Failure to initiate buffer $(F)", PLOG_S(F), self->path); KFileRelease (mfile); } } else pLOGERR (klogErr, rc, "Failure to initiate MD5 summing $(F)", PLOG_S(F), self->path); return rc; }
rc_t BufferQPopBuffer (BufferQ * self, const Buffer ** buff, timeout_t * tm) { rc_t rc = 0; timeout_t t; void * p; LOGMSG (klogDebug10, "BufferQPopBuffer"); assert (self != NULL); assert (buff != NULL); if (tm == NULL) { LOGMSG (klogDebug10, "BufferQPopBuffer tm was NULL"); tm = &t; rc = TimeoutInit (tm, self->timeout); } if (rc == 0) { LOGMSG (klogDebug10, "BufferQPopBuffer call KQueuePop"); rc = KQueuePop (self->q, &p, tm); PLOGMSG (klogDebug10, "BufferQPopBuffer back from KQueuePop $(rc)", PLOG_U32(rc), rc); if (rc == 0) *buff = p; else { *buff = NULL; } } LOGMSG (klogDebug10, "leave BufferQPopBuffer"); return rc; }
rc_t load_consensus_src( con_ctx * sctx, KDirectory * hdf5_src ) { BaseCalls_cmn ConsensusTab; rc_t rc = 0; if ( sctx->lctx->check_src_obj ) rc = check_src_objects( hdf5_src, consensus_groups_to_check, consensus_tables_to_check, false ); if ( rc == 0 ) rc = open_BaseCalls_cmn( hdf5_src, &ConsensusTab, true, "PulseData/ConsensusBaseCalls", sctx->lctx->cache_content, true ); if ( rc == 0 ) { uint64_t total_bases = zmw_total( &ConsensusTab.zmw ); uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ]; KLogLevel tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); PLOGMSG( klogInfo, ( klogInfo, "loading consensus-table ( $(bases) bases / $(spots) spots ):", "bases=%lu,spots=%lu", total_bases, total_spots )); KLogLevelSet( tmp_lvl ); if ( !check_Consensus_totalcount( &ConsensusTab, total_bases ) ) rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid ); else rc = zmw_for_each( &ConsensusTab.zmw, &sctx->lctx->xml_progress, sctx->cursor, sctx->lctx->with_progress, sctx->col_idx, NULL, true, consensus_load_spot, &ConsensusTab ); close_BaseCalls_cmn( &ConsensusTab ); } return rc; }
static rc_t VProdResolveColExpr ( const VProdResolve *self, VProduction **out, VFormatdecl *fd, const SSymExpr *x, bool casting ) { rc_t rc; const SNameOverload *sname; const KSymbol *sym = x -> _sym; BSTree ordered; uint32_t i, count; SColumnBestFit buff [ 16 ], * nodes = buff; /* fail if "fd" has a format */ if ( fd -> fmt != 0 ) { PLOGMSG ( klogWarn, ( klogWarn, "illegal cast of column '$(name)'" , "name=%.*s" , ( int ) sym -> name . size , sym -> name . addr )); return 0; } /* allocate nodes for indexing columns */ sname = sym -> u . obj; count = VectorLength ( & sname -> items ); if ( count > sizeof buff / sizeof buff [ 0 ] ) { nodes = malloc ( sizeof * nodes * count ); if ( nodes == NULL ) return RC ( rcVDB, rcProduction, rcResolving, rcMemory, rcExhausted ); } /* insert columns into ordered tree */ BSTreeInit ( & ordered ); for ( i = VectorStart ( & sname -> items ), count += i; i < count; ++ i ) { /* get SColumn */ nodes [ i ] . scol = ( const void* ) VectorGet ( & sname -> items, i ); /* perform type cast and measure distance */ if ( casting ? VTypedeclCommonAncestor ( & nodes [ i ] . scol -> td, self -> schema, & fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) : VTypedeclToTypedecl ( & nodes [ i ] . scol -> td, self -> schema, & fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) ) { BSTreeInsert ( & ordered, & nodes [ i ] . n, order_column ); } } /* try to resolve each in order */ rc = VProdResolveBestColumn ( self, out, & ordered, x -> alt ); if ( nodes != buff ) free ( nodes ); return rc; }
static rc_t CCCopyDoDirectory (CCCopy * self) { rc_t rc = 0; PLOGMSG (klogInfo, "CCCopyDoDirectory $(d)", PLOG_S(d), self->path); rc = RC (rcExe, rcDirectory, rcCopying, rcParam, rcUnsupported); return rc; }
static rc_t consensus_loader( ld_context *lctx, const char * table_name, bool cache_content ) { uint32_t col_idx[ consensus_tab_count ]; rc_t rc = add_columns( lctx->cursor, consensus_tab_count, -1, col_idx, consensus_tab_names ); if ( rc == 0 ) { rc = VCursorOpen( lctx->cursor ); if ( rc != 0 ) LOGERR( klogErr, rc, "cannot open cursor on consensus-table" ); else { BaseCalls_cmn ConsensusTab; const INSDC_SRA_platform_id platform = SRA_PLATFORM_PACBIO_SMRT; rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_PLATFORM ], sizeof platform * 8, &platform, 0, 1 ); if ( rc != 0 ) LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for platform-column" ); else { const INSDC_SRA_read_filter filter = SRA_READ_FILTER_PASS; rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_READ_FILTER ], sizeof filter * 8, &filter, 0, 1 ); if ( rc != 0 ) LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for read-filter-column" ); } if ( rc == 0 ) rc = open_BaseCalls_cmn( lctx->hdf5_dir, &ConsensusTab, true, "PulseData/ConsensusBaseCalls", cache_content, true ); if ( rc == 0 ) { uint64_t total_bases = zmw_total( &ConsensusTab.zmw ); uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ]; KLogLevel tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); PLOGMSG( klogInfo, ( klogInfo, "loading consensus-table ( $(bases) bases / $(spots) spots ):", "bases=%lu,spots=%lu", total_bases, total_spots )); KLogLevelSet( tmp_lvl ); if ( check_Consensus_totalcount( &ConsensusTab, total_bases ) ) rc = zmw_for_each( &ConsensusTab.zmw, lctx, col_idx, NULL, true, consensus_load_spot, &ConsensusTab ); else rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid ); close_BaseCalls_cmn( &ConsensusTab ); } } } return rc; }
rc_t CopierDoOne (Copier * self) { rc_t rc = 0; const Buffer * b; LOGMSG (klogDebug10, "CopierDoOne"); rc = Quitting(); if (rc == 0) { LOGMSG (klogDebug10, "call BufferQPopBuffer"); rc = BufferQPopBuffer (self->q, &b, NULL); if (rc == 0) { size_t w; size_t z; LOGMSG (klogDebug10, "call BufferContentGetSize"); z = BufferContentGetSize (b); rc = KFileWrite (self->f, self->o, b, z, &w); self->o += w; if (w != z) rc = RC (rcExe, rcFile, rcWriting, rcTransfer, rcIncomplete); else rc = BufferRelease (b); } /* ow this is ugly! */ /* is the rc a "exhausted" on a timeout? */ else if ((GetRCObject(rc) == rcTimeout) && (GetRCState(rc) == rcExhausted)) { rc = 0; LOGMSG (klogDebug10, "CopierDoOne timeout"); /* if so is the queue also sealed? */ if (BufferQSealed (self->q) == true) { LOGMSG (klogDebug10, "CopierDoOne sealed"); /* if both then we are done and so signal */ rc = KFileRelease (self->f); PLOGMSG (klogDebug10, "CopierDoOne back from KFileRelease $(rc)",PLOG_U32(rc),rc); if (rc == 0) { self->f = NULL; rc = BufferQRelease (self->q); if (rc == 0) { self->q = NULL; rc = RC (rcExe, rcNoTarg, rcCopying, rcNoTarg, rcDone ); } } } } else LOGMSG (klogDebug10, "CopierDoOne pop failure"); } else LOGMSG (klogDebug10, "CopierDoOne: quitting"); return rc; }
rc_t CGWriterAlgn_Whack(const CGWriterAlgn* cself, bool commit, uint64_t* rows_1st, uint64_t* rows_2nd) { rc_t rc = 0; if( cself != NULL ) { CGWriterAlgn* self = (CGWriterAlgn*)cself; rc_t rc1 = TableWriterAlgn_Whack(cself->primary, commit, rows_1st); rc_t rc2 = TableWriterAlgn_Whack(cself->secondary, commit, rows_2nd); if( self->forced_pairs_cnt > 0 ) { PLOGMSG(klogInfo, (klogInfo, "$(forced_pairs_cnt) forced pairs to PRIMARY", "forced_pairs_cnt=%lu", self->forced_pairs_cnt)); } if( self->dropped_mates_cnt > 0 ) { PLOGMSG(klogInfo, (klogInfo, "$(dropped_mates_cnt) dropped duplicate mates in SECONDARY", "dropped_mates_cnt=%lu", self->dropped_mates_cnt)); } rc = rc1 ? rc1 : rc2; free(self); } return rc; }
static rc_t passes_load_loop( ld_context *lctx, VCursor * cursor, Passes_src *tab, uint32_t *col_idx ) { rc_t rc = 0; KLogLevel tmp_lvl; pass_block block; pl_progress *progress; uint64_t pos = 0; uint64_t total_passes = tab->AdapterHitBefore.extents[0]; pl_progress_make( &progress, total_passes ); rc = progress_chunk( &lctx->xml_progress, total_passes ); tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); PLOGMSG( klogInfo, ( klogInfo, "loading passes-table ( $(rows) rows ) :", "rows=%lu", total_passes )); KLogLevelSet( tmp_lvl ); while( pos < total_passes && rc == 0 ) { rc = pass_block_read_from_src( tab, total_passes, pos, &block ); if ( rc == 0 ) { uint32_t i; for ( i = 0; i < block.n_read && rc == 0; ++i ) { rc = Quitting(); if ( rc == 0 ) { rc = passes_load_pass( cursor, &block, i, col_idx ); if ( rc == 0 ) { rc = progress_step( lctx->xml_progress ); if ( lctx->with_progress ) pl_progress_increment( progress, 1 ); } } else LOGERR( klogErr, rc, "...loading passes interrupted" ); } pos += block.n_read; } } pl_progress_destroy( progress ); if ( rc == 0 ) { rc = VCursorCommit( cursor ); if ( rc != 0 ) LOGERR( klogErr, rc, "cannot commit cursor on PASSES-tab" ); } return rc; }
rc_t WriteFileMeta(SIndexObj* obj) { rc_t rc = 0; KMDataNode* nd = NULL; PLOGMSG(klogInfo, (klogInfo, "Meta $(f) on index $(i): file size $(s), buffer $(b)", PLOG_4(PLOG_S(f),PLOG_S(i),PLOG_U64(s),PLOG_U32(b)), obj->file, obj->index, obj->file_size, obj->buffer_sz)); if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Format")) == 0 ) { KMDataNode* opt = NULL; rc = KMDataNodeWriteCString(nd, obj->format); if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(nd, &opt, "Options")) == 0 ) { KMDataNode* ond = NULL; if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "accession")) == 0 ) { rc = KMDataNodeWriteCString(ond, g_accession); KMDataNodeRelease(ond); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "minSpotId")) == 0 ) { rc = KMDataNodeWriteB64(ond, &obj->minSpotId); KMDataNodeRelease(ond); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "maxSpotId")) == 0 ) { rc = KMDataNodeWriteB64(ond, &obj->maxSpotId); KMDataNodeRelease(ond); } KMDataNodeRelease(opt); } KMDataNodeRelease(nd); } if( rc == 0 && obj->file_size > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Size")) == 0 ) { rc = KMDataNodeWriteB64(nd, &obj->file_size); KMDataNodeRelease(nd); } if( rc == 0 && obj->buffer_sz > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Buffer")) == 0 ) { rc = KMDataNodeWriteB32(nd, &obj->buffer_sz); KMDataNodeRelease(nd); } if( rc == 0 && strlen(obj->index) > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Index")) == 0 ) { rc = KMDataNodeWriteCString(nd, obj->index); KMDataNodeRelease(nd); } if( rc == 0 && obj->file_size > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "md5")) == 0 ) { char x[5]; int i; for( i = 0; rc == 0 && i < sizeof(obj->md5_digest); i++ ) { int l = snprintf(x, 4, "%02x", obj->md5_digest[i]); rc = KMDataNodeAppend(nd, x, l); } KMDataNodeRelease(nd); } return rc; }
static rc_t XMLThread( const KThread *self, void *data ) { KDirectory *dir = NULL; PLOGMSG(klogInfo, (klogInfo, "XML sync thread started with $(s) sec", PLOG_U32(s), g_xml_sync)); do { rc_t rc = 0; KTime_t dt = 0; DEBUG_MSG(8, ("XML sync thread checking %s\n", g_xml_path)); if( (rc = KDirectoryNativeDir(&dir)) == 0 ) { rc = KDirectoryDate(dir, &dt, "%s", g_xml_path); ReleaseComplain(KDirectoryRelease, dir); } if( rc == 0 ) { if( dt != g_xml_mtime ) { const FSNode* new_root = NULL; PLOGMSG(klogInfo, (klogInfo, "File $(f) changed ($(m) <> $(d)), updating...", PLOG_3(PLOG_S(f),PLOG_I64(m),PLOG_I64(d)), g_xml_path, g_xml_mtime, dt)); if( XML_Open(g_xml_path, &new_root) == 0 ) { if( (rc = XMLLock(true)) == 0 ) { const FSNode* old_root = g_root; g_root = new_root; g_xml_mtime = dt; XMLUnlock(); FSNode_Release(old_root); PLOGMSG(klogInfo, (klogInfo, "Data from $(f) updated successfully", PLOG_S(f), g_xml_path)); } } } else { DEBUG_MSG(8, ("XML sync thread up-to-date %s\n", g_xml_path)); } } else { LOGERR(klogErr, rc, g_xml_path); } SRAList_PostRefresh(); sleep(g_xml_sync); } while( g_xml_sync > 0 ); LOGMSG(klogInfo, "XML sync thread ended"); return 0; }
void get_ref_var_object (KSearch::CVRefVariation& obj, size_t ref_pos, size_t del_len, char const* allele, size_t allele_len, ngs::ReferenceSequence const& ref_seq, std::string & ref_allele) { size_t var_len = allele_len; size_t chunk_size = 5000; // TODO: add the method Reference[Sequence].getChunkSize() to the API size_t chunk_no = ref_pos / chunk_size; size_t ref_pos_in_slice = ref_pos % chunk_size; size_t bases_start = chunk_no * chunk_size; size_t chunk_no_last = ref_seq.getLength() / chunk_size; bool cont = false; size_t chunk_no_start = chunk_no, chunk_no_end = chunk_no; // optimization: first look into the current chunk only (using ngs::StringRef) { ngs::StringRef ref_chunk = ref_seq.getReferenceChunk ( bases_start ); if ( ! check_ref_slice (ref_chunk.data() + ref_pos_in_slice, del_len) ) { PLOGMSG ( klogWarn, ( klogWarn, "The selected reference region [$(REFSLICE)] does not contain valid bases, skipping...", "REFSLICE=%.*s", (int)del_len, ref_chunk.data() + ref_pos_in_slice )); } cont = Common::find_variation_core_step ( obj, g_Params.alg, ref_chunk.data(), ref_chunk.size(), ref_pos_in_slice, allele, var_len, del_len, chunk_size, chunk_no_last, bases_start, chunk_no_start, chunk_no_end ); if ( !cont ) ref_allele.assign (ref_chunk.data() + obj.GetAlleleStartRelative(), obj.GetAlleleLenOnRef()); } // general case - expanding ref_slice to multiple chunks if ( cont ) { ngs::String ref_slice; while ( cont ) { ref_slice = ref_seq.getReferenceBases ( bases_start, (chunk_no_end - chunk_no_start + 1)*chunk_size ); cont = Common::find_variation_core_step ( obj, g_Params.alg, ref_slice.c_str(), ref_slice.size(), ref_pos_in_slice, allele, var_len, del_len, chunk_size, chunk_no_last, bases_start, chunk_no_start, chunk_no_end ); } ref_allele.assign (ref_slice.c_str() + obj.GetAlleleStartRelative(), obj.GetAlleleLenOnRef()); } }
static rc_t Run(const CmdLine* args) { rc_t rc = 0; Db db; SpotIterator it; assert(args); if (!SpotIteratorFileExists(args->file)) { rc = RC(rcExe, rcFile, rcOpening, rcFile, rcNotFound); PLOGERR(klogErr, (klogErr, rc, "Cannot find '$(path)'", "path=%s", args->file)); } else if (!SpotIteratorFileExists(args->table)) { rc = RC(rcExe, rcTable, rcOpening, rcTable, rcNotFound); PLOGERR(klogErr, (klogErr, rc, "Cannot find '$(path)'", "path=%s", args->table)); } { rc_t rc2 = DbInit(rc, args, &db); if (rc == 0) { rc = rc2; } } if (rc == 0) { rc = SpotIteratorInit(args->file, &db, &it); } if (rc == 0) { rc = Work(&db, &it); } if (rc == 0) { PLOGMSG(klogInfo, (klogInfo, "Success: redacted $(redacted) spots out of $(all)", "redacted=%d,all=%d", db.redactedSpots, db.nSpots)); } { rc_t rc2 = SpotIteratorDestroy(&it); if (rc == 0) { rc = rc2; } } { rc_t rc2 = DbDestroy(&db); if (rc == 0) { rc = rc2; } } return rc; }
static void seq_load_info( regions_stat * stat ) { KLogLevel tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); if ( stat->expands_a > 0 ) PLOGMSG( klogInfo, ( klogInfo, "adapter expanded: $(times) ( in $(spots) spots )", "times=%u,spots=%u", stat->expands_a, stat->expands_spots )); if ( stat->expands_i > 0 ) PLOGMSG( klogInfo, ( klogInfo, "insert expanded : $(times) ( in $(spots) spots )", "times=%u,spots=%u", stat->expands_i, stat->expands_spots )); if ( stat->inserts > 0 ) PLOGMSG( klogInfo, ( klogInfo, "regions inserted: $(times) ( in $(spots) spots )", "times=%u,spots=%u", stat->inserts, stat->inserts_spots )); if ( stat->end_gap > 0 ) PLOGMSG( klogInfo, ( klogInfo, "end-gap inserted: $(times)", "times=%u", stat->end_gap )); if ( stat->overlapps > 0 ) PLOGMSG( klogInfo, ( klogInfo, "overlapping rngs: $(times)", "times=%u", stat->overlapps )); if ( stat->removed > 0 ) PLOGMSG( klogInfo, ( klogInfo, "removed rgns : $(times)", "times=%u", stat->removed )); KLogLevelSet( tmp_lvl ); }
int expand_variations (int argc, char** argv) { int ret; try { KApp::CArgs args; args.MakeAndHandle (argc, argv, Options, countof (Options)); if (args.GetOptionCount (OPTION_ALG) == 1) { char const* alg = args.GetOptionValue ( OPTION_ALG, 0 ); if (!strcmp(alg, PARAM_ALG_SW)) g_Params.alg = ::refvarAlgSW; else if (!strcmp(alg, PARAM_ALG_RA)) g_Params.alg = ::refvarAlgRA; else { PLOGMSG ( klogErr, ( klogErr, "Error: Unknown algorithm specified: \"$(ALG)\"", "ALG=%s", alg )); return 3; } } ret = expand_variations_impl (); } catch ( ngs::ErrorMsg const& e ) { PLOGMSG ( klogErr, ( klogErr, "ngs::ErrorMsg: $(WHAT)", "WHAT=%s", e.what() )); ret = 3; } catch (...) { Utils::HandleException (); ret = 3; } return ret; }
rc_t CCCopyDo (CCCopy * self) { rc_t rc = 0; enum KPathType type; assert (self != NULL); assert (self->path != NULL); type = KDirectoryVPathType (self->in, self->path, NULL); switch (type & ~kptAlias) { case kptNotFound: rc = RC (rcExe, rcPath, rcAccessing, rcPath, rcNotFound); break; default: case kptBadPath: rc = RC (rcExe, rcPath, rcAccessing, rcPath, rcInvalid); break; case kptFile: rc = CCCopyDoFile (self); break; case kptDir: #if 0 rc = CCCopyDoDirectory (self); #else rc = PLOGMSG (klogInfo, "Ignoring directory $(p)",PLOG_S(p),self->path); #endif break; case kptCharDev: rc = PLOGMSG (klogInfo, "Ignoring kptCharDev $(p)",PLOG_S(p),self->path); break; case kptBlockDev: rc = PLOGMSG (klogInfo, "Ignoring kptBlockDev $(p)",PLOG_S(p),self->path); break; case kptFIFO: rc = PLOGMSG (klogInfo, "Ignoring kptFIFO $(p)",PLOG_S(p),self->path); break; } return rc; }
/* OpenUpdate * finish create operation */ static rc_t VDatabaseOpenUpdate ( VDatabase *self, const char *decl ) { /* open metadata */ rc_t rc = KDatabaseOpenMetadataUpdate ( self -> kdb, & self -> meta ); if ( rc == 0 ) { /* fetch stored schema */ rc = VDatabaseLoadSchema ( self ); if ( rc == 0 ) { /* fetch requested schema */ const SDatabase *sdb = self -> sdb; if ( decl != NULL && decl [ 0 ] != 0 ) { uint32_t type; const SNameOverload *name; sdb = ( self -> dad != NULL ) ? SDatabaseFind ( self -> dad -> sdb, self -> schema, & name, & type, decl, "VDatabaseOpenUpdate" ): VSchemaFind ( self -> schema, & name, & type, decl, "VDatabaseOpenUpdate", true ); if ( sdb != NULL && type != eDatabase ) { PLOGMSG ( klogWarn, ( klogWarn, "expression '$(expr)' is not a database", "expr=%s", decl )); sdb = NULL; } } /* error if the two definitions differ */ if ( sdb != NULL && self -> sdb != NULL && sdb != self -> sdb ) rc = RC ( rcVDB, rcDatabase, rcOpening, rcSchema, rcIncorrect ); else if ( sdb == NULL && self -> sdb == NULL ) rc = RC ( rcVDB, rcDatabase, rcOpening, rcSchema, rcNotFound ); else { if ( sdb != NULL ) self -> sdb = sdb; /* write schema to metadata */ rc = VDatabaseStoreSchema ( self ); if ( rc == 0 ) return 0; } } } DBGMSG(DBG_VDB, DBG_FLAG(DBG_VDB_VDB), ("VDatabaseOpenUpdate = %d\n", rc)); return rc; }
/* AddRef * Release */ LIB_EXPORT rc_t CC KMemBankAddRef(const KMemBank *cself) { KMemBank *const self = (KMemBank *)cself; if (self != NULL) { int const rslt = KRefcountAdd(&self->refcount, "KMemBank"); assert(rslt == krefOkay); if (rslt != krefOkay) { PLOGMSG(klogFatal, (klogFatal, "LOGIC ERROR - " __FILE__ ":$(line)", "line=%i", __LINE__)); abort(); return RC(rcFS, rcMemory, rcAttaching, rcConstraint, rcViolated); } } return 0; }
static rc_t VProdResolveProdExpr ( const VProdResolve *self, VProduction **out, const KSymbol *sym ) { const SProduction *sprod = sym -> u . obj; if ( ! sprod -> trigger ) return VProdResolveSProduction ( self, out, sprod ); PLOGMSG ( klogWarn, ( klogWarn, "trigger production '$(trig)' used in expression" , "trig=%.*s" , ( int ) sym -> name . size , sym -> name . addr )); return 0; }
static rc_t remove_path( KDirectory * dir, const char * path, bool quiet ) { rc_t rc; if ( !quiet ) { PLOGMSG( klogInfo, ( klogInfo, "removing '$(path)'", "path=%s", path )); } rc = KDirectoryRemove ( dir, true, path ); if ( rc != 0 ) { LOGERR( klogErr, rc, "remove_path:KDirectoryRemove() failed" ); } return rc; }
/* ParamExpr * resolve a simple parameter by name */ LIB_EXPORT rc_t CC VProdResolveParamExpr ( const VProdResolve *self, VProduction **out, const KSymbol *sym ) { const SProduction *sprod = sym -> u . obj; VProduction *vprod = VCursorCacheGet ( self -> cache, & sprod -> cid ); if ( vprod != NULL ) { * out = vprod; return 0; } PLOGMSG ( klogWarn, ( klogWarn, "unknown parameter '$(param)' used in expression" , "param=%.*s" , ( int ) sprod -> name -> name . size , sprod -> name -> name . addr )); return 0; }
static rc_t VProdResolvePhysExpr ( const VProdResolve *self, VProduction **out, const KSymbol *sym ) { if ( self -> chain == chainEncoding ) { assert ( self -> curs -> read_only == false ); PLOGMSG ( klogWarn, ( klogWarn, "illegal access of physical column '$(name)'" , "name=%.*s" , ( int ) sym -> name . size , sym -> name . addr )); return 0; } return VProdResolveSPhysMember ( self, out, sym -> u . obj ); }
static int string_buffer_add(KDataBuffer *const self, unsigned const len, char const name[/* len */]) { size_t const rslt = self->elem_count; size_t const newsize = rslt + len; if (KDataBufferResize(self, newsize) != 0) return -1; #if 1 PLOGMSG(klogInfo, (klogInfo, "New spot group '$(NAME)'", "NAME=%.*s", (int)len, name)); #endif memcpy(((char *)self->base) + rslt, name, len); return (int)rslt; }
void seq_report_totals( ld_context *lctx ) { const char* accession; KLogLevel tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); accession = strrchr( lctx->dst_path, '/' ); if( accession == NULL ) accession = lctx->dst_path; else accession++; PLOGMSG( klogInfo, ( klogInfo, "loaded", "severity=total,status=success,accession=%s,spot_count=%lu,base_count=%lu,bad_spots=0", accession, lctx->total_seq_spots, lctx->total_seq_bases )); KLogLevelSet( tmp_lvl ); }