static void seq_load_info( regions_stat * stat ) { KLogLevel tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); if ( stat->expands_a > 0 ) PLOGMSG( klogInfo, ( klogInfo, "adapter expanded: $(times) ( in $(spots) spots )", "times=%u,spots=%u", stat->expands_a, stat->expands_spots )); if ( stat->expands_i > 0 ) PLOGMSG( klogInfo, ( klogInfo, "insert expanded : $(times) ( in $(spots) spots )", "times=%u,spots=%u", stat->expands_i, stat->expands_spots )); if ( stat->inserts > 0 ) PLOGMSG( klogInfo, ( klogInfo, "regions inserted: $(times) ( in $(spots) spots )", "times=%u,spots=%u", stat->inserts, stat->inserts_spots )); if ( stat->end_gap > 0 ) PLOGMSG( klogInfo, ( klogInfo, "end-gap inserted: $(times)", "times=%u", stat->end_gap )); if ( stat->overlapps > 0 ) PLOGMSG( klogInfo, ( klogInfo, "overlapping rngs: $(times)", "times=%u", stat->overlapps )); if ( stat->removed > 0 ) PLOGMSG( klogInfo, ( klogInfo, "removed rgns : $(times)", "times=%u", stat->removed )); KLogLevelSet( tmp_lvl ); }
rc_t load_consensus_src( con_ctx * sctx, KDirectory * hdf5_src ) { BaseCalls_cmn ConsensusTab; rc_t rc = 0; if ( sctx->lctx->check_src_obj ) rc = check_src_objects( hdf5_src, consensus_groups_to_check, consensus_tables_to_check, false ); if ( rc == 0 ) rc = open_BaseCalls_cmn( hdf5_src, &ConsensusTab, true, "PulseData/ConsensusBaseCalls", sctx->lctx->cache_content, true ); if ( rc == 0 ) { uint64_t total_bases = zmw_total( &ConsensusTab.zmw ); uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ]; KLogLevel tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); PLOGMSG( klogInfo, ( klogInfo, "loading consensus-table ( $(bases) bases / $(spots) spots ):", "bases=%lu,spots=%lu", total_bases, total_spots )); KLogLevelSet( tmp_lvl ); if ( !check_Consensus_totalcount( &ConsensusTab, total_bases ) ) rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid ); else rc = zmw_for_each( &ConsensusTab.zmw, &sctx->lctx->xml_progress, sctx->cursor, sctx->lctx->with_progress, sctx->col_idx, NULL, true, consensus_load_spot, &ConsensusTab ); close_BaseCalls_cmn( &ConsensusTab ); } return rc; }
static rc_t consensus_loader( ld_context *lctx, const char * table_name, bool cache_content ) { uint32_t col_idx[ consensus_tab_count ]; rc_t rc = add_columns( lctx->cursor, consensus_tab_count, -1, col_idx, consensus_tab_names ); if ( rc == 0 ) { rc = VCursorOpen( lctx->cursor ); if ( rc != 0 ) LOGERR( klogErr, rc, "cannot open cursor on consensus-table" ); else { BaseCalls_cmn ConsensusTab; const INSDC_SRA_platform_id platform = SRA_PLATFORM_PACBIO_SMRT; rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_PLATFORM ], sizeof platform * 8, &platform, 0, 1 ); if ( rc != 0 ) LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for platform-column" ); else { const INSDC_SRA_read_filter filter = SRA_READ_FILTER_PASS; rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_READ_FILTER ], sizeof filter * 8, &filter, 0, 1 ); if ( rc != 0 ) LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for read-filter-column" ); } if ( rc == 0 ) rc = open_BaseCalls_cmn( lctx->hdf5_dir, &ConsensusTab, true, "PulseData/ConsensusBaseCalls", cache_content, true ); if ( rc == 0 ) { uint64_t total_bases = zmw_total( &ConsensusTab.zmw ); uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ]; KLogLevel tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); PLOGMSG( klogInfo, ( klogInfo, "loading consensus-table ( $(bases) bases / $(spots) spots ):", "bases=%lu,spots=%lu", total_bases, total_spots )); KLogLevelSet( tmp_lvl ); if ( check_Consensus_totalcount( &ConsensusTab, total_bases ) ) rc = zmw_for_each( &ConsensusTab.zmw, lctx, col_idx, NULL, true, consensus_load_spot, &ConsensusTab ); else rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid ); close_BaseCalls_cmn( &ConsensusTab ); } } } return rc; }
static rc_t passes_load_loop( ld_context *lctx, VCursor * cursor, Passes_src *tab, uint32_t *col_idx ) { rc_t rc = 0; KLogLevel tmp_lvl; pass_block block; pl_progress *progress; uint64_t pos = 0; uint64_t total_passes = tab->AdapterHitBefore.extents[0]; pl_progress_make( &progress, total_passes ); rc = progress_chunk( &lctx->xml_progress, total_passes ); tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); PLOGMSG( klogInfo, ( klogInfo, "loading passes-table ( $(rows) rows ) :", "rows=%lu", total_passes )); KLogLevelSet( tmp_lvl ); while( pos < total_passes && rc == 0 ) { rc = pass_block_read_from_src( tab, total_passes, pos, &block ); if ( rc == 0 ) { uint32_t i; for ( i = 0; i < block.n_read && rc == 0; ++i ) { rc = Quitting(); if ( rc == 0 ) { rc = passes_load_pass( cursor, &block, i, col_idx ); if ( rc == 0 ) { rc = progress_step( lctx->xml_progress ); if ( lctx->with_progress ) pl_progress_increment( progress, 1 ); } } else LOGERR( klogErr, rc, "...loading passes interrupted" ); } pos += block.n_read; } } pl_progress_destroy( progress ); if ( rc == 0 ) { rc = VCursorCommit( cursor ); if ( rc != 0 ) LOGERR( klogErr, rc, "cannot commit cursor on PASSES-tab" ); } return rc; }
rc_t CC KMain (int argc, char * argv[]) { rc_t rc = 0; KDirectory* wd; KLogLevelSet(klogInfo); LogMsg ( klogInfo, "KeyringServer: starting"); rc = KDirectoryNativeDir (&wd); if (rc == 0) { KFile* lockedFile; const char* dataDir; char lockFileName[MAX_PATH]; if (argc < 2 || argv[1] == NULL) dataDir = KeyRingDefaultDataDir; else dataDir = argv[1]; rc = string_printf(lockFileName, sizeof(lockFileName)-1, NULL, "%s/keyring_lock", dataDir); if (rc == 0) { rc = KDirectoryCreateExclusiveAccessFile(wd, &lockedFile, true, 0600, kcmOpen, "%s", lockFileName); if (rc == 0) { KNSManager* mgr; rc = KNSManagerMake(&mgr); if (rc == 0) { rc = string_printf(keyRingFileName, sizeof(keyRingFileName)-1, NULL, "%s/keyring", dataDir); if (rc == 0) rc = Server(mgr); KNSManagerRelease(mgr); } else LogErr(klogErr, rc, "KeyringServer: KNSManagerMake failed"); KFileRelease(lockedFile); LogMsg ( klogInfo, "KeyringServer: removing lock file."); KDirectoryRemove(wd, true, "%s", lockFileName); } else { /*TODO: check for stale lock file*/ LogMsg ( klogInfo, "KeyringServer: another instance appears to be running."); rc = 0; } } else LogErr ( klogErr, rc, "KeyringServer: failed to build the lock file name" ); KDirectoryRelease(wd); } else LogErr(klogErr, rc, "KeyringServer: KDirectoryNativeDir failed"); LogMsg ( klogInfo, "KeyringServer: finishing"); return rc; }
void seq_report_totals( ld_context *lctx ) { const char* accession; KLogLevel tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); accession = strrchr( lctx->dst_path, '/' ); if( accession == NULL ) accession = lctx->dst_path; else accession++; PLOGMSG( klogInfo, ( klogInfo, "loaded", "severity=total,status=success,accession=%s,spot_count=%lu,base_count=%lu,bad_spots=0", accession, lctx->total_seq_spots, lctx->total_seq_bases )); KLogLevelSet( tmp_lvl ); }
void CVDBMgr::x_Init(void) { if ( rc_t rc = VDBManagerMakeRead(x_InitPtr(), 0) ) { *x_InitPtr() = 0; NCBI_THROW2(CSraException, eInitFailed, "Cannot open VDBManager", rc); } uint32_t sdk_ver; if ( rc_t rc = VDBManagerVersion(*this, &sdk_ver) ) { NCBI_THROW2(CSraException, eInitFailed, "Cannot get VDBManager version", rc); } CKNSManager kns_mgr(CVFSManager(*this)); CNcbiOstrstream str; CNcbiApplication* app = CNcbiApplication::Instance(); if ( app ) { str << app->GetAppName() << ": " << app->GetVersion().Print() << "; "; } #if NCBI_PACKAGE str << "Package: " << NCBI_PACKAGE_NAME << ' ' << NCBI_PACKAGE_VERSION << "; "; #endif str << "C++ "; #ifdef NCBI_PRODUCTION_VER str << NCBI_PRODUCTION_VER << "/"; #endif #ifdef NCBI_DEVELOPMENT_VER str << NCBI_DEVELOPMENT_VER; #endif string prefix = CNcbiOstrstreamToString(str); KNSManagerSetUserAgent(kns_mgr, "%s; SRA Toolkit %V", prefix.c_str(), sdk_ver); // redirect VDB log to C++ Toolkit if ( s_GetDiagHandler() ) { KLogInit(); KLogLevelSet(klogDebug); KLogLibHandlerSet(VDBLogWriter, 0); } if ( app ) { string host = app->GetConfig().GetString("CONN", "HTTP_PROXY_HOST", kEmptyStr); int port = app->GetConfig().GetInt("CONN", "HTTP_PROXY_PORT", 0); if ( !host.empty() && port != 0 ) { if ( rc_t rc = KNSManagerSetHTTPProxyPath(kns_mgr, "%s:%d", host.c_str(), port) ) { NCBI_THROW2(CSraException, eInitFailed, "Cannot set KNSManager proxy parameters", rc); } KNSManagerSetHTTPProxyEnabled(kns_mgr, true); } } }
rc_t CC KMain ( int argc, char *argv [] ) { rc_t rc = 0; Args * args; rc = ArgsMakeAndHandle (&args, argc, argv, 0); if (rc == 0) { do { uint32_t pcount; rc = ArgsParamCount (args, &pcount); if (rc) break; if (pcount == 0) { MiniUsage(args); rc = RC(rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid); } else { const char * pc; rc_t exp; uint32_t ix; /* over rule any set log level */ rc = KLogLevelSet (klogInfo); for (ix = 0; ix < pcount; ++ix) { rc = ArgsParamValue (args, ix, (const void **)&pc); if (rc) break; exp = AsciiToU32 (pc, NULL, NULL); rc = LOGERR (klogInfo, exp, pc); } } } while (0); ArgsWhack (args); } return rc; }
rc_t CC KMain ( int argc, char *argv [] ) { XMLLogger const *xlogger = NULL; Args * args; rc_t rc; bool any_failed = false; CheckCorruptConfig config = { -1.0, SA_TABLE_LOOKUP_LIMIT, -1.0, SEQ_TABLE_LOOKUP_LIMIT, PA_LONGER_SA_LIMIT, 0 }; KLogLevelSet(klogInfo); rc = ArgsMakeAndHandle (&args, argc, argv, 2, Options, sizeof (Options) / sizeof (Options[0]), XMLLogger_Args, XMLLogger_ArgsQty); if (rc) LOGERR (klogInt, rc, "failed to parse command line parameters"); else { rc = XMLLogger_Make(&xlogger, NULL, args); if (rc) LOGERR (klogInt, rc, "failed to make xml logger"); else { rc = parseArgs ( args, &config ); if (rc == 0) { uint32_t pcount; rc = ArgsParamCount ( args, &pcount ); if (rc) LOGERR (klogInt, rc, "ArgsParamCount() failed"); else { if ( pcount == 0 ) LOGMSG (klogErr, "no accessions were passed in"); else { for ( uint32_t i = 0; i < pcount; ++i ) { const char * accession; rc = ArgsParamValue ( args, i, (const void **)&accession ); if (rc) { PLOGERR (klogInt, (klogInt, rc, "failed to get $(PARAM_I) accession from command line", "PARAM_I=%d", i)); any_failed = true; } else { if (!checkAccession ( accession, &config )) any_failed = true; } } if (!any_failed) LOGMSG (klogInfo, "All accessions are good!"); } } } XMLLogger_Release(xlogger); } ArgsWhack ( args ); } return rc != 0 || any_failed ? 1 : 0; }
rc_t load_seq_src( seq_ctx * sctx, KDirectory * hdf5_src ) { rc_t rc = 0; if ( sctx->lctx->check_src_obj ) rc = check_src_objects( hdf5_src, seq_groups_to_check, seq_tables_to_check, true ); if ( rc == 0 && !sctx->src_open ) rc = open_BaseCalls( hdf5_src, &sctx->BaseCallsTab, "PulseData/BaseCalls", sctx->lctx->cache_content, &sctx->rgn_present ); if ( rc == 0 ) { /* calculates the total number of bases, according to the zmw-table */ uint64_t total_bases = zmw_total( &sctx->BaseCallsTab.cmn.zmw ); /* calculates the total number of spots, according to the zmw-table */ uint64_t total_spots = sctx->BaseCallsTab.cmn.zmw.NumEvent.extents[ 0 ]; KLogLevel tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); PLOGMSG( klogInfo, ( klogInfo, "loading sequence-table ( $(bases) bases / $(spots) spots ):", "bases=%lu,spots=%lu", total_bases, total_spots )); KLogLevelSet( tmp_lvl ); /* checks that all tables, which are loaded do have the correct number of values (the number that the zmw-table requests) */ if ( !check_BaseCall_totalcount( &sctx->BaseCallsTab, total_bases ) ) rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid ); else { region_type_mapping mapping; if ( sctx->rgn_present ) { const KNamelist *region_types; /* read the meta-data-entry "RegionTypes" of the hdf5-regions-table into a KNamelist */ rc = KArrayFileGetMeta ( sctx->BaseCallsTab.rgn.hdf5_regions.af, "RegionTypes", ®ion_types ); if ( rc != 0 ) { LOGERR( klogErr, rc, "cannot read Regions.RegionTypes" ); } else { /* extract the region-type-mapping out of the read KNamelist */ rc = rgn_extract_type_mappings( region_types, &mapping, false ); KNamelistRelease ( region_types ); if ( rc != 0 ) { LOGERR( klogErr, rc, "cannot map regions-types" ); } } } if ( rc == 0 ) { region_type_mapping *mapping_ptr = NULL; if ( sctx->rgn_present ) mapping_ptr = &mapping; /* call for every spot the function >seq_load_spot< */ rc = zmw_for_each( &sctx->BaseCallsTab.cmn.zmw, &sctx->lctx->xml_progress, sctx->cursor, sctx->lctx->with_progress, sctx->col_idx, mapping_ptr, false, seq_load_spot, &sctx->BaseCallsTab ); } if ( sctx->rgn_present ) seq_load_info( &sctx->BaseCallsTab.rgn.stat ); sctx->lctx->total_seq_bases += total_bases; sctx->lctx->total_seq_spots += total_spots; } close_BaseCalls( &sctx->BaseCallsTab ); sctx->src_open = false; } return rc; }
static rc_t seq_loader( ld_context *lctx, KDirectory * hdf5_src, VCursor * cursor, const char * table_name ) { BaseCalls BaseCallsTab; bool rgn_present; /* opens all hdf5-tables, which are needed to load the sequence-table */ rc_t rc = open_BaseCalls( hdf5_src, &BaseCallsTab, "PulseData/BaseCalls", lctx->cache_content, &rgn_present ); if ( rc == 0 ) { /* calculates the total number of bases, according to the zmw-table */ uint64_t total_bases = zmw_total( &BaseCallsTab.cmn.zmw ); /* calculates the total number of spots, according to the zmw-table */ uint64_t total_spots = BaseCallsTab.cmn.zmw.NumEvent.extents[ 0 ]; KLogLevel tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); PLOGMSG( klogInfo, ( klogInfo, "loading sequence-table ( $(bases) bases / $(spots) spots ):", "bases=%lu,spots=%lu", total_bases, total_spots )); KLogLevelSet( tmp_lvl ); /* checks that all tables, which are loaded do have the correct number of values (the number that the zmw-table requests) */ if ( check_BaseCall_totalcount( &BaseCallsTab, total_bases ) ) { region_type_mapping mapping; if ( rgn_present ) { const KNamelist *region_types; /* read the meta-data-entry "RegionTypes" of the hdf5-regions-table into a KNamelist */ rc = KArrayFileGetMeta ( BaseCallsTab.rgn.hdf5_regions.af, "RegionTypes", ®ion_types ); if ( rc != 0 ) { LOGERR( klogErr, rc, "cannot read Regions.RegionTypes" ); } else { /* extract the region-type-mapping out of the read KNamelist */ rc = rgn_extract_type_mappings( region_types, &mapping, false ); KNamelistRelease ( region_types ); if ( rc != 0 ) { LOGERR( klogErr, rc, "cannot map regions-types" ); } } } if ( rc == 0 ) { /* holds the vdb-column-index for every column we write */ uint32_t col_idx[ seq_tab_count ]; int32_t to_exclude; /* depending on the bit-size of the PulseIndex-table in HDF5 exclude the opposite column from VDB */ if ( BaseCallsTab.PulseIndex.element_bits == PULSE_INDEX_BITSIZE_16 ) to_exclude = seq_tab_PULSE_INDEX_32; else to_exclude = seq_tab_PULSE_INDEX_16; /* add all columns to the vdb-cursor */ rc = add_columns( cursor, seq_tab_count, to_exclude, col_idx, seq_tab_names ); if ( rc == 0 ) { rc = VCursorOpen( cursor ); if ( rc != 0 ) LOGERR( klogErr, rc, "cannot open cursor on seq-table" ); else { const uint8_t platform = SRA_PLATFORM_PACBIO_SMRT; rc = VCursorDefault ( cursor, col_idx[ seq_tab_PLATFORM ], 8, &platform, 0, 1 ); if ( rc != 0 ) LOGERR( klogErr, rc, "cannot set cursor-default on seq-table for platform-column" ); else { region_type_mapping *mapping_ptr = NULL; if ( rgn_present ) { mapping_ptr = &mapping; } /* call for every spot the function >seq_load_spot< */ rc = zmw_for_each( &BaseCallsTab.cmn.zmw, &lctx->xml_progress, cursor, lctx->with_progress, col_idx, mapping_ptr, false, seq_load_spot, &BaseCallsTab ); } } } } if ( rgn_present ) { seq_load_info( &BaseCallsTab.rgn.stat ); if ( mapping.count_of_unknown_rgn_types > 0 ) { KLogLevel tmp_lvl = KLogLevelGet(); KLogLevelSet( klogInfo ); PLOGMSG( klogInfo, ( klogInfo, "$(times) x unknown region types encountered", "times=%i", mapping.count_of_unknown_rgn_types ) ); KLogLevelSet( tmp_lvl ); } } lctx->total_seq_bases += total_bases; lctx->total_seq_spots += total_spots; } else rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid ); close_BaseCalls( &BaseCallsTab ); } return rc; }