static void seq_load_info( regions_stat * stat )
{
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
if ( stat->expands_a > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"adapter expanded: $(times) ( in $(spots) spots )",
"times=%u,spots=%u",
stat->expands_a, stat->expands_spots ));
if ( stat->expands_i > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"insert expanded : $(times) ( in $(spots) spots )",
"times=%u,spots=%u",
stat->expands_i, stat->expands_spots ));
if ( stat->inserts > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"regions inserted: $(times) ( in $(spots) spots )",
"times=%u,spots=%u",
stat->inserts, stat->inserts_spots ));
if ( stat->end_gap > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"end-gap inserted: $(times)",
"times=%u", stat->end_gap ));
if ( stat->overlapps > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"overlapping rngs: $(times)",
"times=%u", stat->overlapps ));
if ( stat->removed > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"removed rgns : $(times)",
"times=%u", stat->removed ));
KLogLevelSet( tmp_lvl );
}
rc_t load_consensus_src( con_ctx * sctx, KDirectory * hdf5_src )
{
BaseCalls_cmn ConsensusTab;
rc_t rc = 0;
if ( sctx->lctx->check_src_obj )
rc = check_src_objects( hdf5_src, consensus_groups_to_check,
consensus_tables_to_check, false );
if ( rc == 0 )
rc = open_BaseCalls_cmn( hdf5_src, &ConsensusTab, true,
"PulseData/ConsensusBaseCalls", sctx->lctx->cache_content, true );
if ( rc == 0 )
{
uint64_t total_bases = zmw_total( &ConsensusTab.zmw );
uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ];
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading consensus-table ( $(bases) bases / $(spots) spots ):",
"bases=%lu,spots=%lu", total_bases, total_spots ));
KLogLevelSet( tmp_lvl );
if ( !check_Consensus_totalcount( &ConsensusTab, total_bases ) )
rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
else
rc = zmw_for_each( &ConsensusTab.zmw, &sctx->lctx->xml_progress, sctx->cursor,
sctx->lctx->with_progress, sctx->col_idx, NULL,
true, consensus_load_spot, &ConsensusTab );
close_BaseCalls_cmn( &ConsensusTab );
}
return rc;
}
static rc_t consensus_loader( ld_context *lctx, const char * table_name,
bool cache_content )
{
uint32_t col_idx[ consensus_tab_count ];
rc_t rc = add_columns( lctx->cursor, consensus_tab_count, -1,
col_idx, consensus_tab_names );
if ( rc == 0 )
{
rc = VCursorOpen( lctx->cursor );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot open cursor on consensus-table" );
else
{
BaseCalls_cmn ConsensusTab;
const INSDC_SRA_platform_id platform = SRA_PLATFORM_PACBIO_SMRT;
rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_PLATFORM ],
sizeof platform * 8, &platform, 0, 1 );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for platform-column" );
else
{
const INSDC_SRA_read_filter filter = SRA_READ_FILTER_PASS;
rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_READ_FILTER ],
sizeof filter * 8, &filter, 0, 1 );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for read-filter-column" );
}
if ( rc == 0 )
rc = open_BaseCalls_cmn( lctx->hdf5_dir, &ConsensusTab, true,
"PulseData/ConsensusBaseCalls", cache_content, true );
if ( rc == 0 )
{
uint64_t total_bases = zmw_total( &ConsensusTab.zmw );
uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ];
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading consensus-table ( $(bases) bases / $(spots) spots ):",
"bases=%lu,spots=%lu", total_bases, total_spots ));
KLogLevelSet( tmp_lvl );
if ( check_Consensus_totalcount( &ConsensusTab, total_bases ) )
rc = zmw_for_each( &ConsensusTab.zmw, lctx, col_idx, NULL,
true, consensus_load_spot, &ConsensusTab );
else
rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
close_BaseCalls_cmn( &ConsensusTab );
}
}
}
return rc;
}
static rc_t passes_load_loop( ld_context *lctx, VCursor * cursor, Passes_src *tab,
uint32_t *col_idx )
{
rc_t rc = 0;
KLogLevel tmp_lvl;
pass_block block;
pl_progress *progress;
uint64_t pos = 0;
uint64_t total_passes = tab->AdapterHitBefore.extents[0];
pl_progress_make( &progress, total_passes );
rc = progress_chunk( &lctx->xml_progress, total_passes );
tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading passes-table ( $(rows) rows ) :",
"rows=%lu", total_passes ));
KLogLevelSet( tmp_lvl );
while( pos < total_passes && rc == 0 )
{
rc = pass_block_read_from_src( tab, total_passes, pos, &block );
if ( rc == 0 )
{
uint32_t i;
for ( i = 0; i < block.n_read && rc == 0; ++i )
{
rc = Quitting();
if ( rc == 0 )
{
rc = passes_load_pass( cursor, &block, i, col_idx );
if ( rc == 0 )
{
rc = progress_step( lctx->xml_progress );
if ( lctx->with_progress )
pl_progress_increment( progress, 1 );
}
}
else
LOGERR( klogErr, rc, "...loading passes interrupted" );
}
pos += block.n_read;
}
}
pl_progress_destroy( progress );
if ( rc == 0 )
{
rc = VCursorCommit( cursor );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot commit cursor on PASSES-tab" );
}
return rc;
}
rc_t CC KMain (int argc, char * argv[])
{
rc_t rc = 0;
KDirectory* wd;
KLogLevelSet(klogInfo);
LogMsg ( klogInfo, "KeyringServer: starting");
rc = KDirectoryNativeDir (&wd);
if (rc == 0)
{
KFile* lockedFile;
const char* dataDir;
char lockFileName[MAX_PATH];
if (argc < 2 || argv[1] == NULL)
dataDir = KeyRingDefaultDataDir;
else
dataDir = argv[1];
rc = string_printf(lockFileName, sizeof(lockFileName)-1, NULL, "%s/keyring_lock", dataDir);
if (rc == 0)
{
rc = KDirectoryCreateExclusiveAccessFile(wd, &lockedFile, true, 0600, kcmOpen, "%s", lockFileName);
if (rc == 0)
{
KNSManager* mgr;
rc = KNSManagerMake(&mgr);
if (rc == 0)
{
rc = string_printf(keyRingFileName, sizeof(keyRingFileName)-1, NULL, "%s/keyring", dataDir);
if (rc == 0)
rc = Server(mgr);
KNSManagerRelease(mgr);
}
else
LogErr(klogErr, rc, "KeyringServer: KNSManagerMake failed");
KFileRelease(lockedFile);
LogMsg ( klogInfo, "KeyringServer: removing lock file.");
KDirectoryRemove(wd, true, "%s", lockFileName);
}
else
{ /*TODO: check for stale lock file*/
LogMsg ( klogInfo, "KeyringServer: another instance appears to be running.");
rc = 0;
}
}
else
LogErr ( klogErr, rc, "KeyringServer: failed to build the lock file name" );
KDirectoryRelease(wd);
}
else
LogErr(klogErr, rc, "KeyringServer: KDirectoryNativeDir failed");
LogMsg ( klogInfo, "KeyringServer: finishing");
return rc;
}
void seq_report_totals( ld_context *lctx )
{
const char* accession;
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
accession = strrchr( lctx->dst_path, '/' );
if( accession == NULL )
accession = lctx->dst_path;
else
accession++;
PLOGMSG( klogInfo, ( klogInfo, "loaded",
"severity=total,status=success,accession=%s,spot_count=%lu,base_count=%lu,bad_spots=0",
accession, lctx->total_seq_spots, lctx->total_seq_bases ));
KLogLevelSet( tmp_lvl );
}
void CVDBMgr::x_Init(void)
{
if ( rc_t rc = VDBManagerMakeRead(x_InitPtr(), 0) ) {
*x_InitPtr() = 0;
NCBI_THROW2(CSraException, eInitFailed,
"Cannot open VDBManager", rc);
}
uint32_t sdk_ver;
if ( rc_t rc = VDBManagerVersion(*this, &sdk_ver) ) {
NCBI_THROW2(CSraException, eInitFailed,
"Cannot get VDBManager version", rc);
}
CKNSManager kns_mgr(CVFSManager(*this));
CNcbiOstrstream str;
CNcbiApplication* app = CNcbiApplication::Instance();
if ( app ) {
str << app->GetAppName() << ": " << app->GetVersion().Print() << "; ";
}
#if NCBI_PACKAGE
str << "Package: " << NCBI_PACKAGE_NAME << ' ' <<
NCBI_PACKAGE_VERSION << "; ";
#endif
str << "C++ ";
#ifdef NCBI_PRODUCTION_VER
str << NCBI_PRODUCTION_VER << "/";
#endif
#ifdef NCBI_DEVELOPMENT_VER
str << NCBI_DEVELOPMENT_VER;
#endif
string prefix = CNcbiOstrstreamToString(str);
KNSManagerSetUserAgent(kns_mgr, "%s; SRA Toolkit %V",
prefix.c_str(),
sdk_ver);
// redirect VDB log to C++ Toolkit
if ( s_GetDiagHandler() ) {
KLogInit();
KLogLevelSet(klogDebug);
KLogLibHandlerSet(VDBLogWriter, 0);
}
if ( app ) {
string host = app->GetConfig().GetString("CONN", "HTTP_PROXY_HOST", kEmptyStr);
int port = app->GetConfig().GetInt("CONN", "HTTP_PROXY_PORT", 0);
if ( !host.empty() && port != 0 ) {
if ( rc_t rc = KNSManagerSetHTTPProxyPath(kns_mgr, "%s:%d", host.c_str(), port) ) {
NCBI_THROW2(CSraException, eInitFailed,
"Cannot set KNSManager proxy parameters", rc);
}
KNSManagerSetHTTPProxyEnabled(kns_mgr, true);
}
}
}
rc_t CC KMain ( int argc, char *argv [] )
{
rc_t rc = 0;
Args * args;
rc = ArgsMakeAndHandle (&args, argc, argv, 0);
if (rc == 0)
{
do
{
uint32_t pcount;
rc = ArgsParamCount (args, &pcount);
if (rc)
break;
if (pcount == 0) {
MiniUsage(args);
rc = RC(rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid);
}
else
{
const char * pc;
rc_t exp;
uint32_t ix;
/* over rule any set log level */
rc = KLogLevelSet (klogInfo);
for (ix = 0; ix < pcount; ++ix)
{
rc = ArgsParamValue (args, ix, (const void **)&pc);
if (rc)
break;
exp = AsciiToU32 (pc, NULL, NULL);
rc = LOGERR (klogInfo, exp, pc);
}
}
} while (0);
ArgsWhack (args);
}
return rc;
}
rc_t CC KMain ( int argc, char *argv [] )
{
XMLLogger const *xlogger = NULL;
Args * args;
rc_t rc;
bool any_failed = false;
CheckCorruptConfig config = { -1.0, SA_TABLE_LOOKUP_LIMIT, -1.0, SEQ_TABLE_LOOKUP_LIMIT, PA_LONGER_SA_LIMIT, 0 };
KLogLevelSet(klogInfo);
rc = ArgsMakeAndHandle (&args, argc, argv, 2, Options,
sizeof (Options) / sizeof (Options[0]),
XMLLogger_Args, XMLLogger_ArgsQty);
if (rc)
LOGERR (klogInt, rc, "failed to parse command line parameters");
else
{
rc = XMLLogger_Make(&xlogger, NULL, args);
if (rc)
LOGERR (klogInt, rc, "failed to make xml logger");
else
{
rc = parseArgs ( args, &config );
if (rc == 0)
{
uint32_t pcount;
rc = ArgsParamCount ( args, &pcount );
if (rc)
LOGERR (klogInt, rc, "ArgsParamCount() failed");
else
{
if ( pcount == 0 )
LOGMSG (klogErr, "no accessions were passed in");
else
{
for ( uint32_t i = 0; i < pcount; ++i )
{
const char * accession;
rc = ArgsParamValue ( args, i, (const void **)&accession );
if (rc)
{
PLOGERR (klogInt, (klogInt, rc, "failed to get $(PARAM_I) accession from command line", "PARAM_I=%d", i));
any_failed = true;
}
else
{
if (!checkAccession ( accession, &config ))
any_failed = true;
}
}
if (!any_failed)
LOGMSG (klogInfo, "All accessions are good!");
}
}
}
XMLLogger_Release(xlogger);
}
ArgsWhack ( args );
}
return rc != 0 || any_failed ? 1 : 0;
}
rc_t load_seq_src( seq_ctx * sctx, KDirectory * hdf5_src )
{
rc_t rc = 0;
if ( sctx->lctx->check_src_obj )
rc = check_src_objects( hdf5_src, seq_groups_to_check, seq_tables_to_check, true );
if ( rc == 0 && !sctx->src_open )
rc = open_BaseCalls( hdf5_src, &sctx->BaseCallsTab, "PulseData/BaseCalls", sctx->lctx->cache_content, &sctx->rgn_present );
if ( rc == 0 )
{
/* calculates the total number of bases, according to the zmw-table */
uint64_t total_bases = zmw_total( &sctx->BaseCallsTab.cmn.zmw );
/* calculates the total number of spots, according to the zmw-table */
uint64_t total_spots = sctx->BaseCallsTab.cmn.zmw.NumEvent.extents[ 0 ];
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading sequence-table ( $(bases) bases / $(spots) spots ):",
"bases=%lu,spots=%lu", total_bases, total_spots ));
KLogLevelSet( tmp_lvl );
/* checks that all tables, which are loaded do have the correct
number of values (the number that the zmw-table requests) */
if ( !check_BaseCall_totalcount( &sctx->BaseCallsTab, total_bases ) )
rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
else
{
region_type_mapping mapping;
if ( sctx->rgn_present )
{
const KNamelist *region_types;
/* read the meta-data-entry "RegionTypes" of the hdf5-regions-table
into a KNamelist */
rc = KArrayFileGetMeta ( sctx->BaseCallsTab.rgn.hdf5_regions.af, "RegionTypes", ®ion_types );
if ( rc != 0 )
{
LOGERR( klogErr, rc, "cannot read Regions.RegionTypes" );
}
else
{
/* extract the region-type-mapping out of the read KNamelist */
rc = rgn_extract_type_mappings( region_types, &mapping, false );
KNamelistRelease ( region_types );
if ( rc != 0 )
{
LOGERR( klogErr, rc, "cannot map regions-types" );
}
}
}
if ( rc == 0 )
{
region_type_mapping *mapping_ptr = NULL;
if ( sctx->rgn_present )
mapping_ptr = &mapping;
/* call for every spot the function >seq_load_spot< */
rc = zmw_for_each( &sctx->BaseCallsTab.cmn.zmw, &sctx->lctx->xml_progress, sctx->cursor,
sctx->lctx->with_progress, sctx->col_idx, mapping_ptr, false,
seq_load_spot, &sctx->BaseCallsTab );
}
if ( sctx->rgn_present )
seq_load_info( &sctx->BaseCallsTab.rgn.stat );
sctx->lctx->total_seq_bases += total_bases;
sctx->lctx->total_seq_spots += total_spots;
}
close_BaseCalls( &sctx->BaseCallsTab );
sctx->src_open = false;
}
return rc;
}
static rc_t seq_loader( ld_context *lctx, KDirectory * hdf5_src, VCursor * cursor, const char * table_name )
{
BaseCalls BaseCallsTab;
bool rgn_present;
/* opens all hdf5-tables, which are needed to load the sequence-table */
rc_t rc = open_BaseCalls( hdf5_src, &BaseCallsTab,
"PulseData/BaseCalls", lctx->cache_content, &rgn_present );
if ( rc == 0 )
{
/* calculates the total number of bases, according to the zmw-table */
uint64_t total_bases = zmw_total( &BaseCallsTab.cmn.zmw );
/* calculates the total number of spots, according to the zmw-table */
uint64_t total_spots = BaseCallsTab.cmn.zmw.NumEvent.extents[ 0 ];
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading sequence-table ( $(bases) bases / $(spots) spots ):",
"bases=%lu,spots=%lu", total_bases, total_spots ));
KLogLevelSet( tmp_lvl );
/* checks that all tables, which are loaded do have the correct
number of values (the number that the zmw-table requests) */
if ( check_BaseCall_totalcount( &BaseCallsTab, total_bases ) )
{
region_type_mapping mapping;
if ( rgn_present )
{
const KNamelist *region_types;
/* read the meta-data-entry "RegionTypes" of the hdf5-regions-table
into a KNamelist */
rc = KArrayFileGetMeta ( BaseCallsTab.rgn.hdf5_regions.af, "RegionTypes", ®ion_types );
if ( rc != 0 )
{
LOGERR( klogErr, rc, "cannot read Regions.RegionTypes" );
}
else
{
/* extract the region-type-mapping out of the read KNamelist */
rc = rgn_extract_type_mappings( region_types, &mapping, false );
KNamelistRelease ( region_types );
if ( rc != 0 )
{
LOGERR( klogErr, rc, "cannot map regions-types" );
}
}
}
if ( rc == 0 )
{
/* holds the vdb-column-index for every column we write */
uint32_t col_idx[ seq_tab_count ];
int32_t to_exclude;
/* depending on the bit-size of the PulseIndex-table in HDF5
exclude the opposite column from VDB */
if ( BaseCallsTab.PulseIndex.element_bits == PULSE_INDEX_BITSIZE_16 )
to_exclude = seq_tab_PULSE_INDEX_32;
else
to_exclude = seq_tab_PULSE_INDEX_16;
/* add all columns to the vdb-cursor */
rc = add_columns( cursor, seq_tab_count, to_exclude, col_idx, seq_tab_names );
if ( rc == 0 )
{
rc = VCursorOpen( cursor );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot open cursor on seq-table" );
else
{
const uint8_t platform = SRA_PLATFORM_PACBIO_SMRT;
rc = VCursorDefault ( cursor, col_idx[ seq_tab_PLATFORM ], 8, &platform, 0, 1 );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot set cursor-default on seq-table for platform-column" );
else
{
region_type_mapping *mapping_ptr = NULL;
if ( rgn_present )
{
mapping_ptr = &mapping;
}
/* call for every spot the function >seq_load_spot< */
rc = zmw_for_each( &BaseCallsTab.cmn.zmw, &lctx->xml_progress, cursor,
lctx->with_progress, col_idx, mapping_ptr, false, seq_load_spot, &BaseCallsTab );
}
}
}
}
if ( rgn_present )
{
seq_load_info( &BaseCallsTab.rgn.stat );
if ( mapping.count_of_unknown_rgn_types > 0 )
{
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"$(times) x unknown region types encountered",
"times=%i", mapping.count_of_unknown_rgn_types ) );
KLogLevelSet( tmp_lvl );
}
}
lctx->total_seq_bases += total_bases;
lctx->total_seq_spots += total_spots;
}
else
rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
close_BaseCalls( &BaseCallsTab );
}
return rc;
}
