rc_t load_consensus_src( con_ctx * sctx, KDirectory * hdf5_src )
{
BaseCalls_cmn ConsensusTab;
rc_t rc = 0;
if ( sctx->lctx->check_src_obj )
rc = check_src_objects( hdf5_src, consensus_groups_to_check,
consensus_tables_to_check, false );
if ( rc == 0 )
rc = open_BaseCalls_cmn( hdf5_src, &ConsensusTab, true,
"PulseData/ConsensusBaseCalls", sctx->lctx->cache_content, true );
if ( rc == 0 )
{
uint64_t total_bases = zmw_total( &ConsensusTab.zmw );
uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ];
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading consensus-table ( $(bases) bases / $(spots) spots ):",
"bases=%lu,spots=%lu", total_bases, total_spots ));
KLogLevelSet( tmp_lvl );
if ( !check_Consensus_totalcount( &ConsensusTab, total_bases ) )
rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
else
rc = zmw_for_each( &ConsensusTab.zmw, &sctx->lctx->xml_progress, sctx->cursor,
sctx->lctx->with_progress, sctx->col_idx, NULL,
true, consensus_load_spot, &ConsensusTab );
close_BaseCalls_cmn( &ConsensusTab );
}
return rc;
}
static void seq_load_info( regions_stat * stat )
{
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
if ( stat->expands_a > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"adapter expanded: $(times) ( in $(spots) spots )",
"times=%u,spots=%u",
stat->expands_a, stat->expands_spots ));
if ( stat->expands_i > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"insert expanded : $(times) ( in $(spots) spots )",
"times=%u,spots=%u",
stat->expands_i, stat->expands_spots ));
if ( stat->inserts > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"regions inserted: $(times) ( in $(spots) spots )",
"times=%u,spots=%u",
stat->inserts, stat->inserts_spots ));
if ( stat->end_gap > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"end-gap inserted: $(times)",
"times=%u", stat->end_gap ));
if ( stat->overlapps > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"overlapping rngs: $(times)",
"times=%u", stat->overlapps ));
if ( stat->removed > 0 )
PLOGMSG( klogInfo, ( klogInfo,
"removed rgns : $(times)",
"times=%u", stat->removed ));
KLogLevelSet( tmp_lvl );
}
/* SetLogging
*/
static
rc_t KDylibSetLogging ( const KDylib *self )
{
rc_t ( CC * set_formatter ) ( KFmtWriter writer, KLogFmtFlags flags, void *data );
rc_t ( CC * set_writer ) ( KWrtWriter writer, void *data );
/* set the current logging level */
rc_t ( CC * set_level ) ( KLogLevel lvl ) = ( void* ) GetProcAddress ( self -> handle, "KLogLevelSet" );
if ( set_level != NULL )
{
KLogLevel lvl = KLogLevelGet ();
( * set_level ) ( lvl );
}
/* determine current library logging */
set_writer = ( void* ) GetProcAddress ( self -> handle, "KOutHandlerSet" );
if ( set_writer != NULL )
{
const KWrtHandler* handler = KOutHandlerGet ();
( * set_writer ) ( handler -> writer, handler -> data );
}
set_formatter = ( void* ) GetProcAddress ( self -> handle, "KLogLibFmtHandlerSet" );
if ( set_formatter != NULL )
{
KLogFmtFlags flags = KLogLibFmtFlagsGet ();
const KFmtHandler* fmt_handler = KLogFmtHandlerGet ();
( * set_formatter ) ( fmt_handler -> formatter, flags, fmt_handler -> data );
}
set_writer = ( void* ) GetProcAddress ( self -> handle, "KLogLibHandlerSet" );
if ( set_writer != NULL )
{
const KWrtHandler* handler = KLogLibHandlerGet ();
( * set_writer ) ( handler -> writer, handler -> data );
}
set_formatter = ( void* ) GetProcAddress ( self -> handle, "KStsLibFmtHandlerSet" );
if ( set_formatter != NULL )
{
KStsFmtFlags flags = KStsLibFmtFlagsGet ();
const KFmtHandler* fmt_handler = KStsFmtHandlerGet ();
( * set_formatter ) ( fmt_handler -> formatter, flags, fmt_handler -> data );
}
set_writer = ( void* ) GetProcAddress ( self -> handle, "KStsLibHandlerSet" );
if ( set_writer != NULL )
{
const KWrtHandler* handler = KStsLibHandlerGet ();
( * set_writer ) ( handler -> writer, handler -> data );
}
#if _DEBUGGING
set_writer = ( void* ) GetProcAddress ( self -> handle, "KDbgHandlerSet" );
if ( set_writer != NULL )
{
const KWrtHandler* handler = KDbgHandlerGet ();
( * set_writer ) ( handler -> writer, handler -> data );
}
#endif
return 0;
}
static rc_t consensus_loader( ld_context *lctx, const char * table_name,
bool cache_content )
{
uint32_t col_idx[ consensus_tab_count ];
rc_t rc = add_columns( lctx->cursor, consensus_tab_count, -1,
col_idx, consensus_tab_names );
if ( rc == 0 )
{
rc = VCursorOpen( lctx->cursor );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot open cursor on consensus-table" );
else
{
BaseCalls_cmn ConsensusTab;
const INSDC_SRA_platform_id platform = SRA_PLATFORM_PACBIO_SMRT;
rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_PLATFORM ],
sizeof platform * 8, &platform, 0, 1 );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for platform-column" );
else
{
const INSDC_SRA_read_filter filter = SRA_READ_FILTER_PASS;
rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_READ_FILTER ],
sizeof filter * 8, &filter, 0, 1 );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for read-filter-column" );
}
if ( rc == 0 )
rc = open_BaseCalls_cmn( lctx->hdf5_dir, &ConsensusTab, true,
"PulseData/ConsensusBaseCalls", cache_content, true );
if ( rc == 0 )
{
uint64_t total_bases = zmw_total( &ConsensusTab.zmw );
uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ];
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading consensus-table ( $(bases) bases / $(spots) spots ):",
"bases=%lu,spots=%lu", total_bases, total_spots ));
KLogLevelSet( tmp_lvl );
if ( check_Consensus_totalcount( &ConsensusTab, total_bases ) )
rc = zmw_for_each( &ConsensusTab.zmw, lctx, col_idx, NULL,
true, consensus_load_spot, &ConsensusTab );
else
rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
close_BaseCalls_cmn( &ConsensusTab );
}
}
}
return rc;
}
static rc_t passes_load_loop( ld_context *lctx, VCursor * cursor, Passes_src *tab,
uint32_t *col_idx )
{
rc_t rc = 0;
KLogLevel tmp_lvl;
pass_block block;
pl_progress *progress;
uint64_t pos = 0;
uint64_t total_passes = tab->AdapterHitBefore.extents[0];
pl_progress_make( &progress, total_passes );
rc = progress_chunk( &lctx->xml_progress, total_passes );
tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading passes-table ( $(rows) rows ) :",
"rows=%lu", total_passes ));
KLogLevelSet( tmp_lvl );
while( pos < total_passes && rc == 0 )
{
rc = pass_block_read_from_src( tab, total_passes, pos, &block );
if ( rc == 0 )
{
uint32_t i;
for ( i = 0; i < block.n_read && rc == 0; ++i )
{
rc = Quitting();
if ( rc == 0 )
{
rc = passes_load_pass( cursor, &block, i, col_idx );
if ( rc == 0 )
{
rc = progress_step( lctx->xml_progress );
if ( lctx->with_progress )
pl_progress_increment( progress, 1 );
}
}
else
LOGERR( klogErr, rc, "...loading passes interrupted" );
}
pos += block.n_read;
}
}
pl_progress_destroy( progress );
if ( rc == 0 )
{
rc = VCursorCommit( cursor );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot commit cursor on PASSES-tab" );
}
return rc;
}
void seq_report_totals( ld_context *lctx )
{
const char* accession;
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
accession = strrchr( lctx->dst_path, '/' );
if( accession == NULL )
accession = lctx->dst_path;
else
accession++;
PLOGMSG( klogInfo, ( klogInfo, "loaded",
"severity=total,status=success,accession=%s,spot_count=%lu,base_count=%lu,bad_spots=0",
accession, lctx->total_seq_spots, lctx->total_seq_bases ));
KLogLevelSet( tmp_lvl );
}
/* KMain - EXTERN
* executable entrypoint "main" is implemented by
* an OS-specific wrapper that takes care of establishing
* signal handlers, logging, etc.
*
* in turn, OS-specific "main" will invoke "KMain" as
* platform independent main entrypoint.
*
* "argc" [ IN ] - the number of textual parameters in "argv"
* should never be < 0, but has been left as a signed int
* for reasons of tradition.
*
* "argv" [ IN ] - array of NUL terminated strings expected
* to be in the shell-native character set: ASCII or UTF-8
* element 0 is expected to be executable identity or path.
*/
rc_t CC KMain ( int argc, char *argv [] )
{
Args * args;
rc_t rc;
default_log_level = KLogLevelGet();
rc = ArgsMakeAndHandle (&args, argc, argv, 1, Options, sizeof Options / sizeof (OptDef));
if (rc == 0)
{
do
{
srakar_parms pb;
KDirectory * pwd;
const char * name;
uint32_t pcount;
char archive_name [256];
rc = KDirectoryNativeDir (&pwd);
if (rc)
break;
pb.lite = false;
pb.force = false;
pb.dir = pwd;
rc = ArgsOptionCount (args, OPTION_LITE, &pcount);
if (rc)
break;
if (pcount == 1)
{
STSMSG (1, ("Using lite option"));
pb.lite = true;
}
#if USE_FORCE
rc = ArgsOptionCount (args, OPTION_FORCE, &pcount);
if (rc)
break;
if (pcount == 1)
{
STSMSG (1, ("Using force option"));
pb.force = true;
}
#endif
rc = ArgsParamCount (args, &pcount);
if (rc)
break;
if (pcount == 0)
{
KOutMsg ("missing source table\n");
rc = MiniUsage (args);
break;
}
else if (pcount > 2)
{
KOutMsg ("Too many parameters\n");
rc = MiniUsage (args);
break;
}
rc = ArgsParamValue (args, 0, &pb.src_path);
if (rc)
{
KOutMsg ("failure to get source path/name\n");
break;
}
pb.dst_path = archive_name;
name = string_rchr (pb.src_path, string_size (pb.src_path), '/');
if (name == NULL)
name = pb.src_path;
if (pcount == 1)
{
size_t size;
rc = string_printf (archive_name, sizeof (archive_name), & size,
"%s%s", name, pb.lite ? ".lite.sra" : ".sra");
if ( rc != 0 )
{
rc = RC (rcExe, rcArgv, rcParsing, rcPath, rcInsufficient);
PLOGERR (klogFatal, (klogFatal, rc, "Failure building archive name $(P)", "P=%s", archive_name));
break;
}
}
else
{
rc = ArgsParamValue (args, 1, &pb.dst_path);
if (rc)
{
LOGERR (klogInt, rc, "failure to get destination path");
break;
}
}
if (rc == 0)
{
KPathType kpt;
kpt = KDirectoryPathType (pwd, "%s", pb.dst_path);
switch (kpt & ~kptAlias)
{
case kptNotFound:
/* found nothing so assume its a valid new file name
* not gonna tweak extensions at this point but that can be upgraded */
break;
case kptFile:
/* got a file name, use it - would need force... */
if (pb.force == false)
{
rc = RC (rcExe, rcArgv, rcParsing, rcPath, rcBusy);
PLOGERR (klogFatal, (klogFatal, rc,
"Output file already exists $(P)",
"P=%s", archive_name));
}
break;
case kptDir:
{
size_t size;
rc = string_printf (archive_name, sizeof (archive_name), & size,
"%s/%s%s", pb.dst_path, name, pb.lite ? ".lite.sra" : ".sra");
if ( rc != 0 )
{
rc = RC (rcExe, rcArgv, rcParsing, rcPath, rcInsufficient);
PLOGERR (klogFatal, (klogFatal, rc, "Failure building archive name $(P)", "P=%s", archive_name));
break;
}
pb.dst_path = archive_name;
break;
}
default:
rc = RC (rcExe, rcArgv, rcParsing, rcPath, rcInvalid);
break;
}
if (rc == 0)
{
STSMSG (1,("Creating archive (%s) from table (%s)\n", pb.dst_path, pb.src_path));
rc = run (&pb);
STSMSG (5, ("Run exits with %d %R\n", rc, rc));
}
}
} while (0);
}
STSMSG (1, ("Exit status %u %R", rc, rc));
return rc;
}
rc_t load_seq_src( seq_ctx * sctx, KDirectory * hdf5_src )
{
rc_t rc = 0;
if ( sctx->lctx->check_src_obj )
rc = check_src_objects( hdf5_src, seq_groups_to_check, seq_tables_to_check, true );
if ( rc == 0 && !sctx->src_open )
rc = open_BaseCalls( hdf5_src, &sctx->BaseCallsTab, "PulseData/BaseCalls", sctx->lctx->cache_content, &sctx->rgn_present );
if ( rc == 0 )
{
/* calculates the total number of bases, according to the zmw-table */
uint64_t total_bases = zmw_total( &sctx->BaseCallsTab.cmn.zmw );
/* calculates the total number of spots, according to the zmw-table */
uint64_t total_spots = sctx->BaseCallsTab.cmn.zmw.NumEvent.extents[ 0 ];
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading sequence-table ( $(bases) bases / $(spots) spots ):",
"bases=%lu,spots=%lu", total_bases, total_spots ));
KLogLevelSet( tmp_lvl );
/* checks that all tables, which are loaded do have the correct
number of values (the number that the zmw-table requests) */
if ( !check_BaseCall_totalcount( &sctx->BaseCallsTab, total_bases ) )
rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
else
{
region_type_mapping mapping;
if ( sctx->rgn_present )
{
const KNamelist *region_types;
/* read the meta-data-entry "RegionTypes" of the hdf5-regions-table
into a KNamelist */
rc = KArrayFileGetMeta ( sctx->BaseCallsTab.rgn.hdf5_regions.af, "RegionTypes", ®ion_types );
if ( rc != 0 )
{
LOGERR( klogErr, rc, "cannot read Regions.RegionTypes" );
}
else
{
/* extract the region-type-mapping out of the read KNamelist */
rc = rgn_extract_type_mappings( region_types, &mapping, false );
KNamelistRelease ( region_types );
if ( rc != 0 )
{
LOGERR( klogErr, rc, "cannot map regions-types" );
}
}
}
if ( rc == 0 )
{
region_type_mapping *mapping_ptr = NULL;
if ( sctx->rgn_present )
mapping_ptr = &mapping;
/* call for every spot the function >seq_load_spot< */
rc = zmw_for_each( &sctx->BaseCallsTab.cmn.zmw, &sctx->lctx->xml_progress, sctx->cursor,
sctx->lctx->with_progress, sctx->col_idx, mapping_ptr, false,
seq_load_spot, &sctx->BaseCallsTab );
}
if ( sctx->rgn_present )
seq_load_info( &sctx->BaseCallsTab.rgn.stat );
sctx->lctx->total_seq_bases += total_bases;
sctx->lctx->total_seq_spots += total_spots;
}
close_BaseCalls( &sctx->BaseCallsTab );
sctx->src_open = false;
}
return rc;
}
static rc_t seq_loader( ld_context *lctx, KDirectory * hdf5_src, VCursor * cursor, const char * table_name )
{
BaseCalls BaseCallsTab;
bool rgn_present;
/* opens all hdf5-tables, which are needed to load the sequence-table */
rc_t rc = open_BaseCalls( hdf5_src, &BaseCallsTab,
"PulseData/BaseCalls", lctx->cache_content, &rgn_present );
if ( rc == 0 )
{
/* calculates the total number of bases, according to the zmw-table */
uint64_t total_bases = zmw_total( &BaseCallsTab.cmn.zmw );
/* calculates the total number of spots, according to the zmw-table */
uint64_t total_spots = BaseCallsTab.cmn.zmw.NumEvent.extents[ 0 ];
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"loading sequence-table ( $(bases) bases / $(spots) spots ):",
"bases=%lu,spots=%lu", total_bases, total_spots ));
KLogLevelSet( tmp_lvl );
/* checks that all tables, which are loaded do have the correct
number of values (the number that the zmw-table requests) */
if ( check_BaseCall_totalcount( &BaseCallsTab, total_bases ) )
{
region_type_mapping mapping;
if ( rgn_present )
{
const KNamelist *region_types;
/* read the meta-data-entry "RegionTypes" of the hdf5-regions-table
into a KNamelist */
rc = KArrayFileGetMeta ( BaseCallsTab.rgn.hdf5_regions.af, "RegionTypes", ®ion_types );
if ( rc != 0 )
{
LOGERR( klogErr, rc, "cannot read Regions.RegionTypes" );
}
else
{
/* extract the region-type-mapping out of the read KNamelist */
rc = rgn_extract_type_mappings( region_types, &mapping, false );
KNamelistRelease ( region_types );
if ( rc != 0 )
{
LOGERR( klogErr, rc, "cannot map regions-types" );
}
}
}
if ( rc == 0 )
{
/* holds the vdb-column-index for every column we write */
uint32_t col_idx[ seq_tab_count ];
int32_t to_exclude;
/* depending on the bit-size of the PulseIndex-table in HDF5
exclude the opposite column from VDB */
if ( BaseCallsTab.PulseIndex.element_bits == PULSE_INDEX_BITSIZE_16 )
to_exclude = seq_tab_PULSE_INDEX_32;
else
to_exclude = seq_tab_PULSE_INDEX_16;
/* add all columns to the vdb-cursor */
rc = add_columns( cursor, seq_tab_count, to_exclude, col_idx, seq_tab_names );
if ( rc == 0 )
{
rc = VCursorOpen( cursor );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot open cursor on seq-table" );
else
{
const uint8_t platform = SRA_PLATFORM_PACBIO_SMRT;
rc = VCursorDefault ( cursor, col_idx[ seq_tab_PLATFORM ], 8, &platform, 0, 1 );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot set cursor-default on seq-table for platform-column" );
else
{
region_type_mapping *mapping_ptr = NULL;
if ( rgn_present )
{
mapping_ptr = &mapping;
}
/* call for every spot the function >seq_load_spot< */
rc = zmw_for_each( &BaseCallsTab.cmn.zmw, &lctx->xml_progress, cursor,
lctx->with_progress, col_idx, mapping_ptr, false, seq_load_spot, &BaseCallsTab );
}
}
}
}
if ( rgn_present )
{
seq_load_info( &BaseCallsTab.rgn.stat );
if ( mapping.count_of_unknown_rgn_types > 0 )
{
KLogLevel tmp_lvl = KLogLevelGet();
KLogLevelSet( klogInfo );
PLOGMSG( klogInfo, ( klogInfo,
"$(times) x unknown region types encountered",
"times=%i", mapping.count_of_unknown_rgn_types ) );
KLogLevelSet( tmp_lvl );
}
}
lctx->total_seq_bases += total_bases;
lctx->total_seq_spots += total_spots;
}
else
rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
close_BaseCalls( &BaseCallsTab );
}
return rc;
}
