/* Open
* open cursor, resolving schema
* for the set of opened columns
*
* NB - there is no corresponding "Close"
* use "Release" instead.
*/
static
rc_t VProdResolveAddShallowTriggers ( const VProdResolve *self, const STable *stbl )
{
rc_t rc;
VCursor *curs;
uint32_t i = VectorStart ( & stbl -> prod );
uint32_t end = i + VectorLength ( & stbl -> prod );
for ( rc = 0, curs = self -> curs; ( rc == 0 || self -> ignore_column_errors ) && i < end; ++ i )
{
SProduction *sprod = VectorGet ( & stbl -> prod, i );
if ( sprod != NULL && sprod -> trigger )
{
VProduction *prod = NULL;
rc = VProdResolveSProduction ( self, & prod, sprod );
if ( rc == 0 && prod != NULL )
rc = VectorAppend ( & curs -> trig, NULL, prod );
}
}
if ( self -> ignore_column_errors )
return 0;
return rc;
}
static rc_t CC report_reference_cb( const char * name, Vector * ranges, void *data )
{
rc_t rc = KOutMsg( "region: <%s>\n", name );
if ( rc == 0 )
{
uint32_t count = VectorLength( ranges );
if ( count > 0 )
{
uint32_t i;
for ( i = VectorStart( ranges ); i < count && rc == 0; ++i )
{
range *r = VectorGet( ranges, i );
if ( r->end == 0 )
{
if ( r->start == 0 )
rc = KOutMsg( "\t[ start ... end ]\n" );
else
rc = KOutMsg( "\t[ %u ... ]\n", r->start );
}
else
rc = KOutMsg( "\t[ %u ... %u ]\n", r->start, r->end );
}
}
}
return rc;
}
static
rc_t VProdResolveColExpr ( const VProdResolve *self, VProduction **out,
VFormatdecl *fd, const SSymExpr *x, bool casting )
{
rc_t rc;
const SNameOverload *sname;
const KSymbol *sym = x -> _sym;
BSTree ordered;
uint32_t i, count;
SColumnBestFit buff [ 16 ], * nodes = buff;
/* fail if "fd" has a format */
if ( fd -> fmt != 0 )
{
PLOGMSG ( klogWarn, ( klogWarn, "illegal cast of column '$(name)'"
, "name=%.*s"
, ( int ) sym -> name . size
, sym -> name . addr ));
return 0;
}
/* allocate nodes for indexing columns */
sname = sym -> u . obj;
count = VectorLength ( & sname -> items );
if ( count > sizeof buff / sizeof buff [ 0 ] )
{
nodes = malloc ( sizeof * nodes * count );
if ( nodes == NULL )
return RC ( rcVDB, rcProduction, rcResolving, rcMemory, rcExhausted );
}
/* insert columns into ordered tree */
BSTreeInit ( & ordered );
for ( i = VectorStart ( & sname -> items ), count += i; i < count; ++ i )
{
/* get SColumn */
nodes [ i ] . scol = ( const void* ) VectorGet ( & sname -> items, i );
/* perform type cast and measure distance */
if ( casting ?
VTypedeclCommonAncestor ( & nodes [ i ] . scol -> td, self -> schema,
& fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) :
VTypedeclToTypedecl ( & nodes [ i ] . scol -> td, self -> schema,
& fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) )
{
BSTreeInsert ( & ordered, & nodes [ i ] . n, order_column );
}
}
/* try to resolve each in order */
rc = VProdResolveBestColumn ( self, out, & ordered, x -> alt );
if ( nodes != buff )
free ( nodes );
return rc;
}
rc_t VCursorListSeededWritableColumns ( VCursor *self, BSTree *columns, const KNamelist *seed )
{
rc_t rc;
KDlset *libs;
struct resolve_phys_data pb;
pb . pr . schema = self -> schema;
pb . pr . ld = self -> tbl -> linker;
pb . pr . stbl = self -> stbl;
pb . pr . curs = self;
pb . pr . cache = & self -> prod;
pb . pr . owned = & self -> owned;
pb . pr . chain = chainEncoding;
pb . pr . blobbing = false;
pb . pr . ignore_column_errors = true;
pb . pr . discover_writable_columns = true;
pb . seed = seed;
if ( seed != NULL )
{
rc = KNamelistCount ( seed, & pb . count );
if ( rc != 0 )
return rc;
}
/* open the dynamic linker libraries */
rc = VLinkerOpen ( pb . pr . ld, & libs );
if ( rc == 0 )
{
pb . pr . libs = libs;
VProdResolveWritableColumns ( & pb , self->suspend_triggers );
KDlsetRelease ( libs );
if ( rc == 0 )
{
/* add columns to list */
uint32_t idx = VectorStart ( & self -> row );
uint32_t end = VectorLength ( & self -> row );
for ( end += idx; idx < end; ++idx )
{
const VColumn* vcol = ( const VColumn* ) VectorGet ( & self -> row, idx );
if ( vcol != NULL )
{
VColumnRef *cref;
rc = VColumnRefMake ( & cref, self -> schema, vcol -> scol );
if ( rc != 0 )
break;
rc = BSTreeInsert ( columns, & cref -> n, VColumnRefSort );
assert ( rc == 0 );
}
}
}
}
return rc;
}
static rc_t vdi_test( const Vector * v )
{
rc_t rc = 0;
uint32_t start = VectorStart( v );
uint32_t len = VectorLength( v );
uint32_t idx;
for ( idx = start; rc == 0 && idx < ( start + len ); ++idx )
rc = KOutMsg( "{%S} ", VectorGet( v, idx ) );
return rc;
}
static
rc_t VProdResolveAddTriggers ( const VProdResolve *self, const STable *stbl )
{
uint32_t i = VectorStart ( & stbl -> overrides );
uint32_t end = VectorLength ( & stbl -> overrides );
for ( end += i; i < end; ++ i )
{
const STable *dad = STableFindOrdAncestor ( stbl, i );
rc_t rc = VProdResolveAddShallowTriggers ( self, dad );
if ( rc != 0 )
return rc;
}
return VProdResolveAddShallowTriggers ( self, stbl );
}
bool filter_by_slices( const Vector * slices, const String * refname, uint64_t pos, uint32_t len )
{
bool res = false;
uint32_t idx;
uint64_t end = pos + len;
uint32_t v_start = VectorStart( slices );
uint32_t v_end = v_start + VectorLength( slices );
for ( idx = v_start; !res && idx < v_end; ++idx )
{
const slice * slice = VectorGet( slices, idx );
if ( slice != NULL )
res = filter_by_slice( slice, refname, pos, end );
}
return res;
}
static
void VProdResolveWritableColumns ( struct resolve_phys_data *pb, bool suspend_triggers )
{
const STable *dad, *stbl = pb -> pr . stbl;
/* walk table schema looking for parents */
uint32_t i = VectorStart ( & stbl -> overrides );
uint32_t end = VectorLength ( & stbl -> overrides );
for ( end += i; i < end; ++ i )
{
dad = STableFindOrdAncestor ( stbl, i );
VectorForEach ( & dad -> phys, false, resolve_writable_sphys, pb );
}
/* walk current table */
VectorForEach ( & stbl -> phys, false, resolve_writable_sphys, pb );
/* add triggers */
if ( !suspend_triggers && pb -> seed == NULL )
{
pb -> pr . chain = chainUncommitted;
VProdResolveAddTriggers ( & pb -> pr, stbl );
}
}
LIB_EXPORT rc_t CC KDyldVLoadLib ( KDyld *self,
KDylib **lib, const char *path, va_list args )
{
rc_t rc;
if ( lib == NULL )
rc = RC ( rcFS, rcDylib, rcLoading, rcParam, rcNull );
else
{
if ( self == NULL )
rc = RC ( rcFS, rcDylib, rcLoading, rcSelf, rcNull );
else if ( path == NULL || path [ 0 ] == 0 )
{
WString pstr;
CONST_WSTRING ( & pstr, "" );
rc = KDylibMake ( lib, & pstr );
if ( rc == 0 )
{
rc = KDyldLoad ( self, * lib, NULL );
if ( rc == 0 )
return 0;
free ( * lib );
}
}
else
{
uint32_t i = VectorStart ( & self -> search );
uint32_t end = i + VectorLength ( & self -> search );
if ( i == end )
{
char name [ 4096 ];
int len = vsnprintf ( name, sizeof name, path, args );
if ( len < 0 || len >= sizeof name )
rc = RC ( rcFS, rcDylib, rcLoading, rcPath, rcExcessive );
else
{
WString pstr;
wchar_t wname [ 4096 ];
size_t wsize = string_cvt_wchar_copy ( wname, sizeof wname, name, len );
WStringInit ( & pstr, wname, wsize * sizeof wname [ 0 ], string_len ( name, len ) );
rc = KDylibMake ( lib, & pstr );
if ( rc == 0 )
{
rc = KDyldLoad ( self, * lib, wname );
if ( rc == 0 )
return 0;
free ( * lib );
}
}
}
else
{
for ( * lib = NULL; i < end; ++ i )
{
const KDirectory *dir;
va_list cpy;
va_copy ( cpy, args );
dir = ( const void* ) VectorGet ( & self -> search, i );
rc = KDyldVTryLoadLib ( self, lib, dir, path, cpy );
va_end ( cpy );
if ( rc == 0 || GetRCState ( rc ) != rcNotFound )
return rc;
}
rc = RC ( rcFS, rcDylib, rcLoading, rcPath, rcNotFound );
}
}
* lib = NULL;
}
return rc;
}
static
rc_t populate_cursors ( VTable *dtbl, VCursor *dcurs, const VCursor *scurs,
vtblcp_column_map *cm, const Vector *v, uint32_t *rd_filt )
{
uint32_t end = VectorLength ( v );
uint32_t i = VectorStart ( v );
BSTree stype_tbl, rftype_tbl;
const VSchema *schema;
rc_t rc = VTableOpenSchema ( dtbl, & schema );
if ( rc != 0 )
{
LOGERR ( klogInt, rc, "failed to open destination table schema" );
return rc;
}
/* populate sensitive type table */
rc = populate_stype_tbl ( & stype_tbl, schema );
if ( rc != 0 )
{
VSchemaRelease ( schema );
return rc;
}
/* populate read filter type table */
rc = populate_rdfilt_tbl ( & rftype_tbl, schema );
if ( rc != 0 )
{
BSTreeWhack ( & stype_tbl, stype_id_whack, NULL );
VSchemaRelease ( schema );
return rc;
}
for ( end += i, rc = 0, * rd_filt = 0; i < end; ++ i )
{
VTypedecl td;
char typedecl [ 128 ];
const char *spec = ( const void* ) VectorGet ( v, i );
/* request column in destination */
rc = VCursorAddColumn ( dcurs, & cm [ i ] . wr, spec );
if ( rc != 0 )
{
PLOGERR ( klogErr, (klogErr, rc, "failed to add '$(spec)' to destination cursor", "spec=%s", spec ));
break;
}
/* always retrieve data type */
rc = VCursorDatatype ( dcurs, cm [ i ] . wr, & td, NULL );
if ( rc != 0 )
{
PLOGERR ( klogInt, (klogInt, rc, "failed to determine datatype of destination column '$(name)'", "name=%s", spec ));
break;
}
/* mark column as sensitive or not */
rc = mark_type_sensitivity ( & stype_tbl, schema, & td, & cm [ i ] );
if ( rc != 0 )
break;
/* if spec is already typed, request it in source */
if ( spec [ 0 ] == '(' )
{
rc = VCursorAddColumn ( scurs, & cm [ i ] . rd, spec );
if ( rc != 0 )
{
PLOGERR ( klogErr, (klogErr, rc, "failed to add '$(spec)' to source cursor", "spec=%s", spec ));
break;
}
}
else
{
rc = VTypedeclToText ( & td, schema, typedecl, sizeof typedecl );
if ( rc != 0 )
{
PLOGERR ( klogInt, (klogInt, rc, "failed to print datatype of destination column '$(name)'", "name=%s", spec ));
break;
}
rc = VCursorAddColumn ( scurs, & cm [ i ] . rd, "(%s)%s", typedecl, spec );
if ( rc != 0 )
{
PLOGERR ( klogErr, (klogErr, rc, "failed to add '$(spec)' to source cursor", "spec=(%s)%s", typedecl, spec ));
break;
}
}
/* check if column is a read filter */
cm [ i ] . rd_filter = false;
if ( ! cm [ i ] . sensitive )
{
if ( BSTreeFind ( & rftype_tbl, & td, stype_id_cmp ) != NULL )
{
if ( * rd_filt != 0 )
{
rc = RC ( rcExe, rcColumn, rcOpening, rcColumn, rcExists );
PLOGERR ( klogInt, (klogInt, rc, "can't use column '$(name)' as read filter", "name=%s", spec ));
break;
}
* rd_filt = cm [ i ] . rd;
cm [ i ] . rd_filter = true;
}
}
}
BSTreeWhack ( & rftype_tbl, stype_id_whack, NULL );
BSTreeWhack ( & stype_tbl, stype_id_whack, NULL );
VSchemaRelease ( schema );
/* add read filter to input if not already there in some way */
if ( * rd_filt == 0 )
{
rc = VCursorAddColumn ( scurs, rd_filt, "RD_FILTER" );
if ( rc != 0 && GetRCState ( rc ) == rcNotFound )
rc = 0;
}
return rc;
}
rc_t execute_tbl_join( KDirectory * dir,
const char * accession_path,
const char * accession_short,
join_stats * stats,
const char * tbl_name,
const struct temp_dir * temp_dir,
struct temp_registry * registry,
size_t cur_cache,
size_t buf_size,
uint32_t num_threads,
bool show_progress,
format_t fmt,
const join_options * join_options )
{
rc_t rc = 0;
if ( show_progress )
rc = KOutMsg( "join :" );
if ( rc == 0 )
{
uint64_t row_count = 0;
rc = extract_sra_row_count( dir, accession_path, tbl_name, cur_cache, &row_count ); /* above */
if ( rc == 0 && row_count > 0 )
{
bool name_column_present;
if ( tbl_name == NULL )
rc = cmn_check_tbl_column( dir, accession_path, "NAME", &name_column_present );
else
rc = cmn_check_db_column( dir, accession_path, tbl_name, "NAME", &name_column_present );
if ( rc == 0 )
{
Vector threads;
int64_t row = 1;
uint32_t thread_id;
uint64_t rows_per_thread;
struct bg_progress * progress = NULL;
struct join_options corrected_join_options; /* helper.h */
VectorInit( &threads, 0, num_threads );
corrected_join_options . rowid_as_name = name_column_present ? join_options -> rowid_as_name : true;
corrected_join_options . skip_tech = join_options -> skip_tech;
corrected_join_options . print_read_nr = join_options -> print_read_nr;
corrected_join_options . print_name = name_column_present;
corrected_join_options . min_read_len = join_options -> min_read_len;
corrected_join_options . filter_bases = join_options -> filter_bases;
corrected_join_options . terminate_on_invalid = join_options -> terminate_on_invalid;
if ( row_count < ( num_threads * 100 ) )
{
num_threads = 1;
rows_per_thread = row_count;
}
else
{
rows_per_thread = ( row_count / num_threads ) + 1;
}
if ( show_progress )
rc = bg_progress_make( &progress, row_count, 0, 0 ); /* progress_thread.c */
for ( thread_id = 0; rc == 0 && thread_id < num_threads; ++thread_id )
{
join_thread_data * jtd = calloc( 1, sizeof * jtd );
if ( jtd != NULL )
{
jtd -> dir = dir;
jtd -> accession_path = accession_path;
jtd -> accession_short = accession_short;
jtd -> tbl_name = tbl_name;
jtd -> first_row = row;
jtd -> row_count = rows_per_thread;
jtd -> cur_cache = cur_cache;
jtd -> buf_size = buf_size;
jtd -> progress = progress;
jtd -> registry = registry;
jtd -> fmt = fmt;
jtd -> join_options = &corrected_join_options;
rc = make_joined_filename( temp_dir, jtd -> part_file, sizeof jtd -> part_file,
accession_short, thread_id ); /* temp_dir.c */
if ( rc == 0 )
{
rc = KThreadMake( &jtd -> thread, cmn_thread_func, jtd );
if ( rc != 0 )
ErrMsg( "KThreadMake( fastq/special #%d ) -> %R", thread_id, rc );
else
{
rc = VectorAppend( &threads, NULL, jtd );
if ( rc != 0 )
ErrMsg( "VectorAppend( sort-thread #%d ) -> %R", thread_id, rc );
}
row += rows_per_thread;
}
}
}
{
/* collect the threads, and add the join_stats */
uint32_t i, n = VectorLength( &threads );
for ( i = VectorStart( &threads ); i < n; ++i )
{
join_thread_data * jtd = VectorGet( &threads, i );
if ( jtd != NULL )
{
rc_t rc_thread;
KThreadWait( jtd -> thread, &rc_thread );
if ( rc_thread != 0 )
rc = rc_thread;
KThreadRelease( jtd -> thread );
add_join_stats( stats, &jtd -> stats );
free( jtd );
}
}
VectorWhack ( &threads, NULL, NULL );
}
bg_progress_release( progress ); /* progress_thread.c ( ignores NULL )*/
}
}
}
return rc;
}
