/* Open * open cursor, resolving schema * for the set of opened columns * * NB - there is no corresponding "Close" * use "Release" instead. */ static rc_t VProdResolveAddShallowTriggers ( const VProdResolve *self, const STable *stbl ) { rc_t rc; VCursor *curs; uint32_t i = VectorStart ( & stbl -> prod ); uint32_t end = i + VectorLength ( & stbl -> prod ); for ( rc = 0, curs = self -> curs; ( rc == 0 || self -> ignore_column_errors ) && i < end; ++ i ) { SProduction *sprod = VectorGet ( & stbl -> prod, i ); if ( sprod != NULL && sprod -> trigger ) { VProduction *prod = NULL; rc = VProdResolveSProduction ( self, & prod, sprod ); if ( rc == 0 && prod != NULL ) rc = VectorAppend ( & curs -> trig, NULL, prod ); } } if ( self -> ignore_column_errors ) return 0; return rc; }
static rc_t CC report_reference_cb( const char * name, Vector * ranges, void *data ) { rc_t rc = KOutMsg( "region: <%s>\n", name ); if ( rc == 0 ) { uint32_t count = VectorLength( ranges ); if ( count > 0 ) { uint32_t i; for ( i = VectorStart( ranges ); i < count && rc == 0; ++i ) { range *r = VectorGet( ranges, i ); if ( r->end == 0 ) { if ( r->start == 0 ) rc = KOutMsg( "\t[ start ... end ]\n" ); else rc = KOutMsg( "\t[ %u ... ]\n", r->start ); } else rc = KOutMsg( "\t[ %u ... %u ]\n", r->start, r->end ); } } } return rc; }
static rc_t VProdResolveColExpr ( const VProdResolve *self, VProduction **out, VFormatdecl *fd, const SSymExpr *x, bool casting ) { rc_t rc; const SNameOverload *sname; const KSymbol *sym = x -> _sym; BSTree ordered; uint32_t i, count; SColumnBestFit buff [ 16 ], * nodes = buff; /* fail if "fd" has a format */ if ( fd -> fmt != 0 ) { PLOGMSG ( klogWarn, ( klogWarn, "illegal cast of column '$(name)'" , "name=%.*s" , ( int ) sym -> name . size , sym -> name . addr )); return 0; } /* allocate nodes for indexing columns */ sname = sym -> u . obj; count = VectorLength ( & sname -> items ); if ( count > sizeof buff / sizeof buff [ 0 ] ) { nodes = malloc ( sizeof * nodes * count ); if ( nodes == NULL ) return RC ( rcVDB, rcProduction, rcResolving, rcMemory, rcExhausted ); } /* insert columns into ordered tree */ BSTreeInit ( & ordered ); for ( i = VectorStart ( & sname -> items ), count += i; i < count; ++ i ) { /* get SColumn */ nodes [ i ] . scol = ( const void* ) VectorGet ( & sname -> items, i ); /* perform type cast and measure distance */ if ( casting ? VTypedeclCommonAncestor ( & nodes [ i ] . scol -> td, self -> schema, & fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) : VTypedeclToTypedecl ( & nodes [ i ] . scol -> td, self -> schema, & fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) ) { BSTreeInsert ( & ordered, & nodes [ i ] . n, order_column ); } } /* try to resolve each in order */ rc = VProdResolveBestColumn ( self, out, & ordered, x -> alt ); if ( nodes != buff ) free ( nodes ); return rc; }
rc_t VCursorListSeededWritableColumns ( VCursor *self, BSTree *columns, const KNamelist *seed ) { rc_t rc; KDlset *libs; struct resolve_phys_data pb; pb . pr . schema = self -> schema; pb . pr . ld = self -> tbl -> linker; pb . pr . stbl = self -> stbl; pb . pr . curs = self; pb . pr . cache = & self -> prod; pb . pr . owned = & self -> owned; pb . pr . chain = chainEncoding; pb . pr . blobbing = false; pb . pr . ignore_column_errors = true; pb . pr . discover_writable_columns = true; pb . seed = seed; if ( seed != NULL ) { rc = KNamelistCount ( seed, & pb . count ); if ( rc != 0 ) return rc; } /* open the dynamic linker libraries */ rc = VLinkerOpen ( pb . pr . ld, & libs ); if ( rc == 0 ) { pb . pr . libs = libs; VProdResolveWritableColumns ( & pb , self->suspend_triggers ); KDlsetRelease ( libs ); if ( rc == 0 ) { /* add columns to list */ uint32_t idx = VectorStart ( & self -> row ); uint32_t end = VectorLength ( & self -> row ); for ( end += idx; idx < end; ++idx ) { const VColumn* vcol = ( const VColumn* ) VectorGet ( & self -> row, idx ); if ( vcol != NULL ) { VColumnRef *cref; rc = VColumnRefMake ( & cref, self -> schema, vcol -> scol ); if ( rc != 0 ) break; rc = BSTreeInsert ( columns, & cref -> n, VColumnRefSort ); assert ( rc == 0 ); } } } } return rc; }
static rc_t vdi_test( const Vector * v ) { rc_t rc = 0; uint32_t start = VectorStart( v ); uint32_t len = VectorLength( v ); uint32_t idx; for ( idx = start; rc == 0 && idx < ( start + len ); ++idx ) rc = KOutMsg( "{%S} ", VectorGet( v, idx ) ); return rc; }
static rc_t VProdResolveAddTriggers ( const VProdResolve *self, const STable *stbl ) { uint32_t i = VectorStart ( & stbl -> overrides ); uint32_t end = VectorLength ( & stbl -> overrides ); for ( end += i; i < end; ++ i ) { const STable *dad = STableFindOrdAncestor ( stbl, i ); rc_t rc = VProdResolveAddShallowTriggers ( self, dad ); if ( rc != 0 ) return rc; } return VProdResolveAddShallowTriggers ( self, stbl ); }
bool filter_by_slices( const Vector * slices, const String * refname, uint64_t pos, uint32_t len ) { bool res = false; uint32_t idx; uint64_t end = pos + len; uint32_t v_start = VectorStart( slices ); uint32_t v_end = v_start + VectorLength( slices ); for ( idx = v_start; !res && idx < v_end; ++idx ) { const slice * slice = VectorGet( slices, idx ); if ( slice != NULL ) res = filter_by_slice( slice, refname, pos, end ); } return res; }
static void VProdResolveWritableColumns ( struct resolve_phys_data *pb, bool suspend_triggers ) { const STable *dad, *stbl = pb -> pr . stbl; /* walk table schema looking for parents */ uint32_t i = VectorStart ( & stbl -> overrides ); uint32_t end = VectorLength ( & stbl -> overrides ); for ( end += i; i < end; ++ i ) { dad = STableFindOrdAncestor ( stbl, i ); VectorForEach ( & dad -> phys, false, resolve_writable_sphys, pb ); } /* walk current table */ VectorForEach ( & stbl -> phys, false, resolve_writable_sphys, pb ); /* add triggers */ if ( !suspend_triggers && pb -> seed == NULL ) { pb -> pr . chain = chainUncommitted; VProdResolveAddTriggers ( & pb -> pr, stbl ); } }
LIB_EXPORT rc_t CC KDyldVLoadLib ( KDyld *self, KDylib **lib, const char *path, va_list args ) { rc_t rc; if ( lib == NULL ) rc = RC ( rcFS, rcDylib, rcLoading, rcParam, rcNull ); else { if ( self == NULL ) rc = RC ( rcFS, rcDylib, rcLoading, rcSelf, rcNull ); else if ( path == NULL || path [ 0 ] == 0 ) { WString pstr; CONST_WSTRING ( & pstr, "" ); rc = KDylibMake ( lib, & pstr ); if ( rc == 0 ) { rc = KDyldLoad ( self, * lib, NULL ); if ( rc == 0 ) return 0; free ( * lib ); } } else { uint32_t i = VectorStart ( & self -> search ); uint32_t end = i + VectorLength ( & self -> search ); if ( i == end ) { char name [ 4096 ]; int len = vsnprintf ( name, sizeof name, path, args ); if ( len < 0 || len >= sizeof name ) rc = RC ( rcFS, rcDylib, rcLoading, rcPath, rcExcessive ); else { WString pstr; wchar_t wname [ 4096 ]; size_t wsize = string_cvt_wchar_copy ( wname, sizeof wname, name, len ); WStringInit ( & pstr, wname, wsize * sizeof wname [ 0 ], string_len ( name, len ) ); rc = KDylibMake ( lib, & pstr ); if ( rc == 0 ) { rc = KDyldLoad ( self, * lib, wname ); if ( rc == 0 ) return 0; free ( * lib ); } } } else { for ( * lib = NULL; i < end; ++ i ) { const KDirectory *dir; va_list cpy; va_copy ( cpy, args ); dir = ( const void* ) VectorGet ( & self -> search, i ); rc = KDyldVTryLoadLib ( self, lib, dir, path, cpy ); va_end ( cpy ); if ( rc == 0 || GetRCState ( rc ) != rcNotFound ) return rc; } rc = RC ( rcFS, rcDylib, rcLoading, rcPath, rcNotFound ); } } * lib = NULL; } return rc; }
static rc_t populate_cursors ( VTable *dtbl, VCursor *dcurs, const VCursor *scurs, vtblcp_column_map *cm, const Vector *v, uint32_t *rd_filt ) { uint32_t end = VectorLength ( v ); uint32_t i = VectorStart ( v ); BSTree stype_tbl, rftype_tbl; const VSchema *schema; rc_t rc = VTableOpenSchema ( dtbl, & schema ); if ( rc != 0 ) { LOGERR ( klogInt, rc, "failed to open destination table schema" ); return rc; } /* populate sensitive type table */ rc = populate_stype_tbl ( & stype_tbl, schema ); if ( rc != 0 ) { VSchemaRelease ( schema ); return rc; } /* populate read filter type table */ rc = populate_rdfilt_tbl ( & rftype_tbl, schema ); if ( rc != 0 ) { BSTreeWhack ( & stype_tbl, stype_id_whack, NULL ); VSchemaRelease ( schema ); return rc; } for ( end += i, rc = 0, * rd_filt = 0; i < end; ++ i ) { VTypedecl td; char typedecl [ 128 ]; const char *spec = ( const void* ) VectorGet ( v, i ); /* request column in destination */ rc = VCursorAddColumn ( dcurs, & cm [ i ] . wr, spec ); if ( rc != 0 ) { PLOGERR ( klogErr, (klogErr, rc, "failed to add '$(spec)' to destination cursor", "spec=%s", spec )); break; } /* always retrieve data type */ rc = VCursorDatatype ( dcurs, cm [ i ] . wr, & td, NULL ); if ( rc != 0 ) { PLOGERR ( klogInt, (klogInt, rc, "failed to determine datatype of destination column '$(name)'", "name=%s", spec )); break; } /* mark column as sensitive or not */ rc = mark_type_sensitivity ( & stype_tbl, schema, & td, & cm [ i ] ); if ( rc != 0 ) break; /* if spec is already typed, request it in source */ if ( spec [ 0 ] == '(' ) { rc = VCursorAddColumn ( scurs, & cm [ i ] . rd, spec ); if ( rc != 0 ) { PLOGERR ( klogErr, (klogErr, rc, "failed to add '$(spec)' to source cursor", "spec=%s", spec )); break; } } else { rc = VTypedeclToText ( & td, schema, typedecl, sizeof typedecl ); if ( rc != 0 ) { PLOGERR ( klogInt, (klogInt, rc, "failed to print datatype of destination column '$(name)'", "name=%s", spec )); break; } rc = VCursorAddColumn ( scurs, & cm [ i ] . rd, "(%s)%s", typedecl, spec ); if ( rc != 0 ) { PLOGERR ( klogErr, (klogErr, rc, "failed to add '$(spec)' to source cursor", "spec=(%s)%s", typedecl, spec )); break; } } /* check if column is a read filter */ cm [ i ] . rd_filter = false; if ( ! cm [ i ] . sensitive ) { if ( BSTreeFind ( & rftype_tbl, & td, stype_id_cmp ) != NULL ) { if ( * rd_filt != 0 ) { rc = RC ( rcExe, rcColumn, rcOpening, rcColumn, rcExists ); PLOGERR ( klogInt, (klogInt, rc, "can't use column '$(name)' as read filter", "name=%s", spec )); break; } * rd_filt = cm [ i ] . rd; cm [ i ] . rd_filter = true; } } } BSTreeWhack ( & rftype_tbl, stype_id_whack, NULL ); BSTreeWhack ( & stype_tbl, stype_id_whack, NULL ); VSchemaRelease ( schema ); /* add read filter to input if not already there in some way */ if ( * rd_filt == 0 ) { rc = VCursorAddColumn ( scurs, rd_filt, "RD_FILTER" ); if ( rc != 0 && GetRCState ( rc ) == rcNotFound ) rc = 0; } return rc; }
rc_t execute_tbl_join( KDirectory * dir, const char * accession_path, const char * accession_short, join_stats * stats, const char * tbl_name, const struct temp_dir * temp_dir, struct temp_registry * registry, size_t cur_cache, size_t buf_size, uint32_t num_threads, bool show_progress, format_t fmt, const join_options * join_options ) { rc_t rc = 0; if ( show_progress ) rc = KOutMsg( "join :" ); if ( rc == 0 ) { uint64_t row_count = 0; rc = extract_sra_row_count( dir, accession_path, tbl_name, cur_cache, &row_count ); /* above */ if ( rc == 0 && row_count > 0 ) { bool name_column_present; if ( tbl_name == NULL ) rc = cmn_check_tbl_column( dir, accession_path, "NAME", &name_column_present ); else rc = cmn_check_db_column( dir, accession_path, tbl_name, "NAME", &name_column_present ); if ( rc == 0 ) { Vector threads; int64_t row = 1; uint32_t thread_id; uint64_t rows_per_thread; struct bg_progress * progress = NULL; struct join_options corrected_join_options; /* helper.h */ VectorInit( &threads, 0, num_threads ); corrected_join_options . rowid_as_name = name_column_present ? join_options -> rowid_as_name : true; corrected_join_options . skip_tech = join_options -> skip_tech; corrected_join_options . print_read_nr = join_options -> print_read_nr; corrected_join_options . print_name = name_column_present; corrected_join_options . min_read_len = join_options -> min_read_len; corrected_join_options . filter_bases = join_options -> filter_bases; corrected_join_options . terminate_on_invalid = join_options -> terminate_on_invalid; if ( row_count < ( num_threads * 100 ) ) { num_threads = 1; rows_per_thread = row_count; } else { rows_per_thread = ( row_count / num_threads ) + 1; } if ( show_progress ) rc = bg_progress_make( &progress, row_count, 0, 0 ); /* progress_thread.c */ for ( thread_id = 0; rc == 0 && thread_id < num_threads; ++thread_id ) { join_thread_data * jtd = calloc( 1, sizeof * jtd ); if ( jtd != NULL ) { jtd -> dir = dir; jtd -> accession_path = accession_path; jtd -> accession_short = accession_short; jtd -> tbl_name = tbl_name; jtd -> first_row = row; jtd -> row_count = rows_per_thread; jtd -> cur_cache = cur_cache; jtd -> buf_size = buf_size; jtd -> progress = progress; jtd -> registry = registry; jtd -> fmt = fmt; jtd -> join_options = &corrected_join_options; rc = make_joined_filename( temp_dir, jtd -> part_file, sizeof jtd -> part_file, accession_short, thread_id ); /* temp_dir.c */ if ( rc == 0 ) { rc = KThreadMake( &jtd -> thread, cmn_thread_func, jtd ); if ( rc != 0 ) ErrMsg( "KThreadMake( fastq/special #%d ) -> %R", thread_id, rc ); else { rc = VectorAppend( &threads, NULL, jtd ); if ( rc != 0 ) ErrMsg( "VectorAppend( sort-thread #%d ) -> %R", thread_id, rc ); } row += rows_per_thread; } } } { /* collect the threads, and add the join_stats */ uint32_t i, n = VectorLength( &threads ); for ( i = VectorStart( &threads ); i < n; ++i ) { join_thread_data * jtd = VectorGet( &threads, i ); if ( jtd != NULL ) { rc_t rc_thread; KThreadWait( jtd -> thread, &rc_thread ); if ( rc_thread != 0 ) rc = rc_thread; KThreadRelease( jtd -> thread ); add_join_stats( stats, &jtd -> stats ); free( jtd ); } } VectorWhack ( &threads, NULL, NULL ); } bg_progress_release( progress ); /* progress_thread.c ( ignores NULL )*/ } } } return rc; }