rc_t KU64IndexFindAll_v3( const KU64Index_v3* self, uint64_t offset, rc_t (CC*f)(uint64_t key, uint64_t key_size, int64_t id, uint64_t id_qty, void* data), void* data) { KU64Index_GrepData d; memset(&d, 0, sizeof(KU64Index_GrepData)); d.func = f; d.data = data; d.search.key = offset; BSTreeDoUntil(&self->tree, false, KU64Index_Grep, &d); return d.rc; }
rc_t foreach_ref_region( BSTree * regions, rc_t ( CC * on_region ) ( const char * name, const struct reference_range * range, void *data ), void *data ) { foreach_ref_region_func func; func.on_region = on_region; func.data = data; func.rc = 0; BSTreeDoUntil ( regions, false, foreach_ref_region_wrapper, &func ); return func.rc; }
rc_t KU64IndexFind_v3( const KU64Index_v3* self, uint64_t offset, uint64_t* key, uint64_t* key_size, int64_t* id, uint64_t* id_qty ) { KU64Index_GrepData d; memset(&d, 0, sizeof(KU64Index_GrepData)); d.search.key = offset; d.key = key; d.key_size = key_size; d.id = id; d.id_qty = id_qty; if( !BSTreeDoUntil(&self->tree, false, KU64Index_Grep, &d) ) { d.rc = RC(rcDB, rcIndex, rcSelecting, rcId, rcNotFound); } return d.rc; }
/* ---------------------------------------------------------------------- */ static rc_t extract () { rc_data data; bool failed; /* done sequentially - this will cause back ups on reads if both * containers and their contents are extracted * * we are also using a DoUntil approach that quits at the first failed * extract */ failed = BSTreeDoUntil (&options.pathtree, false, extract_one, &data); if (failed) LOGERR (klogErr, data.rc, "failure extracting a file"); return data.rc; }
static rc_t Load( SParam* param ) { rc_t rc = 0, rc1 = 0; BSTree slides, evidence; param->map_dir = NULL; param->asm_dir = NULL; param->output_dir = ( KDirectory * )param->input_dir; BSTreeInit( &slides ); BSTreeInit( &evidence ); rc = open_dir_or_tar( param->input_dir, ¶m->map_dir, param->map_path ); if ( rc == 0 ) { DirVisit_Data dv; dv.param = param; dv.tree = &slides; dv.dir = param->map_dir; rc = KDirectoryVisit( param->map_dir, true, DirVisitor, &dv, NULL ); if ( rc == 0 ) { if ( param->asm_path != NULL ) { rc_t rc2 = open_dir_or_tar( param->input_dir, ¶m->asm_dir, param->asm_path ); if ( rc2 == 0 ) { dv.tree = &evidence; dv.dir = param->asm_dir; rc = KDirectoryVisit( param->asm_dir, true, DirVisitor, &dv, NULL ); } } if ( rc == 0 ) { /* SHOULD HAVE A BSTreeEmpty FUNCTION OR SOMETHING... MAKE ONE HERE - WITH KNOWLEDGE THAT TREE IS NOT NULL: */ #ifndef BSTreeEmpty #define BSTreeEmpty( bst ) \ ( ( bst ) -> root == NULL ) #endif if ( BSTreeEmpty ( & slides ) && BSTreeEmpty ( & evidence ) ) rc = RC( rcExe, rcFile, rcReading, rcData, rcInsufficient ); else { /* CORRECTED SETTING OF "rc" IN "FGroupMAP_Validate" */ assert ( rc == 0 ); BSTreeForEach( &slides, false, FGroupMAP_Validate, &rc ); BSTreeForEach( &evidence, false, FGroupMAP_Validate, &rc ); } if ( rc == 0 ) { FGroupMAP_LoadData data; PLOGMSG( klogInfo, ( klogInfo, "file set validation complete", "severity=status" ) ); memset( &data, 0, sizeof( data ) ); data.rc = 0; data.param = param; data.reads = &slides; rc = DB_Init( param, &data.db ); if ( rc == 0 ) { BSTreeDoUntil( &slides, false, FGroupMAP_LoadReads, &data ); rc = data.rc; if ( rc == 0 ) { PLOGMSG( klogInfo, ( klogInfo, "MAP loaded", "severity=status" ) ); BSTreeDoUntil( &evidence, false, FGroupMAP_LoadEvidence, &data ); rc = data.rc; if ( rc == 0 ) PLOGMSG( klogInfo, ( klogInfo, "ASM loaded", "severity=status" ) ); } } rc1 = DB_Fini( param, &data.db, rc != 0 ); if ( rc == 0 ) rc = rc1; } } } /* copy the extra library ( file or recursive directory ) */ if ( rc == 0 && param->library != NULL ) { const KDirectory *lib_src; rc = open_dir_or_tar( param->input_dir, &lib_src, param->library ); if ( rc == 0 ) { rc = copy_library( param->input_dir, param->output_dir, param->library, param->out ); if ( rc == 0 ) STSMSG( 0, ( "extra lib copied" ) ); else LOGERR( klogErr, rc, "failed to copy extra library" ); KDirectoryRelease( lib_src ); } /* else { rc = copy_library( param->input_dir, param->output_dir, ".", param->out ); if ( rc == 0 ) STSMSG( 0, ( "extra lib copied" ) ); else LOGERR( klogErr, rc, "failed to copy extra library" ); } */ } KDirectoryRelease( param->map_dir ); KDirectoryRelease( param->asm_dir ); } BSTreeWhack( &slides, FGroupMAP_Whack, NULL ); BSTreeWhack( &evidence, FGroupMAP_Whack, NULL ); return rc; }
bool CC FGroupMAP_LoadEvidence( BSTNode *node, void *data ) { FGroupMAP* n = (FGroupMAP*)node; FGroupMAP_LoadData* d = (FGroupMAP_LoadData*)data; DEBUG_MSG(5, ("' started\n", FGroupKey_Validate(&n->key))); while( d->rc == 0 ) { if( (d->rc = CGLoaderFile_GetEvidenceIntervals(n->seq, d->db.ev_int)) == 0 ) { int64_t evint_rowid; if( n->align != NULL ) { d->rc = CGLoaderFile_GetEvidenceDnbs(n->align, d->db.ev_int->interval_id, d->db.ev_dnb); } else { d->db.ev_dnb->qty = 0; /***weird, but the easiest place to fix ***/ } /* interval written 1st than dnbs which uses interval as reference */ if( d->rc == 0 ) { d->rc = CGWriterEvdInt_Write(d->db.wev_int, d->db.ev_dnb, &evint_rowid); } if( d->rc == 0 && n->align != NULL ) { /* attach dnbs to reads */ uint16_t i; FGroupMAP_FindData found; found.key.type = cg_eFileType_READS; d->rc = CGLoaderFile_GetAssemblyId(n->align, &found.key.assembly_id); for(i = 0; d->rc == 0 && i < d->db.ev_dnb->qty; i++) { found.key.u.map.slide = d->db.ev_dnb->dnbs[i].slide; found.key.u.map.lane = d->db.ev_dnb->dnbs[i].lane; found.key.u.map.batch_file_number = &d->db.ev_dnb->dnbs[i].file_num_in_lane; if( BSTreeDoUntil(d->reads, false, FGroupMAP_FindRowId, &found) ) { d->rc = CGWriterEvdDnbs_SetSEQ(d->db.wev_dnb, i, found.rowid); } else { d->rc = RC(rcExe, rcFile, rcWriting, rcData, rcInconsistent); } } } if( d->rc == 0 && n->align != NULL ) { d->rc = CGWriterEvdDnbs_Write(d->db.wev_dnb, d->db.ev_int, evint_rowid); } } if( GetRCState(d->rc) == rcDone && GetRCObject(d->rc) == rcData ) { bool eof = false; d->rc = 0; if( n->align == NULL || ((d->rc = CGLoaderFile_IsEof(n->align, &eof)) == 0 && eof) ) { /* dnbs file EOF detected ok */ DEBUG_MSG(5, ("' done\n", FGroupKey_Validate(&n->key))); break; } else if( d->rc == 0 ) { /* not EOF */ d->rc = RC(rcExe, rcFile, rcReading, rcData, rcUnexpected); CGLoaderFile_LOG(n->align, klogErr, d->rc, "extra dnbs, possible that corresponding intervals file is truncated", NULL); } } d->rc = d->rc ? d->rc : Quitting(); } if( d->rc != 0 ) { CGLoaderFile_LOG(n->seq, klogErr, d->rc, NULL, NULL); CGLoaderFile_LOG(n->align, klogErr, d->rc, NULL, NULL); } FGroupMAP_CloseFiles(n); return d->rc != 0; }
/* BSTreePersist * write a b-tree to some storage location * * the b-tree is persisted by making between one and three passes * over its nodes, see description of "write" parameter. * * the first pass examines internal tree structure and invokes * a user-supplied function to determine overall size. * * the second pass persists the internal structure in a packed * format, using the user-supplied generic "write" function. * * the third pass invokes another user-supplied function to write * auxiliary node data to output. * * "num_writ" returns the number of bytes written as a result of * persisting the b-tree. this will be the actual bytes written * regardless of return status. * * "write" is a generic output streaming function used for all * operations. if NULL, then the function will exit after its * first pass with the number of bytes required in "num_writ". * * "aux" is a specialized function for streaming auxiliary node * data to output using the supplied "write" function. it is invoked * during the first pass with a NULL write function for gathering * size data, and during the third pass with a non-NULL write function. */ KLIB_EXTERN rc_t CC BSTreePersist ( const BSTree *bt, size_t *num_writ, PTWriteFunc write, void *write_param, PTAuxFunc aux, void *aux_param ) { PBSTreeData pb; if ( num_writ != NULL ) * num_writ = 0; if ( bt == NULL ) return RC ( rcCont, rcTree, rcPersisting, rcSelf, rcNull ); if ( aux == NULL ) return RC ( rcCont, rcTree, rcPersisting, rcFunction, rcNull ); pb . num_writ = 0; /* handle the trivial case */ if ( bt -> root == NULL ) { P_BSTree pt; if ( write == NULL ) { pb . num_writ = sizeof pt . num_nodes; pb . rc = 0; } else { pt . num_nodes = 0; pb . rc = ( * write ) ( write_param, & pt, sizeof pt . num_nodes, & pb . num_writ ); } } else { /* initialize callback param block */ pb . write = write; pb . write_param = write_param; pb . aux = aux; pb . aux_param = aux_param; pb . rc = 0; /* count nodes and tally data size */ pb . pt = NULL; pb . data_size = 0; pb . num_nodes = 0; BSTreeDoUntil ( bt, 0, PBSTreeGatherInfo, & pb ); if ( pb . rc == 0 ) { size_t pt_size; /* determine object size */ if ( pb . data_size <= 256 ) { pt_size = 1; pb . record = PBSTreeRecordU8; } else if ( pb . data_size <= 65536 ) { pt_size = 2; pb . record = PBSTreeRecordU16; } else { pt_size = 4; pb . record = PBSTreeRecordU32; } pt_size = sizeof * pb . pt - sizeof pb . pt -> data_idx + pb . num_nodes * pt_size; if ( write == NULL ) pb . num_writ = pt_size + pb . data_size; else { pb . pt = malloc ( pt_size ); if ( pb . pt != NULL ) { pb . pt -> num_nodes = pb . num_nodes; pb . pt -> data_size = ( uint32_t ) pb . data_size; /* record node offsets */ pb . data_size = 0; pb . num_nodes = 0; BSTreeDoUntil ( bt, 0, PBSTreeGatherInfo, & pb ); if ( pb . rc != 0 ) free ( pb . pt ); else { pb . rc = ( write ) ( write_param, pb . pt, pt_size, & pb . num_writ ); free ( pb . pt ); if ( pb . rc == 0 ) BSTreeDoUntil ( bt, 0, PBSTreeWriteNodes, & pb ); } } } } } if ( num_writ != NULL ) * num_writ = pb . num_writ; return pb . rc; }
KNSProxies * KNSManagerKNSProxiesMake ( struct KNSManager * mgr, const KConfig * kfg ) { rc_t rc = 0; int i = 0; int n = 2; typedef enum { eEnv, eKfg, } EType; EType type [ 2 ] = { eKfg, eEnv }; KNSProxies * self = calloc ( 1, sizeof * self ); if ( self == NULL ) return NULL; assert ( self ); BSTreeInit ( & self -> proxie_tree ); rc = KConfigReadBool ( kfg, "/http/proxy/enabled", & self -> http_proxy_enabled ); if ( rc != 0 ) { if ( GetRCState ( rc ) == rcNotFound ) rc = 0; else { KNSManagerSetHTTPProxyPath ( mgr, NULL ); assert ( self -> http_proxy_enabled == false ); } } else if ( ! self -> http_proxy_enabled ) return self; { bool proxy_only = false; rc_t rc = KConfigReadBool ( kfg, "/http/proxy/only", & proxy_only ); if ( rc == 0 && proxy_only ) self-> http_proxy_only = true; } { String * result = NULL; rc = KConfigReadString ( kfg, "/http/proxy/use", & result ); if ( rc == 0 ) { if ( StringCmp ( result, "env") ) { n = 1; type [ 0 ] = eEnv; } else if ( StringCmp ( result, "kfg") ) { n = 1; type [ 0 ] = eKfg; } else if ( StringCmp ( result, "none") ) { n = 0; } else if ( StringCmp ( result, "env,kfg") ) { n = 2; type [ 0 ] = eEnv; type [ 1 ] = eKfg; } else if ( StringCmp ( result, "kfg,env") ) { n = 2; type [ 0 ] = eKfg; type [ 1 ] = eEnv; } } RELEASE ( String, result ); } for ( i = 0; i < n; ++ i ) { switch ( type [ i ] ) { case eEnv: KNSProxiesHttpProxyInitFromEnv ( self ); break; case eKfg: KNSProxiesHttpProxyInitFromKfg ( self, kfg ); break; default: assert ( 0 ); break; } } BSTreeForEach ( & self -> proxie_tree, false, KNSProxiesBSTreeCount, & self -> http_proxies_cnt ); if ( self -> http_proxies_cnt > 0 ) { self -> http_proxies = calloc ( self -> http_proxies_cnt, sizeof * self -> http_proxies ); if ( self -> http_proxies == NULL ) return NULL; DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_PROXY ), ( "Will use %zu proxy spec%s%s\n", self -> http_proxies_cnt, self -> http_proxies_cnt > 1 ? "s" : "", self -> http_proxy_only ? "" : " and direct connection") ); } self -> tmpS = 0; n = self -> http_proxies_cnt; srand ( time ( NULL ) ); while ( n > 0 ) { self -> rand = rand () % n; self -> tmpI = 0; if ( ! BSTreeDoUntil ( & self -> proxie_tree, false, KNSProxiesBSTreeSetRand, self ) ) { BSTreeForEach ( & self -> proxie_tree, false, KNSProxiesBSTreeInit, self ) ; n = 0; } else { const BSTItem * item = ( BSTItem * ) self -> tmpB; self -> http_proxies [ self -> tmpS ++ ] = item -> proxy; BSTreeUnlink ( & self -> proxie_tree, self -> tmpB ); BSTItemWhack ( self -> tmpB, NULL ); self -> tmpB = NULL; -- n; } } /* BSTreeForEach ( & self -> proxie_tree, false, KNSProxiesBSTreeInit, self );*/ for ( self -> tmpS = 1; self -> tmpS < self ->http_proxies_cnt; ++ self -> tmpS ) { self -> http_proxies [ self -> tmpS - 1 ] -> next = self -> http_proxies [ self -> tmpS ]; } return self; }