rc_t KU64IndexFindAll_v3( const KU64Index_v3* self, uint64_t offset,
rc_t (CC*f)(uint64_t key, uint64_t key_size, int64_t id, uint64_t id_qty, void* data), void* data)
{
KU64Index_GrepData d;
memset(&d, 0, sizeof(KU64Index_GrepData));
d.func = f;
d.data = data;
d.search.key = offset;
BSTreeDoUntil(&self->tree, false, KU64Index_Grep, &d);
return d.rc;
}
rc_t foreach_ref_region( BSTree * regions,
rc_t ( CC * on_region ) ( const char * name, const struct reference_range * range, void *data ),
void *data )
{
foreach_ref_region_func func;
func.on_region = on_region;
func.data = data;
func.rc = 0;
BSTreeDoUntil ( regions, false, foreach_ref_region_wrapper, &func );
return func.rc;
}
rc_t KU64IndexFind_v3( const KU64Index_v3* self, uint64_t offset, uint64_t* key, uint64_t* key_size, int64_t* id, uint64_t* id_qty )
{
KU64Index_GrepData d;
memset(&d, 0, sizeof(KU64Index_GrepData));
d.search.key = offset;
d.key = key;
d.key_size = key_size;
d.id = id;
d.id_qty = id_qty;
if( !BSTreeDoUntil(&self->tree, false, KU64Index_Grep, &d) ) {
d.rc = RC(rcDB, rcIndex, rcSelecting, rcId, rcNotFound);
}
return d.rc;
}
/* ----------------------------------------------------------------------
*/
static
rc_t extract ()
{
rc_data data;
bool failed;
/* done sequentially - this will cause back ups on reads if both
* containers and their contents are extracted
*
* we are also using a DoUntil approach that quits at the first failed
* extract
*/
failed = BSTreeDoUntil (&options.pathtree, false, extract_one, &data);
if (failed)
LOGERR (klogErr, data.rc, "failure extracting a file");
return data.rc;
}
static rc_t Load( SParam* param )
{
rc_t rc = 0, rc1 = 0;
BSTree slides, evidence;
param->map_dir = NULL;
param->asm_dir = NULL;
param->output_dir = ( KDirectory * )param->input_dir;
BSTreeInit( &slides );
BSTreeInit( &evidence );
rc = open_dir_or_tar( param->input_dir, ¶m->map_dir, param->map_path );
if ( rc == 0 )
{
DirVisit_Data dv;
dv.param = param;
dv.tree = &slides;
dv.dir = param->map_dir;
rc = KDirectoryVisit( param->map_dir, true, DirVisitor, &dv, NULL );
if ( rc == 0 )
{
if ( param->asm_path != NULL )
{
rc_t rc2 = open_dir_or_tar( param->input_dir, ¶m->asm_dir, param->asm_path );
if ( rc2 == 0 )
{
dv.tree = &evidence;
dv.dir = param->asm_dir;
rc = KDirectoryVisit( param->asm_dir, true, DirVisitor, &dv, NULL );
}
}
if ( rc == 0 )
{
/* SHOULD HAVE A BSTreeEmpty FUNCTION OR SOMETHING...
MAKE ONE HERE - WITH KNOWLEDGE THAT TREE IS NOT NULL: */
#ifndef BSTreeEmpty
#define BSTreeEmpty( bst ) \
( ( bst ) -> root == NULL )
#endif
if ( BSTreeEmpty ( & slides ) && BSTreeEmpty ( & evidence ) )
rc = RC( rcExe, rcFile, rcReading, rcData, rcInsufficient );
else
{
/* CORRECTED SETTING OF "rc" IN "FGroupMAP_Validate" */
assert ( rc == 0 );
BSTreeForEach( &slides, false, FGroupMAP_Validate, &rc );
BSTreeForEach( &evidence, false, FGroupMAP_Validate, &rc );
}
if ( rc == 0 )
{
FGroupMAP_LoadData data;
PLOGMSG( klogInfo, ( klogInfo, "file set validation complete", "severity=status" ) );
memset( &data, 0, sizeof( data ) );
data.rc = 0;
data.param = param;
data.reads = &slides;
rc = DB_Init( param, &data.db );
if ( rc == 0 )
{
BSTreeDoUntil( &slides, false, FGroupMAP_LoadReads, &data );
rc = data.rc;
if ( rc == 0 )
{
PLOGMSG( klogInfo, ( klogInfo, "MAP loaded", "severity=status" ) );
BSTreeDoUntil( &evidence, false, FGroupMAP_LoadEvidence, &data );
rc = data.rc;
if ( rc == 0 )
PLOGMSG( klogInfo, ( klogInfo, "ASM loaded", "severity=status" ) );
}
}
rc1 = DB_Fini( param, &data.db, rc != 0 );
if ( rc == 0 )
rc = rc1;
}
}
}
/* copy the extra library ( file or recursive directory ) */
if ( rc == 0 && param->library != NULL )
{
const KDirectory *lib_src;
rc = open_dir_or_tar( param->input_dir, &lib_src, param->library );
if ( rc == 0 )
{
rc = copy_library( param->input_dir, param->output_dir,
param->library, param->out );
if ( rc == 0 )
STSMSG( 0, ( "extra lib copied" ) );
else
LOGERR( klogErr, rc, "failed to copy extra library" );
KDirectoryRelease( lib_src );
}
/*
else
{
rc = copy_library( param->input_dir, param->output_dir,
".", param->out );
if ( rc == 0 )
STSMSG( 0, ( "extra lib copied" ) );
else
LOGERR( klogErr, rc, "failed to copy extra library" );
}
*/
}
KDirectoryRelease( param->map_dir );
KDirectoryRelease( param->asm_dir );
}
BSTreeWhack( &slides, FGroupMAP_Whack, NULL );
BSTreeWhack( &evidence, FGroupMAP_Whack, NULL );
return rc;
}
bool CC FGroupMAP_LoadEvidence( BSTNode *node, void *data )
{
FGroupMAP* n = (FGroupMAP*)node;
FGroupMAP_LoadData* d = (FGroupMAP_LoadData*)data;
DEBUG_MSG(5, ("' started\n", FGroupKey_Validate(&n->key)));
while( d->rc == 0 ) {
if( (d->rc = CGLoaderFile_GetEvidenceIntervals(n->seq, d->db.ev_int)) == 0 ) {
int64_t evint_rowid;
if( n->align != NULL ) {
d->rc = CGLoaderFile_GetEvidenceDnbs(n->align, d->db.ev_int->interval_id, d->db.ev_dnb);
} else {
d->db.ev_dnb->qty = 0; /***weird, but the easiest place to fix ***/
}
/* interval written 1st than dnbs which uses interval as reference */
if( d->rc == 0 ) {
d->rc = CGWriterEvdInt_Write(d->db.wev_int, d->db.ev_dnb, &evint_rowid);
}
if( d->rc == 0 && n->align != NULL ) {
/* attach dnbs to reads */
uint16_t i;
FGroupMAP_FindData found;
found.key.type = cg_eFileType_READS;
d->rc = CGLoaderFile_GetAssemblyId(n->align, &found.key.assembly_id);
for(i = 0; d->rc == 0 && i < d->db.ev_dnb->qty; i++) {
found.key.u.map.slide = d->db.ev_dnb->dnbs[i].slide;
found.key.u.map.lane = d->db.ev_dnb->dnbs[i].lane;
found.key.u.map.batch_file_number = &d->db.ev_dnb->dnbs[i].file_num_in_lane;
if( BSTreeDoUntil(d->reads, false, FGroupMAP_FindRowId, &found) ) {
d->rc = CGWriterEvdDnbs_SetSEQ(d->db.wev_dnb, i, found.rowid);
} else {
d->rc = RC(rcExe, rcFile, rcWriting, rcData, rcInconsistent);
}
}
}
if( d->rc == 0 && n->align != NULL ) {
d->rc = CGWriterEvdDnbs_Write(d->db.wev_dnb, d->db.ev_int, evint_rowid);
}
}
if( GetRCState(d->rc) == rcDone && GetRCObject(d->rc) == rcData ) {
bool eof = false;
d->rc = 0;
if( n->align == NULL || ((d->rc = CGLoaderFile_IsEof(n->align, &eof)) == 0 && eof) ) {
/* dnbs file EOF detected ok */
DEBUG_MSG(5, ("' done\n", FGroupKey_Validate(&n->key)));
break;
} else if( d->rc == 0 ) {
/* not EOF */
d->rc = RC(rcExe, rcFile, rcReading, rcData, rcUnexpected);
CGLoaderFile_LOG(n->align, klogErr, d->rc,
"extra dnbs, possible that corresponding intervals file is truncated", NULL);
}
}
d->rc = d->rc ? d->rc : Quitting();
}
if( d->rc != 0 ) {
CGLoaderFile_LOG(n->seq, klogErr, d->rc, NULL, NULL);
CGLoaderFile_LOG(n->align, klogErr, d->rc, NULL, NULL);
}
FGroupMAP_CloseFiles(n);
return d->rc != 0;
}
/* BSTreePersist
* write a b-tree to some storage location
*
* the b-tree is persisted by making between one and three passes
* over its nodes, see description of "write" parameter.
*
* the first pass examines internal tree structure and invokes
* a user-supplied function to determine overall size.
*
* the second pass persists the internal structure in a packed
* format, using the user-supplied generic "write" function.
*
* the third pass invokes another user-supplied function to write
* auxiliary node data to output.
*
* "num_writ" returns the number of bytes written as a result of
* persisting the b-tree. this will be the actual bytes written
* regardless of return status.
*
* "write" is a generic output streaming function used for all
* operations. if NULL, then the function will exit after its
* first pass with the number of bytes required in "num_writ".
*
* "aux" is a specialized function for streaming auxiliary node
* data to output using the supplied "write" function. it is invoked
* during the first pass with a NULL write function for gathering
* size data, and during the third pass with a non-NULL write function.
*/
KLIB_EXTERN rc_t CC BSTreePersist ( const BSTree *bt, size_t *num_writ,
PTWriteFunc write, void *write_param, PTAuxFunc aux, void *aux_param )
{
PBSTreeData pb;
if ( num_writ != NULL )
* num_writ = 0;
if ( bt == NULL )
return RC ( rcCont, rcTree, rcPersisting, rcSelf, rcNull );
if ( aux == NULL )
return RC ( rcCont, rcTree, rcPersisting, rcFunction, rcNull );
pb . num_writ = 0;
/* handle the trivial case */
if ( bt -> root == NULL )
{
P_BSTree pt;
if ( write == NULL )
{
pb . num_writ = sizeof pt . num_nodes;
pb . rc = 0;
}
else
{
pt . num_nodes = 0;
pb . rc = ( * write ) ( write_param,
& pt, sizeof pt . num_nodes, & pb . num_writ );
}
}
else
{
/* initialize callback param block */
pb . write = write;
pb . write_param = write_param;
pb . aux = aux;
pb . aux_param = aux_param;
pb . rc = 0;
/* count nodes and tally data size */
pb . pt = NULL;
pb . data_size = 0;
pb . num_nodes = 0;
BSTreeDoUntil ( bt, 0, PBSTreeGatherInfo, & pb );
if ( pb . rc == 0 )
{
size_t pt_size;
/* determine object size */
if ( pb . data_size <= 256 )
{
pt_size = 1;
pb . record = PBSTreeRecordU8;
}
else if ( pb . data_size <= 65536 )
{
pt_size = 2;
pb . record = PBSTreeRecordU16;
}
else
{
pt_size = 4;
pb . record = PBSTreeRecordU32;
}
pt_size = sizeof * pb . pt - sizeof pb . pt -> data_idx +
pb . num_nodes * pt_size;
if ( write == NULL )
pb . num_writ = pt_size + pb . data_size;
else
{
pb . pt = malloc ( pt_size );
if ( pb . pt != NULL )
{
pb . pt -> num_nodes = pb . num_nodes;
pb . pt -> data_size = ( uint32_t ) pb . data_size;
/* record node offsets */
pb . data_size = 0;
pb . num_nodes = 0;
BSTreeDoUntil ( bt, 0, PBSTreeGatherInfo, & pb );
if ( pb . rc != 0 )
free ( pb . pt );
else
{
pb . rc = ( write ) ( write_param,
pb . pt, pt_size, & pb . num_writ );
free ( pb . pt );
if ( pb . rc == 0 )
BSTreeDoUntil ( bt, 0, PBSTreeWriteNodes, & pb );
}
}
}
}
}
if ( num_writ != NULL )
* num_writ = pb . num_writ;
return pb . rc;
}
KNSProxies * KNSManagerKNSProxiesMake ( struct KNSManager * mgr,
const KConfig * kfg )
{
rc_t rc = 0;
int i = 0;
int n = 2;
typedef enum {
eEnv,
eKfg,
} EType;
EType type [ 2 ] = { eKfg, eEnv };
KNSProxies * self = calloc ( 1, sizeof * self );
if ( self == NULL )
return NULL;
assert ( self );
BSTreeInit ( & self -> proxie_tree );
rc = KConfigReadBool
( kfg, "/http/proxy/enabled", & self -> http_proxy_enabled );
if ( rc != 0 ) {
if ( GetRCState ( rc ) == rcNotFound )
rc = 0;
else {
KNSManagerSetHTTPProxyPath ( mgr, NULL );
assert ( self -> http_proxy_enabled == false );
}
}
else if ( ! self -> http_proxy_enabled )
return self;
{
bool proxy_only = false;
rc_t rc = KConfigReadBool ( kfg, "/http/proxy/only", & proxy_only );
if ( rc == 0 && proxy_only )
self-> http_proxy_only = true;
}
{
String * result = NULL;
rc = KConfigReadString ( kfg, "/http/proxy/use", & result );
if ( rc == 0 ) {
if ( StringCmp ( result, "env") ) {
n = 1;
type [ 0 ] = eEnv;
} else if ( StringCmp ( result, "kfg") ) {
n = 1;
type [ 0 ] = eKfg;
} else if ( StringCmp ( result, "none") ) {
n = 0;
} else if ( StringCmp ( result, "env,kfg") ) {
n = 2;
type [ 0 ] = eEnv;
type [ 1 ] = eKfg;
} else if ( StringCmp ( result, "kfg,env") ) {
n = 2;
type [ 0 ] = eKfg;
type [ 1 ] = eEnv;
}
}
RELEASE ( String, result );
}
for ( i = 0; i < n; ++ i ) {
switch ( type [ i ] ) {
case eEnv:
KNSProxiesHttpProxyInitFromEnv ( self );
break;
case eKfg:
KNSProxiesHttpProxyInitFromKfg ( self, kfg );
break;
default:
assert ( 0 );
break;
}
}
BSTreeForEach ( & self -> proxie_tree, false, KNSProxiesBSTreeCount,
& self -> http_proxies_cnt );
if ( self -> http_proxies_cnt > 0 ) {
self -> http_proxies = calloc ( self -> http_proxies_cnt,
sizeof * self -> http_proxies );
if ( self -> http_proxies == NULL )
return NULL;
DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_PROXY ),
( "Will use %zu proxy spec%s%s\n", self -> http_proxies_cnt,
self -> http_proxies_cnt > 1 ? "s" : "",
self -> http_proxy_only ? "" : " and direct connection") );
}
self -> tmpS = 0;
n = self -> http_proxies_cnt;
srand ( time ( NULL ) );
while ( n > 0 ) {
self -> rand = rand () % n;
self -> tmpI = 0;
if ( ! BSTreeDoUntil ( & self -> proxie_tree, false,
KNSProxiesBSTreeSetRand, self ) )
{
BSTreeForEach ( & self -> proxie_tree, false,
KNSProxiesBSTreeInit, self ) ;
n = 0;
}
else {
const BSTItem * item = ( BSTItem * ) self -> tmpB;
self -> http_proxies [ self -> tmpS ++ ] = item -> proxy;
BSTreeUnlink ( & self -> proxie_tree, self -> tmpB );
BSTItemWhack ( self -> tmpB, NULL );
self -> tmpB = NULL;
-- n;
}
}
/* BSTreeForEach ( & self -> proxie_tree, false, KNSProxiesBSTreeInit, self );*/
for ( self -> tmpS = 1; self -> tmpS < self ->http_proxies_cnt;
++ self -> tmpS )
{
self -> http_proxies [ self -> tmpS - 1 ] -> next
= self -> http_proxies [ self -> tmpS ];
}
return self;
}
