/* ----------------------------------------------------------------------
* KTocEntryNewDirectory
*
* [RET] rc_t 0 for success; anything else for a failure
* see itf/klib/rc.h for general details
* [OUT] KTocEntry ** new_entry where to put a pointer to the new TOC Entry
* [IN] const char * name name of the directory (not assumed to be ASCIZ)
* [IN] size_t name_size length of name
* [IN] uint32_t access unix/posix style permission flags
*/
rc_t KTocEntryNewDirectory ( KTocEntry ** new_entry,
const char * name,
size_t name_size,
KTime_t mtime,
uint32_t access )
{
rc_t rc;
/* -----
* get rid of trailing '/' characters in a file name
*/
while (name[name_size-1] == '/')
--name_size;
if ((rc = KTocEntryNew (new_entry, name, name_size, mtime, access,
sizeof(KTocEntry)
- sizeof(union KTocEntryUnion)
+ sizeof(struct KTocEntryDir)))
!= 0)
{
return rc;
}
(*new_entry)->type = ktocentrytype_dir;
BSTreeInit(&(*new_entry)->u.dir.tree); /* start with an empty tree */
return 0;
}
static
rc_t populate_rdfilt_tbl ( BSTree *rftype_tbl, const VSchema *schema )
{
rc_t rc;
uint32_t i;
const char *rftypes [] =
{
"NCBI:SRA:read_filter",
"INSDC:SRA:read_filter"
};
BSTreeInit ( rftype_tbl );
for ( rc = 0, i = 0; i < sizeof rftypes / sizeof rftypes [ 0 ]; ++ i )
{
VTypedecl td;
const char *decl = rftypes [ i ];
rc = VSchemaResolveTypedecl ( schema, & td, decl );
if ( rc == 0 )
{
BSTNode *exist;
stype_id *n = malloc ( sizeof * n );
if ( n == NULL )
{
rc = RC ( rcExe, rcNode, rcAllocating, rcMemory, rcExhausted );
LOGERR ( klogInt, rc, "failed to record read_filter data type" );
break;
}
n -> redact_value = NULL;
n -> type_id = td . type_id;
n -> elem_size = 8;
rc = BSTreeInsertUnique ( rftype_tbl, & n -> n, & exist, stype_id_sort );
if ( rc != 0 )
{
free ( n );
if ( GetRCState ( rc ) != rcExists )
{
LOGERR ( klogInt, rc, "failed to record read_filter data type" );
break;
}
rc = 0;
}
}
else if ( GetRCState ( rc ) == rcNotFound )
{
rc = 0;
}
else
{
break;
}
}
return rc;
}
static
rc_t VProdResolveColExpr ( const VProdResolve *self, VProduction **out,
VFormatdecl *fd, const SSymExpr *x, bool casting )
{
rc_t rc;
const SNameOverload *sname;
const KSymbol *sym = x -> _sym;
BSTree ordered;
uint32_t i, count;
SColumnBestFit buff [ 16 ], * nodes = buff;
/* fail if "fd" has a format */
if ( fd -> fmt != 0 )
{
PLOGMSG ( klogWarn, ( klogWarn, "illegal cast of column '$(name)'"
, "name=%.*s"
, ( int ) sym -> name . size
, sym -> name . addr ));
return 0;
}
/* allocate nodes for indexing columns */
sname = sym -> u . obj;
count = VectorLength ( & sname -> items );
if ( count > sizeof buff / sizeof buff [ 0 ] )
{
nodes = malloc ( sizeof * nodes * count );
if ( nodes == NULL )
return RC ( rcVDB, rcProduction, rcResolving, rcMemory, rcExhausted );
}
/* insert columns into ordered tree */
BSTreeInit ( & ordered );
for ( i = VectorStart ( & sname -> items ), count += i; i < count; ++ i )
{
/* get SColumn */
nodes [ i ] . scol = ( const void* ) VectorGet ( & sname -> items, i );
/* perform type cast and measure distance */
if ( casting ?
VTypedeclCommonAncestor ( & nodes [ i ] . scol -> td, self -> schema,
& fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) :
VTypedeclToTypedecl ( & nodes [ i ] . scol -> td, self -> schema,
& fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) )
{
BSTreeInsert ( & ordered, & nodes [ i ] . n, order_column );
}
}
/* try to resolve each in order */
rc = VProdResolveBestColumn ( self, out, & ordered, x -> alt );
if ( nodes != buff )
free ( nodes );
return rc;
}
/* Make - PRIVATE
*/
rc_t VTableMake ( VTable **tblp,
const VDBManager *mgr, const VDatabase *db, const VSchema *schema )
{
rc_t rc;
/* create a structure */
VTable *tbl = calloc ( sizeof * tbl, 1 );
if ( tbl == NULL )
rc = RC ( rcVDB, rcTable, rcConstructing, rcMemory, rcExhausted );
else
{
/* since the object has been zeroed, Whack will work properly */
rc = VSchemaMake ( ( VSchema** ) & tbl -> schema, schema );
if ( rc == 0 )
{
tbl -> mgr = VDBManagerAttach ( mgr );
tbl -> db = VDatabaseAttach ( db );
BSTreeInit ( & tbl -> read_col_cache );
BSTreeInit ( & tbl -> write_col_cache );
KRefcountInit ( & tbl -> refcount, 1, "VTable", "make", "vtbl" );
tbl -> cmode = ( uint8_t ) kcmOpen;
tbl -> checksum = ( uint8_t ) kcsNone;
rc = VLinkerMake ( & tbl -> linker, mgr -> linker, mgr -> linker -> dl );
if ( rc == 0 )
{
* tblp = tbl;
return 0;
}
}
VTableWhack ( tbl );
}
* tblp = NULL;
return rc;
}
rc_t make_statistic( statistic *self,
uint32_t gc_window,
bool has_alignments )
{
memset( self, 0, sizeof *self );
BSTreeInit( &self->spotgroups );
self->last_used_spotgroup = NULL;
self->gc_window = gc_window > N_GC_VALUES ? N_GC_VALUES : gc_window;
return 0;
}
/* ----------------------------------------------------------------------
*/
static
rc_t build_tree ()
{
rc_t rc;
rc_data data;
bool did_until = false;
BSTreeInit (&options.pathtree);
VectorDoUntil (options.pathpath, false, build_tree_add, &data);
}
LIB_EXPORT
rc_t CC
XFSOwpInit ( struct XFSOwp * self )
{
if ( self == NULL ) {
return XFS_RC ( rcNull );
}
BSTreeInit ( ( BSTree * ) self );
return 0;
} /* XFSOwpInit () */
rc_t make_ref_exclude( ref_exclude *exclude, KDirectory *dir,
const char * path, bool info )
{
rc_t rc;
BSTreeInit( &exclude->ref_nodes );
BSTreeInit( &exclude->translations );
exclude->last_used_ref_node = NULL;
exclude->info = info;
rc = VDBManagerMakeUpdate ( &exclude->mgr, dir );
if ( rc != 0 )
{
LogErr( klogInt, rc, "VDBManagerMakeUpdate() in make_ref_exclude() failed\n" );
}
else
{
insert_trans_nodes( &exclude->translations );
exclude->path = string_dup_measure ( path, NULL );
}
return rc;
}
static
rc_t RefPosMake ( RefPos **objp, const VTable *tbl, const VCursor *native_curs )
{
rc_t rc;
/* create the object */
RefPos *obj = malloc ( sizeof * obj );
if ( obj == NULL ) {
rc = RC ( rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted );
} else {
obj->curs=NULL;
BSTreeInit(&obj->tr_range);
/* open the reference table cursor*/
if( (rc = AlignRefTableCursor(tbl, native_curs, &obj->curs, NULL)) == 0 ) {
uint32_t itmp;
if( (rc = VCursorAddColumn(obj->curs, &itmp, "(U32)MAX_SEQ_LEN")) == 0 || GetRCState(rc) == rcExists) {
const void *base;
uint32_t row_len;
rc = VCursorCellDataDirect(obj->curs, 1, itmp, NULL, &base, NULL, &row_len);
if(rc == 0) {
assert(row_len == 1);
memcpy(&obj->max_seq_len, base, 4);
}
}
if( GetRCObject(rc) == rcColumn && GetRCState(rc) == rcNotFound ) {
/*** no MAX_SEQ_LEN means that REF_POS==REF_START **/
VCursorRelease(obj->curs);
obj->curs = NULL;
obj->max_seq_len = 0;
obj->name_range_idx = 0;
obj->name_idx = 0;
rc = 0;
} else if( rc == 0 ) {
/* add columns to cursor */
rc = VCursorAddColumn(obj->curs, &obj->name_idx, "(utf8)NAME");
if(rc == 0 || GetRCState(rc) == rcExists)
rc = VCursorAddColumn(obj->curs, &obj->name_range_idx, "NAME_RANGE");
if(GetRCState(rc) == rcExists)
rc = 0;
}
}
if( rc == 0 ) {
*objp = obj;
} else {
VCursorRelease(obj->curs);
free(obj);
}
}
return rc;
}
static rc_t XmlMetaInit(rc_t rc, XmlMeta* meta,
const AppCtx* ctx, const char* xml)
{
const KFile* f = NULL;
const KXMLDoc* doc = NULL;
const KXMLNodeset* ns = NULL;
const KXMLNode* node = NULL;
const char path[] = "/Run";
memset(meta, 0, sizeof *meta);
if (rc) {
return rc;
}
if (rc == 0) {
rc = KDirectoryOpenFileRead(ctx->dir, &f, xml);
DISP_RC(rc, xml);
}
if (rc == 0) {
rc = KXMLMgrMakeDocRead(ctx->x_mgr, &doc, f);
DISP_RC(rc, xml);
}
if (rc == 0) {
rc = KXMLDocOpenNodesetRead(doc, &ns, path);
DISP_RC(rc, path);
}
if (rc == 0) {
rc = KXMLNodesetGetNodeRead(ns, &node, 0);
DISP_RC(rc, path);
}
if (rc == 0) {
rc = ParseSraMetaNode(node, path, NULL, &meta->root);
}
if (rc == 0 && !meta->root.found) {
meta->tr = calloc(1, sizeof *meta->tr);
if (meta->tr == NULL) {
rc = RC(rcExe, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
else {
BSTreeInit(meta->tr);
rc = XmlMetaInitMembers(meta->tr, node);
}
}
RELEASE(KXMLNode, node);
RELEASE(KXMLNodeset, ns);
RELEASE(KXMLDoc, doc);
RELEASE(KFile, f);
return rc;
}
/* ----------------------------------------------------------------------
*/
static
rc_t build_tree_then_run ()
{
rc_data data;
data.rc = 0;
BSTreeInit (&options.pathtree);
VectorForEach (&options.pathvpath, false, handle_path, &data);
if (data.rc == 0)
data.rc = extract();
BSTreeWhack (&options.pathtree, extnode_whack, NULL);
return data.rc;
}
rc_t KU64IndexOpen_v3(KU64Index_v3* self, struct KMMap const *mm, bool byteswap)
{
rc_t rc = 0;
const char* maddr;
size_t msize;
PBSTree* ptree = NULL;
self->rc = 0;
BSTreeInit(&self->tree);
/* when opened for create, there will be no existing index */
if( mm == NULL ) {
return 0;
}
/* open the prior index in persisted mode */
rc = KMMapAddrRead(mm, (const void**)&maddr);
if( rc != 0 ) {
return rc;
}
rc = KMMapSize(mm, &msize);
if( rc != 0 ) {
return rc;
}
if( msize <= sizeof(struct KIndexFileHeader_v3) ) {
return 0;
}
rc = PBSTreeMake(&ptree, (const void**)(maddr + sizeof(struct KIndexFileHeader_v3)),
msize - sizeof(struct KIndexFileHeader_v3), byteswap);
if( rc != 0 ) {
return rc;
}
PBSTreeDoUntil(ptree, false, KU64Index_UnrollPersisted, self);
rc = self->rc;
PBSTreeWhack(ptree);
if( rc != 0 ) {
KU64IndexWhack_v3(self);
}
return rc;
}
struct skiplist * skiplist_make( BSTree * regions )
{
struct skiplist *res = calloc( 1, sizeof *res );
if ( res != NULL )
{
BSTreeInit( &(res->nodes) );
res->current = NULL;
res->node_count = 0;
/* walk the given regions-tree to generate the skip-list */
BSTreeForEach ( regions, false, visit_region_node_for_skiplist, res );
if ( res->node_count == 0 )
{
skiplist_release( res );
res = NULL;
}
}
return res;
}
LIB_EXPORT rc_t CC VTableListSeededWritableColumns ( VTable *self,
KNamelist **names, const KNamelist *seed )
{
rc_t rc;
if ( names == NULL )
rc = RC ( rcVDB, rcTable, rcListing, rcParam, rcNull );
else
{
* names = NULL;
if ( self == NULL )
rc = RC ( rcVDB, rcTable, rcListing, rcSelf, rcNull );
else if ( seed == NULL )
rc = RC ( rcVDB, rcTable, rcListing, rcParam, rcNull );
else
{
BSTree cache;
BSTreeInit ( & cache );
rc = 0;
if ( ! self -> read_only )
{
VCursor *curs;
rc = VTableCreateCursorWriteInt ( self, & curs, kcmInsert, false );
if ( rc == 0 )
{
/* let this private VCursor-function list the columns */
rc = VCursorListSeededWritableColumns ( curs, & cache, seed );
VCursorRelease ( curs );
}
}
if ( rc == 0 )
rc = make_column_namelist ( & cache, names );
BSTreeWhack ( & cache, VColumnRefWhack, NULL );
}
}
return rc;
}
rc_t make_statistic( statistic *data,
uint32_t gc_window,
bool ignore_mismatches )
{
rc_t rc = 0;
memset( data, 0, sizeof *data );
BSTreeInit( &data->spotgroups );
data->last_used_spotgroup = NULL;
data->ignore_mismatches = ignore_mismatches;
data->gc_window = gc_window;
if ( data->gc_window >= N_GC_VALUES )
{
data->gc_window = ( N_GC_VALUES - 1 );
}
if ( rc == 0 )
{
rc = make_vector( (void**)&data->case_vector, &data->case_vector_len, 512 );
}
return rc;
}
/* Whack
*/
rc_t KColumnIdx1Whack ( KColumnIdx1 *self )
{
rc_t rc = KFileRelease ( self -> fidx );
if ( rc == 0 )
{
self -> fidx = NULL;
rc = KFileRelease ( self -> f );
if ( rc == 0 )
{
self -> f = NULL;
#if USE_BSTREE_IN_COLUMN_IDX1
BSTreeWhack ( & self -> bst, KColumnIdx1NodeWhack, NULL );
BSTreeInit ( & self -> bst );
#else
free ( self -> data );
self -> data = NULL;
#endif
}
}
return rc;
}
static rc_t print_headers_by_recalculating( const samdump_opts * opts, input_files * ifs )
{
rc_t rc = KOutMsg( "@HD\tVN:1.3\n" );
if ( rc == 0 )
{
BSTree tree;
/* collect sequenc-id's and names and their lengths, unique by sequence-id */
BSTreeInit( &tree );
rc = build_seq_id_tree( &tree, ifs );
if ( rc == 0 )
{
hdr_print_ctx hctx;
hctx.rc = 0;
hctx.use_seq_id = opts->use_seqid_as_refname;
/* ptrint it */
BSTreeForEach( &tree, false, print_header_callback, &hctx );
rc = hctx.rc;
}
free_seq_id_tree( &tree );
/* collect spot-groups ( unique ) */
if ( rc == 0 )
{
VNamelist * spotgroups;
rc = VNamelistMake( &spotgroups, 10 );
if ( rc == 0 )
{
bool from_stats = false;
rc = extract_spotgroups( spotgroups, ifs, from_stats );
if ( rc == 0 )
rc = print_spotgroups( spotgroups, from_stats );
else
rc = 0; /* otherwise the tool would be not able to handle something that has no headers */
VNamelistRelease( spotgroups );
}
}
}
return rc;
}
LIB_EXPORT rc_t CC KDyldRegisterBuiltin ( const char *c_name, void *addr )
{
String name;
if (builtins == NULL) {
rc_t rc = 0;
BSTree *scope;
builtins = malloc ( sizeof * builtins );
if ( builtins == NULL ) {
return RC ( rcFS, rcDylib, rcRegistering, rcMemory, rcExhausted );
}
rc = KSymTableInit ( builtins, NULL );
if (rc == 0) {
scope = malloc ( sizeof * scope );
if ( scope == NULL ) {
rc = RC ( rcFS, rcDylib, rcRegistering, rcMemory, rcExhausted );
} else {
BSTreeInit ( scope );
rc = KSymTablePushScope ( builtins, scope );
if (rc != 0) {
free ( scope );
}
}
}
if ( rc != 0 ) {
free ( builtins );
builtins = NULL;
return rc;
}
}
StringInitCString ( & name, c_name );
return KSymTableCreateSymbol ( builtins, NULL, & name, 0, addr );
}
static rc_t Load( SParam* param )
{
rc_t rc = 0, rc1 = 0;
BSTree slides, evidence;
param->map_dir = NULL;
param->asm_dir = NULL;
param->output_dir = ( KDirectory * )param->input_dir;
BSTreeInit( &slides );
BSTreeInit( &evidence );
rc = open_dir_or_tar( param->input_dir, ¶m->map_dir, param->map_path );
if ( rc == 0 )
{
DirVisit_Data dv;
dv.param = param;
dv.tree = &slides;
dv.dir = param->map_dir;
rc = KDirectoryVisit( param->map_dir, true, DirVisitor, &dv, NULL );
if ( rc == 0 )
{
if ( param->asm_path != NULL )
{
rc_t rc2 = open_dir_or_tar( param->input_dir, ¶m->asm_dir, param->asm_path );
if ( rc2 == 0 )
{
dv.tree = &evidence;
dv.dir = param->asm_dir;
rc = KDirectoryVisit( param->asm_dir, true, DirVisitor, &dv, NULL );
}
}
if ( rc == 0 )
{
/* SHOULD HAVE A BSTreeEmpty FUNCTION OR SOMETHING...
MAKE ONE HERE - WITH KNOWLEDGE THAT TREE IS NOT NULL: */
#ifndef BSTreeEmpty
#define BSTreeEmpty( bst ) \
( ( bst ) -> root == NULL )
#endif
if ( BSTreeEmpty ( & slides ) && BSTreeEmpty ( & evidence ) )
rc = RC( rcExe, rcFile, rcReading, rcData, rcInsufficient );
else
{
/* CORRECTED SETTING OF "rc" IN "FGroupMAP_Validate" */
assert ( rc == 0 );
BSTreeForEach( &slides, false, FGroupMAP_Validate, &rc );
BSTreeForEach( &evidence, false, FGroupMAP_Validate, &rc );
}
if ( rc == 0 )
{
FGroupMAP_LoadData data;
PLOGMSG( klogInfo, ( klogInfo, "file set validation complete", "severity=status" ) );
memset( &data, 0, sizeof( data ) );
data.rc = 0;
data.param = param;
data.reads = &slides;
rc = DB_Init( param, &data.db );
if ( rc == 0 )
{
BSTreeDoUntil( &slides, false, FGroupMAP_LoadReads, &data );
rc = data.rc;
if ( rc == 0 )
{
PLOGMSG( klogInfo, ( klogInfo, "MAP loaded", "severity=status" ) );
BSTreeDoUntil( &evidence, false, FGroupMAP_LoadEvidence, &data );
rc = data.rc;
if ( rc == 0 )
PLOGMSG( klogInfo, ( klogInfo, "ASM loaded", "severity=status" ) );
}
}
rc1 = DB_Fini( param, &data.db, rc != 0 );
if ( rc == 0 )
rc = rc1;
}
}
}
/* copy the extra library ( file or recursive directory ) */
if ( rc == 0 && param->library != NULL )
{
const KDirectory *lib_src;
rc = open_dir_or_tar( param->input_dir, &lib_src, param->library );
if ( rc == 0 )
{
rc = copy_library( param->input_dir, param->output_dir,
param->library, param->out );
if ( rc == 0 )
STSMSG( 0, ( "extra lib copied" ) );
else
LOGERR( klogErr, rc, "failed to copy extra library" );
KDirectoryRelease( lib_src );
}
/*
else
{
rc = copy_library( param->input_dir, param->output_dir,
".", param->out );
if ( rc == 0 )
STSMSG( 0, ( "extra lib copied" ) );
else
LOGERR( klogErr, rc, "failed to copy extra library" );
}
*/
}
KDirectoryRelease( param->map_dir );
KDirectoryRelease( param->asm_dir );
}
BSTreeWhack( &slides, FGroupMAP_Whack, NULL );
BSTreeWhack( &evidence, FGroupMAP_Whack, NULL );
return rc;
}
rc_t CC KMain ( int argc, char *argv [] )
{
Args * args;
rc_t rc;
rc = ArgsMakeAndHandle (&args, argc, argv, 1, Options, sizeof Options / sizeof (OptDef));
if (rc == 0)
{
uint32_t pcount;
const char * pc;
const char * directory;
const char * archive;
do
{
op_mode mode;
uint32_t ix;
BSTreeInit (&pnames);
rc = ArgsParamCount (args, &pcount);
if (rc)
break;
for (ix = 0; ix < pcount; ++ix)
{
rc = ArgsParamValue (args, ix, (const void **)&pc);
if (rc)
break;
rc = pnamesInsert (pc);
if (rc)
{
PLOGERR (klogInt, (klogInt, rc, "failure to process filter parameter [%s]", pc));
break;
}
}
if (rc)
break;
rc = ArgsOptionCount (args, OPTION_ALIGN, &pcount);
if (rc)
break;
if (pcount == 0)
pc = NULL;
else
{
rc = ArgsOptionValue (args, OPTION_ALIGN, 0, (const void **)&pc);
if (rc)
break;
}
alignment = get_alignment (pc);
if (alignment == sraAlignInvalid)
{
rc = RC (rcExe, rcArgv, rcParsing, rcParam, rcInvalid);
PLOGERR (klogFatal, (klogFatal, rc,
"Parameter for alignment [$(A)] is invalid: must be a power of two bytes",
PLOG_S(A), pc));
break;
}
rc = ArgsOptionCount (args, OPTION_LONGLIST, &pcount);
if (rc)
break;
long_list = (pcount != 0);
rc = ArgsOptionCount (args, OPTION_FORCE, &pcount);
if (rc)
break;
force = (pcount != 0);
mode = OM_NONE;
rc = ArgsOptionCount (args, OPTION_DIRECTORY, &pcount);
if (rc)
break;
if (pcount)
{
rc = ArgsOptionValue (args, OPTION_DIRECTORY, 0, (const void **)&directory);
if (rc)
break;
}
else
directory = NULL;
rc = ArgsOptionCount (args, OPTION_CREATE, &pcount);
if (rc)
break;
if (pcount)
{
rc = ArgsOptionValue (args, OPTION_CREATE, 0, (const void **)&archive);
if (rc)
break;
mode = OM_CREATE;
}
rc = ArgsOptionCount (args, OPTION_TEST, &pcount);
if (rc)
break;
if (pcount)
{
if (mode == OM_NONE)
{
mode = OM_TEST;
rc = ArgsOptionValue (args, OPTION_TEST, 0, (const void **)&archive);
if (rc)
break;
}
else
mode = OM_ERROR;
}
rc = ArgsOptionCount (args, OPTION_LIST, &pcount);
if (rc)
break;
if (pcount)
{
if (mode == OM_NONE)
{
mode = OM_TEST;
rc = ArgsOptionValue (args, OPTION_LIST, 0, (const void **)&archive);
if (rc)
break;
}
else
mode = OM_ERROR;
}
rc = ArgsOptionCount (args, OPTION_EXTRACT, &pcount);
if (rc)
break;
if (pcount)
{
if (mode == OM_NONE)
{
mode = OM_EXTRACT;
rc = ArgsOptionValue (args, OPTION_EXTRACT, 0, (const void **)&archive);
if (rc)
break;
}
else
mode = OM_ERROR;
}
if (mode == OM_NONE)
{
MiniUsage(args);
rc = RC(rcApp, rcArgv, rcParsing, rcParam, rcInsufficient);
break;
}
/* -----
* Mount the native filesystem as root
*/
rc = KDirectoryNativeDir (&kdir);
if (rc != 0)
{
LOGMSG (klogFatal, "Failed to open native KDirectory");
}
else
{
switch (mode)
{
default: /* OM_NONE presumably */
/* assert (mode == OM_NONE); */
/* rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcEmpty); */
/* LOGERR (klogFatal, rc, "Must provide a single operating mode and archive: Create, Test (list) or eXtract"); */
pc = NULL;
break;
case OM_ERROR:
rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcAmbiguous);
LOGERR (klogFatal, rc, "Must provide a single operating mode and archive: Create, Test (list) or eXtract");
break;
case OM_CREATE:
STSMSG (2, ("Create Mode %s %s", archive, directory));
if (directory == NULL)
{
rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcNull);
LOGERR (klogFatal, rc, "Must provide a directory for create mode");
}
else
rc = run_kar_create (archive, directory);
break;
case OM_TEST:
STSMSG (2, ("Test Mode %s", archive));
rc = run_kar_test (archive);
break;
case OM_EXTRACT:
STSMSG (2, ("Extract Mode %s %s", archive, directory));
if (directory == NULL)
{
rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcNull);
LOGERR (klogFatal, rc, "Must provide a directory for extract mode");
}
else
rc = run_kar_extract (archive, directory);
break;
}
KDirectoryRelease (kdir);
}
} while (0);
ArgsWhack (args);
}
if (rc)
LOGERR (klogWarn, rc, "Failure exiting kar");
else
STSMSG (1, ("Success: Exiting kar\n"));
return rc;
}
/* Open
*/
rc_t KColumnIdx1OpenRead ( KColumnIdx1 *self, const KDirectory *dir,
uint64_t *data_eof, uint32_t *idx0_count, uint64_t *idx2_eof,
size_t *pgsize, int32_t *checksum )
{
rc_t rc;
#if USE_BSTREE_IN_COLUMN_IDX1
BSTreeInit ( & self -> bst );
#else
self -> last_found = 0;
self -> data = NULL;
#endif
self -> fidx = NULL;
self -> count = 0;
self -> vers = 0;
#if LAZY_LOAD_COLUMN_IDX1
self -> load_off = 0;
self -> load_rc = 0;
#endif
self -> bswap = false;
self -> loaded = false;
rc = KDirectoryVOpenFileRead ( dir, & self -> f, "idx1", NULL );
if ( rc == 0 )
{
KColumnHdr hdr;
size_t num_bytes;
rc = KFileReadAll ( self -> f, 0, & hdr, sizeof hdr, & num_bytes );
if ( rc == 0 )
{
if ( num_bytes == 0 )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcNotFound );
else if ( num_bytes < KColumnHdrMinSize ( hdr ) )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
rc = KDBHdrValidate ( & hdr . dad, num_bytes, 1, KCOL_CURRENT_VERSION );
if ( GetRCState ( rc ) == rcIncorrect && GetRCObject ( rc ) == rcByteOrder )
{
self -> bswap = true;
hdr . dad . endian = bswap_32 ( hdr . dad . endian );
hdr . dad . version = bswap_32 ( hdr . dad . version );
rc = KDBHdrValidate ( & hdr . dad, num_bytes, 1, KCOL_CURRENT_VERSION );
}
if ( rc == 0 )
{
uint32_t off, count;
switch ( hdr . dad . version )
{
case 1:
if ( num_bytes < KColumnHdrOffset ( hdr, v1 ) )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
if ( self -> bswap )
{
hdr . u . v1 . data_eof = bswap_64 ( hdr . u . v1 . data_eof );
hdr . u . v1 . idx2_eof = bswap_64 ( hdr . u . v1 . idx2_eof );
hdr . u . v1 . num_blocks = bswap_32 ( hdr . u . v1 . num_blocks );
hdr . u . v1 . page_size = bswap_32 ( hdr . u . v1 . page_size );
}
* data_eof = hdr . u . v1 . data_eof;
* idx2_eof = hdr . u . v1 . idx2_eof;
* pgsize = hdr . u . v1 . page_size;
* checksum = hdr . u . v1 . checksum;
count = hdr . u . v1 . num_blocks;
off = KColumnHdrOffset ( hdr, v1 );
/* not supported in v1 */
* idx0_count = 0;
}
break;
default:
rc = KDirectoryVOpenFileRead ( dir, ( const KFile** ) & self -> fidx, "idx", NULL );
if ( rc == 0 )
{
off = sizeof hdr . dad;
rc = KFileReadAll ( self -> fidx, 0, & hdr, sizeof hdr, & num_bytes );
if ( rc == 0 )
{
if ( num_bytes < KColumnHdrOffset ( hdr, v2 ) )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
rc = KDBHdrValidate ( & hdr . dad,
num_bytes, 2, KCOL_CURRENT_VERSION );
if ( GetRCState ( rc ) == rcIncorrect && GetRCObject ( rc ) == rcByteOrder )
{
if ( ! self->bswap ) /* catch mis-matched endianess */
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
hdr . dad . endian = bswap_32 ( hdr . dad . endian );
hdr . dad . version = bswap_32 ( hdr . dad . version );
rc = KDBHdrValidate ( & hdr . dad, num_bytes, 1, KCOL_CURRENT_VERSION );
}
}
else if ( self -> bswap ) /* catch mis-matched endianess */
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
if ( rc == 0 ) switch ( hdr . dad . version )
{
case 2:
if ( self -> bswap )
{
hdr. u . v2 . data_eof = bswap_64 ( hdr. u . v2 . data_eof );
hdr. u . v2 . idx2_eof = bswap_64 ( hdr. u . v2 . idx2_eof );
hdr. u . v2 . num_blocks = bswap_32 ( hdr. u . v2 . num_blocks );
hdr. u . v2 . page_size = bswap_32 ( hdr. u . v2 . page_size );
}
* data_eof = hdr . u . v2 . data_eof;
* idx2_eof = hdr . u . v2 . idx2_eof;
* pgsize = hdr . u . v2 . page_size;
* checksum = hdr . u . v2 . checksum;
count = hdr . u . v2 . num_blocks;
* idx0_count = 0;
break;
case 3:
if ( self -> bswap )
{
hdr. u . v3 . data_eof = bswap_64 ( hdr. u . v3 . data_eof );
hdr. u . v3 . idx2_eof = bswap_64 ( hdr. u . v3 . idx2_eof );
hdr. u . v3 . idx0_count = bswap_32 ( hdr. u . v3 . idx0_count );
hdr. u . v3 . num_blocks = bswap_32 ( hdr. u . v3 . num_blocks );
hdr. u . v3 . page_size = bswap_32 ( hdr. u . v3 . page_size );
}
* data_eof = hdr . u . v3 . data_eof;
* idx2_eof = hdr . u . v3 . idx2_eof;
* idx0_count = hdr . u . v3 . idx0_count;
* pgsize = hdr . u . v3 . page_size;
* checksum = hdr . u . v3 . checksum;
count = hdr . u . v3 . num_blocks;
break;
default:
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcBadVersion );
}
}
}
}
}
if ( rc == 0 )
{
self -> vers = hdr . dad . version;
#if LAZY_LOAD_COLUMN_IDX1
self -> load_off = off;
self -> count = count;
#else
rc = KColumnIdx1Init ( self, off, count );
if ( rc == 0 )
#endif
return rc;
}
KFileRelease ( self -> fidx );
self -> fidx = NULL;
}
}
}
KFileRelease ( self -> f );
self -> f = NULL;
}
return rc;
}
static
rc_t CC
_TeleportInit ()
{
rc_t RCt;
RCt = 0;
if ( _sTeleportInited == true ) {
return 0;
}
BSTreeInit ( & _sTeleport );
_sTeleportInited = true;
/* Here we are adding Teleports */
do {
RCt = _TeleportAdd ( CACHE_NAME, XFSCacheProvider );
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd ( FILE_NAME, XFSFileProvider );
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd (
ENCRYPTED_FILE_NAME,
XFSEncryptedFileProvider
);
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd ( DIRECTORY_NAME, XFSDirectoryProvider );
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd ( README_NAME, XFSReadMeProvider );
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd (
KART_COLLECTION_NAME,
XFSKartCollectionProvider
);
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd (
KART_NAME,
XFSKartProvider
);
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd ( LINK_NAME, XFSLinkProvider );
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd (
LOCAL_REPOSITORY_NAME,
XFSLocalRepositoryProvider
);
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd (
REMOTE_REPOSITORY_NAME,
XFSRemoteRepositoryProvider
);
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd ( WORKSPACE_NAME, XFSWorkspaceProvider );
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd (
SIMPLE_CONTAINER_NAME,
XFSSimpleContainerProvider
);
if ( RCt != 0 ) {
break;
}
RCt = _TeleportAdd (
TAR_ARCHIVE_NAME,
XFSTarArchiveProvider
);
if ( RCt != 0 ) {
break;
}
} while ( false );
return RCt;
} /* _TeleportInit () */
static
rc_t populate_stype_tbl ( BSTree *stype_tbl, const VSchema *schema )
{
rc_t rc;
uint32_t i;
static struct
{
const char *typename;
const char *redact_value;
} sensitive_types [] =
{
/* original SRA types */
{ "INSDC:fasta", "N" },
{ "INSDC:csfasta", "." },
{ "NCBI:2na", "\x00" },
{ "NCBI:2cs", "\x00" },
{ "NCBI:4na", "\xFF" },
{ "NCBI:qual1", "\x00" },
{ "NCBI:qual4", "\xFB\xFB\xFB\xFB" },
{ "NCBI:isamp1", "\x00\x00\x00" },
{ "NCBI:isamp4", "\x00\x00\x00" },
{ "NCBI:fsamp1", "\x00\x00\x00" },
{ "NCBI:fsamp4", "\x00\x00\x00" },
{ "INSDC:dna:text", "N" },
{ "INSDC:dna:bin", "\x04" },
{ "INSDC:dna:4na", "\xFF" },
{ "INSDC:dna:2na", "\x00" },
{ "INSDC:color:text", "." },
{ "INSDC:color:bin", "\x04" },
{ "INSDC:color:2cs", "\x00" },
{ "INSDC:quality:phred", "\x00" },
{ "INSDC:quality:log_odds", "\x00\x00\x00" }
/* signal types TBD */
};
BSTreeInit ( stype_tbl );
for ( rc = 0, i = 0; i < sizeof sensitive_types / sizeof sensitive_types [ 0 ]; ++ i )
{
VTypedecl td;
const char *decl = sensitive_types [ i ] . typename;
rc = VSchemaResolveTypedecl ( schema, & td, decl );
if ( rc == 0 )
{
stype_id *n;
BSTNode *exist;
VTypedesc desc;
rc = VSchemaDescribeTypedecl ( schema, & desc, & td );
if ( rc != 0 )
{
PLOGERR ( klogInt, (klogInt, rc, "failed to describe type '$(type)'", "type=%s", decl ));
break;
}
n = malloc ( sizeof * n );
if ( n == NULL )
{
rc = RC ( rcExe, rcNode, rcAllocating, rcMemory, rcExhausted );
LOGERR ( klogInt, rc, "failed to record sensitive data type" );
break;
}
n -> redact_value = sensitive_types [ i ] . redact_value;
n -> type_id = td . type_id;
n -> elem_size = VTypedescSizeof ( & desc );
rc = BSTreeInsertUnique ( stype_tbl, & n -> n, & exist, stype_id_sort );
if ( rc != 0 )
{
free ( n );
if ( GetRCState ( rc ) != rcExists )
{
LOGERR ( klogInt, rc, "failed to record sensitive data type" );
break;
}
rc = 0;
}
}
else if ( GetRCState ( rc ) == rcNotFound )
{
rc = 0;
}
else
{
break;
}
}
return rc;
}
KNSProxies * KNSManagerKNSProxiesMake ( struct KNSManager * mgr,
const KConfig * kfg )
{
rc_t rc = 0;
int i = 0;
int n = 2;
typedef enum {
eEnv,
eKfg,
} EType;
EType type [ 2 ] = { eKfg, eEnv };
KNSProxies * self = calloc ( 1, sizeof * self );
if ( self == NULL )
return NULL;
assert ( self );
BSTreeInit ( & self -> proxie_tree );
rc = KConfigReadBool
( kfg, "/http/proxy/enabled", & self -> http_proxy_enabled );
if ( rc != 0 ) {
if ( GetRCState ( rc ) == rcNotFound )
rc = 0;
else {
KNSManagerSetHTTPProxyPath ( mgr, NULL );
assert ( self -> http_proxy_enabled == false );
}
}
else if ( ! self -> http_proxy_enabled )
return self;
{
bool proxy_only = false;
rc_t rc = KConfigReadBool ( kfg, "/http/proxy/only", & proxy_only );
if ( rc == 0 && proxy_only )
self-> http_proxy_only = true;
}
{
String * result = NULL;
rc = KConfigReadString ( kfg, "/http/proxy/use", & result );
if ( rc == 0 ) {
if ( StringCmp ( result, "env") ) {
n = 1;
type [ 0 ] = eEnv;
} else if ( StringCmp ( result, "kfg") ) {
n = 1;
type [ 0 ] = eKfg;
} else if ( StringCmp ( result, "none") ) {
n = 0;
} else if ( StringCmp ( result, "env,kfg") ) {
n = 2;
type [ 0 ] = eEnv;
type [ 1 ] = eKfg;
} else if ( StringCmp ( result, "kfg,env") ) {
n = 2;
type [ 0 ] = eKfg;
type [ 1 ] = eEnv;
}
}
RELEASE ( String, result );
}
for ( i = 0; i < n; ++ i ) {
switch ( type [ i ] ) {
case eEnv:
KNSProxiesHttpProxyInitFromEnv ( self );
break;
case eKfg:
KNSProxiesHttpProxyInitFromKfg ( self, kfg );
break;
default:
assert ( 0 );
break;
}
}
BSTreeForEach ( & self -> proxie_tree, false, KNSProxiesBSTreeCount,
& self -> http_proxies_cnt );
if ( self -> http_proxies_cnt > 0 ) {
self -> http_proxies = calloc ( self -> http_proxies_cnt,
sizeof * self -> http_proxies );
if ( self -> http_proxies == NULL )
return NULL;
DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_PROXY ),
( "Will use %zu proxy spec%s%s\n", self -> http_proxies_cnt,
self -> http_proxies_cnt > 1 ? "s" : "",
self -> http_proxy_only ? "" : " and direct connection") );
}
self -> tmpS = 0;
n = self -> http_proxies_cnt;
srand ( time ( NULL ) );
while ( n > 0 ) {
self -> rand = rand () % n;
self -> tmpI = 0;
if ( ! BSTreeDoUntil ( & self -> proxie_tree, false,
KNSProxiesBSTreeSetRand, self ) )
{
BSTreeForEach ( & self -> proxie_tree, false,
KNSProxiesBSTreeInit, self ) ;
n = 0;
}
else {
const BSTItem * item = ( BSTItem * ) self -> tmpB;
self -> http_proxies [ self -> tmpS ++ ] = item -> proxy;
BSTreeUnlink ( & self -> proxie_tree, self -> tmpB );
BSTItemWhack ( self -> tmpB, NULL );
self -> tmpB = NULL;
-- n;
}
}
/* BSTreeForEach ( & self -> proxie_tree, false, KNSProxiesBSTreeInit, self );*/
for ( self -> tmpS = 1; self -> tmpS < self ->http_proxies_cnt;
++ self -> tmpS )
{
self -> http_proxies [ self -> tmpS - 1 ] -> next
= self -> http_proxies [ self -> tmpS ];
}
return self;
}
