static
DictionaryEntry * MakeNode ( SRA_Statistics * self, ctx_t ctx, const char * path )
{
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
size_t path_size = string_size ( path );
DictionaryEntry * node = malloc ( sizeof ( * node ) + path_size );
if ( node == NULL )
{
SYSTEM_ERROR ( xcNoMemory, "allocating dictionary item" );
}
else
{
rc_t rc;
string_copy ( node -> path, path_size + 1, path, path_size );
/*TODO: decide whether to allow overwriting (not allowed now) */
rc = BSTreeInsertUnique ( & self -> dictionary, & node -> dad, NULL, DictionaryEntryCompare );
if ( rc == 0 )
{
return node;
}
INTERNAL_ERROR ( xcUnexpected, "inserting dictionary item '%s' rc = %R", node -> path, rc );
free ( node );
}
return NULL;
}
static
rc_t populate_rdfilt_tbl ( BSTree *rftype_tbl, const VSchema *schema )
{
rc_t rc;
uint32_t i;
const char *rftypes [] =
{
"NCBI:SRA:read_filter",
"INSDC:SRA:read_filter"
};
BSTreeInit ( rftype_tbl );
for ( rc = 0, i = 0; i < sizeof rftypes / sizeof rftypes [ 0 ]; ++ i )
{
VTypedecl td;
const char *decl = rftypes [ i ];
rc = VSchemaResolveTypedecl ( schema, & td, decl );
if ( rc == 0 )
{
BSTNode *exist;
stype_id *n = malloc ( sizeof * n );
if ( n == NULL )
{
rc = RC ( rcExe, rcNode, rcAllocating, rcMemory, rcExhausted );
LOGERR ( klogInt, rc, "failed to record read_filter data type" );
break;
}
n -> redact_value = NULL;
n -> type_id = td . type_id;
n -> elem_size = 8;
rc = BSTreeInsertUnique ( rftype_tbl, & n -> n, & exist, stype_id_sort );
if ( rc != 0 )
{
free ( n );
if ( GetRCState ( rc ) != rcExists )
{
LOGERR ( klogInt, rc, "failed to record read_filter data type" );
break;
}
rc = 0;
}
}
else if ( GetRCState ( rc ) == rcNotFound )
{
rc = 0;
}
else
{
break;
}
}
return rc;
}
/* you cannot addref to this dir object cause it's created on stack silently */
static
rc_t CC DirVisitor(const KDirectory *dir, uint32_t type, const char *name, void *data)
{
rc_t rc = 0;
DirVisit_Data* d = (DirVisit_Data*)data;
if( (type & ~kptAlias) == kptFile ) {
if (strcmp(&name[strlen(name) - 4], ".tsv") == 0 ||
strcmp(&name[strlen(name) - 8], ".tsv.bz2") == 0 ||
strcmp(&name[strlen(name) - 7], ".tsv.gz") == 0)
{
char buf[4096];
const CGLoaderFile* file;
FGroupKey key;
if( (rc = KDirectoryResolvePath(dir, true, buf, sizeof(buf), name)) == 0 &&
(rc = CGLoaderFile_Make(&file, d->dir, buf, NULL, !d->param->no_read_ahead)) == 0 &&
(rc = FGroupKey_Make(&key, file, d->param)) == 0 ) {
FGroupMAP* found = (FGroupMAP*)BSTreeFind(d->tree, &key, FGroupMAP_Cmp);
DEBUG_MSG(5, ("file %s recognized\n", name));
if( found != NULL ) {
rc = FGroupMAP_Set(found, file);
} else {
FGroupMAP* x = calloc(1, sizeof(*x));
if( x == NULL ) {
rc = RC(rcExe, rcFile, rcInserting, rcMemory, rcExhausted);
} else {
memcpy(&x->key, &key, sizeof(key));
if( (rc = FGroupMAP_Set(x, file)) == 0 ) {
rc = BSTreeInsertUnique(d->tree, &x->dad, NULL, FGroupMAP_Sort);
}
}
}
} else if( GetRCObject(rc) == rcItem && GetRCState(rc) == rcIgnored ) {
DEBUG_MSG(5, ("file %s ignored\n", name));
rc = CGLoaderFile_Release(file, true);
file = NULL;
}
if( rc != 0 && file != NULL ) {
CGLoaderFile_LOG(file, klogErr, rc, NULL, NULL);
CGLoaderFile_Release(file, true);
}
} else if( strcmp(&name[strlen(name) - 4], ".tar") == 0 ) {
const KDirectory* tmp = d->dir;
if( (rc = KDirectoryOpenArcDirRead(dir, &d->dir, true, name, tocKFile, KArcParseTAR, NULL, NULL)) == 0 ) {
rc = KDirectoryVisit(d->dir, true, DirVisitor, d, ".");
KDirectoryRelease(d->dir);
}
d->dir = tmp;
}
}
return rc;
}
rc_t KU64IndexInsert_v3(KU64Index_v3* self, bool unique, uint64_t key, uint64_t key_size, int64_t id, uint64_t id_qty)
{
KU64Index_Node* node = calloc(1, sizeof(KU64Index_Node));
self->rc = 0;
if( node == NULL ) {
self->rc = RC(rcExe, rcNode, rcConstructing, rcMemory, rcInsufficient);
} else {
node->key = key;
node->key_size = key_size;
node->id = id;
node->id_qty = id_qty;
if( unique ) {
self->rc = BSTreeInsertUnique(&self->tree, &node->node, NULL, KU64Index_NodeSortUnique);
} else {
self->rc = BSTreeInsert(&self->tree, &node->node, KU64Index_NodeSort);
}
}
if( self->rc != 0 ) {
free(node);
}
return self->rc;
}
static
rc_t KColumnIdx1Inflate ( KColumnIdx1 *self,
const KColBlockLoc *buffer, uint32_t count )
{
uint32_t i;
for ( i = 0; i < count; ++ i )
{
KColumnIdx1Node *exist, *n = malloc ( sizeof * n );
if ( n == NULL )
return RC ( rcDB, rcColumn, rcConstructing, rcMemory, rcExhausted );
n -> loc = buffer [ i ];
if ( BSTreeInsertUnique ( & self -> bst,
& n -> n, ( BSTNode** ) & exist, KColumnIdx1NodeSort ) )
{
free ( n );
return RC ( rcDB, rcColumn, rcConstructing, rcIndex, rcCorrupt );
}
++ self -> count;
}
return 0;
}
static
rc_t populate_stype_tbl ( BSTree *stype_tbl, const VSchema *schema )
{
rc_t rc;
uint32_t i;
static struct
{
const char *typename;
const char *redact_value;
} sensitive_types [] =
{
/* original SRA types */
{ "INSDC:fasta", "N" },
{ "INSDC:csfasta", "." },
{ "NCBI:2na", "\x00" },
{ "NCBI:2cs", "\x00" },
{ "NCBI:4na", "\xFF" },
{ "NCBI:qual1", "\x00" },
{ "NCBI:qual4", "\xFB\xFB\xFB\xFB" },
{ "NCBI:isamp1", "\x00\x00\x00" },
{ "NCBI:isamp4", "\x00\x00\x00" },
{ "NCBI:fsamp1", "\x00\x00\x00" },
{ "NCBI:fsamp4", "\x00\x00\x00" },
{ "INSDC:dna:text", "N" },
{ "INSDC:dna:bin", "\x04" },
{ "INSDC:dna:4na", "\xFF" },
{ "INSDC:dna:2na", "\x00" },
{ "INSDC:color:text", "." },
{ "INSDC:color:bin", "\x04" },
{ "INSDC:color:2cs", "\x00" },
{ "INSDC:quality:phred", "\x00" },
{ "INSDC:quality:log_odds", "\x00\x00\x00" }
/* signal types TBD */
};
BSTreeInit ( stype_tbl );
for ( rc = 0, i = 0; i < sizeof sensitive_types / sizeof sensitive_types [ 0 ]; ++ i )
{
VTypedecl td;
const char *decl = sensitive_types [ i ] . typename;
rc = VSchemaResolveTypedecl ( schema, & td, decl );
if ( rc == 0 )
{
stype_id *n;
BSTNode *exist;
VTypedesc desc;
rc = VSchemaDescribeTypedecl ( schema, & desc, & td );
if ( rc != 0 )
{
PLOGERR ( klogInt, (klogInt, rc, "failed to describe type '$(type)'", "type=%s", decl ));
break;
}
n = malloc ( sizeof * n );
if ( n == NULL )
{
rc = RC ( rcExe, rcNode, rcAllocating, rcMemory, rcExhausted );
LOGERR ( klogInt, rc, "failed to record sensitive data type" );
break;
}
n -> redact_value = sensitive_types [ i ] . redact_value;
n -> type_id = td . type_id;
n -> elem_size = VTypedescSizeof ( & desc );
rc = BSTreeInsertUnique ( stype_tbl, & n -> n, & exist, stype_id_sort );
if ( rc != 0 )
{
free ( n );
if ( GetRCState ( rc ) != rcExists )
{
LOGERR ( klogInt, rc, "failed to record sensitive data type" );
break;
}
rc = 0;
}
}
else if ( GetRCState ( rc ) == rcNotFound )
{
rc = 0;
}
else
{
break;
}
}
return rc;
}
