static
rc_t CC VPhysicalLoadSchema ( VPhysical *self,
VTypedecl *td, VSchema *schema, const KMDataNode *node )
{
rc_t rc;
KMDataNodeSchemaFillData pb;
pb . node = node;
pb . pos = 0;
pb . add_v0 = false;
/* add stored declaration to cursor schema */
rc = VSchemaParseTextCallback ( schema,
"VPhysicalLoadSchema", KMDataNodeFillSchema, & pb );
if ( rc == 0 )
{
/* retrieve fully-resolved type attribute */
rc = KMDataNodeReadAttr ( node, "type",
pb . buff, sizeof pb . buff, & pb . pos );
if ( rc == 0 )
rc = VSchemaResolveTypedecl ( schema, td, pb . buff );
if ( rc == 0 )
{
/* get encoding expression */
rc = KMDataNodeReadAttr ( node, "expr",
pb . buff, sizeof pb . buff, & pb . pos );
if ( rc == 0 )
{
VTypedecl etd;
/* create a new expression object */
rc = VSchemaImplicitPhysEncExpr ( schema, & etd,
& self -> enc, pb . buff, "VPhysicalLoadSchema" );
if ( rc != 0 )
{
PLOGERR ( klogInt, ( klogInt, rc, "failed to establish column type from '$(expr)'",
"expr=%s", pb . buff ));
}
/* match SPhysical type against stated type */
else if ( ! VTypedeclToTypedecl ( & etd, schema, td, NULL, NULL ) )
{
rc = RC ( rcVDB, rcColumn, rcLoading, rcType, rcInconsistent );
PLOGERR ( klogInt, ( klogInt, rc, "inconsistent metadata for column '$(name)'"
, "name=%.*s"
, ( int ) self -> smbr -> name -> name . size
, self -> smbr -> name -> name . addr ));
}
}
else if ( GetRCState ( rc ) == rcNotFound )
{
rc = 0;
}
}
}
KMDataNodeRelease ( node );
return rc;
}
static
rc_t populate_rdfilt_tbl ( BSTree *rftype_tbl, const VSchema *schema )
{
rc_t rc;
uint32_t i;
const char *rftypes [] =
{
"NCBI:SRA:read_filter",
"INSDC:SRA:read_filter"
};
BSTreeInit ( rftype_tbl );
for ( rc = 0, i = 0; i < sizeof rftypes / sizeof rftypes [ 0 ]; ++ i )
{
VTypedecl td;
const char *decl = rftypes [ i ];
rc = VSchemaResolveTypedecl ( schema, & td, decl );
if ( rc == 0 )
{
BSTNode *exist;
stype_id *n = malloc ( sizeof * n );
if ( n == NULL )
{
rc = RC ( rcExe, rcNode, rcAllocating, rcMemory, rcExhausted );
LOGERR ( klogInt, rc, "failed to record read_filter data type" );
break;
}
n -> redact_value = NULL;
n -> type_id = td . type_id;
n -> elem_size = 8;
rc = BSTreeInsertUnique ( rftype_tbl, & n -> n, & exist, stype_id_sort );
if ( rc != 0 )
{
free ( n );
if ( GetRCState ( rc ) != rcExists )
{
LOGERR ( klogInt, rc, "failed to record read_filter data type" );
break;
}
rc = 0;
}
}
else if ( GetRCState ( rc ) == rcNotFound )
{
rc = 0;
}
else
{
break;
}
}
return rc;
}
static bool vdcd_type_cmp( const VSchema *my_schema, VTypedecl * typedecl, const char * to_check )
{
VTypedecl type_to_check;
rc_t rc = VSchemaResolveTypedecl ( my_schema, &type_to_check, to_check );
if ( rc == 0 )
{
return VTypedeclToTypedecl ( typedecl, my_schema, &type_to_check, NULL, NULL );
}
return false;
}
static rc_t redactable_types_2_type_id_vector( const VSchema * s,
const char * redactable_types,
Vector * id_vector )
{
const KNamelist *r_types;
rc_t rc;
if ( redactable_types == NULL || s == NULL || id_vector == NULL )
return RC( rcExe, rcNoTarg, rcResolving, rcParam, rcNull );
rc = nlt_make_namelist_from_string( &r_types, redactable_types );
if ( rc == 0 )
{
uint32_t count, idx;
rc = KNamelistCount( r_types, &count );
if ( rc == 0 && count > 0 )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char *name;
rc = KNamelistGet( r_types, idx, &name );
if ( rc == 0 )
{
VTypedecl td;
rc = VSchemaResolveTypedecl ( s, &td, "%s", name );
if ( rc == 0 )
{
uint32_t *id = malloc( sizeof *id );
if ( id != NULL )
{
*id = td.type_id;
rc = VectorAppend ( id_vector, NULL, id );
}
else
rc = RC( rcExe, rcNoTarg, rcResolving, rcMemory, rcExhausted );
}
}
}
KNamelistRelease( r_types );
}
return rc;
}
static rc_t matcher_append_type( const char *name, const bool dflt,
const uint32_t order,
const VSchema *schema, Vector *v )
{
rc_t rc = 0;
p_mtype t = matcher_init_type( name, dflt, order );
if ( t == NULL )
rc = RC( rcExe, rcNoTarg, rcResolving, rcMemory, rcExhausted );
if ( rc == 0 )
{
rc = VectorAppend( v, NULL, t );
if ( rc == 0 )
{
rc = VSchemaResolveTypedecl( schema, &(t->type_decl), "%s", name );
if ( rc == 0 )
{
rc = VSchemaDescribeTypedecl( schema, &(t->type_desc), &(t->type_decl) );
}
}
}
return rc;
}
/* checks if a column with the given name has at least one type
that is also in the given typelist... */
bool matcher_src_has_type( const matcher* self, const VSchema * s,
const char *name, const Vector *id_vector )
{
bool res = false;
p_mcol col;
VTypedecl td;
if ( self == NULL || s == NULL || name == NULL || id_vector == NULL )
return res;
col = matcher_find_col( self, name );
if ( col == NULL ) return res; /* column not found */
if ( col->type_cast == NULL ) return res; /* column has no typecast */
/* we use the destination-type-cast */
if ( VSchemaResolveTypedecl ( s, &td, "%s", col->type_cast->dst->name ) == 0 )
res = match_type_with_id_vector( s, &td, id_vector );
/*
if ( res )
KOutMsg( "redact-type found on (%s)%s\n", col->type_cast->dst->name, name );
*/
return res;
}
/* ResolveTypename
* convert a type id into its type definition
*
* "resolved" [ OUT ] - resolved type definition
*
* "typname" [ IN ] - NUL-terminated type name
*/
LIB_EXPORT rc_t CC VDatatypesResolveTypename ( const VDatatypes *self,
VTypedef *resolved, const char *typname )
{
rc_t rc;
if ( resolved == NULL )
rc = RC ( rcSRA, rcSchema, rcResolving, rcParam, rcNull );
else
{
VTypedecl td;
rc = VSchemaResolveTypedecl ( ( const VSchema* ) self, & td, typname );
if ( rc == 0 )
{
rc = VSchemaDescribeTypedef ( ( const VSchema* ) self, resolved, td . type_id );
if ( rc == 0 )
return rc;
}
memset ( resolved, 0, sizeof * resolved );
}
return rc;
}
static
rc_t populate_stype_tbl ( BSTree *stype_tbl, const VSchema *schema )
{
rc_t rc;
uint32_t i;
static struct
{
const char *typename;
const char *redact_value;
} sensitive_types [] =
{
/* original SRA types */
{ "INSDC:fasta", "N" },
{ "INSDC:csfasta", "." },
{ "NCBI:2na", "\x00" },
{ "NCBI:2cs", "\x00" },
{ "NCBI:4na", "\xFF" },
{ "NCBI:qual1", "\x00" },
{ "NCBI:qual4", "\xFB\xFB\xFB\xFB" },
{ "NCBI:isamp1", "\x00\x00\x00" },
{ "NCBI:isamp4", "\x00\x00\x00" },
{ "NCBI:fsamp1", "\x00\x00\x00" },
{ "NCBI:fsamp4", "\x00\x00\x00" },
{ "INSDC:dna:text", "N" },
{ "INSDC:dna:bin", "\x04" },
{ "INSDC:dna:4na", "\xFF" },
{ "INSDC:dna:2na", "\x00" },
{ "INSDC:color:text", "." },
{ "INSDC:color:bin", "\x04" },
{ "INSDC:color:2cs", "\x00" },
{ "INSDC:quality:phred", "\x00" },
{ "INSDC:quality:log_odds", "\x00\x00\x00" }
/* signal types TBD */
};
BSTreeInit ( stype_tbl );
for ( rc = 0, i = 0; i < sizeof sensitive_types / sizeof sensitive_types [ 0 ]; ++ i )
{
VTypedecl td;
const char *decl = sensitive_types [ i ] . typename;
rc = VSchemaResolveTypedecl ( schema, & td, decl );
if ( rc == 0 )
{
stype_id *n;
BSTNode *exist;
VTypedesc desc;
rc = VSchemaDescribeTypedecl ( schema, & desc, & td );
if ( rc != 0 )
{
PLOGERR ( klogInt, (klogInt, rc, "failed to describe type '$(type)'", "type=%s", decl ));
break;
}
n = malloc ( sizeof * n );
if ( n == NULL )
{
rc = RC ( rcExe, rcNode, rcAllocating, rcMemory, rcExhausted );
LOGERR ( klogInt, rc, "failed to record sensitive data type" );
break;
}
n -> redact_value = sensitive_types [ i ] . redact_value;
n -> type_id = td . type_id;
n -> elem_size = VTypedescSizeof ( & desc );
rc = BSTreeInsertUnique ( stype_tbl, & n -> n, & exist, stype_id_sort );
if ( rc != 0 )
{
free ( n );
if ( GetRCState ( rc ) != rcExists )
{
LOGERR ( klogInt, rc, "failed to record sensitive data type" );
break;
}
rc = 0;
}
}
else if ( GetRCState ( rc ) == rcNotFound )
{
rc = 0;
}
else
{
break;
}
}
return rc;
}
/* LoadSchema
* looks in metadata for stored schema
*/
static
rc_t CC VPhysicalLoadV1Schema ( VPhysical *self,
VTypedecl *td, VSchema *schema, const KMDataNode *node )
{
rc_t rc;
KMDataNodeSchemaFillData pb;
pb . node = node;
pb . pos = 0;
pb . add_v0 = true;
/* add stored declaration to cursor schema */
rc = VSchemaParseTextCallback ( schema,
"VPhysicalLoadV1Schema", KMDataNodeFillSchema, & pb );
if ( rc == 0 )
{
size_t size;
char type_expr [ 256 ];
/* retrieve and resolve "type" attribute */
rc = KMDataNodeReadAttr ( node, "type",
type_expr, sizeof type_expr, & size );
if ( rc == 0 )
rc = VSchemaResolveTypedecl ( schema, td, type_expr );
/* if a decoding schema exists */
if ( rc == 0 && pb . pos != 0 )
{
char sphysical_name [ 512 ];
/* preserve schema function expression */
size_t decoding_xsize;
char decoding_expr [ 256 ];
rc = KMDataNodeReadAttr ( node, "schema",
decoding_expr, sizeof decoding_expr, & decoding_xsize );
if ( rc == 0 )
{
/* look for "encoding" */
const KMDataNode *enc;
rc = KMetadataOpenNodeRead ( self -> meta, & enc, "encoding" );
if ( rc == 0 )
{
#if ALLOW_V1_UPDATE
if ( ! self -> read_only )
{
/* add stored declaration to cursor schema */
pb . node = enc;
pb . pos = 0;
pb . add_v0 = true;
rc = VSchemaParseTextCallback ( schema,
"VPhysicalLoadV1Schema", KMDataNodeFillSchema, & pb );
}
if ( rc == 0 )
#endif
{
/* preserve schema function expression */
size_t encoding_xsize;
char encoding_expr [ 256 ], enc_type [ 256 ];
rc = KMDataNodeReadAttr ( enc, "schema",
encoding_expr, sizeof encoding_expr, & encoding_xsize );
if ( rc == 0 )
{
rc = KMDataNodeReadAttr ( enc, "type",
enc_type, sizeof enc_type, & size );
}
if ( rc == 0 )
{
#if ALLOW_V1_UPDATE
if ( self -> read_only )
{
#endif
/* build sphysical name */
sprintf ( sphysical_name, "%s_only", decoding_expr );
/* build physical decl */
pb . pos = sprintf ( pb . buff, "version 1;"
"physical %s %s:phys#1"
"{decode{%s k=@;return %s(k);}}"
, type_expr
, sphysical_name
, enc_type
, decoding_expr
);
#if ALLOW_V1_UPDATE
}
else
{
/* strip off common namespace */
size_t i, ns_size;
string_match ( decoding_expr, decoding_xsize,
encoding_expr, encoding_xsize, -1, & ns_size );
if ( ns_size != 0 )
{
char *p = string_rchr ( decoding_expr, ns_size, ':' );
ns_size = ( p == NULL ) ? 0U : ( uint32_t ) ( p - decoding_expr ) + 1U;
}
/* build sphysical name */
sprintf ( sphysical_name, "%s_%s", decoding_expr, & encoding_expr [ ns_size ] );
for ( i = ns_size; sphysical_name [ i ] != 0; ++ i )
{
if ( sphysical_name [ i ] == ':' )
sphysical_name [ i ] = '_';
}
/* build physical decl */
pb . pos = sprintf ( pb . buff, "version 1;"
"physical %s %s:phys#1"
"{encode{return %s(@);}"
"decode{%s k=@;return %s(k);}}"
, type_expr
, sphysical_name
, encoding_expr
, enc_type
, decoding_expr
);
}
#endif
}
}
KMDataNodeRelease ( enc );
}
else if ( GetRCState ( rc ) == rcNotFound )
{
/* build sphysical name */
sprintf ( sphysical_name, "%s_only", decoding_expr );
/* build decode-only physical decl */
pb . pos = sprintf ( pb . buff, "version 1;"
"physical %s %s:phys#1"
"{decode{opaque k=@;return %s(k);}}"
, type_expr
, sphysical_name
, decoding_expr
);
rc = 0;
}
if ( rc == 0 )
{
/* parse synthesized schema into cursor VSchema */
rc = VSchemaParseText ( schema,
"VPhysicalLoadV1Schema", pb . buff, pb . pos );
if ( rc == 0 )
{
VTypedecl etd;
/* create a new expression object */
sprintf ( pb . buff, "%s:phys#1", sphysical_name );
rc = VSchemaImplicitPhysEncExpr ( schema, & etd,
& self -> enc, pb . buff, "VPhysicalLoadV1Schema" );
if ( rc != 0 )
{
PLOGERR ( klogInt, ( klogInt, rc, "failed to establish column type from '$(expr)'",
"expr=%s", pb . buff ));
}
else if ( self -> smbr != NULL && self -> smbr -> type == NULL )
{
/* back-patch schema */
( ( SPhysMember* ) self -> smbr ) -> type = self -> enc;
atomic32_inc ( & ( ( SExpression* ) self -> enc ) -> refcount );
}
}
}
}
}
}
KMDataNodeRelease ( node );
return rc;
}
