rc_t write_statistic_into_tab( KDirectory *dir, statistic * data,
const KNamelist *schema_list, const char *output_file_path,
uint64_t * written, bool show_progress )
{
VDBManager *my_manager;
rc_t rc;
if ( written != NULL ) *written = 0;
rc = VDBManagerMakeUpdate ( &my_manager, dir );
if ( rc != 0 )
LogErr( klogInt, rc, "VDBManagerMakeUpdate() failed\n" );
else
{
VSchema * my_schema;
rc = make_schema( schema_list, my_manager, &my_schema );
if ( rc == 0 )
{
VTable * my_table;
rc = VDBManagerCreateTable( my_manager, &my_table, my_schema,
"NCBI:align:tbl:qstat",
kcmInit | kcmParents,
output_file_path );
if ( rc != 0 )
LogErr( klogInt, rc, "VDBManagerCreateTable() failed\n" );
else
{
rc = write_statistic_cmn( my_table, data, written, show_progress );
VTableRelease( my_table );
}
VSchemaRelease( my_schema );
}
VDBManagerRelease( my_manager );
}
return rc;
}
static
rc_t open_dst_table ( const vtblcp_parms *pb,
VDBManager *mgr, const VSchema *schema, VTable **tblp )
{
return VDBManagerCreateTable ( mgr, tblp, schema,
pb -> dst_type, kcmInit, pb -> dst_path );
}
LIB_EXPORT rc_t CC SRAMgrVCreateTable ( SRAMgr *self, SRATable **rslt,
const char *typespec, const char *spec, va_list args )
{
rc_t rc;
if ( rslt == NULL )
rc = RC ( rcSRA, rcTable, rcConstructing, rcParam, rcNull );
else
{
if ( self == NULL )
rc = RC ( rcSRA, rcTable, rcConstructing, rcSelf, rcNull );
else if ( spec == NULL )
rc = RC ( rcSRA, rcTable, rcConstructing, rcString, rcNull );
else if ( spec [ 0 ] == 0 )
rc = RC ( rcSRA, rcTable, rcConstructing, rcString, rcEmpty );
else
{
char path [ 4097 ];
size_t act_size = 0;
rc = string_vprintf(path, 4097, &act_size, spec, args);
path[act_size] = '\0';
if ( rc == 0 )
{
VTable *vtbl;
rc = VDBManagerCreateTable ( self -> vmgr, & vtbl, self -> schema,
typespec, ( self -> mode & kcmBitMask ) | kcmCreate, "%s", path );
if ( rc == 0 )
{
rc = VTableColumnCreateParams ( vtbl, kcmCreate, kcsCRC32, 0 );
if ( rc == 0 )
{
SRATable *tbl = calloc ( 1, sizeof * tbl );
if ( tbl == NULL )
rc = RC ( rcSRA, rcTable, rcConstructing, rcMemory, rcExhausted );
else
{
tbl -> vtbl = vtbl;
rc = VTableOpenMetadataUpdate ( vtbl, & tbl -> meta );
if ( rc == 0 )
rc = KMetadataVersion ( tbl -> meta, & tbl -> metavers );
if ( rc == 0 )
rc = VTableCreateCursorWrite ( vtbl, & tbl -> curs, kcmInsert );
if ( rc == 0 )
{
tbl -> mgr = SRAMgrAttach ( self );
tbl -> mode = self -> mode;
tbl -> read_only = false;
KRefcountInit ( & tbl -> refcount, 1, "SRATable", "OpenTableUpdate", path );
VectorInit ( & tbl -> wcol, 0, 16 );
* rslt = tbl;
return 0;
}
vtbl = NULL;
SRATableWhack ( tbl );
}
}
VTableRelease ( vtbl );
}
}
}
* rslt = NULL;
}
return rc;
}
static rc_t static_write(void)
{
const char *schema_text = "version 1; include 'vdb/vdb.vschema'; table foo #1 { column ascii bar { read = .bar; } physical ascii .bar = bar; }";
bool force = 1;
rc_t rc;
VDBManager *vmgr;
rc = VDBManagerMakeUpdate ( & vmgr, NULL );
if ( rc != 0 )
LOGERR ( klogInt, rc, "failed to make VDB manager" );
else
{
VSchema *schema;
rc = VDBManagerMakeSchema ( vmgr, & schema );
if ( rc != 0 )
LOGERR ( klogInt, rc, "failed to make empty schema" );
else
{
rc = VSchemaParseText( schema, "static_schema", schema_text, strlen(schema_text) );
if ( rc != 0 )
PLOGERR ( klogErr, (klogErr, rc, "failed to parse schema '$(text)'", "test=%s", schema_text ));
else
{
VTable *vtbl;
rc = VDBManagerCreateTable ( vmgr, & vtbl, schema, "foo", force ? kcmInit : kcmCreate, table_path );
if ( rc != 0 )
{
PLOGERR ( klogErr, (klogErr, rc, "failed to $(cmode) table '$(path)'"
, "cmode=%s,path=%s"
, force ? "create or replace" : "create"
, table_path
));
}
else
{
VCursor *curs;
rc = VTableCreateCursorWrite ( vtbl, & curs, kcmInsert );
if ( rc != 0 )
LOGERR ( klogInt, rc, "failed to create cursor" );
else
{
uint32_t idx;
rc = VCursorAddColumn ( curs, &idx, "bar" );
if ( rc != 0 )
{
PLOGERR ( klogErr, (klogErr, rc, "failed to add column '$(col)' to cursor"
, "col=bar"
));
}
else {
rc = VCursorOpen ( curs );
if ( rc != 0 )
LOGERR ( klogErr, rc, "failed to open cursor" );
else
{
int i;
for ( i = 0; i != 10 && rc == 0; ++i )
{
rc = VCursorOpenRow ( curs );
if ( rc != 0 )
LOGERR ( klogErr, rc, "failed to open cursor row" );
else
{
rc_t rc2;
uint32_t count = sizeof(buff) - 1;
rc = VCursorWrite ( curs, idx, 8, buff, 0, count );
if ( rc != 0 )
{
int64_t rid = 0;
VCursorRowId ( curs, & rid );
PLOGERR ( klogInt, (klogInt, rc, "failed to write data to row $(row_id)'"
, "row_id=%ld"
, rid
));
break;
}
if ( rc == 0 )
{
rc = VCursorCommitRow ( curs );
if ( rc != 0 )
LOGERR ( klogErr, rc, "failed to commit row" );
}
rc2 = VCursorCloseRow ( curs );
if ( rc2 != 0 )
{
LOGERR ( klogErr, rc2, "failed to close cursor row" );
if ( rc == 0 )
rc = rc2;
}
}
}
if ( GetRCState ( rc ) == rcDone )
rc = 0;
if ( rc == 0 )
rc = VCursorCommit ( curs );
}
}
VCursorRelease ( curs );
}
#if 1
if ( rc == 0 )
rc = VTableReindex ( vtbl );
#endif
VTableRelease ( vtbl );
}
}
VSchemaRelease ( schema );
}
VDBManagerRelease ( vmgr );
}
return rc;
}
rc_t run (const char * table_path, uint64_t N )
{
static const char *colInf[] = {
"C1: Same value, same length",
"C2: Var. value, same length",
"C3: Var. value, var. legnth",
"C4: Same value except I row",
"C5: Same value except L row" };
rc_t rc;
uint64_t row = 0;
VDBManager *mgr = NULL;
VSchema *schema = NULL;
VTable *table = NULL;
VCursor *cursor;
uint32_t idx[COLUMNS];
uint64_t total[COLUMNS];
int i = 0, j = 0, prev = 0;
char *buffer[BUFFERS];
/* Initialize arrays */
memset(&idx, 0, sizeof idx);
memset(&total, 0, sizeof total);
for (i = 0; i < BUFFERS; ++i) {
char c;
size_t sz = ROWLEN + 1;
if (i == (BUFFERS - 1))
sz += ROWLEN;
buffer[i] = malloc(sz);
for (j = 0, c = 0; j < sz - 1; ++j, ++c) {
if (c >= ROWLEN)
c -= ROWLEN;
buffer[i][j] = '0' + c;
}
buffer[i][j] = '\0';
}
/* Create manager */
rc = VDBManagerMakeUpdate(&mgr, NULL);
if (rc != 0) {
LOGERR(klogInt, rc, "failed to open vdb library");
}
/* Initialize schema */
if (rc == 0) {
rc = VDBManagerMakeSchema(mgr, &schema);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create empty schema");
}
if (rc == 0) {
char text[512] = "table Table #1 {\n";
char col [128];
for (i = 1; i <= COLUMNS; ++i) {
sprintf(col,
" column ascii C%d = .C%d; physical ascii .C%d = C%d;\n",
i, i, i, i);
strcat(text, col);
}
strcat(text, "};");
STSMSG(1,("Parsing schema:\n%s", text));
rc = VSchemaParseText(schema, "Schema", text, strlen(text));
if (rc != 0)
LOGERR(klogInt, rc, "failed to parse schema");
}
/* Create table */
if (rc == 0) {
STSMSG(1,("Creating %s", tablePath));
rc = VDBManagerCreateTable(mgr, &table, schema, "Table", kcmInit,
tablePath);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create table");
}
/* Initialize cursor */
if (rc == 0) {
rc = VTableCreateCursorWrite(table, &cursor, kcmInsert);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create cursor");
}
for (i = 0; rc == 0 && i < COLUMNS; ++i) {
char col[3];
sprintf(col, "C%d", i + 1);
STSMSG(2,("Adding column %s to cursor", col));
rc = VCursorAddColumn(cursor, &idx[i], col);
if (rc != 0)
PLOGERR(klogInt, (klogInt, rc,
"failed to add $(c) to cursor", "c=%s", col));
}
if (rc == 0) {
rc = VCursorOpen(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to open cursor");
}
/* Write data */
for (row = 0; row < N && rc == 0; ++row) {
int max = 2 * ROWLEN - 1;
int sz = 0;
if ((row % 2) == 0) {
int min = 1;
sz = min + (int) (max * (rand() / (RAND_MAX + 1.0)));
prev = sz;
buffer[1][0] = '1';
}
else {
sz = max + 1 - prev;
buffer[1][0] = '2';
}
rc = Quitting();
if (rc == 0) {
KStsLevel lvl = 2;
if (row > 0 && ((row % ROWS) == 0)) {
lvl = 1;
}
STSMSG (lvl, ("Writing row %ji / %ji",
row + 1, N));
rc = VCursorOpenRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to open row");
}
for (j = 0; j < COLUMNS && rc == 0; ++j) {
uint32_t count = 0;
int buf = j;
switch (j) {
case 0:
case 1:
count = strlen(buffer[j]);
break;
case 2:
count = sz;
break;
case 3:
buf = 0;
if (row == 0)
buf = 1;
count = strlen(buffer[buf]);
break;
case 4:
buf = 0;
if (row == (N - 1))
buf = 1;
count = strlen(buffer[buf]);
break;
default:
assert(0);
break;
}
STSMSG (3, ("Row %ji/Col.%d: %sd %.*s\n",
row + 1, j + 1, count, count, buffer[buf]));
rc = VCursorWrite
(cursor, idx[j], 8, buffer[buf], 0, count);
if (rc != 0)
PLOGERR(klogInt, (klogInt, rc, "failed to write row[$i]", "i=%d", j + 1));
total[j] += count;
}
if (rc == 0) {
rc = VCursorCommitRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to commit row");
}
if (rc == 0) {
rc = VCursorCloseRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to close row");
}
}
if (rc == 0) {
STSMSG (1, ("Commiting cursor\n"));
rc = VCursorCommit(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to commit cursor");
}
/* Cleanup */
VCursorRelease(cursor);
VTableRelease(table);
VSchemaRelease(schema);
VDBManagerRelease(mgr);
for (i = 0; i < BUFFERS; ++i) {
free(buffer[i]);
}
/* Log */
if (rc == 0) {
PLOGMSG(klogInfo, (klogInfo, "$(t)", "t=%s", tablePath));
PLOGMSG(klogInfo,(klogInfo,
"$(n)($(N)) rows written - $(b) bytes per row",
PLOG_I64(n) "," PLOG_X64(N) ",b=%d", N, N, ROWLEN));
for (i = 0; i < COLUMNS; ++i) {
PLOGMSG(klogInfo,(klogInfo,
"$(i): $(n)($(N)) bytes",
"i=%s," PLOG_I64(n) "," PLOG_X64(N),
colInf[i], total[i], total[i]));
}
}
if (rc == 0) {
KDirectory *dir = NULL;
uint64_t sizes[COLUMNS];
memset(&sizes, 0, sizeof sizes);
rc = KDirectoryNativeDir(&dir);
if (rc != 0)
LOGERR(klogInt, rc, "failed to KDirectoryNativeDir");
else {
for (i = 1; i <= COLUMNS; ++i) {
uint64_t size = 0;
#define FORMAT "%s/col/%s/data"
#define KFORMAT "$(d)/col/$(n)/data", "d=%s,n=%s"
#define STATUS(action) (action FORMAT, tablePath, name)
char name[3];
sprintf(name, "C%d", i);
STSMSG (1, STATUS("checking "));
rc = KDirectoryFileSize(dir, &size, FORMAT, tablePath, name);
if (rc != 0) {
if (GetRCState(rc) == rcNotFound) {
STSMSG (2, STATUS("not found "));
rc = 0;
}
else
PLOGERR(klogInt, (klogInt, rc,
"failed to check " KFORMAT, tablePath, name));
}
else {
STSMSG (2, STATUS("found "));
}
PLOGMSG(klogInfo, (klogInfo, "Size of $(d)/col/$(n)/data = $(s)",
"d=%s,n=%s," PLOG_I64(s), tablePath, name, size));
sizes[i - 1] = size;
}
}
KDirectoryRelease(dir);
if (rc == 0) {
puts("");
KOutMsg("%ld rows, %d bytes per row:\n", N, ROWLEN);
for (i = 0; i < COLUMNS; ++i) {
puts(colInf[i]);
}
puts("");
for (i = 0; i < COLUMNS; ++i) {
int64_t over = sizes[i] - total[i];
KOutMsg("C%d: %9ld bytes written; "
"%9ld in 'data'", i + 1, total[i], sizes[i]);
if (over > 0) {
double p = 100.0 * over / sizes[i];
printf(": %7ld extra bytes (%.4f%%)\n", over, p);
}
else {
puts("");
}
}
}
}
return rc;
}
rc_t open (param_block * pb)
{
rc_t rc;
rc = KDirectoryNativeDir (&pb->pwd);
if (rc)
LOGERR (klogFatal, rc, "Failed to open file system");
else
{
rc = KDirectoryOpenFileRead (pb->pwd, &pb->file, "%s", pb->file_path);
if (rc)
LOGERR (klogFatal, rc, "Failed to open input file");
else
{
rc = KMMapMakeRead (&pb->mmap, pb->file);
if (rc)
LOGERR (klogFatal, rc, "unable to map file");
else
{
rc = VDBManagerMakeUpdate (&pb->mgr, pb->pwd);
if (rc)
LOGERR (klogFatal, rc, "Failed to open DB Manager");
else
{
rc = VDBManagerMakeSchema (pb->mgr, &pb->schema);
if (rc)
LOGERR (klogFatal, rc, "Failed to create a schema object");
else
{
VSchemaAddIncludePath (pb->schema, "interfaces");
rc = VSchemaParseFile (pb->schema, "%s", pb->schema_path);
if (rc)
LOGERR (klogFatal, rc, "Failed to parse schema");
else
{
rc = VDBManagerCreateTable (pb->mgr, &pb->table, pb->schema,
TYPESPEC, kcmCreate, "%s", pb->table_path);
if (rc)
LOGERR (klogFatal, rc, "Failed to create table");
else
{
rc = VTableCreateCursorWrite (pb->table, &pb->cursor, kcmCreate);
if (rc)
LOGERR (klogFatal, rc, "Failed to create cursor");
else
{
rc = VCursorAddColumn (pb->cursor, &pb->idx, "READ");
if (rc)
LOGERR (klogFatal, rc, "Failed to add READ to cursor");
else
{
rc = VCursorOpen (pb->cursor);
if (rc)
LOGERR (klogFatal, rc, "Failed to open cursor");
else
{
rc = write_rows (pb);
if (rc == 0)
VCursorCommit (pb->cursor);
}
}
VCursorRelease (pb->cursor);
}
VTableRelease (pb->table);
}
}
VSchemaRelease (pb->schema);
}
VDBManagerRelease (pb->mgr);
}
KMMapRelease (pb->mmap);
}
KFileRelease (pb->file);
}
KDirectoryRelease (pb->pwd);
}
return rc;
}
rc_t matcher_execute( matcher* self, const p_matcher_input in )
{
VSchema * dflt_schema;
const VTable * src_table;
rc_t rc;
if ( self == NULL )
return RC( rcExe, rcNoTarg, rcResolving, rcSelf, rcNull );
if ( in->manager == NULL || in->add_schemas == NULL ||
in->cfg == NULL || in->columns == NULL ||
in->src_path == NULL || in->dst_path == NULL ||
in->dst_tabname == NULL )
return RC( rcExe, rcNoTarg, rcResolving, rcParam, rcNull );
rc = matcher_build_column_vector( self, in->columns );
if ( rc != 0 ) return rc;
rc = matcher_exclude_columns( self, in->excluded_columns );
if ( rc != 0 ) return rc;
rc = helper_parse_schema( in->manager, &dflt_schema, in->add_schemas );
if ( rc != 0 ) return rc;
rc = VDBManagerOpenTableRead( in->manager, &src_table, dflt_schema, "%s", in->src_path );
if ( rc == 0 )
{
const VSchema * src_schema;
rc = VTableOpenSchema ( src_table, &src_schema );
if ( rc == 0 )
{
rc = matcher_read_src_types( self, src_table, src_schema );
if ( rc == 0 )
{
if ( in->legacy_schema != NULL )
rc = VSchemaParseFile ( dflt_schema, "%s", in->legacy_schema );
if ( rc == 0 )
{
VTable * dst_table;
KCreateMode cmode = kcmParents;
const VSchema * dst_schema = src_schema;
if ( in->legacy_schema != NULL )
dst_schema = dflt_schema;
if ( in->force_unlock )
VDBManagerUnlock ( in->manager, "%s", in->dst_path );
if ( in->force_kcmInit )
cmode |= kcmInit;
else
cmode |= kcmCreate;
rc = VDBManagerCreateTable( in->manager, &dst_table,
dst_schema, in->dst_tabname, cmode, "%s", in->dst_path );
if ( rc == 0 )
{
rc = matcher_read_dst_types( self, dst_table, dst_schema );
if ( rc == 0 )
{
rc = matcher_make_type_matrix( self );
if ( rc == 0 )
rc = matcher_match_matrix( self, src_schema, in->cfg );
}
VTableRelease( dst_table );
if ( !(in->force_kcmInit) )
KDirectoryRemove ( in->dir, true, "%s", in->dst_path );
}
}
}
VSchemaRelease( src_schema );
}
VTableRelease( src_table );
}
VSchemaRelease( dflt_schema );
return rc;
}
