static
rc_t SaveObjects ( const ObjectTable* data, VDatabase* db )
{
VTable* tbl;
rc_t rc = VDatabaseCreateTable(db, &tbl, "object_inst", kcmCreate | kcmMD5, "OBJECTS");
if (rc == 0)
{
rc_t rc2;
VCursor *cur;
rc = VTableCreateCursorWrite( tbl, &cur, kcmInsert );
if (rc == 0)
{
uint32_t id_idx, name_idx, proj_idx, dname_idx, size_idx, csum_idx, enc_idx;
if (rc == 0) rc = VCursorAddColumn( cur, &id_idx, "id" );
if (rc == 0) rc = VCursorAddColumn( cur, &name_idx, "name" );
if (rc == 0) rc = VCursorAddColumn( cur, &proj_idx, "project" );
if (rc == 0) rc = VCursorAddColumn( cur, &dname_idx, "display_name" );
if (rc == 0) rc = VCursorAddColumn( cur, &size_idx, "size" );
if (rc == 0) rc = VCursorAddColumn( cur, &csum_idx, "checksum" );
if (rc == 0) rc = VCursorAddColumn( cur, &enc_idx, "encryption_key" );
if (rc == 0)
{
rc = VCursorOpen( cur );
if (rc == 0)
{
const Object* obj = (const Object*)BSTreeFirst(data);
while (rc == 0 && obj != NULL)
{
rc = VCursorOpenRow( cur );
if (rc == 0) rc = VCursorWrite( cur, id_idx, sizeof(obj->id) * 8, &obj->id, 0, 1);
if (rc == 0) rc = VCursorWrite( cur, name_idx, StringLength(obj->name) * 8, obj->name->addr, 0, 1);
if (rc == 0) rc = VCursorWrite( cur, proj_idx, StringLength(obj->project) * 8, obj->project->addr, 0, 1);
if (rc == 0) rc = VCursorWrite( cur, dname_idx, StringLength(obj->display_name) * 8, obj->display_name->addr, 0, 1);
if (rc == 0) rc = VCursorWrite( cur, size_idx, sizeof(obj->size) * 8, &obj->size, 0, 1);
if (rc == 0) rc = VCursorWrite( cur, csum_idx, StringLength(obj->encryption_key) * 8, obj->encryption_key->addr, 0, 1);
if (rc == 0) rc = VCursorWrite( cur, enc_idx, StringLength(obj->encryption_key) * 8, obj->encryption_key->addr, 0, 1);
if (rc == 0) rc = VCursorCommitRow( cur );
if (rc == 0) rc = VCursorCloseRow( cur );
obj = (const Object*)BSTNodeNext(&obj->dad);
}
if (rc == 0) rc = VCursorCommit( cur );
}
}
rc2 = VCursorRelease(cur);
if (rc == 0)
rc = rc2;
}
rc2 = VTableRelease(tbl);
if (rc == 0)
rc = rc2;
}
return rc;
}
static rc_t seq_load_spot( VCursor *cursor, const uint32_t *col_idx,
region_type_mapping *mapping, zmw_row * spot,
void * data )
{
BaseCalls *tab = (BaseCalls *)data;
rc_t rc = VCursorOpenRow( cursor );
if ( rc != 0 )
PLOGERR( klogErr, ( klogErr, rc, "cannot open seq-row on spot# $(spotnr)",
"spotnr=%u", spot->spot_nr ) );
if ( rc == 0 )
rc = vdb_write_uint32( cursor, col_idx[ seq_tab_HOLE_NUMBER ],
spot->HoleNumber, "seq.HOLE_NUMBER" );
if ( rc == 0 )
rc = vdb_write_uint8( cursor, col_idx[ seq_tab_HOLE_STATUS ],
spot->HoleStatus, "seq.HOLE_STATUS" );
if ( rc == 0 )
rc = vdb_write_value( cursor, col_idx[ seq_tab_HOLE_XY ],
&spot->HoleXY, HOLE_XY_BITSIZE, 2, "seq.HOLE_XY" );
if ( rc == 0 )
{
/* we load the bases / quality-values and other data belonging
to this hole(bacbio)/spot(ncbi) */
if ( spot->NumEvent > 0 )
rc = seq_load_spot_bases( cursor, tab, col_idx, spot );
else
rc = seq_load_zero_bases( cursor, tab, col_idx );
/* we try to divide the spot into regions(pacbio)/reads(ncbi) */
if ( rc == 0 )
{
if ( mapping != NULL )
rc = seq_load_read_desc( cursor, col_idx, mapping, spot, tab );
else
rc = seq_load_one_spot( cursor, col_idx, spot, tab );
}
}
if ( rc == 0 )
{
rc = VCursorCommitRow( cursor );
if ( rc != 0 )
PLOGERR( klogErr, ( klogErr, rc, "cannot commit seq-row on spot# $(spotnr)",
"spotnr=%u", spot->spot_nr ) );
}
if ( rc == 0 )
{
rc = VCursorCloseRow( cursor );
if ( rc != 0 )
PLOGERR( klogErr, ( klogErr, rc, "cannot close seq-row on spot# $(spotnr)",
"spotnr=%u", spot->spot_nr ) );
}
return rc;
}
/* CloseSpot
* closes a spot opened with either NewSpot or OpenSpot
*/
LIB_EXPORT rc_t CC SRATableCloseSpot( SRATable *self ) {
rc_t rc;
if (self == NULL)
return RC(RC_MODULE, RC_TARGET, rcClosing, rcSelf, rcNull);
rc = VCursorCommitRow(self->curs);
if (rc == 0)
return VCursorCloseRow(self->curs);
VCursorCloseRow(self->curs);
return rc;
}
static
rc_t SaveProjects( const ProjectTable* data, VDatabase* db )
{
VTable* tbl;
rc_t rc = VDatabaseCreateTable(db, &tbl, "project_inst", kcmCreate | kcmMD5, "PROJECTS");
if (rc == 0)
{
rc_t rc2;
VCursor *cur;
rc = VTableCreateCursorWrite( tbl, &cur, kcmInsert );
if (rc == 0)
{
uint32_t id_idx, name_idx, dl_idx, enc_idx;
rc = VCursorAddColumn( cur, &id_idx, "id" );
rc = VCursorAddColumn( cur, &name_idx, "name" );
rc = VCursorAddColumn( cur, &dl_idx, "download_ticket" );
rc = VCursorAddColumn( cur, &enc_idx, "encryption_key" );
if (rc == 0)
{
rc = VCursorOpen( cur );
if (rc == 0)
{
const Project* p = (const Project*)BSTreeFirst(data);
while (rc == 0 && p != NULL)
{
rc = VCursorOpenRow( cur );
if (rc == 0) rc = VCursorWrite( cur, id_idx, sizeof(p->id) * 8, &p->id, 0, 1);
if (rc == 0) rc = VCursorWrite( cur, name_idx, StringLength(p->name) * 8, p->name->addr, 0, 1);
if (rc == 0) rc = VCursorWrite( cur, dl_idx, StringLength(p->download_ticket) * 8, p->download_ticket->addr, 0, 1);
if (rc == 0) rc = VCursorWrite( cur, enc_idx, StringLength(p->encryption_key) * 8, p->encryption_key->addr, 0, 1);
if (rc == 0) rc = VCursorCommitRow( cur );
if (rc == 0) rc = VCursorCloseRow( cur );
p = (const Project*)BSTNodeNext(&p->dad);
}
if (rc == 0)
rc = VCursorCommit( cur );
}
}
rc2 = VCursorRelease(cur);
if (rc == 0)
rc = rc2;
}
rc2 = VTableRelease(tbl);
if (rc == 0)
rc = rc2;
}
return rc;
}
static rc_t passes_load_pass( VCursor *cursor, pass_block *block,
const uint32_t idx, uint32_t *col_idx )
{
rc_t rc = VCursorOpenRow( cursor );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot open passes-row" );
if ( rc == 0 )
rc = vdb_write_uint8( cursor,
col_idx[ passes_tab_ADAPTER_HIT_BEFORE ],
block->AdapterHitBefore[ idx ],
"passes.AdapterHitBefore" );
if ( rc == 0 )
rc = vdb_write_uint8( cursor,
col_idx[ passes_tab_ADAPTER_HIT_AFTER ],
block->AdapterHitAfter[ idx ],
"passes.AdapterHitAfter" );
if ( rc == 0 )
rc = vdb_write_uint8( cursor,
col_idx[ passes_tab_PASS_DIRECTION ],
block->PassDirection[ idx ],
"passes.PassDirection" );
if ( rc == 0 )
rc = vdb_write_uint32( cursor,
col_idx[ passes_tab_PASS_NUM_BASES ],
block->PassNumBases[ idx ],
"passes.PassNumBases" );
if ( rc == 0 )
rc = vdb_write_uint32( cursor,
col_idx[ passes_tab_PASS_START_BASE ],
block->PassStartBase[ idx ],
"passes.PassStartBase" );
if ( rc == 0 )
{
rc = VCursorCommitRow( cursor );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot commit passes-row" );
}
if ( rc == 0 )
{
rc = VCursorCloseRow( cursor );
if ( rc != 0 )
LOGERR( klogErr, rc, "cannot close passes-row" );
}
return rc;
}
rc_t write_rows (param_block * pb)
{
rc_t rc = 0;
pb->seq = NULL;
do
{
rc = get_a_sequence (pb);
if (rc)
{
LOGERR (klogFatal, rc, "Failed to read a sequence");
}
else
{
if (pb->seq == NULL)
break;
rc = VCursorOpenRow (pb->cursor);
if (rc)
{
LOGERR (klogFatal, rc, "Failed to open row");
break;
}
else
{
rc_t rc2;
rc = VCursorWrite (pb->cursor, pb->idx, 8, pb->seq, 0, pb->seq_size);
if (rc)
LOGERR (klogFatal, rc, "Failed to write row");
else
rc = VCursorCommitRow (pb->cursor);
rc2 = VCursorCloseRow (pb->cursor);
if (rc == 0)
rc = rc2;
}
}
} while (rc == 0);
return rc;
}
static bool CC qstats_write(stat_row const *row, void *ctx)
{
writer_ctx_t *self = ctx;
rc_t rc = VCursorOpenRow(self->curs);
while (rc == 0) {
rc = VCursorWrite(self->curs, self->cid[0], 8, row->spotgroup, 0, strlen(row->spotgroup)); if (rc) break;
rc = VCursorWrite(self->curs, self->cid[1], 32, &row->base_pos, 0, 1); if (rc) break;
rc = VCursorWrite(self->curs, self->cid[2], 8, row->dimer, 0, 2); if (rc) break;
rc = VCursorWrite(self->curs, self->cid[3], 8, &row->hp_run, 0, 1); if (rc) break;
rc = VCursorWrite(self->curs, self->cid[4], 8, &row->gc_content, 0, 1); if (rc) break;
rc = VCursorWrite(self->curs, self->cid[5], 8, &row->quality, 0, 1); if (rc) break;
rc = VCursorWrite(self->curs, self->cid[6], 32, &row->count, 0, 1); if (rc) break;
if (self->alignMode) {
rc = VCursorWrite(self->curs, self->cid[7], 32, &row->mismatch_count, 0, 1); if (rc) break;
rc = VCursorWrite(self->curs, self->cid[8], 32, &row->insert_count, 0, 1); if (rc) break;
rc = VCursorWrite(self->curs, self->cid[9], 32, &row->delete_count, 0, 1); if (rc) break;
}
rc = VCursorCommitRow(self->curs); if (rc) break;
rc = VCursorCloseRow(self->curs);
}
return rc == 0;
}
rc_t CC KMain(int argc, char* argv[])
{
const char table[] = "/home/klymenka/REDACT-IN";
const char name[] = "READ_FILTER";
rc_t rc = 0;
bool locked = false;
VDBManager* mgr;
VTable *tbl;
const VCursor *rCursor = NULL;
int i;
LogLevelSet("info");
for (i = 1; i < argc; ++i) {
if (!strcmp(argv[i], "-+")) {
if (++i <= argc) {
#if _DEBUGGING
KDbgSetString(argv[i]);
#endif
}
}
}
/*KDbgSetString("VDB");*/
if (rc == 0) {
/* +01: ManagerMake */
LOGMSG(klogInfo, "VDBManagerMakeUpdate");
rc = VDBManagerMakeUpdate(&mgr, NULL);
DISP_RC_INT(rc, "while calling VDBManagerMakeUpdate");
}
if (rc == 0) {
rc = VDBManagerWritable(mgr, table);
if (GetRCState(rc) == rcLocked) {
LOGMSG(klogInfo, "VDBManagerUnlock");
rc = VDBManagerUnlock(mgr, table);
DISP_RC_INT(rc, "while calling VDBManagerUnlock");
locked = true;
}
}
if (rc == 0) {
/* +02: OpenTable */
PLOGMSG(klogInfo, (klogInfo,
"VDBManagerOpenTableUpdate(\"$(t)\")", "t=%s", table));
rc = VDBManagerOpenTableUpdate
(mgr, &tbl, NULL, table);
if (rc != 0) {
PLOGERR(klogErr, (klogErr, rc,
"while opening VTable '$(path)'", "path=%s", table));
}
}
if (rc == 0) {
/* +03: CreateCursorRead */
LOGMSG(klogInfo, "VDBManagerUnlock");
rc = VTableCreateCursorRead(tbl, &rCursor);
DISP_RC_INT(rc, "while creating read cursor");
#if 1
if (rc == 0) {
uint32_t idx;
PLOGMSG(klogInfo, (klogInfo,
"VCursorAddColumn(read cursor, \"$(n)\")", "n=%s", name));
rc = VCursorAddColumn(rCursor, &idx, name);
if (rc != 0) {
PLOGERR(klogErr, (klogErr, rc,
"while adding $(name) to read cursor", "name=%s", name));
}
}
#endif
if (rc == 0) {
LOGMSG(klogInfo, "VCursorOpen(read cursor)");
rc = VCursorOpen(rCursor);
DISP_RC_INT(rc, "while opening read cursor");
}
}
if (rc == 0) {
VCursor *cursor;
uint32_t idx;
/* +04: CreateCursorWrite */
LOGMSG(klogInfo, "VTableCreateCursorWrite");
rc = VTableCreateCursorWrite(tbl, &cursor, kcmInsert);
DISP_RC_INT(rc, "while creating write cursor");
if (rc == 0) {
PLOGMSG(klogInfo, (klogInfo,
"VCursorAddColumn(write cursor, \"$(n)\")", "n=%s", name));
rc = VCursorAddColumn(cursor, &idx, name);
if (rc != 0) {
PLOGERR(klogErr, (klogErr, rc,
"while adding $(name) to write cursor", "name=%s", name));
}
}
if (rc == 0) {
LOGMSG(klogInfo, "VCursorOpen(write cursor)");
rc = VCursorOpen(cursor);
DISP_RC_INT(rc, "while opening write cursor");
}
#if 1
for (i = 0; i < 3 && rc == 0; ++i) {
if (rc == 0) {
PLOGMSG(klogInfo, (klogInfo,
"VCursorOpenRow(write cursor) $(i)", "i=%d", i));
rc = VCursorOpenRow(cursor);
DISP_RC_INT(rc, "while opening row to write");
}
if (rc == 0) {
char buffer[1];
char b;
switch (i) {
case 0:
buffer[0] = SRA_READ_FILTER_CRITERIA;
buffer[0] = SRA_READ_FILTER_REJECT;
break;
case 1:
buffer[0] = SRA_READ_FILTER_REJECT;
buffer[0] = SRA_READ_FILTER_CRITERIA;
break;
case 2:
buffer[0] = SRA_READ_FILTER_REDACTED;
break;
}
buffer[0] = SRA_READ_FILTER_PASS;
b = buffer[0];
PLOGMSG(klogInfo, (klogInfo,
"VCursorWrite('$(v)') $(i)", "v=%s,i=%d",
b == SRA_READ_FILTER_REDACTED ? "SRA_READ_FILTER_REDACTED" :
"?",
i));
rc = VCursorWrite(cursor, idx, 8, buffer, 0, 1);
DISP_RC_INT(rc, "while writing");
}
if (rc == 0) {
PLOGMSG(klogInfo, (klogInfo,
"VCursorCommitRow(write cursor) $(i)", "i=%d", i));
rc = VCursorCommitRow(cursor);
DISP_RC_INT(rc, "while committing row");
}
PLOGMSG(klogInfo, (klogInfo,
"VCursorCloseRow(write cursor) $(i)", "i=%d", i));
{
rc_t rc2 = VCursorCloseRow(cursor);
DISP_RC_INT(rc2, "while closing row");
if (rc == 0)
{ rc = rc2; }
}
}
#endif
LOGMSG(klogInfo, "VCursorRelease(read cursor)");
/* -03: CreateCursorRead */
VCursorRelease(rCursor);
if (rc == 0) {
LOGMSG(klogInfo, "VCursorCommit(write cursor)");
rc = VCursorCommit(cursor);
DISP_RC_INT(rc, "while committing cursor");
}
LOGMSG(klogInfo, "VCursorRelease(write cursor)");
/* -04: CreateCursorWrite */
VCursorRelease(cursor);
}
LOGMSG(klogInfo, "VTableRelease");
/* -02: OpenTable */
VTableRelease(tbl);
LOGMSG(klogInfo, "VDBManagerLock");
if (locked) {
rc_t rc2 = VDBManagerLock(mgr, table);
DISP_RC_INT(rc2, "while VDBManagerLock");
}
/* -01: ManagerMake */
LOGMSG(klogInfo, "VDBManagerRelease");
VDBManagerRelease(mgr);
if (rc == 0) {
LOGMSG(klogInfo, "SUCCESS");
}
else { LOGMSG(klogInfo, "FAILURE"); }
return rc;
}
static rc_t static_write(void)
{
const char *schema_text = "version 1; include 'vdb/vdb.vschema'; table foo #1 { column ascii bar { read = .bar; } physical ascii .bar = bar; }";
bool force = 1;
rc_t rc;
VDBManager *vmgr;
rc = VDBManagerMakeUpdate ( & vmgr, NULL );
if ( rc != 0 )
LOGERR ( klogInt, rc, "failed to make VDB manager" );
else
{
VSchema *schema;
rc = VDBManagerMakeSchema ( vmgr, & schema );
if ( rc != 0 )
LOGERR ( klogInt, rc, "failed to make empty schema" );
else
{
rc = VSchemaParseText( schema, "static_schema", schema_text, strlen(schema_text) );
if ( rc != 0 )
PLOGERR ( klogErr, (klogErr, rc, "failed to parse schema '$(text)'", "test=%s", schema_text ));
else
{
VTable *vtbl;
rc = VDBManagerCreateTable ( vmgr, & vtbl, schema, "foo", force ? kcmInit : kcmCreate, table_path );
if ( rc != 0 )
{
PLOGERR ( klogErr, (klogErr, rc, "failed to $(cmode) table '$(path)'"
, "cmode=%s,path=%s"
, force ? "create or replace" : "create"
, table_path
));
}
else
{
VCursor *curs;
rc = VTableCreateCursorWrite ( vtbl, & curs, kcmInsert );
if ( rc != 0 )
LOGERR ( klogInt, rc, "failed to create cursor" );
else
{
uint32_t idx;
rc = VCursorAddColumn ( curs, &idx, "bar" );
if ( rc != 0 )
{
PLOGERR ( klogErr, (klogErr, rc, "failed to add column '$(col)' to cursor"
, "col=bar"
));
}
else {
rc = VCursorOpen ( curs );
if ( rc != 0 )
LOGERR ( klogErr, rc, "failed to open cursor" );
else
{
int i;
for ( i = 0; i != 10 && rc == 0; ++i )
{
rc = VCursorOpenRow ( curs );
if ( rc != 0 )
LOGERR ( klogErr, rc, "failed to open cursor row" );
else
{
rc_t rc2;
uint32_t count = sizeof(buff) - 1;
rc = VCursorWrite ( curs, idx, 8, buff, 0, count );
if ( rc != 0 )
{
int64_t rid = 0;
VCursorRowId ( curs, & rid );
PLOGERR ( klogInt, (klogInt, rc, "failed to write data to row $(row_id)'"
, "row_id=%ld"
, rid
));
break;
}
if ( rc == 0 )
{
rc = VCursorCommitRow ( curs );
if ( rc != 0 )
LOGERR ( klogErr, rc, "failed to commit row" );
}
rc2 = VCursorCloseRow ( curs );
if ( rc2 != 0 )
{
LOGERR ( klogErr, rc2, "failed to close cursor row" );
if ( rc == 0 )
rc = rc2;
}
}
}
if ( GetRCState ( rc ) == rcDone )
rc = 0;
if ( rc == 0 )
rc = VCursorCommit ( curs );
}
}
VCursorRelease ( curs );
}
#if 1
if ( rc == 0 )
rc = VTableReindex ( vtbl );
#endif
VTableRelease ( vtbl );
}
}
VSchemaRelease ( schema );
}
VDBManagerRelease ( vmgr );
}
return rc;
}
rc_t run (const char * table_path, uint64_t N )
{
static const char *colInf[] = {
"C1: Same value, same length",
"C2: Var. value, same length",
"C3: Var. value, var. legnth",
"C4: Same value except I row",
"C5: Same value except L row" };
rc_t rc;
uint64_t row = 0;
VDBManager *mgr = NULL;
VSchema *schema = NULL;
VTable *table = NULL;
VCursor *cursor;
uint32_t idx[COLUMNS];
uint64_t total[COLUMNS];
int i = 0, j = 0, prev = 0;
char *buffer[BUFFERS];
/* Initialize arrays */
memset(&idx, 0, sizeof idx);
memset(&total, 0, sizeof total);
for (i = 0; i < BUFFERS; ++i) {
char c;
size_t sz = ROWLEN + 1;
if (i == (BUFFERS - 1))
sz += ROWLEN;
buffer[i] = malloc(sz);
for (j = 0, c = 0; j < sz - 1; ++j, ++c) {
if (c >= ROWLEN)
c -= ROWLEN;
buffer[i][j] = '0' + c;
}
buffer[i][j] = '\0';
}
/* Create manager */
rc = VDBManagerMakeUpdate(&mgr, NULL);
if (rc != 0) {
LOGERR(klogInt, rc, "failed to open vdb library");
}
/* Initialize schema */
if (rc == 0) {
rc = VDBManagerMakeSchema(mgr, &schema);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create empty schema");
}
if (rc == 0) {
char text[512] = "table Table #1 {\n";
char col [128];
for (i = 1; i <= COLUMNS; ++i) {
sprintf(col,
" column ascii C%d = .C%d; physical ascii .C%d = C%d;\n",
i, i, i, i);
strcat(text, col);
}
strcat(text, "};");
STSMSG(1,("Parsing schema:\n%s", text));
rc = VSchemaParseText(schema, "Schema", text, strlen(text));
if (rc != 0)
LOGERR(klogInt, rc, "failed to parse schema");
}
/* Create table */
if (rc == 0) {
STSMSG(1,("Creating %s", tablePath));
rc = VDBManagerCreateTable(mgr, &table, schema, "Table", kcmInit,
tablePath);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create table");
}
/* Initialize cursor */
if (rc == 0) {
rc = VTableCreateCursorWrite(table, &cursor, kcmInsert);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create cursor");
}
for (i = 0; rc == 0 && i < COLUMNS; ++i) {
char col[3];
sprintf(col, "C%d", i + 1);
STSMSG(2,("Adding column %s to cursor", col));
rc = VCursorAddColumn(cursor, &idx[i], col);
if (rc != 0)
PLOGERR(klogInt, (klogInt, rc,
"failed to add $(c) to cursor", "c=%s", col));
}
if (rc == 0) {
rc = VCursorOpen(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to open cursor");
}
/* Write data */
for (row = 0; row < N && rc == 0; ++row) {
int max = 2 * ROWLEN - 1;
int sz = 0;
if ((row % 2) == 0) {
int min = 1;
sz = min + (int) (max * (rand() / (RAND_MAX + 1.0)));
prev = sz;
buffer[1][0] = '1';
}
else {
sz = max + 1 - prev;
buffer[1][0] = '2';
}
rc = Quitting();
if (rc == 0) {
KStsLevel lvl = 2;
if (row > 0 && ((row % ROWS) == 0)) {
lvl = 1;
}
STSMSG (lvl, ("Writing row %ji / %ji",
row + 1, N));
rc = VCursorOpenRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to open row");
}
for (j = 0; j < COLUMNS && rc == 0; ++j) {
uint32_t count = 0;
int buf = j;
switch (j) {
case 0:
case 1:
count = strlen(buffer[j]);
break;
case 2:
count = sz;
break;
case 3:
buf = 0;
if (row == 0)
buf = 1;
count = strlen(buffer[buf]);
break;
case 4:
buf = 0;
if (row == (N - 1))
buf = 1;
count = strlen(buffer[buf]);
break;
default:
assert(0);
break;
}
STSMSG (3, ("Row %ji/Col.%d: %sd %.*s\n",
row + 1, j + 1, count, count, buffer[buf]));
rc = VCursorWrite
(cursor, idx[j], 8, buffer[buf], 0, count);
if (rc != 0)
PLOGERR(klogInt, (klogInt, rc, "failed to write row[$i]", "i=%d", j + 1));
total[j] += count;
}
if (rc == 0) {
rc = VCursorCommitRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to commit row");
}
if (rc == 0) {
rc = VCursorCloseRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to close row");
}
}
if (rc == 0) {
STSMSG (1, ("Commiting cursor\n"));
rc = VCursorCommit(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to commit cursor");
}
/* Cleanup */
VCursorRelease(cursor);
VTableRelease(table);
VSchemaRelease(schema);
VDBManagerRelease(mgr);
for (i = 0; i < BUFFERS; ++i) {
free(buffer[i]);
}
/* Log */
if (rc == 0) {
PLOGMSG(klogInfo, (klogInfo, "$(t)", "t=%s", tablePath));
PLOGMSG(klogInfo,(klogInfo,
"$(n)($(N)) rows written - $(b) bytes per row",
PLOG_I64(n) "," PLOG_X64(N) ",b=%d", N, N, ROWLEN));
for (i = 0; i < COLUMNS; ++i) {
PLOGMSG(klogInfo,(klogInfo,
"$(i): $(n)($(N)) bytes",
"i=%s," PLOG_I64(n) "," PLOG_X64(N),
colInf[i], total[i], total[i]));
}
}
if (rc == 0) {
KDirectory *dir = NULL;
uint64_t sizes[COLUMNS];
memset(&sizes, 0, sizeof sizes);
rc = KDirectoryNativeDir(&dir);
if (rc != 0)
LOGERR(klogInt, rc, "failed to KDirectoryNativeDir");
else {
for (i = 1; i <= COLUMNS; ++i) {
uint64_t size = 0;
#define FORMAT "%s/col/%s/data"
#define KFORMAT "$(d)/col/$(n)/data", "d=%s,n=%s"
#define STATUS(action) (action FORMAT, tablePath, name)
char name[3];
sprintf(name, "C%d", i);
STSMSG (1, STATUS("checking "));
rc = KDirectoryFileSize(dir, &size, FORMAT, tablePath, name);
if (rc != 0) {
if (GetRCState(rc) == rcNotFound) {
STSMSG (2, STATUS("not found "));
rc = 0;
}
else
PLOGERR(klogInt, (klogInt, rc,
"failed to check " KFORMAT, tablePath, name));
}
else {
STSMSG (2, STATUS("found "));
}
PLOGMSG(klogInfo, (klogInfo, "Size of $(d)/col/$(n)/data = $(s)",
"d=%s,n=%s," PLOG_I64(s), tablePath, name, size));
sizes[i - 1] = size;
}
}
KDirectoryRelease(dir);
if (rc == 0) {
puts("");
KOutMsg("%ld rows, %d bytes per row:\n", N, ROWLEN);
for (i = 0; i < COLUMNS; ++i) {
puts(colInf[i]);
}
puts("");
for (i = 0; i < COLUMNS; ++i) {
int64_t over = sizes[i] - total[i];
KOutMsg("C%d: %9ld bytes written; "
"%9ld in 'data'", i + 1, total[i], sizes[i]);
if (over > 0) {
double p = 100.0 * over / sizes[i];
printf(": %7ld extra bytes (%.4f%%)\n", over, p);
}
else {
puts("");
}
}
}
}
return rc;
}
static bool CC write_cb( stat_row * row, void * data )
{
writer_ctx * ctx = ( writer_ctx * ) data;
col * cols = ( col * )&ctx->writer->wr_col;
VCursor * cursor = ctx->writer->cursor;
rc_t rc = VCursorOpenRow( cursor );
if ( rc != 0 )
LogErr( klogInt, rc, "VCursorOpen() failed\n" );
if ( rc == 0 )
rc = write_to_cursor( cursor, cols[ WIDX_SPOT_GROUP ].idx, 8,
row->spotgroup, string_size( row->spotgroup ),
widx_names[ WIDX_SPOT_GROUP ] );
if ( rc == 0 )
rc = write_to_cursor( cursor, cols[ WIDX_KMER ].idx, 8,
row->dimer, string_size( row->dimer ),
widx_names[ WIDX_KMER ] );
if ( rc == 0 )
rc = write_to_cursor( cursor, cols[ WIDX_ORIG_QUAL ].idx, 8,
&row->quality, 1, widx_names[ WIDX_ORIG_QUAL ] );
if ( rc == 0 )
rc = write_to_cursor( cursor, cols[ WIDX_CYCLE ].idx, 32,
&row->base_pos, 1, widx_names[ WIDX_CYCLE ] );
if ( rc == 0 )
rc = write_to_cursor( cursor, cols[ WIDX_TOTAL_COUNT ].idx, 32,
&row->count, 1, widx_names[ WIDX_TOTAL_COUNT ] );
if ( rc == 0 )
rc = write_to_cursor( cursor, cols[ WIDX_MISMATCH_COUNT ].idx, 32,
&row->mismatch_count, 1,
widx_names[ WIDX_MISMATCH_COUNT ] );
if ( rc == 0 )
rc = write_to_cursor( cursor, cols[ WIDX_HPRUN ].idx, 32,
&row->hp_run, 1, widx_names[ WIDX_HPRUN ] );
if ( rc == 0 )
rc = write_to_cursor( cursor, cols[ WIDX_GC_CONTENT ].idx, 32,
&row->gc_content, 1, widx_names[ WIDX_GC_CONTENT ] );
if ( rc == 0 )
rc = write_to_cursor( cursor, cols[ WIDX_MAX_QUAL ].idx, 8,
&row->max_qual_value, 1, widx_names[ WIDX_MAX_QUAL ] );
if ( rc == 0 )
rc = write_to_cursor( cursor, cols[ WIDX_NREAD ].idx, 8,
&row->n_read, 1, widx_names[ WIDX_NREAD ] );
if ( rc == 0 )
{
rc = VCursorCommitRow( cursor );
if ( rc != 0 )
LogErr( klogInt, rc, "VCursorCommitRow() failed\n" );
}
if ( rc == 0 )
{
rc = VCursorCloseRow( cursor );
if ( rc != 0 )
LogErr( klogInt, rc, "VCursorCloseRow() failed\n" );
}
ctx->rc = rc;
if ( ctx->progress != NULL && rc == 0 )
{
uint32_t percent = progressbar_percent( ctx->entries, ++( ctx->n ), ctx->fract_digits );
update_progressbar( ctx->progress, ctx->fract_digits, percent );
}
return ( rc == 0 );
}
static rc_t Work(Db* db, SpotIterator* it)
{
rc_t rc = 0;
bool toRedact = false;
int64_t row_id = 0;
spotid_t nSpots = 0;
spotid_t redactedSpots = 0;
uint8_t filter[64];
memset(filter, SRA_READ_FILTER_REDACTED, sizeof filter);
assert(it);
while (rc == 0 && SpotIteratorNext(it, &rc, &row_id, &toRedact)) {
uint8_t nreads = 0;
char bufferIn[64];
void* buffer = NULL;
uint32_t row_len = 0;
rc = Quitting();
++nSpots;
if (rc == 0) {
uint32_t elem_bits = 8;
rc = VCursorReadDirect(db->rCursor, row_id, db->rFilterIdx,
elem_bits, bufferIn, sizeof bufferIn, &row_len);
DISP_RC(rc, "while reading READ_FILTER");
nreads = row_len;
}
if (toRedact) {
buffer = filter;
++redactedSpots;
DBGMSG(DBG_APP,DBG_COND_1,
("Redacting spot %d: %d reads\n",row_id,nreads));
}
else {
buffer = bufferIn;
}
if (rc == 0) {
rc = VCursorOpenRow(db->wCursor);
DISP_RC(rc, "while opening row to write");
if (rc == 0) {
rc = VCursorWrite
(db->wCursor, db->wIdx, 8 * nreads, buffer, 0, 1);
DISP_RC(rc, "while writing READ_FILTER");
}
if (rc == 0) {
rc = VCursorCommitRow(db->wCursor);
DISP_RC(rc, "while committing row");
}
if (rc == 0) {
rc = VCursorCloseRow(db->wCursor);
DISP_RC(rc, "while closing row");
}
}
}
db->nSpots = nSpots;
db->redactedSpots = redactedSpots;
if (rc == 0) {
rc = VCursorCommit(db->wCursor);
DISP_RC(rc, "while committing cursor");
}
return rc;
}
static rc_t consensus_load_spot( VCursor *cursor, const uint32_t *col_idx,
region_type_mapping *mapping, zmw_row * spot,
void * data )
{
BaseCalls_cmn *tab = (BaseCalls_cmn *)data;
rc_t rc = VCursorOpenRow( cursor );
if ( rc != 0 )
PLOGERR( klogErr, ( klogErr, rc, "cannot open consensus-row on spot# $(spotnr)",
"spotnr=%u", spot->spot_nr ) );
if ( rc == 0 )
rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_HOLE_NUMBER ],
spot->HoleNumber, "consensus.HOLE_NUMBER" );
if ( rc == 0 )
rc = vdb_write_uint8( cursor, col_idx[ consensus_tab_HOLE_STATUS ],
spot->HoleStatus, "consensus.HOLE_STATUS" );
if ( rc == 0 )
rc = vdb_write_value( cursor, col_idx[ consensus_tab_HOLE_XY ],
&spot->HoleXY, HOLE_XY_BITSIZE, 2, "consensus.HOLE_XY" );
/* has to be read ... from "PulseData/ConsensusBaesCalls/Passes/NumPasses" */
if ( rc == 0 )
rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_NUM_PASSES ],
spot->NumPasses, "consensus.NUM_PASSES" );
if ( rc == 0 )
{
if ( spot->NumEvent > 0 )
rc = consensus_load_spot_bases( cursor, tab, col_idx, spot );
else
rc = consensus_load_zero_bases( cursor, col_idx );
}
if ( rc == 0 )
rc = vdb_write_uint8( cursor, col_idx[ consensus_tab_NREADS ],
1, "consensus.NREADS" );
if ( rc == 0 )
rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_READ_START ],
0, "consensus.READ_START" );
if ( rc == 0 )
rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_READ_LEN ],
spot->NumEvent, "consensus.READ_LEN" );
if ( rc == 0 )
rc = vdb_write_uint8( cursor, col_idx[ consensus_tab_READ_TYPE ],
SRA_READ_TYPE_BIOLOGICAL, "consensus.READ_TYPE" );
if ( rc == 0 )
{
rc = VCursorCommitRow( cursor );
if ( rc != 0 )
PLOGERR( klogErr, ( klogErr, rc, "cannot commit consensus-row on spot# $(spotnr)",
"spotnr=%u", spot->spot_nr ) );
}
if ( rc == 0 )
{
rc = VCursorCloseRow( cursor );
if ( rc != 0 )
PLOGERR( klogErr, ( klogErr, rc, "cannot close consensus-row on spot# $(spotnr)",
"spotnr=%u", spot->spot_nr ) );
}
return rc;
}
static
rc_t copy_row ( const vtblcp_parms *pb, VCursor *dcurs, const VCursor *scurs,
const vtblcp_column_map *cm, uint32_t count, uint32_t rdfilt_idx, int64_t row )
{
rc_t rc = VCursorOpenRow ( scurs );
if ( rc != 0 )
PLOGERR ( klogErr, (klogErr, rc, "failed to open source row '$(row)'", "row=%" LD64, row ));
else
{
rc_t rc2;
rc = VCursorOpenRow ( dcurs );
if ( rc != 0 )
PLOGERR ( klogErr, (klogErr, rc, "failed to open destination row '$(row)'", "row=%" LD64, row ));
else
{
uint32_t i;
const void *base;
uint32_t elem_bits, boff, row_len;
bool redact = false;
if ( rdfilt_idx != 0 && pb -> redact_sensitive )
{
rc = VCursorCellData ( scurs, rdfilt_idx,
& elem_bits, & base, & boff, & row_len );
if ( rc != 0 )
{
PLOGERR ( klogErr, (klogErr, rc, "failed to read cell data for read filter, row '$(row)'",
"row=%" LD64, row ));
}
else
{
const uint8_t *rd_filter = base;
for ( i = 0; i < row_len; ++ i )
{
if ( rd_filter [ i ] == 3 )
{
redact = pb -> redact_sensitive;
break;
}
}
}
}
for ( i = 0; i < count && rc != 0; ++ i )
{
/* get column data */
rc = VCursorCellData ( scurs, cm [ i ] . rd,
& elem_bits, & base, & boff, & row_len );
if ( rc != 0 )
{
PLOGERR ( klogErr, (klogErr, rc, "failed to read cell data for column '$(idx)', row '$(row)'",
"idx=%u,row=%" LD64, i, row ));
break;
}
if ( redact && cm [ i ] . sensitive )
{
uint32_t j;
/* substitute base pointer with redact value */
base = cm [ i ] . redact_value;
/* redact destination */
for ( j = 0; j < row_len; ++ j )
{
rc = VCursorWrite ( dcurs, cm [ i ] . wr, elem_bits, base, 0, 1 );
if ( rc != 0 )
{
PLOGERR ( klogErr, (klogErr, rc, "failed to redact cell data for column '$(idx)', row '$(row)'",
"idx=%u,row=%" LD64, i, row ));
break;
}
}
}
else
{
/* write to destination */
rc = VCursorWrite ( dcurs, cm [ i ] . wr, elem_bits, base, boff, row_len );
if ( rc != 0 )
{
PLOGERR ( klogErr, (klogErr, rc, "failed to write cell data for column '$(idx)', row '$(row)'",
"idx=%u,row=%" LD64, i, row ));
break;
}
}
}
/* commit row */
if ( rc == 0 )
rc = VCursorCommitRow ( dcurs );
rc2 = VCursorCloseRow ( dcurs );
if ( rc == 0 )
rc = rc2;
}
rc2 = VCursorCloseRow ( scurs );
if ( rc == 0 )
rc = rc2;
}
return rc;
}
rc_t run_test (VTable * table, test_params * pb)
{
VCursor * cursor;
const VCursor * rcursor;
int64_t ix;
int64_t rowid;
uint32_t dat;
uint32_t len;
uint32_t clen;
uint32_t plen;
rc_t rc;
rc_t orc;
uint8_t b [BUFSIZE];
cursor = NULL;
do
{
if (verbose)
printf ("%s call open_write_cursot\n", __func__);
rc = open_write_cursor (table, &cursor,
&dat, &len,
pb->dat_name, pb->len_name);
if (rc)
{
LOGERR (klogDebug1, rc, "failed to create write cursor");
cursor = NULL;
break;
}
for (ix = 0; ix < Limit; ++ix)
{
if (verbose)
printf ("%s call VCursorOpenRow\n", __func__);
rc = VCursorOpenRow (cursor);
if (rc)
{
LOGERR (klogErr, rc, "Failed to Open Cursor");
break;
}
else
{
uint32_t c[1];
pb->func(ix, b, c);
if (verbose)
printf ("%s call VCursorWrite %" LD64 "\n", __func__, ix);
rc = VCursorWrite (cursor, dat, pb->bits, b, 0, *c);
if (rc)
{
pLOGERR (klogErr, rc, "Write fail dat row $(R)", PLOG_I64(R), ix);
break;
}
if (verbose)
printf ("%s call VCursorWrite %" LD64 "\n", __func__, ix);
rc = VCursorWrite (cursor, len, 32, &c, 0, 1);
if (rc)
{
pLOGERR (klogErr, rc, "Write fail len row $(R)", PLOG_I64(R), ix);
break;
}
if (verbose)
printf ("%s call VCursorCommitRow\n", __func__);
rc = VCursorCommitRow (cursor);
if (rc)
{
pLOGERR (klogErr, rc, "Commit fail row $(R)", PLOG_I64(R), ix);
break;
}
if (verbose)
printf ("%s call VCursorCloseRow\n", __func__);
rc = VCursorCloseRow (cursor);
if (rc)
{
pLOGERR (klogErr, orc, "Commit fail row $(R)", PLOG_I64(R), ix);
break;
}
}
if (rc)
break;
} /* for (ix = 0; ix < Limit; ++ix) */
if (ix != Limit)
fprintf (stderr, "Quit early %d\n", (int)ix);
if (rc)
{
pLOGERR (klogInfo, rc, "failed in loop $(T) $(R)",
PLOG_2(PLOG_S(T),PLOG_I64(R)), pb->test_name, ix);
}
else
{
if (verbose)
printf ("%s call VCursorCommit\n", __func__);
orc = VCursorCommit (cursor);
if (orc && (rc == 0))
rc = orc;
}
if (verbose)
printf ("%s call VCursorRelease\n", __func__);
orc = VCursorRelease (cursor);
if (orc && (rc == 0))
rc = orc;
if (rc)
break;
if (verbose)
printf ("%s call open_read_cursor\n",__func__);
rc = open_read_cursor (table, &rcursor,
&len, &plen, &clen, &dat,
pb->len_name, pb->plen_name, pb->clen_name, pb->dat_name);
if (rc)
{
LOGERR (klogErr, rc, "failed to open read cursor");
break;
}
for (ix = 0; ix < Limit; ++ix)
{
uint32_t l;
uint32_t p;
uint32_t c;
uint32_t r;
uint32_t x;
rc = VCursorRowId (rcursor, &rowid);
if (rc)
{
pLOGERR (klogErr, rc, "failed to get rowid $(R)", PLOG_I64(R), ix);
break;
}
if (rowid != ix+1)
{
fprintf (stderr, "ROWID failure %" LD64 ":%" LD64 "\n", ix, rowid);
failed = true;
}
rc = VCursorOpenRow (rcursor);
if (rc)
{
pLOGERR (klogErr, rc, "failed to open row $(R)", PLOG_I64(R), ix);
break;
}
rc = VCursorRead (rcursor, len, 32, &l, 1, &r);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->len_name, ix);
break;
}
rc = VCursorRead (rcursor, clen, 32, &c, 1, &r);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->clen_name, ix);
break;
}
rc = VCursorRead (rcursor, plen, 32, &p, 1, &r);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->plen_name, ix);
break;
}
/* rc = VCursorReadBits (rcursor, dat, pb->bits, 0, b, 0, (BUFSIZE*8)/pb->bits, &r, &x);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->dat_name, ix);
break;
}
*/
VCursorCloseRow (rcursor);
if (l != p)
{
fprintf (stderr, "error in physical column row_len() %u != %u\n", l, p);
failed = true;
}
if (l != c)
{
fprintf (stderr, "error in physical column row_len() %u != %u\n", l, c);
failed = true;
}
}
if (verbose)
printf ("%s call VCursorRelease\n",__func__);
orc = VCursorRelease (rcursor);
if (orc)
{
LOGERR (klogErr, rc, "release was funky");
}
if (orc && (rc == 0))
rc = orc;
if (rc)
break;
}
while (0);
return rc;
}