static rc_t MaxNReadsValidator_GetKey( const SRASplitter* cself,
const char** key, spotid_t spot, readmask_t* readmask )
{
rc_t rc = 0;
MaxNReadsValidator* self = ( MaxNReadsValidator* )cself;
if ( self == NULL || key == NULL )
{
rc = RC( rcSRA, rcNode, rcExecuting, rcParam, rcNull );
}
else
{
const void* nreads = NULL;
bitsz_t o = 0, sz = 0;
uint64_t nn = 0;
*key = "";
if ( self->col != NULL )
{
rc = SRAColumnRead( self->col, spot, &nreads, &o, &sz );
if ( rc == 0 )
{
switch( sz )
{
case 8:
nn = *((const uint8_t*)nreads);
break;
case 16:
nn = *((const uint16_t*)nreads);
break;
case 32:
nn = *((const uint32_t*)nreads);
break;
case 64:
nn = *((const uint64_t*)nreads);
break;
default:
rc = RC( rcSRA, rcNode, rcExecuting, rcData, rcUnexpected );
break;
}
if ( nn > nreads_max )
{
clear_readmask( readmask );
PLOGMSG(klogWarn, (klogWarn, "too many reads $(nreads) at spot id $(row), maximum $(max) supported, skipped",
PLOG_3(PLOG_U64(nreads),PLOG_I64(row),PLOG_U32(max)), nn, spot, nreads_max));
}
else if ( nn == nreads_max - 1 )
{
PLOGMSG(klogWarn, (klogWarn, "too many reads $(nreads) at spot id $(row), truncated to $(max)",
PLOG_3(PLOG_U64(nreads),PLOG_I64(row),PLOG_U32(max)), nn + 1, spot, nreads_max));
}
}
}
}
return rc;
}
static
rc_t XMLThread( const KThread *self, void *data )
{
KDirectory *dir = NULL;
PLOGMSG(klogInfo, (klogInfo, "XML sync thread started with $(s) sec", PLOG_U32(s), g_xml_sync));
do {
rc_t rc = 0;
KTime_t dt = 0;
DEBUG_MSG(8, ("XML sync thread checking %s\n", g_xml_path));
if( (rc = KDirectoryNativeDir(&dir)) == 0 ) {
rc = KDirectoryDate(dir, &dt, "%s", g_xml_path);
ReleaseComplain(KDirectoryRelease, dir);
}
if( rc == 0 ) {
if( dt != g_xml_mtime ) {
const FSNode* new_root = NULL;
PLOGMSG(klogInfo, (klogInfo, "File $(f) changed ($(m) <> $(d)), updating...",
PLOG_3(PLOG_S(f),PLOG_I64(m),PLOG_I64(d)), g_xml_path, g_xml_mtime, dt));
if( XML_Open(g_xml_path, &new_root) == 0 ) {
if( (rc = XMLLock(true)) == 0 ) {
const FSNode* old_root = g_root;
g_root = new_root;
g_xml_mtime = dt;
XMLUnlock();
FSNode_Release(old_root);
PLOGMSG(klogInfo, (klogInfo, "Data from $(f) updated successfully", PLOG_S(f), g_xml_path));
}
}
} else {
DEBUG_MSG(8, ("XML sync thread up-to-date %s\n", g_xml_path));
}
} else {
LOGERR(klogErr, rc, g_xml_path);
}
SRAList_PostRefresh();
sleep(g_xml_sync);
} while( g_xml_sync > 0 );
LOGMSG(klogInfo, "XML sync thread ended");
return 0;
}
rc_t run (const char * table_path, uint64_t N )
{
static const char *colInf[] = {
"C1: Same value, same length",
"C2: Var. value, same length",
"C3: Var. value, var. legnth",
"C4: Same value except I row",
"C5: Same value except L row" };
rc_t rc;
uint64_t row = 0;
VDBManager *mgr = NULL;
VSchema *schema = NULL;
VTable *table = NULL;
VCursor *cursor;
uint32_t idx[COLUMNS];
uint64_t total[COLUMNS];
int i = 0, j = 0, prev = 0;
char *buffer[BUFFERS];
/* Initialize arrays */
memset(&idx, 0, sizeof idx);
memset(&total, 0, sizeof total);
for (i = 0; i < BUFFERS; ++i) {
char c;
size_t sz = ROWLEN + 1;
if (i == (BUFFERS - 1))
sz += ROWLEN;
buffer[i] = malloc(sz);
for (j = 0, c = 0; j < sz - 1; ++j, ++c) {
if (c >= ROWLEN)
c -= ROWLEN;
buffer[i][j] = '0' + c;
}
buffer[i][j] = '\0';
}
/* Create manager */
rc = VDBManagerMakeUpdate(&mgr, NULL);
if (rc != 0) {
LOGERR(klogInt, rc, "failed to open vdb library");
}
/* Initialize schema */
if (rc == 0) {
rc = VDBManagerMakeSchema(mgr, &schema);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create empty schema");
}
if (rc == 0) {
char text[512] = "table Table #1 {\n";
char col [128];
for (i = 1; i <= COLUMNS; ++i) {
sprintf(col,
" column ascii C%d = .C%d; physical ascii .C%d = C%d;\n",
i, i, i, i);
strcat(text, col);
}
strcat(text, "};");
STSMSG(1,("Parsing schema:\n%s", text));
rc = VSchemaParseText(schema, "Schema", text, strlen(text));
if (rc != 0)
LOGERR(klogInt, rc, "failed to parse schema");
}
/* Create table */
if (rc == 0) {
STSMSG(1,("Creating %s", tablePath));
rc = VDBManagerCreateTable(mgr, &table, schema, "Table", kcmInit,
tablePath);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create table");
}
/* Initialize cursor */
if (rc == 0) {
rc = VTableCreateCursorWrite(table, &cursor, kcmInsert);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create cursor");
}
for (i = 0; rc == 0 && i < COLUMNS; ++i) {
char col[3];
sprintf(col, "C%d", i + 1);
STSMSG(2,("Adding column %s to cursor", col));
rc = VCursorAddColumn(cursor, &idx[i], col);
if (rc != 0)
PLOGERR(klogInt, (klogInt, rc,
"failed to add $(c) to cursor", "c=%s", col));
}
if (rc == 0) {
rc = VCursorOpen(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to open cursor");
}
/* Write data */
for (row = 0; row < N && rc == 0; ++row) {
int max = 2 * ROWLEN - 1;
int sz = 0;
if ((row % 2) == 0) {
int min = 1;
sz = min + (int) (max * (rand() / (RAND_MAX + 1.0)));
prev = sz;
buffer[1][0] = '1';
}
else {
sz = max + 1 - prev;
buffer[1][0] = '2';
}
rc = Quitting();
if (rc == 0) {
KStsLevel lvl = 2;
if (row > 0 && ((row % ROWS) == 0)) {
lvl = 1;
}
STSMSG (lvl, ("Writing row %ji / %ji",
row + 1, N));
rc = VCursorOpenRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to open row");
}
for (j = 0; j < COLUMNS && rc == 0; ++j) {
uint32_t count = 0;
int buf = j;
switch (j) {
case 0:
case 1:
count = strlen(buffer[j]);
break;
case 2:
count = sz;
break;
case 3:
buf = 0;
if (row == 0)
buf = 1;
count = strlen(buffer[buf]);
break;
case 4:
buf = 0;
if (row == (N - 1))
buf = 1;
count = strlen(buffer[buf]);
break;
default:
assert(0);
break;
}
STSMSG (3, ("Row %ji/Col.%d: %sd %.*s\n",
row + 1, j + 1, count, count, buffer[buf]));
rc = VCursorWrite
(cursor, idx[j], 8, buffer[buf], 0, count);
if (rc != 0)
PLOGERR(klogInt, (klogInt, rc, "failed to write row[$i]", "i=%d", j + 1));
total[j] += count;
}
if (rc == 0) {
rc = VCursorCommitRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to commit row");
}
if (rc == 0) {
rc = VCursorCloseRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to close row");
}
}
if (rc == 0) {
STSMSG (1, ("Commiting cursor\n"));
rc = VCursorCommit(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to commit cursor");
}
/* Cleanup */
VCursorRelease(cursor);
VTableRelease(table);
VSchemaRelease(schema);
VDBManagerRelease(mgr);
for (i = 0; i < BUFFERS; ++i) {
free(buffer[i]);
}
/* Log */
if (rc == 0) {
PLOGMSG(klogInfo, (klogInfo, "$(t)", "t=%s", tablePath));
PLOGMSG(klogInfo,(klogInfo,
"$(n)($(N)) rows written - $(b) bytes per row",
PLOG_I64(n) "," PLOG_X64(N) ",b=%d", N, N, ROWLEN));
for (i = 0; i < COLUMNS; ++i) {
PLOGMSG(klogInfo,(klogInfo,
"$(i): $(n)($(N)) bytes",
"i=%s," PLOG_I64(n) "," PLOG_X64(N),
colInf[i], total[i], total[i]));
}
}
if (rc == 0) {
KDirectory *dir = NULL;
uint64_t sizes[COLUMNS];
memset(&sizes, 0, sizeof sizes);
rc = KDirectoryNativeDir(&dir);
if (rc != 0)
LOGERR(klogInt, rc, "failed to KDirectoryNativeDir");
else {
for (i = 1; i <= COLUMNS; ++i) {
uint64_t size = 0;
#define FORMAT "%s/col/%s/data"
#define KFORMAT "$(d)/col/$(n)/data", "d=%s,n=%s"
#define STATUS(action) (action FORMAT, tablePath, name)
char name[3];
sprintf(name, "C%d", i);
STSMSG (1, STATUS("checking "));
rc = KDirectoryFileSize(dir, &size, FORMAT, tablePath, name);
if (rc != 0) {
if (GetRCState(rc) == rcNotFound) {
STSMSG (2, STATUS("not found "));
rc = 0;
}
else
PLOGERR(klogInt, (klogInt, rc,
"failed to check " KFORMAT, tablePath, name));
}
else {
STSMSG (2, STATUS("found "));
}
PLOGMSG(klogInfo, (klogInfo, "Size of $(d)/col/$(n)/data = $(s)",
"d=%s,n=%s," PLOG_I64(s), tablePath, name, size));
sizes[i - 1] = size;
}
}
KDirectoryRelease(dir);
if (rc == 0) {
puts("");
KOutMsg("%ld rows, %d bytes per row:\n", N, ROWLEN);
for (i = 0; i < COLUMNS; ++i) {
puts(colInf[i]);
}
puts("");
for (i = 0; i < COLUMNS; ++i) {
int64_t over = sizes[i] - total[i];
KOutMsg("C%d: %9ld bytes written; "
"%9ld in 'data'", i + 1, total[i], sizes[i]);
if (over > 0) {
double p = 100.0 * over / sizes[i];
printf(": %7ld extra bytes (%.4f%%)\n", over, p);
}
else {
puts("");
}
}
}
}
return rc;
}
static rc_t ReadFilterSplitter_GetKeySet( const SRASplitter* cself,
const SRASplitter_Keys** key, uint32_t* keys, spotid_t spot, const readmask_t* readmask )
{
rc_t rc = 0;
ReadFilterSplitter* self = ( ReadFilterSplitter* )cself;
if ( self == NULL || key == NULL )
{
rc = RC( rcSRA, rcNode, rcExecuting, rcParam, rcNull );
}
else
{
const INSDC_SRA_read_filter* rdf;
bitsz_t o = 0, sz = 0;
*keys = 0;
if ( self->col_rdf != NULL )
{
rc = SRAColumnRead( self->col_rdf, spot, (const void **)&rdf, &o, &sz );
if ( rc == 0 )
{
int32_t j, i = sz / sizeof( INSDC_SRA_read_filter ) / 8;
*key = self->keys;
*keys = sizeof( self->keys ) / sizeof( self->keys[ 0 ] );
for ( j = 0; j < *keys; j++ )
{
clear_readmask( self->keys[ j ].readmask );
}
while ( i > 0 )
{
i--;
if ( self->read_filter != 0xFF && self->read_filter != rdf[i] )
{
/* skip by filter value != to command line */
}
else if ( rdf[ i ] == SRA_READ_FILTER_PASS )
{
set_readmask( self->keys[ EReadFilterSplitter_pass ].readmask, i );
}
else if ( rdf[ i ] == SRA_READ_FILTER_REJECT )
{
set_readmask( self->keys[ EReadFilterSplitter_reject ].readmask, i );
}
else if( rdf[ i ] == SRA_READ_FILTER_CRITERIA )
{
set_readmask( self->keys[ EReadFilterSplitter_criteria ].readmask, i );
}
else if( rdf[ i ] == SRA_READ_FILTER_REDACTED )
{
set_readmask( self->keys[ EReadFilterSplitter_redacted ].readmask, i );
}
else
{
set_readmask( self->keys[ EReadFilterSplitter_unknown ].readmask, i );
PLOGMSG( klogWarn, ( klogWarn,
"unknown READ_FILTER value $(value) at spot id $(row)",
PLOG_2( PLOG_U8( value ), PLOG_I64( row ) ), rdf[ i ], spot ) );
}
}
}
}
}
return rc;
}
rc_t run_test (VTable * table, test_params * pb)
{
VCursor * cursor;
const VCursor * rcursor;
int64_t ix;
int64_t rowid;
uint32_t dat;
uint32_t len;
uint32_t clen;
uint32_t plen;
rc_t rc;
rc_t orc;
uint8_t b [BUFSIZE];
cursor = NULL;
do
{
if (verbose)
printf ("%s call open_write_cursot\n", __func__);
rc = open_write_cursor (table, &cursor,
&dat, &len,
pb->dat_name, pb->len_name);
if (rc)
{
LOGERR (klogDebug1, rc, "failed to create write cursor");
cursor = NULL;
break;
}
for (ix = 0; ix < Limit; ++ix)
{
if (verbose)
printf ("%s call VCursorOpenRow\n", __func__);
rc = VCursorOpenRow (cursor);
if (rc)
{
LOGERR (klogErr, rc, "Failed to Open Cursor");
break;
}
else
{
uint32_t c[1];
pb->func(ix, b, c);
if (verbose)
printf ("%s call VCursorWrite %" LD64 "\n", __func__, ix);
rc = VCursorWrite (cursor, dat, pb->bits, b, 0, *c);
if (rc)
{
pLOGERR (klogErr, rc, "Write fail dat row $(R)", PLOG_I64(R), ix);
break;
}
if (verbose)
printf ("%s call VCursorWrite %" LD64 "\n", __func__, ix);
rc = VCursorWrite (cursor, len, 32, &c, 0, 1);
if (rc)
{
pLOGERR (klogErr, rc, "Write fail len row $(R)", PLOG_I64(R), ix);
break;
}
if (verbose)
printf ("%s call VCursorCommitRow\n", __func__);
rc = VCursorCommitRow (cursor);
if (rc)
{
pLOGERR (klogErr, rc, "Commit fail row $(R)", PLOG_I64(R), ix);
break;
}
if (verbose)
printf ("%s call VCursorCloseRow\n", __func__);
rc = VCursorCloseRow (cursor);
if (rc)
{
pLOGERR (klogErr, orc, "Commit fail row $(R)", PLOG_I64(R), ix);
break;
}
}
if (rc)
break;
} /* for (ix = 0; ix < Limit; ++ix) */
if (ix != Limit)
fprintf (stderr, "Quit early %d\n", (int)ix);
if (rc)
{
pLOGERR (klogInfo, rc, "failed in loop $(T) $(R)",
PLOG_2(PLOG_S(T),PLOG_I64(R)), pb->test_name, ix);
}
else
{
if (verbose)
printf ("%s call VCursorCommit\n", __func__);
orc = VCursorCommit (cursor);
if (orc && (rc == 0))
rc = orc;
}
if (verbose)
printf ("%s call VCursorRelease\n", __func__);
orc = VCursorRelease (cursor);
if (orc && (rc == 0))
rc = orc;
if (rc)
break;
if (verbose)
printf ("%s call open_read_cursor\n",__func__);
rc = open_read_cursor (table, &rcursor,
&len, &plen, &clen, &dat,
pb->len_name, pb->plen_name, pb->clen_name, pb->dat_name);
if (rc)
{
LOGERR (klogErr, rc, "failed to open read cursor");
break;
}
for (ix = 0; ix < Limit; ++ix)
{
uint32_t l;
uint32_t p;
uint32_t c;
uint32_t r;
uint32_t x;
rc = VCursorRowId (rcursor, &rowid);
if (rc)
{
pLOGERR (klogErr, rc, "failed to get rowid $(R)", PLOG_I64(R), ix);
break;
}
if (rowid != ix+1)
{
fprintf (stderr, "ROWID failure %" LD64 ":%" LD64 "\n", ix, rowid);
failed = true;
}
rc = VCursorOpenRow (rcursor);
if (rc)
{
pLOGERR (klogErr, rc, "failed to open row $(R)", PLOG_I64(R), ix);
break;
}
rc = VCursorRead (rcursor, len, 32, &l, 1, &r);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->len_name, ix);
break;
}
rc = VCursorRead (rcursor, clen, 32, &c, 1, &r);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->clen_name, ix);
break;
}
rc = VCursorRead (rcursor, plen, 32, &p, 1, &r);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->plen_name, ix);
break;
}
/* rc = VCursorReadBits (rcursor, dat, pb->bits, 0, b, 0, (BUFSIZE*8)/pb->bits, &r, &x);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->dat_name, ix);
break;
}
*/
VCursorCloseRow (rcursor);
if (l != p)
{
fprintf (stderr, "error in physical column row_len() %u != %u\n", l, p);
failed = true;
}
if (l != c)
{
fprintf (stderr, "error in physical column row_len() %u != %u\n", l, c);
failed = true;
}
}
if (verbose)
printf ("%s call VCursorRelease\n",__func__);
orc = VCursorRelease (rcursor);
if (orc)
{
LOGERR (klogErr, rc, "release was funky");
}
if (orc && (rc == 0))
rc = orc;
if (rc)
break;
}
while (0);
return rc;
}
