rc_t open_read_cursor (const VTable * table, const VCursor ** cursor,
uint32_t * idx1, uint32_t * idx2, uint32_t * idx3, uint32_t * idx4,
const char * name1, const char * name2, const char * name3, const char * name4)
{
rc_t rc;
rc = VTableCreateCursorRead (table, cursor);
if (rc)
{
LOGERR (klogErr, rc, "Failed to create Cursor");
}
else
{
do
{
rc = VCursorAddColumn (*cursor, idx1, name1);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), name1);
break;
}
rc = VCursorAddColumn (*cursor, idx2, name2);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), name2);
break;
}
rc = VCursorAddColumn (*cursor, idx3, name3);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), name3);
break;
}
rc = VCursorAddColumn (*cursor, idx4, name4);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), name4);
break;
}
rc = VCursorOpen (*cursor);
if (rc)
{
LOGERR (klogErr, rc, "Failed to open cursor");
break;
}
return 0;
}
while (0);
VCursorRelease (*cursor);
}
*idx1 = *idx2 = *idx3 = *idx4 = 0;
*cursor = NULL;
return rc;
}
rc_t ProcessOneDoFile (ProcessOne * self)
{
rc_t rc = 0;
KFile * mfile;
PLOGMSG (klogInfo, "ProcessOneDoFile: $(F)", PLOG_S(F), self->path);
rc = KFileMakeNewMD5Read (&mfile, self->file, self->md5, self->path);
if (rc == 0)
{
const KFile * bfile;
rc = KFileMakeBuf (&bfile, mfile, 64*1024);
if (rc == 0)
{
/* add more here */
KFileRelease (bfile);
}
else
{
pLOGERR (klogErr, rc, "Failure to initiate buffer $(F)", PLOG_S(F), self->path);
KFileRelease (mfile);
}
}
else
pLOGERR (klogErr, rc, "Failure to initiate MD5 summing $(F)", PLOG_S(F), self->path);
return rc;
}
rc_t CCCopyMake (CCCopy ** p, const KDirectory * in, KDirectory * out,
KDirectory * xml, bool force, KMD5SumFmt * md5,
CCFileFormat * ff, CCTree * tree, const char * path)
{
rc_t rc;
size_t pathlen;
CCCopy * self;
char relpath [4096];
assert (in != NULL);
assert (out != NULL);
assert (xml != NULL);
assert (path != NULL);
rc = KDirectoryVResolvePath (in, false, relpath, sizeof relpath, path, NULL);
if (rc != 0)
{
pLOGERR (klogErr, rc, "unable to resolve path $(P)", PLOG_S(P), path);
return rc;
}
if ((relpath[0] == '.') && (relpath[1] == '.') && (relpath[2] == '/'))
{
rc = RC (rcExe, rcDirectory, rcResolving, rcPath, rcOutOfKDirectory);
pLOGERR (klogErr, rc, "Path must resolve to current directory or subdirectories $(P)",
PLOG_S(P), relpath);
return rc;
}
pathlen = strlen(relpath);
self = malloc (sizeof (*self) - sizeof (*self->path) + pathlen + 1);
if (self == NULL)
rc = RC (rcExe, rcNoTarg, rcAllocating, rcMemory, rcExhausted);
else
{
atomic32_set (&self->refcount, 1);
KDirectoryAddRef (in);
KDirectoryAddRef (out);
KDirectoryAddRef (xml);
KMD5SumFmtAddRef (md5);
CCFileFormatAddRef (ff);
self->in = in;
self->out = out;
self->xml = xml;
self->force = force;
self->md5 = md5;
self->ff = ff;
self->tree = tree;
memcpy (self->path, relpath, pathlen+1);
*p = self;
}
return rc;
}
rc_t open_write_cursor (VTable * table, VCursor ** cursor,
uint32_t * dat, uint32_t * len,
const char * dat_name, const char* len_name)
{
rc_t rc;
rc = VTableCreateCursorWrite (table, cursor, kcmInsert);
if (rc)
{
LOGERR (klogErr, rc, "Failed to create Cursor");
*cursor = NULL;
}
else
{
do
{
rc = VCursorAddColumn (*cursor, dat, dat_name);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), dat_name);
break;
}
rc = VCursorAddColumn (*cursor, len, len_name);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), len_name);
break;
}
rc = VCursorOpen (*cursor);
if (rc)
{
LOGERR (klogErr, rc, "Failed to open cursor");
break;
}
return 0;
}
while (0);
}
VCursorRelease (*cursor);
*dat = 0;
*len = 0;
*cursor = NULL;
return rc;
}
rc_t KMain (int argc, char *argv[])
{
int ix;
rc_t rc;
if (argc != 1)
{
for (ix = 1; ix < argc; ++ix)
{
/* might add logging and verbose options in the future */
if ((strcmp (argv[ix], "-V") == 0) ||
(strcmp (argv[ix], "--version") == 0))
{
rc = KAppVersion();
}
else if ((strcmp (argv[ix], "-?") == 0) ||
(strcmp (argv[ix], "-h") == 0) ||
(strcmp (argv[ix], "--help") == 0))
{
Usage (argv[0]);
}
if ((strcmp (argv[ix], "-v") == 0) ||
(strcmp (argv[ix], "--verbose") == 0))
{
verbose = true;
rc = KAppVersion();
}
/*
if ((strcmp (argv[ix], "-k") == 0) ||
(strcmp (argv[ix], "--iterations") == 0))
{
Limit = atol (
verbose = true;
rc = KAppVersion();
}
*/
else
{
rc = RC (rcExe, rcArgv, rcReading, rcParam, rcUnknown);
pLOGERR (klogErr, rc, "unknown parameter $(p)", PLOG_S(p), argv[ix]);
return rc;
}
}
}
return run_tests ();
}
rc_t KMain ( int argc, char *argv [] )
{
VDBManager *vmgr;
rc_t rc = VDBManagerMakeUpdate ( & vmgr, NULL );
if ( rc != 0 )
LOGERR ( klogInt, rc, "failed to make update manager" );
else
{
VSchema *vschema;
rc = VDBManagerMakeSchema ( vmgr, & vschema );
if ( rc != 0 )
LOGERR ( klogInt, rc, "failed to make empty schema" );
else
{
const char *file = "align/align.vschema";
#if _DEBUGGING
VSchemaListSymtab ( vschema );
#endif
rc = VSchemaParseFile ( vschema, "align/align.vschema" );
#if _DEBUGGING
VSchemaListSymtab ( vschema );
#endif
if ( rc != 0 )
{
pLOGERR ( klogErr, rc, "failed to parse schema file '$(file)'"
, "file=%s"
, file );
}
else
{
VDatabase *vdb;
const char *path = "test-ac-db";
const char *typespec = "NCBI:align:db:alignment";
rc = VDBManagerCreateDB ( vmgr, & vdb, vschema, typespec, kcmInit, path );
if ( rc != 0 )
{
pLOGERR ( klogInt, rc, "failed to create '$(typespec)' db at '$(path)'"
, "typespec=%s,path=%s"
, typespec
, path
);
}
else
{
VTable *vtbl;
const char *member = "align";
path = member;
rc = VDatabaseCreateTable ( vdb, & vtbl, member, kcmInit, path );
if ( rc != 0 )
{
pLOGERR ( klogInt, rc, "failed to create '$(member)' table at '$(path)'"
, "member=%s,path=%s"
, member
, path
);
}
else
{
printf ( "done\n" );
VTableRelease ( vtbl );
}
VDatabaseRelease ( vdb );
}
}
VSchemaRelease ( vschema );
}
VDBManagerRelease ( vmgr );
}
return rc;
}
rc_t run_tests (void)
{
VDBManager * mgr;
rc_t rc;
if (verbose)
printf("%s call VDBManagerMakeUpdate\n", __func__);
rc = VDBManagerMakeUpdate (&mgr, NULL);
if (rc)
{
LOGERR (klogInt, rc, "Failed to open VDBManager");
return rc;
}
else
{
VSchema * schema;
if (verbose)
printf("%s call VDBManagerMakeSchema\n", __func__);
rc = VDBManagerMakeSchema (mgr, &schema);
printf("%s schema == %p\n", __func__, (void*)schema);
if (rc)
LOGERR (klogInt, rc, "Failed to make empty schema");
else
{
if(verbose)
printf("%s call VSchemaParseText\n", __func__);
rc = VSchemaParseText (schema, "rowlen_schema", schema_text, string_size (schema_text));
if (rc)
LOGERR (klogInt, rc, "Failed to parse internal schema");
else
{
int ix;
for ( ix = 0; ix < 6; ++ix)
{
VTable * table;
rc_t orc;
if (verbose)
printf("%s call open_table\n", __func__);
rc = open_table (mgr, schema, &table);
if (rc)
{
LOGERR (klogErr, rc, "Failed to open table");
break;
}
if (verbose)
printf("%s call run_test\n", __func__);
rc = run_test (table, &tests[ix]);
if (rc)
{
pLOGERR (klogErr, rc, "Failed $(D)", PLOG_S(D), tests[ix].test_name);
}
if (verbose)
printf("%s call VTableRelease\n", __func__);
orc = VTableRelease (table);
if (orc)
{
LOGERR (klogErr, rc, "failed to close table");
}
if (orc && (rc == 0))
rc = orc;
if (rc)
break;
}
}
if (verbose)
printf("%s call VSchemaRelease\n", __func__);
VSchemaRelease (schema);
}
if (verbose)
printf("%s call VDBManagerRelease\n", __func__);
VDBManagerRelease (mgr);
}
return rc;
}
rc_t run_test (VTable * table, test_params * pb)
{
VCursor * cursor;
const VCursor * rcursor;
int64_t ix;
int64_t rowid;
uint32_t dat;
uint32_t len;
uint32_t clen;
uint32_t plen;
rc_t rc;
rc_t orc;
uint8_t b [BUFSIZE];
cursor = NULL;
do
{
if (verbose)
printf ("%s call open_write_cursot\n", __func__);
rc = open_write_cursor (table, &cursor,
&dat, &len,
pb->dat_name, pb->len_name);
if (rc)
{
LOGERR (klogDebug1, rc, "failed to create write cursor");
cursor = NULL;
break;
}
for (ix = 0; ix < Limit; ++ix)
{
if (verbose)
printf ("%s call VCursorOpenRow\n", __func__);
rc = VCursorOpenRow (cursor);
if (rc)
{
LOGERR (klogErr, rc, "Failed to Open Cursor");
break;
}
else
{
uint32_t c[1];
pb->func(ix, b, c);
if (verbose)
printf ("%s call VCursorWrite %" LD64 "\n", __func__, ix);
rc = VCursorWrite (cursor, dat, pb->bits, b, 0, *c);
if (rc)
{
pLOGERR (klogErr, rc, "Write fail dat row $(R)", PLOG_I64(R), ix);
break;
}
if (verbose)
printf ("%s call VCursorWrite %" LD64 "\n", __func__, ix);
rc = VCursorWrite (cursor, len, 32, &c, 0, 1);
if (rc)
{
pLOGERR (klogErr, rc, "Write fail len row $(R)", PLOG_I64(R), ix);
break;
}
if (verbose)
printf ("%s call VCursorCommitRow\n", __func__);
rc = VCursorCommitRow (cursor);
if (rc)
{
pLOGERR (klogErr, rc, "Commit fail row $(R)", PLOG_I64(R), ix);
break;
}
if (verbose)
printf ("%s call VCursorCloseRow\n", __func__);
rc = VCursorCloseRow (cursor);
if (rc)
{
pLOGERR (klogErr, orc, "Commit fail row $(R)", PLOG_I64(R), ix);
break;
}
}
if (rc)
break;
} /* for (ix = 0; ix < Limit; ++ix) */
if (ix != Limit)
fprintf (stderr, "Quit early %d\n", (int)ix);
if (rc)
{
pLOGERR (klogInfo, rc, "failed in loop $(T) $(R)",
PLOG_2(PLOG_S(T),PLOG_I64(R)), pb->test_name, ix);
}
else
{
if (verbose)
printf ("%s call VCursorCommit\n", __func__);
orc = VCursorCommit (cursor);
if (orc && (rc == 0))
rc = orc;
}
if (verbose)
printf ("%s call VCursorRelease\n", __func__);
orc = VCursorRelease (cursor);
if (orc && (rc == 0))
rc = orc;
if (rc)
break;
if (verbose)
printf ("%s call open_read_cursor\n",__func__);
rc = open_read_cursor (table, &rcursor,
&len, &plen, &clen, &dat,
pb->len_name, pb->plen_name, pb->clen_name, pb->dat_name);
if (rc)
{
LOGERR (klogErr, rc, "failed to open read cursor");
break;
}
for (ix = 0; ix < Limit; ++ix)
{
uint32_t l;
uint32_t p;
uint32_t c;
uint32_t r;
uint32_t x;
rc = VCursorRowId (rcursor, &rowid);
if (rc)
{
pLOGERR (klogErr, rc, "failed to get rowid $(R)", PLOG_I64(R), ix);
break;
}
if (rowid != ix+1)
{
fprintf (stderr, "ROWID failure %" LD64 ":%" LD64 "\n", ix, rowid);
failed = true;
}
rc = VCursorOpenRow (rcursor);
if (rc)
{
pLOGERR (klogErr, rc, "failed to open row $(R)", PLOG_I64(R), ix);
break;
}
rc = VCursorRead (rcursor, len, 32, &l, 1, &r);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->len_name, ix);
break;
}
rc = VCursorRead (rcursor, clen, 32, &c, 1, &r);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->clen_name, ix);
break;
}
rc = VCursorRead (rcursor, plen, 32, &p, 1, &r);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->plen_name, ix);
break;
}
/* rc = VCursorReadBits (rcursor, dat, pb->bits, 0, b, 0, (BUFSIZE*8)/pb->bits, &r, &x);
if (rc)
{
pLOGERR (klogErr, rc, "failed to read column $(N) $(R)", PLOG_2(PLOG_S(N),PLOG_I64(R)), pb->dat_name, ix);
break;
}
*/
VCursorCloseRow (rcursor);
if (l != p)
{
fprintf (stderr, "error in physical column row_len() %u != %u\n", l, p);
failed = true;
}
if (l != c)
{
fprintf (stderr, "error in physical column row_len() %u != %u\n", l, c);
failed = true;
}
}
if (verbose)
printf ("%s call VCursorRelease\n",__func__);
orc = VCursorRelease (rcursor);
if (orc)
{
LOGERR (klogErr, rc, "release was funky");
}
if (orc && (rc == 0))
rc = orc;
if (rc)
break;
}
while (0);
return rc;
}
static
rc_t CCCopyDoFile (CCCopy * self)
{
const KFile * original;
rc_t rc = 0;
enum KCreateMode mode;
PLOGMSG (klogDebug9, "CCCopyDoFile $(f)", PLOG_S(f), self->path);
if (! self->force)
{
/* if not forced replace mode we fail on existing file */
mode = kcmCreate;
}
else
{
uint32_t tt;
tt = KDirectoryPathType (self->out, self->path);
switch (tt)
{
default:
PLOGMSG (klogWarn, "File exists and will be replaced in output directory $(f)",
PLOG_S(f), self->path);
break;
/* if the path to the file or the file do not exist no warning */
case kptNotFound:
case kptBadPath:
break;
}
tt = KDirectoryPathType (self->xml, self->path);
switch (tt)
{
default:
PLOGMSG (klogWarn, "File exists and might be replaced in xml directory $(f)",
PLOG_S(f), self->path);
break;
/* if the path to the file or the file do not exist no warning */
case kptNotFound:
case kptBadPath:
break;
}
/* forced mode we create with init instead of create forcing a delete/create effect */
mode = kcmInit;
}
/* open original source for read */
rc = KDirectoryVOpenFileRead (self->in, &original, self->path, NULL);
if (rc == 0)
{
KFile * copy;
/* create copy output for write */
rc = KDirectoryVCreateFile (self->out, ©, false, 0644, mode|kcmParents,
self->path, NULL);
if (rc == 0)
{
KFile * fm;
/* create parallel <path>.md5 */
rc = KDirectoryCreateFile (self->out, &fm, true, 0644, mode, "%s.md5",
self->path);
if (rc == 0)
{
KMD5SumFmt * md5f;
/* make the .md5 an MD5 sum format file */
rc = KMD5SumFmtMakeUpdate (&md5f, fm);
if (rc == 0)
{
union u
{
KFile * kf;
KMD5File * mf;
} outf;
/* combine the copy and MD5 file into our special KFile */
rc = KMD5FileMakeWrite (&outf.mf, copy, md5f, self->path);
if (rc == 0)
{
const KFile * inf;
/* release this outside reference to the MD5SumFMT leaving
* only the one internal to the KMD5File */
KMD5SumFmtRelease (md5f);
/* create the KTeeFile that copies reads from the
* original as writes to the copy. Reads will be
* made by the cataloging process */
rc = KFileMakeTeeRead (&inf, original, outf.kf);
if (rc == 0)
{
CCCat * po;
KTime_t mtime;
/* try to get a modification time for this pathname */
rc = KDirectoryVDate (self->in, &mtime, self->path, NULL);
if (rc != 0)
mtime = 0; /* default to ? 0? */
/* create the cataloger giving it the infile which
* is the KTeeFile, Indirectory, the XML directory,
* and the original path for the file */
rc = CCCatMake (&po, self->in, self->xml, inf, self->md5,
self->ff, mtime, self->tree, false, 0, self->path);
if (rc == 0)
{
/* do the catalog (and thus copy) */
rc = CCCatDo(po);
/* release the cataloger object */
CCCatRelease (po);
}
else
pLOGERR (klogDebug6, rc, "failure in CCCatMake $(P)",
PLOG_S(P), self->path);
/* release the infile which will complete a copy
* regardless of the state of the cataloger */
KFileRelease (inf);
/* return rc; */
}
else
{
KFileRelease (outf.kf);
KFileRelease (original);
pLOGERR (klogDebug4, rc, "failure with kfilemaketeeread $(P)",
PLOG_S(P), self->path);
} /* rc = KFileMakeTeeRead (&inf, original, outf.kf);*/
}
else
{
KFileRelease (copy);
KMD5SumFmtRelease (md5f);
pLOGERR (klogDebug4, rc, "failure with KMD5FileMakeWrite $(P)",
PLOG_S(P), self->path);
} /* KMD5FileMakeWrite (&outf.mf, copy, md5f, self->path); */
} /* KDirectoryCreateFile (self->out, &fm, true, 0644, mode, "%s.md5", */
else
pLOGERR (klogDebug4, rc, "failure with KMD5SumFmtMakeUpdate $(P)",
PLOG_S(P), self->path);
KFileRelease (fm);
} /* KDirectoryCreateFile (self->out, &fm, true, 0644, mode, "%s.md5", */
else
pLOGERR (klogDebug4, rc, "failure with KDirectoryCreateFile $(P).md5",
PLOG_S(P), self->path);
KFileRelease (copy);
} /* rc = KDirectoryVCreateFile (self->out, ©, false, 0644, mode|kcmParents, */
else
pLOGERR (klogDebug4, rc, "failure with KDirectoryVCreateFile $(P)",
PLOG_S(P), self->path);
KFileRelease (original);
} /* rc = KDirectoryVOpenFileRead (self->in, &original, self->path, NULL); */
else
pLOGERR (klogDebug4, rc, "failure with KDirectoryVOpenFileRead $(pP)",
PLOG_S(P), self->path);
return rc;
}
