static
rc_t SRAReader_ColumnsOpen(SRAReader *self)
{
rc_t rc = 0;
int i = 0;
while(self->cols[i].name != NULL) {
if( self->cols[i].opt == SRAREADER_COL_MANDATORY || (self->options & self->cols[i].opt) ) {
rc = SRATableOpenColumnRead(self->table, &self->cols[i].col, self->cols[i].name, self->cols[i].datatype);
if( rc != 0 ) {
if( ((self->cols[i].opt & SRAREADER_COL_OPTIONAL) && GetRCState(rc) == rcNotFound) ) {
PLOGERR(klogInfo, (klogWarn, rc, "column $(c)", PLOG_S(c), self->cols[i].name));
rc = 0;
} else if( GetRCState(rc) == rcExists ) {
rc = 0;
} else {
PLOGERR(klogInfo, (klogErr, rc, "column $(c)", PLOG_S(c), self->cols[i].name));
break;
}
}
}
i++;
}
return rc;
}
rc_t open_read_cursor (const VTable * table, const VCursor ** cursor,
uint32_t * idx1, uint32_t * idx2, uint32_t * idx3, uint32_t * idx4,
const char * name1, const char * name2, const char * name3, const char * name4)
{
rc_t rc;
rc = VTableCreateCursorRead (table, cursor);
if (rc)
{
LOGERR (klogErr, rc, "Failed to create Cursor");
}
else
{
do
{
rc = VCursorAddColumn (*cursor, idx1, name1);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), name1);
break;
}
rc = VCursorAddColumn (*cursor, idx2, name2);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), name2);
break;
}
rc = VCursorAddColumn (*cursor, idx3, name3);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), name3);
break;
}
rc = VCursorAddColumn (*cursor, idx4, name4);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), name4);
break;
}
rc = VCursorOpen (*cursor);
if (rc)
{
LOGERR (klogErr, rc, "Failed to open cursor");
break;
}
return 0;
}
while (0);
VCursorRelease (*cursor);
}
*idx1 = *idx2 = *idx3 = *idx4 = 0;
*cursor = NULL;
return rc;
}
rc_t ProcessOneDo (ProcessOne * self)
{
static const char F[] = PLOG_2(PLOG_S(p),PLOG_S(t));
enum KPathType type;
rc_t rc = 0;
type = KDirectoryPathType (self->dir, self->path);
switch (type)
{
case kptFile:
rc = ProcessOneDoFile (self);
break;
/* case kptDir: */
/* break; */
/* case kptAlias|kptFile: */
/* break; */
/* case kptAlias|kptDir: */
/* break; */
default:
PLOGMSG (klogInfo, "+ Skipping $(p) of type $(t)", F, self->path, typeToString(type));
break;
}
return rc;
}
static
rc_t open_file_as_dir (const char * path, const KDirectory ** dir)
{
rc_t rc;
KPathType kpt;
const KDirectory * d;
rc = 0;
kpt = KDirectoryPathType (kdir, path);
switch (kpt & ~kptAlias)
{
case kptNotFound:
rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcNotFound);
PLOGERR (klogFatal, (klogFatal, rc, "Archive [$(A)] must exist", PLOG_S(A), path));
return rc;
case kptDir:
default:
rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcInvalid);
PLOGERR (klogFatal, (klogFatal, rc, "Archive [$(A)] must be a file", PLOG_S(A), path));
return rc;
case kptFile:
break;
}
if (rc == 0)
{
STSMSG (1, ("Opening %s\n", path));
rc = KDirectoryOpenArcDirRead (kdir, &d, /*chroot*/false, path,
tocKFile, KArcParseSRA,
/*filter*/pnamesFilter, /*filt.param*/NULL);
*dir = (rc == 0) ? d : NULL;
}
return rc;
}
rc_t ProcessOneDoFile (ProcessOne * self)
{
rc_t rc = 0;
KFile * mfile;
PLOGMSG (klogInfo, "ProcessOneDoFile: $(F)", PLOG_S(F), self->path);
rc = KFileMakeNewMD5Read (&mfile, self->file, self->md5, self->path);
if (rc == 0)
{
const KFile * bfile;
rc = KFileMakeBuf (&bfile, mfile, 64*1024);
if (rc == 0)
{
/* add more here */
KFileRelease (bfile);
}
else
{
pLOGERR (klogErr, rc, "Failure to initiate buffer $(F)", PLOG_S(F), self->path);
KFileRelease (mfile);
}
}
else
pLOGERR (klogErr, rc, "Failure to initiate MD5 summing $(F)", PLOG_S(F), self->path);
return rc;
}
static
rc_t XML_Open(const char* path, const FSNode** tree)
{
rc_t rc = 0;
char errmsg[4096] = "";
KDirectory *dir = NULL;
PLOGMSG(klogInfo, (klogInfo, "Reading XML file '$(x)'", PLOG_S(x), path));
if( (rc = KDirectoryNativeDir(&dir)) == 0 ) {
const KFile* file = NULL;
if( (rc = KDirectoryOpenFileRead(dir, &file, "%s", path)) == 0 ) {
if( (rc = FSNode_Make((FSNode**)tree, "ROOT", &RootNode_vtbl)) == 0 ) {
const KXMLDoc* xmldoc = NULL;
if( (rc = KXMLMgrMakeDocRead(g_xmlmgr, &xmldoc, file)) == 0 ) {
const KXMLNodeset* ns = NULL;
if( (rc = KXMLDocOpenNodesetRead(xmldoc, &ns, "/FUSE/*")) == 0 ) {
uint32_t count = 0;
if( (rc = KXMLNodesetCount(ns, &count)) == 0 ) {
if( count == 0 ) {
rc = RC(rcExe, rcDoc, rcValidating, rcData, rcEmpty);
} else {
uint32_t i = 0;
while(rc == 0 && i < count) {
const KXMLNode* n = NULL;
if( (rc = KXMLNodesetGetNodeRead(ns, &n, i++)) == 0 ) {
SRAConfigFlags flags = ~0;
errmsg[0] = '\0';
rc = XML_ValidateNode((FSNode*)*tree, n, flags, errmsg);
ReleaseComplain(KXMLNodeRelease, n);
}
}
if( rc == 0 ) {
rc = SRAList_NextVersion();
}
}
}
ReleaseComplain(KXMLNodesetRelease, ns);
}
ReleaseComplain(KXMLDocRelease, xmldoc);
}
if( rc != 0 ) {
FSNode_Release(*tree);
*tree = NULL;
}
}
ReleaseComplain(KFileRelease, file);
}
ReleaseComplain(KDirectoryRelease, dir);
}
if( rc == 0 ) {
PLOGMSG(klogInfo, (klogInfo, "XML file '$(x)' ok", PLOG_S(x), path));
} else {
if( strlen(errmsg) < 1 ) {
strcpy(errmsg, path);
}
LOGERR(klogErr, rc, errmsg);
}
return rc;
}
rc_t WriteFileMeta(SIndexObj* obj)
{
rc_t rc = 0;
KMDataNode* nd = NULL;
PLOGMSG(klogInfo, (klogInfo, "Meta $(f) on index $(i): file size $(s), buffer $(b)",
PLOG_4(PLOG_S(f),PLOG_S(i),PLOG_U64(s),PLOG_U32(b)), obj->file, obj->index, obj->file_size, obj->buffer_sz));
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Format")) == 0 ) {
KMDataNode* opt = NULL;
rc = KMDataNodeWriteCString(nd, obj->format);
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(nd, &opt, "Options")) == 0 ) {
KMDataNode* ond = NULL;
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "accession")) == 0 ) {
rc = KMDataNodeWriteCString(ond, g_accession);
KMDataNodeRelease(ond);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "minSpotId")) == 0 ) {
rc = KMDataNodeWriteB64(ond, &obj->minSpotId);
KMDataNodeRelease(ond);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "maxSpotId")) == 0 ) {
rc = KMDataNodeWriteB64(ond, &obj->maxSpotId);
KMDataNodeRelease(ond);
}
KMDataNodeRelease(opt);
}
KMDataNodeRelease(nd);
}
if( rc == 0 && obj->file_size > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Size")) == 0 ) {
rc = KMDataNodeWriteB64(nd, &obj->file_size);
KMDataNodeRelease(nd);
}
if( rc == 0 && obj->buffer_sz > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Buffer")) == 0 ) {
rc = KMDataNodeWriteB32(nd, &obj->buffer_sz);
KMDataNodeRelease(nd);
}
if( rc == 0 && strlen(obj->index) > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Index")) == 0 ) {
rc = KMDataNodeWriteCString(nd, obj->index);
KMDataNodeRelease(nd);
}
if( rc == 0 && obj->file_size > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "md5")) == 0 ) {
char x[5];
int i;
for( i = 0; rc == 0 && i < sizeof(obj->md5_digest); i++ ) {
int l = snprintf(x, 4, "%02x", obj->md5_digest[i]);
rc = KMDataNodeAppend(nd, x, l);
}
KMDataNodeRelease(nd);
}
return rc;
}
rc_t CCCopyMake (CCCopy ** p, const KDirectory * in, KDirectory * out,
KDirectory * xml, bool force, KMD5SumFmt * md5,
CCFileFormat * ff, CCTree * tree, const char * path)
{
rc_t rc;
size_t pathlen;
CCCopy * self;
char relpath [4096];
assert (in != NULL);
assert (out != NULL);
assert (xml != NULL);
assert (path != NULL);
rc = KDirectoryVResolvePath (in, false, relpath, sizeof relpath, path, NULL);
if (rc != 0)
{
pLOGERR (klogErr, rc, "unable to resolve path $(P)", PLOG_S(P), path);
return rc;
}
if ((relpath[0] == '.') && (relpath[1] == '.') && (relpath[2] == '/'))
{
rc = RC (rcExe, rcDirectory, rcResolving, rcPath, rcOutOfKDirectory);
pLOGERR (klogErr, rc, "Path must resolve to current directory or subdirectories $(P)",
PLOG_S(P), relpath);
return rc;
}
pathlen = strlen(relpath);
self = malloc (sizeof (*self) - sizeof (*self->path) + pathlen + 1);
if (self == NULL)
rc = RC (rcExe, rcNoTarg, rcAllocating, rcMemory, rcExhausted);
else
{
atomic32_set (&self->refcount, 1);
KDirectoryAddRef (in);
KDirectoryAddRef (out);
KDirectoryAddRef (xml);
KMD5SumFmtAddRef (md5);
CCFileFormatAddRef (ff);
self->in = in;
self->out = out;
self->xml = xml;
self->force = force;
self->md5 = md5;
self->ff = ff;
self->tree = tree;
memcpy (self->path, relpath, pathlen+1);
*p = self;
}
return rc;
}
static
rc_t DB_Init(const SParam* p, DB_Handle* h)
{
rc_t rc;
if( (rc = VDBManagerMakeUpdate(&h->mgr, NULL)) != 0 ) {
LOGERR(klogErr, rc, "failed to create VDB Manager");
}
else if( (rc = VDBManagerMakeSchema(h->mgr, &h->schema)) != 0 ) {
LOGERR(klogErr, rc, "failed to create schema");
}
else if( (rc = VSchemaParseFile(h->schema, p->schema)) != 0 ) {
PLOGERR(klogErr, (klogErr, rc, "failed to parse schema file '$(schema)'", PLOG_S(schema), p->schema));
}
else if( (rc = VDBManagerCreateDB(h->mgr, &h->db, h->schema, "NCBI:align:db:alignment_evidence",
p->force ? kcmInit : kcmCreate, p->out)) != 0 ) {
PLOGERR(klogErr, (klogErr, rc, "failed to create database at '$(path)'", PLOG_S(path), p->out));
}
else if( (rc = ReferenceMgr_Make(&h->rmgr, h->db, h->mgr, (p->force_refw ? ewrefmgr_co_allREADs : 0),
p->refseqcfg, p->refseqpath, p->refseq_chunk, 350 * 1024 * 1024, 0)) != 0 ) {
LOGERR(klogErr, rc, "failed to create reference manager");
}
else if( (rc = CGWriterAlgn_Make(&h->walgn, &h->mappings, h->db, h->rmgr, p->min_mapq, p->single_mate, p->cluster_size)) != 0 ) {
LOGERR(klogErr, rc, "failed to create alignment writer");
}
else if( (rc = CGWriterSeq_Make(&h->wseq, &h->reads, h->db, (p->force_readw ? ewseq_co_SaveRead : 0) |
(p->no_spot_group ? 0 : ewseq_co_SpotGroup), p->qual_quant)) != 0 ) {
LOGERR(klogErr, rc, "failed to create sequence writer");
} else if( p->asm_path && (rc = CGWriterEvdInt_Make(&h->wev_int, &h->ev_int, h->db, h->rmgr, 0)) != 0 ) {
LOGERR(klogErr, rc, "failed to create evidence intervals writer");
} else if( p->asm_path && (rc = CGWriterEvdDnbs_Make(&h->wev_dnb, &h->ev_dnb, h->db, h->rmgr, 0)) != 0 ) {
LOGERR(klogErr, rc, "failed to create evidence dnbs writer");
} else {
const char** r = p->refFiles;
while( rc == 0 && *r != NULL ) {
if( (rc = ReferenceMgr_FastaPath(h->rmgr, *r++)) != 0 ) {
PLOGERR(klogInfo, (klogInfo, rc, "fasta file '$(file)'", "file=%s", r[-1]));
}
}
}
return rc;
}
static rc_t SRADumper_ArgsValidate( const char* prog, const SRADumperFmt* fmt )
{
rc_t rc = 0;
int k, i;
/* set default log level */
const char* default_log_level = "warn";
rc = LogLevelSet( default_log_level );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc, "default log level to '$(lvl)'",
PLOG_S( lvl ), default_log_level ) );
CoreUsage( prog, fmt, true, EXIT_FAILURE );
}
for ( i = 0; KMainArgs[ i ].abbr != NULL; i++ )
{
for ( k = 0; fmt->arg_desc != NULL && fmt->arg_desc[ k ].abbr != NULL; k++ )
{
if ( strcmp( fmt->arg_desc[ k ].abbr, KMainArgs[ i ].abbr ) == 0 ||
( fmt->arg_desc[ k ].full != NULL && strcmp( fmt->arg_desc[ k ].full, KMainArgs[ i ].full ) == 0 ) )
{
rc = RC(rcExe, rcArgv, rcValidating, rcParam, rcDuplicate);
}
}
}
return rc;
}
void FGroup_Validate( SLNode *node, void *data )
{
rc_t* rc = (rc_t*)data;
static EIlluminaNativeFileType mask = eIlluminaNativeFileTypeNotSet;
FGroup* n = (FGroup*)node;
IlluminaFileInfo* file = n->files;
DEBUG_MSG(3, ("==> group: '%s'\n", n->key.data));
while( file != NULL ) {
DEBUG_MSG(3, ("file: type %u '%s'\n", file->type, file->name.data));
if( mask == eIlluminaNativeFileTypeNotSet ) {
mask = n->mask;
if( !(mask & (eIlluminaNativeFileTypeFasta | eIlluminaNativeFileTypeQSeq)) ) {
*rc = RC(rcSRA, rcFormatter, rcReading, rcData, rcNotFound);
SRALoaderFile_LOG(file->file, klogErr, *rc, "file group '$(p)*': sequence data", PLOG_S(p) , n->key.data);
}
if( (mask & eIlluminaNativeFileTypeFasta) && (mask & eIlluminaNativeFileTypeQSeq) ) {
*rc = RC(rcSRA, rcFormatter, rcReading, rcData, rcDuplicate);
SRALoaderFile_LOG(file->file, klogErr, *rc, "file group '$(p)*': _seq and _qseq", PLOG_S(p) , n->key.data);
}
if( !(mask & eIlluminaNativeFileTypeQuality4) && !(mask & eIlluminaNativeFileTypeQSeq) ) {
*rc = RC(rcSRA, rcFormatter, rcReading, rcData, rcNotFound);
SRALoaderFile_LOG(file->file, klogErr, *rc, "file group '$(p)*': quality data", PLOG_S(p) , n->key.data);
}
} else if( mask != n->mask ) {
*rc = RC(rcSRA, rcFormatter, rcReading, rcData, rcInconsistent);
SRALoaderFile_LOG(file->file, klogErr, *rc, "file group '$(p)*': no match in spot names on 1st lines across files in group",
PLOG_S(p) , n->key.data);
}
file = file->next;
}
DEBUG_MSG(3, ("<== group: '%s'\n", n->key.data));
}
rc_t open_write_cursor (VTable * table, VCursor ** cursor,
uint32_t * dat, uint32_t * len,
const char * dat_name, const char* len_name)
{
rc_t rc;
rc = VTableCreateCursorWrite (table, cursor, kcmInsert);
if (rc)
{
LOGERR (klogErr, rc, "Failed to create Cursor");
*cursor = NULL;
}
else
{
do
{
rc = VCursorAddColumn (*cursor, dat, dat_name);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), dat_name);
break;
}
rc = VCursorAddColumn (*cursor, len, len_name);
if (rc)
{
pLOGERR (klogErr, rc, "Failed to add column $(C)", PLOG_S(C), len_name);
break;
}
rc = VCursorOpen (*cursor);
if (rc)
{
LOGERR (klogErr, rc, "Failed to open cursor");
break;
}
return 0;
}
while (0);
}
VCursorRelease (*cursor);
*dat = 0;
*len = 0;
*cursor = NULL;
return rc;
}
void CC ascii_to_u64_error_handler ( const char * arg, void * data )
{
rc_t * prc = data;
rc_t rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcIncorrect);
PLOGERR (klogFatal, (klogFatal, rc, "numeric range option unparsable $(S)", PLOG_S(S), arg));
*prc = rc;
}
static
rc_t CCCopyDoDirectory (CCCopy * self)
{
rc_t rc = 0;
PLOGMSG (klogInfo, "CCCopyDoDirectory $(d)", PLOG_S(d), self->path);
rc = RC (rcExe, rcDirectory, rcCopying, rcParam, rcUnsupported);
return rc;
}
/* this could be modified to accept stdout */
static
rc_t open_out_file (const char * path, KFile ** fout)
{
rc_t rc;
KPathType kpt;
KCreateMode mode;
rc = 0;
mode = kcmParents|kcmCreate;
kpt = KDirectoryPathType (kdir, path);
switch (kpt & ~kptAlias)
{
case kptDir:
/* -----
* this version will fail but perhaps with directory
* take last facet <F> from input path and use that as a name
* to be <F>.sra in this directory
*/
default:
rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcInvalid);
PLOGERR (klogFatal, (klogFatal, rc, "archive [$(A)] must be a file", PLOG_S(A), path));
break;
case kptFile:
if (force)
mode = kcmParents|kcmInit;
else
{
rc = RC (rcExe, rcNoTarg, rcParsing, rcFile, rcExists);
PLOGERR (klogFatal, (klogFatal, rc, "archive [$(A)] already exists", PLOG_S(A), path));
}
break;
case kptNotFound:
break;
}
if (rc == 0)
{
rc = KDirectoryVCreateFile (kdir, fout, false, 0664,
mode, path, NULL);
if (rc)
PLOGERR (klogFatal, (klogFatal, rc, "unable to create archive [$(A)]", PLOG_S(A), path));
}
return rc;
}
static
rc_t KLoaderFile_Open(KLoaderFile* self)
{
rc_t rc = 0;
DBG(("%s opening %s\n", __func__, self->realname));
if( (rc = KDirectoryOpenFileRead(self->dir, &self->kfile, "%s", self->realname)) == 0 ) {
if( self->has_md5 ) {
const KFile *md5File = NULL;
DBG(("%s opening as md5 wrapped %s\n", __func__, self->realname));
if( (rc = KFileMakeMD5Read(&md5File, self->file, self->md5_digest)) == 0) {
self->file = md5File;
}
}
if( rc == 0 ) {
const KFile *z = NULL;
switch(self->compress_type) {
case compress_none:
break;
case compress_gzip:
DBG(("%s opening as gzip wrapped %s\n", __func__, self->realname));
if( (rc = KFileMakeGzipForRead(&z, self->file)) == 0 ) {
KFileRelease(self->file);
self->file = z;
}
break;
case compress_bzip2:
DBG(("%s opening as bzip2 wrapped %s\n", __func__, self->realname));
if( (rc = KFileMakeBzip2ForRead(&z, self->file)) == 0 ) {
KFileRelease(self->file);
self->file = z;
}
break;
default:
rc = RC(rcApp, rcFile, rcOpening, rcType, rcUnexpected);
break;
}
#if ! WINDOWS
if( rc == 0 && self->ahead ) {
const KFile *z = NULL;
if( (rc = KQueueFileMakeRead(&z, self->pos, self->file,
self->buffer_size * 10, self->buffer_size, 0)) == 0 ) {
KFileRelease(self->file);
self->file = z;
}
}
#endif
}
}
if( rc != 0 ) {
PLOGERR(klogErr, (klogErr, rc, "opening $(file)", PLOG_S(file), self->filename));
KFileRelease(self->file);
}
return rc;
}
rc_t CCCopyDo (CCCopy * self)
{
rc_t rc = 0;
enum KPathType type;
assert (self != NULL);
assert (self->path != NULL);
type = KDirectoryVPathType (self->in, self->path, NULL);
switch (type & ~kptAlias)
{
case kptNotFound:
rc = RC (rcExe, rcPath, rcAccessing, rcPath, rcNotFound);
break;
default:
case kptBadPath:
rc = RC (rcExe, rcPath, rcAccessing, rcPath, rcInvalid);
break;
case kptFile:
rc = CCCopyDoFile (self);
break;
case kptDir:
#if 0
rc = CCCopyDoDirectory (self);
#else
rc = PLOGMSG (klogInfo, "Ignoring directory $(p)",PLOG_S(p),self->path);
#endif
break;
case kptCharDev:
rc = PLOGMSG (klogInfo, "Ignoring kptCharDev $(p)",PLOG_S(p),self->path);
break;
case kptBlockDev:
rc = PLOGMSG (klogInfo, "Ignoring kptBlockDev $(p)",PLOG_S(p),self->path);
break;
case kptFIFO:
rc = PLOGMSG (klogInfo, "Ignoring kptFIFO $(p)",PLOG_S(p),self->path);
break;
}
return rc;
}
rc_t KMain (int argc, char *argv[])
{
int ix;
rc_t rc;
if (argc != 1)
{
for (ix = 1; ix < argc; ++ix)
{
/* might add logging and verbose options in the future */
if ((strcmp (argv[ix], "-V") == 0) ||
(strcmp (argv[ix], "--version") == 0))
{
rc = KAppVersion();
}
else if ((strcmp (argv[ix], "-?") == 0) ||
(strcmp (argv[ix], "-h") == 0) ||
(strcmp (argv[ix], "--help") == 0))
{
Usage (argv[0]);
}
if ((strcmp (argv[ix], "-v") == 0) ||
(strcmp (argv[ix], "--verbose") == 0))
{
verbose = true;
rc = KAppVersion();
}
/*
if ((strcmp (argv[ix], "-k") == 0) ||
(strcmp (argv[ix], "--iterations") == 0))
{
Limit = atol (
verbose = true;
rc = KAppVersion();
}
*/
else
{
rc = RC (rcExe, rcArgv, rcReading, rcParam, rcUnknown);
pLOGERR (klogErr, rc, "unknown parameter $(p)", PLOG_S(p), argv[ix]);
return rc;
}
}
}
return run_tests ();
}
static
rc_t SFFLoaderFmt_ReadBlock(SFFLoaderFmt* self, const SRALoaderFile* file, size_t size, const char* location, bool silent)
{
size_t read = 0;
rc_t rc = SRALoaderFileRead(file, self->file_advance, size, (const void**)&self->file_buf, &read);
self->file_advance = 0;
if( rc == 0 && (size > 0 && (self->file_buf == NULL || read < size)) ) {
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcInsufficient);
}
if( rc != 0 && !silent ) {
SRALoaderFile_LOG(file, klogErr, rc, "$(l), needed $(needed) got $(got) bytes",
PLOG_3(PLOG_S(l),PLOG_U32(needed),PLOG_U32(got)), location, size, read);
}
return rc;
}
static rc_t pnamesNodeMake (pnamesNode ** pself, const char * path)
{
size_t len;
pnamesNode * self;
len = strlen (path);
self = malloc (sizeof (pnamesNode));
if (self == NULL)
{
rc_t rc = RC (rcExe, rcNoTarg, rcAllocating, rcMemory, rcExhausted);
PLOGERR (klogErr, (klogErr, rc, "out of memory allocating node for $(P)", PLOG_S(P), path));
return rc;
}
self->name = path;
self->name_len = len;
*pself = self;
return 0;
}
/* Fill
* fill buffer as far as possible, shift unread data in buffer to buffer start
*/
static
rc_t KLoaderFile_Fill(KLoaderFile *self)
{
rc_t rc = 0;
if (self->kfile == NULL) {
rc = KLoaderFile_Open(self);
}
if( rc == 0 ) {
/* determine space in buffer available */
size_t to_read = self->buffer_size - self->avail;
if( to_read > 0 ) {
#if _DEBUGGING
if( to_read < self->buffer_size * 0.5 ) {
self->small_reads++;
if( self->small_reads > 10 ) {
PLOGMSG(klogWarn, (klogWarn, "$(filename) INEFFECTIVE READING: $(times) times, now $(bytes) bytes",
PLOG_3(PLOG_S(filename),PLOG_U32(times),PLOG_U32(bytes)), self->filename, self->small_reads, to_read));
}
}
#endif
/* shift left unread data */
memmove(self->buffer, self->buffer_pos, self->avail);
/* skip read chunk in buffer */
self->pos += self->buffer_pos - self->buffer;
/* reset pointer */
self->buffer_pos = self->buffer;
do { /* fill buffer up to eof */
size_t num_read = 0;
if( (rc = KFileRead(self->file, self->pos + self->avail,
&self->buffer[self->avail], to_read, &num_read)) == 0 ) {
self->eof = (num_read == 0);
self->avail += (uint32_t) num_read;
to_read -= num_read;
DBG(("KLoaderFile read %s from %lu %u bytes%s\n",
self->filename, self->pos + self->avail - num_read, num_read, self->eof ? " EOF" : ""));
}
} while( rc == 0 && to_read > 0 && !self->eof );
}
}
return rc;
}
static
rc_t XMLThread( const KThread *self, void *data )
{
KDirectory *dir = NULL;
PLOGMSG(klogInfo, (klogInfo, "XML sync thread started with $(s) sec", PLOG_U32(s), g_xml_sync));
do {
rc_t rc = 0;
KTime_t dt = 0;
DEBUG_MSG(8, ("XML sync thread checking %s\n", g_xml_path));
if( (rc = KDirectoryNativeDir(&dir)) == 0 ) {
rc = KDirectoryDate(dir, &dt, "%s", g_xml_path);
ReleaseComplain(KDirectoryRelease, dir);
}
if( rc == 0 ) {
if( dt != g_xml_mtime ) {
const FSNode* new_root = NULL;
PLOGMSG(klogInfo, (klogInfo, "File $(f) changed ($(m) <> $(d)), updating...",
PLOG_3(PLOG_S(f),PLOG_I64(m),PLOG_I64(d)), g_xml_path, g_xml_mtime, dt));
if( XML_Open(g_xml_path, &new_root) == 0 ) {
if( (rc = XMLLock(true)) == 0 ) {
const FSNode* old_root = g_root;
g_root = new_root;
g_xml_mtime = dt;
XMLUnlock();
FSNode_Release(old_root);
PLOGMSG(klogInfo, (klogInfo, "Data from $(f) updated successfully", PLOG_S(f), g_xml_path));
}
}
} else {
DEBUG_MSG(8, ("XML sync thread up-to-date %s\n", g_xml_path));
}
} else {
LOGERR(klogErr, rc, g_xml_path);
}
SRAList_PostRefresh();
sleep(g_xml_sync);
} while( g_xml_sync > 0 );
LOGMSG(klogInfo, "XML sync thread ended");
return 0;
}
rc_t ProcessOneRelease (const ProcessOne * cself)
{
ProcessOne * self = (ProcessOne *)cself;
rc_t rc = 0;
if (self != NULL)
{
PLOGMSG (klogDebug10, "ProcessOneRelease $(f)", PLOG_S(f), self->path);
if (atomic32_dec_and_test (&self->refcount))
{
KDirectoryRelease (self->dir);
KDirectoryRelease (self->xml);
KMD5SumFmtRelease (self->md5);
CCFileFormatRelease (self->ff);
KFileRelease (self->file);
free (self);
}
}
return rc;
}
rc_t ProcessOneMake (ProcessOne ** ppo, const KDirectory * dir, KDirectory * xml,
const KFile * file, KMD5SumFmt *md5, CCFileFormat * ff,
const char * path)
{
ProcessOne * self;
rc_t rc = 0;
size_t pathlen;
PLOGMSG (klogDebug10, "ProcessOneMake $(f)", PLOG_S(f), path);
/* legit seeming inputs? these could be replaced with RC returns */
assert (ppo != NULL);
assert (file != NULL);
assert (path != NULL);
/* allocate the object */
pathlen = strlen (path);
self = malloc (sizeof (*self) - sizeof(self->path) + pathlen + 1);
if (self == NULL)
{
rc = RC (rcExe, rcNoTarg, rcAllocating, rcMemory, rcExhausted);
}
else
{
atomic32_set (&self->refcount, 1);
KDirectoryAddRef (dir);
KDirectoryAddRef (xml);
KMD5SumFmtAddRef (md5);
CCFileFormatAddRef (ff);
KFileAddRef (file);
self->dir = dir;
self->xml = xml;
self->md5 = md5;
self->file = file;
self->ff = ff;
memcpy (self->path, path, pathlen);
self->path[pathlen] = '\0';
rc = 0;
}
*ppo = self;
return rc;
}
static
rc_t open_dir_as_archive (const char * path, const KFile ** file)
{
rc_t rc;
KPathType kpt;
const KDirectory * d;
assert (path != NULL);
assert (path[0] != '\0');
assert (file != NULL);
rc = 0;
kpt = KDirectoryPathType (kdir, path);
switch (kpt)
{
case kptFile:
STSMSG (1, ("Opening as archive %s\n", path));
rc = KDirectoryOpenSraArchiveRead (kdir, &d, false, path);
if (rc != 0)
rc = KDirectoryOpenTarArchiveRead (kdir, &d, false, path);
if (rc != 0)
rc = RC (rcExe, rcParam, rcOpening, rcDirectory, rcInvalid);
break;
default:
rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcInvalid);
PLOGERR (klogFatal, (klogFatal, rc, "Parameter [$(P)] must be a directory", PLOG_S(P), path));
return rc;
case kptDir:
/* KOutMsg ("%s: opening dir\n",__func__); */
rc = KDirectoryVOpenDirRead (kdir, &d, false, path, NULL);
}
if (rc == 0)
{
/* KOutMsg ("%s: dir to archive\n",__func__); */
rc = KDirectoryOpenTocFileRead (d, file, alignment, pnamesFilter, NULL, sort_size_then_rel_path );
KDirectoryRelease (d);
}
return rc;
}
static
void CC FGroupMAP_Validate( BSTNode *n, void *data )
{
const FGroupMAP* g = (const FGroupMAP*)n;
rc_t rc = 0, *rc_out = (rc_t*)data;
const char* rnm = NULL, *mnm = NULL;
rc = FGroupKey_Validate(&g->key);
if( g->seq != NULL ) {
CGLoaderFile_Filename(g->seq, &rnm);
rnm = rnm ? strrchr(rnm, '/') : rnm;
DEBUG_MSG(5, (" READS(%s)", rnm));
}
if( g->align ) {
CGLoaderFile_Filename(g->align, &mnm);
mnm = mnm ? strrchr(mnm, '/') : mnm;
DEBUG_MSG(5, (" MAPPINGS(%s)", mnm));
}
DEBUG_MSG(5, ("\n"));
if( rc == 0 && g->seq == NULL ) {
rc = RC(rcExe, rcQueue, rcValidating, rcItem, rcIncomplete);
}
/* THIS USED TO WIPE OUT THE "rc" ON EACH ENTRY */
if( rc != 0) {
PLOGERR(klogErr, (klogErr, rc, "file pair $(f1)[mandatory], $(f2)[optional]", PLOG_2(PLOG_S(f1),PLOG_S(f2)), rnm, mnm));
if ( * rc_out == 0 )
*rc_out = rc;
#if 0
} else {
*rc_out = RC(0, 0, 0, 0, 0);
#endif
}
}
static
rc_t DB_Fini(const SParam* p, DB_Handle* h, bool drop)
{
rc_t rc = 0, rc2;
/* THIS FUNCTION MAKES NO ATTEMPT TO PRESERVE INITIAL ERROR CODES
EACH SUCCESSIVE ERROR OVERWRITES THE PREVIOUS CODE */
if( h != NULL ) {
PLOGMSG(klogInfo, (klogInfo, "Fini SEQUENCE", "severity=status"));
if( (rc2 = CGWriterSeq_Whack(h->wseq, !drop, NULL)) != 0 && !drop ) {
drop = true;
rc = rc2;
}
h->wseq = NULL;
h->reads = NULL;
PLOGMSG(klogInfo, (klogInfo, "Fini (PRI&SEC)_ALIGNMENT", "severity=status"));
if( (rc2 = CGWriterAlgn_Whack(h->walgn, !drop, NULL, NULL)) != 0 && !drop ) {
drop = true;
rc = rc2;
}
h->walgn = NULL;
h->mappings = NULL;
PLOGMSG(klogInfo, (klogInfo, "Fini EVIDENCE_INTERVAL", "severity=status"));
if( (rc2 = CGWriterEvdInt_Whack(h->wev_int, !drop, NULL)) != 0 && !drop ) {
drop = true;
rc = rc2;
}
h->wev_int = NULL;
h->ev_int = NULL;
PLOGMSG(klogInfo, (klogInfo, "Fini EVIDENCE_ALIGNMENT", "severity=status"));
if( (rc2 = CGWriterEvdDnbs_Whack(h->wev_dnb, !drop, NULL)) != 0 && !drop ) {
drop = true;
rc = rc2;
}
h->wev_dnb = NULL;
h->ev_dnb = NULL;
PLOGMSG(klogInfo, (klogInfo, "Fini calculating reference coverage", "severity=status"));
if( (rc2 = ReferenceMgr_Release(h->rmgr, !drop, NULL, drop ? false : true)) != 0 && !drop ) {
drop = true;
rc = rc2;
LOGERR(klogErr, rc, "Failed calculating reference coverage");
}
h->rmgr = NULL;
if( rc == 0 )
{
KMetadata* meta;
if( (rc = VDatabaseOpenMetadataUpdate(h->db, &meta)) == 0 ) {
KMDataNode *node;
if( (rc = KMetadataOpenNodeUpdate(meta, &node, "/")) == 0 ) {
if( (rc = KLoaderMeta_Write(node, p->argv0, __DATE__, "Complete Genomics", KAppVersion())) != 0 ) {
LOGERR(klogErr, rc, "Cannot update loader meta");
}
KMDataNodeRelease(node);
}
KMetadataRelease(meta);
}
}
PLOGMSG(klogInfo, (klogInfo, "Fini VDatabaseRelease", "severity=status"));
VDatabaseRelease(h->db);
h->db = NULL;
VSchemaRelease(h->schema);
h->schema = NULL;
if( drop || rc != 0 ) {
rc2 = VDBManagerDrop(h->mgr, kptDatabase, p->out);
if( GetRCState(rc2) == rcNotFound ) {
/* WHAT WOULD BE THE POINT OF RESETTING "rc" TO ZERO? */
rc = 0;
} else if( rc2 != 0 ) {
if ( rc == 0 )
rc = rc2;
PLOGERR(klogErr, (klogErr, rc2, "cannot drop db at '$(path)'", PLOG_S(path), p->out));
}
}
VDBManagerRelease(h->mgr);
h->mgr = NULL;
}
return rc;
}
rc_t CC KMain ( int argc, char *argv[] )
{
rc_t rc = 0;
Args * args;
rc = ArgsMakeStandardOptions (&args);
if (rc == 0)
{
do
{
uint32_t pcount;
rc = ArgsAddOptionArray (args, MyOptions, sizeof MyOptions / sizeof (OptDef));
if (rc)
break;
rc = ArgsParse (args, argc, argv);
if (rc)
break;
/* quirky way default path is generated means this comes
* before standard argument handling */
rc = ArgsOptionCount (args, OPTION_TABLE, &pcount);
if (rc)
break;
if (pcount == 0)
{
static char * default_name = "RowWriteTestOutTable";
char * user;
user = getenv ("USER");
if (user)
snprintf (tablePath, sizeof (tablePath),
"/home/%s/%s", user, default_name);
else
strncpy (tablePath, default_name, sizeof (tablePath));
}
else
{
const char * pc;
ArgsOptionValue (args, OPTION_TABLE, 0, &pc);
strncpy (tablePath, pc, sizeof (tablePath));
}
rc = ArgsHandleStandardOptions (args);
if (rc)
break;
rc = ArgsParamCount (args, &pcount);
if (rc)
break;
if (pcount)
{
const char * pc;
rc = ArgsArgvValue (args, 0, &pc);
if (rc)
break;
rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcExcessive);
PLOGERR (klogFatal, (klogFatal, rc, "$(P) takes no parameters", PLOG_S(P), pc));
break;
}
rc = ArgsOptionCount (args, OPTION_ROW, &pcount);
if (rc == 0)
{
uint64_t row_count;
if (pcount)
{
const char * pc;
rc = ArgsOptionValue (args, OPTION_ROW, 0, &pc);
if (rc)
break;
row_count = AsciiToU32 (pc, NULL, NULL);
}
else
row_count = ROWS;
rc = run (tablePath, row_count);
}
} while (0);
ArgsWhack (args);
}
return rc;
}
rc_t IlluminaSpot_Add(IlluminaSpot* spot, const pstring* name, const pstring* barcode, const IlluminaRead* read)
{
rc_t rc = 0;
if( spot->nreads == 0 ) {
rc = IlluminaSpot_Set(spot, spot->nreads++, name, barcode, read);
} else if( pstring_cmp(spot->name, name) == 0 ) {
/* look if same read_id was already seen in this spot */
int32_t k;
for(k = 0; k < spot->nreads; k++) {
if( spot->reads[k].read_id == read->read_id ) {
const char* field;
rc = IlluminaSpot_Append(spot, k, barcode, read, &field);
if( GetRCState(rc) == rcDuplicate && read->read_id == ILLUMINAWRITER_READID_NONE ) {
/* may be it is the case when readids are missing on defline and these are separate reads */
k = spot->nreads + 1;
rc = 0;
} else if( rc != 0 ) {
PLOGERR(klogErr, (klogErr, rc, "$(field) for spot '$(s)'", PLOG_2(PLOG_S(field),PLOG_S(s)), field, spot->name->data));
}
break;
}
}
if( rc == 0 && k >= spot->nreads ) {
/* read was not found, adddind new read to this spot */
rc = IlluminaSpot_Set(spot, spot->nreads++, name, barcode, read);
}
} else {
rc = RC(rcSRA, rcFormatter, rcReading, rcData, rcIgnored);
}
return rc;
}
static
rc_t run_kar_create(const char * archive, const char * directory)
{
rc_t rc;
const KFile * fin;
KFile * fout;
rc = open_out_file (archive, &fout);
if (rc == 0)
{
char * directorystr;
rc = derive_directory_name (&directorystr, archive, directory);
if (rc != 0)
LOGERR (klogErr, rc,"failed to derive directory name");
else
{
assert (directorystr != NULL);
assert (directorystr[0] != '\0');
STSMSG (4, ("start creation of archive"));
/* Jira ticket: SRA-1876
Date: September 13, 2013
raw usage of "directorystr" causes tests to fail within libs/kfs/arc.c */
{
char full [ 4096 ];
rc = KDirectoryResolvePath ( kdir, true, full, sizeof full, "%s", directorystr );
if ( rc == 0 )
{
/* string should be non-empty based upon behavior of
"derive_directory_name" ( also fixed today ) */
assert ( full [ 0 ] != 0 );
/* eliminate double-slashes */
if ( full [ 1 ] != 0 )
{
uint32_t i, j;
/* ALLOW double slashes at the very start
set starting index to 2 for that reason */
for ( i = j = 2; full [ i ] != 0; ++ i )
{
if ( ( full [ j ] = full [ i ] ) != '/' || full [ j - 1 ] != '/' )
++ j;
}
full [ j ] = 0;
}
rc = open_dir_as_archive ( full, & fin );
}
}
if (rc != 0)
PLOGERR (klogErr, (klogErr, rc,"failed to open directory '$(D)' as archive",
PLOG_S(D),directorystr));
else
{
assert (fin != NULL);
assert (fout != NULL);
STSMSG (4, ("start copy_file"));
rc = copy_file (fin, fout);
if (rc != 0)
LOGERR (klogErr, rc, "failed copy file in create");
KFileRelease (fin);
}
}
KFileRelease (fout);
free (directorystr);
if (rc)
{
remove_out_file (archive);
}
}
return rc;
}