void
gkStore::gkStore_deletePartitions(void) {
char path[FILENAME_MAX];
sprintf(path, "%s/partitions/map", gkStore_path());
if (AS_UTL_fileExists(path, false, false) == false)
return;
// How many partitions?
FILE *F = fopen(path, "r");
if (errno)
fprintf(stderr, "ERROR: failed to open partition meta data '%s': %s\n", path, strerror(errno)), exit(1);
fread(&_numberOfPartitions, sizeof(uint32), 1, F);
fclose(F);
// Yay! Delete!
AS_UTL_unlink(path);
for (uint32 ii=0; ii<_numberOfPartitions; ii++) {
sprintf(path, "%s/partitions/reads.%04u", gkStore_path(), ii+1); AS_UTL_unlink(path);
sprintf(path, "%s/partitions/blobs.%04u", gkStore_path(), ii+1); AS_UTL_unlink(path);
}
}
void
gkStore::gkStore_delete(void) {
char path[FILENAME_MAX];
delete [] _libraries;
delete [] _reads;
_libraries = NULL;
_reads = NULL;
gkStore_deletePartitions();
sprintf(path, "%s/info", gkStore_path()); AS_UTL_unlink(path);
sprintf(path, "%s/libraries", gkStore_path()); AS_UTL_unlink(path);
sprintf(path, "%s/reads", gkStore_path()); AS_UTL_unlink(path);
sprintf(path, "%s/blobs", gkStore_path()); AS_UTL_unlink(path);
AS_UTL_unlink(path);
}
int
main(int argc, char **argv) {
char *ovlName = NULL;
uint32 maxJob = 0;
bool deleteIntermediates = true;
bool doExplicitTest = false;
bool doFixes = false;
char name[FILENAME_MAX];
argc = AS_configure(argc, argv);
int err=0;
int arg=1;
while (arg < argc) {
if (strcmp(argv[arg], "-O") == 0) {
ovlName = argv[++arg];
} else if (strcmp(argv[arg], "-F") == 0) {
maxJob = atoi(argv[++arg]);
} else if (strcmp(argv[arg], "-f") == 0) {
doFixes = true;
} else if (strcmp(argv[arg], "-t") == 0) {
doExplicitTest = true;
ovlName = argv[++arg];
} else if (strcmp(argv[arg], "-nodelete") == 0) {
deleteIntermediates = false;
} else {
fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]);
}
arg++;
}
if (ovlName == NULL)
err++;
if ((maxJob == 0) && (doExplicitTest == false))
err++;
if (err) {
fprintf(stderr, "usage: %s ...\n", argv[0]);
fprintf(stderr, " -O x.ovlStore path to overlap store to build the final index for\n");
fprintf(stderr, " -F s number of slices used in bucketizing/sorting\n");
fprintf(stderr, "\n");
fprintf(stderr, " -t x.ovlStore explicitly test a previously constructed index\n");
fprintf(stderr, " -f when testing, also create a new 'idx.fixed' which might\n");
fprintf(stderr, " resolve rare problems\n");
fprintf(stderr, "\n");
fprintf(stderr, " -nodelete do not remove intermediate files when the index is\n");
fprintf(stderr, " successfully created\n");
fprintf(stderr, "\n");
fprintf(stderr, " DANGER DO NOT USE DO NOT USE DO NOT USE DANGER\n");
fprintf(stderr, " DANGER DANGER\n");
fprintf(stderr, " DANGER This command is difficult to run by hand. DANGER\n");
fprintf(stderr, " DANGER Use ovStoreCreate instead. DANGER\n");
fprintf(stderr, " DANGER DANGER\n");
fprintf(stderr, " DANGER DO NOT USE DO NOT USE DO NOT USE DANGER\n");
fprintf(stderr, "\n");
if (ovlName == NULL)
fprintf(stderr, "ERROR: No overlap store (-O) supplied.\n");
if ((maxJob == 0) && (doExplicitTest == false))
fprintf(stderr, "ERROR: One of -F (number of slices) or -t (test a store) must be supplied.\n");
exit(1);
}
// Do the test, and maybe fix things up.
if (doExplicitTest == true) {
bool passed = testIndex(ovlName, doFixes);
exit((passed == true) ? 0 : 1);
}
// Check that all segments are present. Every segment should have an info file.
uint32 cntJob = 0;
for (uint32 i=1; i<=maxJob; i++) {
uint32 complete = 0;
sprintf(name, "%s/%04d", ovlName, i);
if (AS_UTL_fileExists(name, FALSE, FALSE) == true)
complete++;
else
fprintf(stderr, "ERROR: Segment "F_U32" data not present (%s)\n", i, name);
sprintf(name, "%s/%04d.info", ovlName, i);
if (AS_UTL_fileExists(name, FALSE, FALSE) == true)
complete++;
else
fprintf(stderr, "ERROR: Segment "F_U32" info not present (%s)\n", i, name);
sprintf(name, "%s/%04d.index", ovlName, i);
if (AS_UTL_fileExists(name, FALSE, FALSE) == true)
complete++;
else
fprintf(stderr, "ERROR: Segment "F_U32" index not present (%s)\n", i, name);
if (complete == 3)
cntJob++;
}
if (cntJob != maxJob) {
fprintf(stderr, "ERROR: Expected "F_U32" segments, only found "F_U32".\n", maxJob, cntJob);
exit(1);
}
// Merge the stuff.
mergeInfoFiles(ovlName, maxJob);
// Diagnostics.
if (testIndex(ovlName, false) == false) {
fprintf(stderr, "ERROR: index failed tests.\n");
exit(1);
}
// Remove intermediates. For the buckets, we keep going until there are 10 in a row not present.
// During testing, on a microbe using 2850 buckets, some buckets were empty.
if (deleteIntermediates == false) {
fprintf(stderr, "\n");
fprintf(stderr, "Not removing intermediate files. Finished.\n");
exit(0);
}
fprintf(stderr, "\n");
fprintf(stderr, "Removing intermediate files.\n");
// Removing indices is easy, beacuse we know how many there are.
for (uint32 i=1; i<=maxJob; i++) {
sprintf(name, "%s/%04u.index", ovlName, i);
AS_UTL_unlink(name);
sprintf(name, "%s/%04u.info", ovlName, i);
AS_UTL_unlink(name);
}
// We don't know how many buckets there are, so we remove until we fail to find ten
// buckets in a row.
for (uint32 missing=0, i=1; missing<10; i++) {
sprintf(name, "%s/bucket%04d", ovlName, i);
if (AS_UTL_fileExists(name, TRUE, FALSE) == FALSE) {
missing++;
continue;
}
missing = 0;
sprintf(name, "%s/bucket%04d/sliceSizes", ovlName, i);
AS_UTL_unlink(name);
sprintf(name, "%s/bucket%04d", ovlName, i);
rmdir(name);
}
fprintf(stderr, "Finished.\n");
exit(0);
}
void
ovStore::addEvalues(uint32 bgnID, uint32 endID, uint16 *evalues, uint64 evaluesLen) {
char name[FILENAME_MAX];
sprintf(name, "%s/evalues", _storePath);
// If we have an opened memory mapped file, and it isn't open for writing, close it.
if ((_evaluesMap) && (_evaluesMap->type() == memoryMappedFile_readOnly)) {
fprintf(stderr, "WARNING: closing read-only evalues file.\n");
delete _evaluesMap;
_evaluesMap = NULL;
_evalues = NULL;
}
// Remove a bogus evalues file if one exists.
if ((AS_UTL_fileExists(name) == true) &&
(AS_UTL_sizeOfFile(name) != (sizeof(uint16) * _info._numOverlapsTotal))) {
fprintf(stderr, "WARNING: existing evalues file is incorrect size: should be "F_U64" bytes, is "F_U64" bytes. Removing.\n",
(sizeof(uint16) * _info._numOverlapsTotal), AS_UTL_sizeOfFile(name));
AS_UTL_unlink(name);
}
// Make a new evalues file if one doesn't exist.
if (AS_UTL_fileExists(name) == false) {
fprintf(stderr, "Creating evalues file for "F_U64" overlaps.\r", _info._numOverlapsTotal);
errno = 0;
FILE *F = fopen(name, "w");
if (errno)
fprintf(stderr, "Failed to make evalues file '%s': %s\n", name, strerror(errno)), exit(1);
uint16 *Z = new uint16 [1048576];
uint64 Zn = 0;
memset(Z, 0, sizeof(uint16) * 1048576);
while (Zn < _info._numOverlapsTotal) {
uint64 S = (Zn + 1048576 < _info._numOverlapsTotal) ? 1048576 : _info._numOverlapsTotal - Zn;
AS_UTL_safeWrite(F, Z, "zero evalues", sizeof(uint16), S);
Zn += S;
fprintf(stderr, "Creating evalues file for "F_U64" overlaps....%07.3f%%\r",
_info._numOverlapsTotal, 100.0 * Zn / _info._numOverlapsTotal);
}
fprintf(stderr, "Creating evalues file for "F_U64" overlaps....%07.3f%%\n",
_info._numOverlapsTotal, 100.0 * Zn / _info._numOverlapsTotal);
fclose(F);
}
// Open the evalues file if it isn't already opened
if (_evalues == NULL) {
_evaluesMap = new memoryMappedFile(name, memoryMappedFile_readWrite);
_evalues = (uint16 *)_evaluesMap->get(0);
}
// Figure out the overlap ID for the first overlap associated with bgnID
setRange(bgnID, endID);
// Load the evalues from 'evalues'
for (uint64 ii=0; ii<evaluesLen; ii++)
_evalues[_offt._overlapID + ii] = evalues[ii];
// That's it. Deleting the ovStore object will close the memoryMappedFile. It's left open
// for more updates.
}
