void gkStore::gkStore_deletePartitions(void) { char path[FILENAME_MAX]; sprintf(path, "%s/partitions/map", gkStore_path()); if (AS_UTL_fileExists(path, false, false) == false) return; // How many partitions? FILE *F = fopen(path, "r"); if (errno) fprintf(stderr, "ERROR: failed to open partition meta data '%s': %s\n", path, strerror(errno)), exit(1); fread(&_numberOfPartitions, sizeof(uint32), 1, F); fclose(F); // Yay! Delete! AS_UTL_unlink(path); for (uint32 ii=0; ii<_numberOfPartitions; ii++) { sprintf(path, "%s/partitions/reads.%04u", gkStore_path(), ii+1); AS_UTL_unlink(path); sprintf(path, "%s/partitions/blobs.%04u", gkStore_path(), ii+1); AS_UTL_unlink(path); } }
void gkStore::gkStore_delete(void) { char path[FILENAME_MAX]; delete [] _libraries; delete [] _reads; _libraries = NULL; _reads = NULL; gkStore_deletePartitions(); sprintf(path, "%s/info", gkStore_path()); AS_UTL_unlink(path); sprintf(path, "%s/libraries", gkStore_path()); AS_UTL_unlink(path); sprintf(path, "%s/reads", gkStore_path()); AS_UTL_unlink(path); sprintf(path, "%s/blobs", gkStore_path()); AS_UTL_unlink(path); AS_UTL_unlink(path); }
int main(int argc, char **argv) { char *ovlName = NULL; uint32 maxJob = 0; bool deleteIntermediates = true; bool doExplicitTest = false; bool doFixes = false; char name[FILENAME_MAX]; argc = AS_configure(argc, argv); int err=0; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-O") == 0) { ovlName = argv[++arg]; } else if (strcmp(argv[arg], "-F") == 0) { maxJob = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-f") == 0) { doFixes = true; } else if (strcmp(argv[arg], "-t") == 0) { doExplicitTest = true; ovlName = argv[++arg]; } else if (strcmp(argv[arg], "-nodelete") == 0) { deleteIntermediates = false; } else { fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); } arg++; } if (ovlName == NULL) err++; if ((maxJob == 0) && (doExplicitTest == false)) err++; if (err) { fprintf(stderr, "usage: %s ...\n", argv[0]); fprintf(stderr, " -O x.ovlStore path to overlap store to build the final index for\n"); fprintf(stderr, " -F s number of slices used in bucketizing/sorting\n"); fprintf(stderr, "\n"); fprintf(stderr, " -t x.ovlStore explicitly test a previously constructed index\n"); fprintf(stderr, " -f when testing, also create a new 'idx.fixed' which might\n"); fprintf(stderr, " resolve rare problems\n"); fprintf(stderr, "\n"); fprintf(stderr, " -nodelete do not remove intermediate files when the index is\n"); fprintf(stderr, " successfully created\n"); fprintf(stderr, "\n"); fprintf(stderr, " DANGER DO NOT USE DO NOT USE DO NOT USE DANGER\n"); fprintf(stderr, " DANGER DANGER\n"); fprintf(stderr, " DANGER This command is difficult to run by hand. DANGER\n"); fprintf(stderr, " DANGER Use ovStoreCreate instead. DANGER\n"); fprintf(stderr, " DANGER DANGER\n"); fprintf(stderr, " DANGER DO NOT USE DO NOT USE DO NOT USE DANGER\n"); fprintf(stderr, "\n"); if (ovlName == NULL) fprintf(stderr, "ERROR: No overlap store (-O) supplied.\n"); if ((maxJob == 0) && (doExplicitTest == false)) fprintf(stderr, "ERROR: One of -F (number of slices) or -t (test a store) must be supplied.\n"); exit(1); } // Do the test, and maybe fix things up. if (doExplicitTest == true) { bool passed = testIndex(ovlName, doFixes); exit((passed == true) ? 0 : 1); } // Check that all segments are present. Every segment should have an info file. uint32 cntJob = 0; for (uint32 i=1; i<=maxJob; i++) { uint32 complete = 0; sprintf(name, "%s/%04d", ovlName, i); if (AS_UTL_fileExists(name, FALSE, FALSE) == true) complete++; else fprintf(stderr, "ERROR: Segment "F_U32" data not present (%s)\n", i, name); sprintf(name, "%s/%04d.info", ovlName, i); if (AS_UTL_fileExists(name, FALSE, FALSE) == true) complete++; else fprintf(stderr, "ERROR: Segment "F_U32" info not present (%s)\n", i, name); sprintf(name, "%s/%04d.index", ovlName, i); if (AS_UTL_fileExists(name, FALSE, FALSE) == true) complete++; else fprintf(stderr, "ERROR: Segment "F_U32" index not present (%s)\n", i, name); if (complete == 3) cntJob++; } if (cntJob != maxJob) { fprintf(stderr, "ERROR: Expected "F_U32" segments, only found "F_U32".\n", maxJob, cntJob); exit(1); } // Merge the stuff. mergeInfoFiles(ovlName, maxJob); // Diagnostics. if (testIndex(ovlName, false) == false) { fprintf(stderr, "ERROR: index failed tests.\n"); exit(1); } // Remove intermediates. For the buckets, we keep going until there are 10 in a row not present. // During testing, on a microbe using 2850 buckets, some buckets were empty. if (deleteIntermediates == false) { fprintf(stderr, "\n"); fprintf(stderr, "Not removing intermediate files. Finished.\n"); exit(0); } fprintf(stderr, "\n"); fprintf(stderr, "Removing intermediate files.\n"); // Removing indices is easy, beacuse we know how many there are. for (uint32 i=1; i<=maxJob; i++) { sprintf(name, "%s/%04u.index", ovlName, i); AS_UTL_unlink(name); sprintf(name, "%s/%04u.info", ovlName, i); AS_UTL_unlink(name); } // We don't know how many buckets there are, so we remove until we fail to find ten // buckets in a row. for (uint32 missing=0, i=1; missing<10; i++) { sprintf(name, "%s/bucket%04d", ovlName, i); if (AS_UTL_fileExists(name, TRUE, FALSE) == FALSE) { missing++; continue; } missing = 0; sprintf(name, "%s/bucket%04d/sliceSizes", ovlName, i); AS_UTL_unlink(name); sprintf(name, "%s/bucket%04d", ovlName, i); rmdir(name); } fprintf(stderr, "Finished.\n"); exit(0); }
void ovStore::addEvalues(uint32 bgnID, uint32 endID, uint16 *evalues, uint64 evaluesLen) { char name[FILENAME_MAX]; sprintf(name, "%s/evalues", _storePath); // If we have an opened memory mapped file, and it isn't open for writing, close it. if ((_evaluesMap) && (_evaluesMap->type() == memoryMappedFile_readOnly)) { fprintf(stderr, "WARNING: closing read-only evalues file.\n"); delete _evaluesMap; _evaluesMap = NULL; _evalues = NULL; } // Remove a bogus evalues file if one exists. if ((AS_UTL_fileExists(name) == true) && (AS_UTL_sizeOfFile(name) != (sizeof(uint16) * _info._numOverlapsTotal))) { fprintf(stderr, "WARNING: existing evalues file is incorrect size: should be "F_U64" bytes, is "F_U64" bytes. Removing.\n", (sizeof(uint16) * _info._numOverlapsTotal), AS_UTL_sizeOfFile(name)); AS_UTL_unlink(name); } // Make a new evalues file if one doesn't exist. if (AS_UTL_fileExists(name) == false) { fprintf(stderr, "Creating evalues file for "F_U64" overlaps.\r", _info._numOverlapsTotal); errno = 0; FILE *F = fopen(name, "w"); if (errno) fprintf(stderr, "Failed to make evalues file '%s': %s\n", name, strerror(errno)), exit(1); uint16 *Z = new uint16 [1048576]; uint64 Zn = 0; memset(Z, 0, sizeof(uint16) * 1048576); while (Zn < _info._numOverlapsTotal) { uint64 S = (Zn + 1048576 < _info._numOverlapsTotal) ? 1048576 : _info._numOverlapsTotal - Zn; AS_UTL_safeWrite(F, Z, "zero evalues", sizeof(uint16), S); Zn += S; fprintf(stderr, "Creating evalues file for "F_U64" overlaps....%07.3f%%\r", _info._numOverlapsTotal, 100.0 * Zn / _info._numOverlapsTotal); } fprintf(stderr, "Creating evalues file for "F_U64" overlaps....%07.3f%%\n", _info._numOverlapsTotal, 100.0 * Zn / _info._numOverlapsTotal); fclose(F); } // Open the evalues file if it isn't already opened if (_evalues == NULL) { _evaluesMap = new memoryMappedFile(name, memoryMappedFile_readWrite); _evalues = (uint16 *)_evaluesMap->get(0); } // Figure out the overlap ID for the first overlap associated with bgnID setRange(bgnID, endID); // Load the evalues from 'evalues' for (uint64 ii=0; ii<evaluesLen; ii++) _evalues[_offt._overlapID + ii] = evalues[ii]; // That's it. Deleting the ovStore object will close the memoryMappedFile. It's left open // for more updates. }