Hmm *hmm_loadFromFile(const char *fileName) { FILE *fH = fopen(fileName, "r"); char *string = stFile_getLineFromFile(fH); stList *tokens = stString_split(string); if (stList_length(tokens) < 2) { st_errAbort("Got an empty line in the input state machine file %s\n", fileName); } int type; int64_t j = sscanf(stList_get(tokens, 0), "%i", &type); if (j != 1) { st_errAbort("Failed to parse state number (int) from string: %s\n", string); } Hmm *hmm = hmm_constructEmpty(0.0, type); if (stList_length(tokens) != hmm->stateNumber * hmm->stateNumber + 2) { st_errAbort( "Got the wrong number of transitions in the input state machine file %s, got %" PRIi64 " instead of %" PRIi64 "\n", fileName, stList_length(tokens), hmm->stateNumber * hmm->stateNumber + 2); } for (int64_t i = 0; i < hmm->stateNumber * hmm->stateNumber; i++) { j = sscanf(stList_get(tokens, i + 1), "%lf", &(hmm->transitions[i])); if (j != 1) { st_errAbort("Failed to parse transition prob (float) from string: %s\n", string); } } j = sscanf(stList_get(tokens, stList_length(tokens) - 1), "%lf", &(hmm->likelihood)); if (j != 1) { st_errAbort("Failed to parse likelihood (float) from string: %s\n", string); } //Cleanup transitions line free(string); stList_destruct(tokens); //Now parse the emissions line string = stFile_getLineFromFile(fH); tokens = stString_split(string); if (stList_length(tokens) != hmm->stateNumber * SYMBOL_NUMBER_NO_N * SYMBOL_NUMBER_NO_N) { st_errAbort( "Got the wrong number of emissions in the input state machine file %s, got %" PRIi64 " instead of %" PRIi64 "\n", fileName, stList_length(tokens), hmm->stateNumber * SYMBOL_NUMBER_NO_N * SYMBOL_NUMBER_NO_N); } for (int64_t i = 0; i < hmm->stateNumber * SYMBOL_NUMBER_NO_N * SYMBOL_NUMBER_NO_N; i++) { j = sscanf(stList_get(tokens, i), "%lf", &(hmm->emissions[i])); if (j != 1) { st_errAbort("Failed to parse emission prob (float) from string: %s\n", string); } } //Final cleanup free(string); stList_destruct(tokens); fclose(fH); return hmm; }
/* * A thread is trivial if all the segments it contains come from blocks containing only a reference segment. * These reference only segments represent scaffold gaps. At the same time, it processes the thread string * to remove the boolean values use to indicate if a thread is trivial or not. */ static bool isTrivialString(char **threadString) { stList *strings = stString_split(*threadString); //Split splits into individual segments. bool trivialString = 1; for(int64_t i=0; i<stList_length(strings); i++) { char *segmentString = stList_get(strings, i); int64_t j = strlen(segmentString)-1; //Location of the boolean value within a segment. assert(j > 0); assert(segmentString[j] == '0' || segmentString[j] == '1'); if(segmentString[j] == '1') { //Found a non-trivial segment, hence the thread is non-trivial. trivialString = 0; } segmentString[j] = '\0'; } free(*threadString); //Free old thread string. Doing it this way is a bit more memory efficient, as we don't keep two copies of the string around. *threadString = stString_join2("", strings); //Concatenation makes one sequence, now without the booleans. stList_destruct(strings); return trivialString; }
stList *lineTokensFromFile(const char *filePath, int64_t getLine) { FILE *fH = fopen(filePath, "r"); int64_t lineCount = 1; stList *tokens; char *string = stFile_getLineFromFile(fH); while (string != NULL) { string = stFile_getLineFromFile(fH); if (lineCount == getLine) { tokens = stString_split(string); return tokens; } else { lineCount++; free(string); } } fclose(fH); return stList_construct3(0, &free); }
int main(int argc, char *argv[]) { /* * Open the database. * Construct a flower. * Construct an event tree representing the species tree. * For each sequence contruct two ends each containing an cap. * Make a file for the sequence. * Link the two caps. * Finish! */ int64_t key, j; Group *group; Flower_EndIterator *endIterator; End *end; bool makeEventHeadersAlphaNumeric = 0; /* * Arguments/options */ char * logLevelString = NULL; char * speciesTree = NULL; char * outgroupEvents = NULL; /////////////////////////////////////////////////////////////////////////// // (0) Parse the inputs handed by genomeCactus.py / setup stuff. /////////////////////////////////////////////////////////////////////////// while (1) { static struct option long_options[] = { { "logLevel", required_argument, 0, 'a' }, { "cactusDisk", required_argument, 0, 'b' }, { "speciesTree", required_argument, 0, 'f' }, { "outgroupEvents", required_argument, 0, 'g' }, { "help", no_argument, 0, 'h' }, { "makeEventHeadersAlphaNumeric", no_argument, 0, 'i' }, { 0, 0, 0, 0 } }; int option_index = 0; key = getopt_long(argc, argv, "a:b:f:hg:i", long_options, &option_index); if (key == -1) { break; } switch (key) { case 'a': logLevelString = optarg; break; case 'b': cactusDiskDatabaseString = optarg; break; case 'f': speciesTree = optarg; break; case 'g': outgroupEvents = optarg; break; case 'h': usage(); return 0; case 'i': makeEventHeadersAlphaNumeric = 1; break; default: usage(); return 1; } } /////////////////////////////////////////////////////////////////////////// // (0) Check the inputs. /////////////////////////////////////////////////////////////////////////// //assert(logLevelString == NULL || strcmp(logLevelString, "CRITICAL") == 0 || strcmp(logLevelString, "INFO") == 0 || strcmp(logLevelString, "DEBUG") == 0); assert(cactusDiskDatabaseString != NULL); assert(speciesTree != NULL); ////////////////////////////////////////////// //Set up logging ////////////////////////////////////////////// st_setLogLevelFromString(logLevelString); ////////////////////////////////////////////// //Log (some of) the inputs ////////////////////////////////////////////// st_logInfo("Flower disk name : %s\n", cactusDiskDatabaseString); for (j = optind; j < argc; j++) { st_logInfo("Sequence file/directory %s\n", argv[j]); } ////////////////////////////////////////////// //Load the database ////////////////////////////////////////////// stKVDatabaseConf *kvDatabaseConf = kvDatabaseConf = stKVDatabaseConf_constructFromString(cactusDiskDatabaseString); if (stKVDatabaseConf_getType(kvDatabaseConf) == stKVDatabaseTypeTokyoCabinet || stKVDatabaseConf_getType(kvDatabaseConf) == stKVDatabaseTypeKyotoTycoon) { assert(stKVDatabaseConf_getDir(kvDatabaseConf) != NULL); cactusDisk = cactusDisk_construct2(kvDatabaseConf, "cactusSequences"); } else { cactusDisk = cactusDisk_construct(kvDatabaseConf, 1); } st_logInfo("Set up the flower disk\n"); ////////////////////////////////////////////// //Construct the flower ////////////////////////////////////////////// if (cactusDisk_getFlower(cactusDisk, 0) != NULL) { cactusDisk_destruct(cactusDisk); st_logInfo("The first flower already exists\n"); return 0; } flower = flower_construct2(0, cactusDisk); assert(flower_getName(flower) == 0); st_logInfo("Constructed the flower\n"); ////////////////////////////////////////////// //Construct the event tree ////////////////////////////////////////////// st_logInfo("Going to build the event tree with newick string: %s\n", speciesTree); stTree *tree = stTree_parseNewickString(speciesTree); st_logInfo("Parsed the tree\n"); if (makeEventHeadersAlphaNumeric) { makeEventHeadersAlphaNumericFn(tree); } stTree_setBranchLength(tree, INT64_MAX); checkBranchLengthsAreDefined(tree); eventTree = eventTree_construct2(flower); //creates the event tree and the root even totalEventNumber = 1; st_logInfo("Constructed the basic event tree\n"); // Construct a set of outgroup names so that ancestral outgroups // get recognized. stSet *outgroupNameSet = stSet_construct3(stHash_stringKey, stHash_stringEqualKey, free); if(outgroupEvents != NULL) { stList *outgroupNames = stString_split(outgroupEvents); for(int64_t i = 0; i < stList_length(outgroupNames); i++) { char *outgroupName = stList_get(outgroupNames, i); stSet_insert(outgroupNameSet, stString_copy(outgroupName)); } stList_destruct(outgroupNames); } //now traverse the tree j = optind; assignEventsAndSequences(eventTree_getRootEvent(eventTree), tree, outgroupNameSet, argv, &j); char *eventTreeString = eventTree_makeNewickString(eventTree); st_logInfo( "Constructed the initial flower with %" PRIi64 " sequences and %" PRIi64 " events with string: %s\n", totalSequenceNumber, totalEventNumber, eventTreeString); assert(event_getSubTreeBranchLength(eventTree_getRootEvent(eventTree)) >= 0.0); free(eventTreeString); //assert(0); ////////////////////////////////////////////// //Label any outgroup events. ////////////////////////////////////////////// if (outgroupEvents != NULL) { stList *outgroupEventsList = stString_split(outgroupEvents); for (int64_t i = 0; i < stList_length(outgroupEventsList); i++) { char *outgroupEvent = makeEventHeadersAlphaNumeric ? makeAlphaNumeric(stList_get(outgroupEventsList, i)) : stString_copy(stList_get(outgroupEventsList, i)); Event *event = eventTree_getEventByHeader(eventTree, outgroupEvent); if (event == NULL) { st_errAbort("Got an outgroup string that does not match an event, outgroup string %s", outgroupEvent); } assert(!event_isOutgroup(event)); event_setOutgroupStatus(event, 1); assert(event_isOutgroup(event)); free(outgroupEvent); } stList_destruct(outgroupEventsList); } ////////////////////////////////////////////// //Construct the terminal group. ////////////////////////////////////////////// if (flower_getEndNumber(flower) > 0) { group = group_construct2(flower); endIterator = flower_getEndIterator(flower); while ((end = flower_getNextEnd(endIterator)) != NULL) { end_setGroup(end, group); } flower_destructEndIterator(endIterator); assert(group_isLeaf(group)); // Create a one link chain if there is only one pair of attached ends.. group_constructChainForLink(group); assert(!flower_builtBlocks(flower)); } else { flower_setBuiltBlocks(flower, 1); } /////////////////////////////////////////////////////////////////////////// // Write the flower to disk. /////////////////////////////////////////////////////////////////////////// //flower_check(flower); cactusDisk_write(cactusDisk); st_logInfo("Updated the flower on disk\n"); /////////////////////////////////////////////////////////////////////////// // Cleanup. /////////////////////////////////////////////////////////////////////////// cactusDisk_destruct(cactusDisk); return 0; //Exit without clean up is quicker, enable cleanup when doing memory leak detection. stSet_destruct(outgroupNameSet); stTree_destruct(tree); stKVDatabaseConf_destruct(kvDatabaseConf); return 0; }
int main(int argc, char *argv[]) { char * logLevelString = NULL; char * cactusDiskDatabaseString = NULL; int64_t i, j; int64_t spanningTrees = 10; int64_t maximumLength = 1500; bool useProgressiveMerging = 0; float matchGamma = 0.5; bool useBanding = 0; int64_t k; stList *listOfEndAlignmentFiles = NULL; char *endAlignmentsToPrecomputeOutputFile = NULL; bool calculateWhichEndsToComputeSeparately = 0; int64_t largeEndSize = 1000000; int64_t chainLengthForBigFlower = 1000000; int64_t longChain = 2; char *ingroupCoverageFilePath = NULL; int64_t minimumSizeToRescue = 1; double minimumCoverageToRescue = 0.0; PairwiseAlignmentParameters *pairwiseAlignmentBandingParameters = pairwiseAlignmentBandingParameters_construct(); /* * Setup the input parameters for cactus core. */ bool pruneOutStubAlignments = 0; /* * Parse the options. */ while (1) { static struct option long_options[] = { { "logLevel", required_argument, 0, 'a' }, { "cactusDisk", required_argument, 0, 'b' }, { "help", no_argument, 0, 'h' }, { "spanningTrees", required_argument, 0, 'i' }, { "maximumLength", required_argument, 0, 'j' }, { "useBanding", no_argument, 0, 'k' }, { "gapGamma", required_argument, 0, 'l' }, { "matchGamma", required_argument, 0, 'L' }, { "splitMatrixBiggerThanThis", required_argument, 0, 'o' }, { "anchorMatrixBiggerThanThis", required_argument, 0, 'p' }, { "repeatMaskMatrixBiggerThanThis", required_argument, 0, 'q' }, { "diagonalExpansion", required_argument, 0, 'r' }, { "constraintDiagonalTrim", required_argument, 0, 't' }, { "minimumDegree", required_argument, 0, 'u' }, { "alignAmbiguityCharacters", no_argument, 0, 'w' }, { "pruneOutStubAlignments", no_argument, 0, 'y' }, { "minimumIngroupDegree", required_argument, 0, 'A' }, { "minimumOutgroupDegree", required_argument, 0, 'B' }, { "precomputedAlignments", required_argument, 0, 'D' }, { "endAlignmentsToPrecomputeOutputFile", required_argument, 0, 'E' }, { "useProgressiveMerging", no_argument, 0, 'F' }, { "calculateWhichEndsToComputeSeparately", no_argument, 0, 'G' }, { "largeEndSize", required_argument, 0, 'I' }, {"ingroupCoverageFile", required_argument, 0, 'J'}, {"minimumSizeToRescue", required_argument, 0, 'K'}, {"minimumCoverageToRescue", required_argument, 0, 'M'}, { "minimumNumberOfSpecies", required_argument, 0, 'N' }, { 0, 0, 0, 0 } }; int option_index = 0; int key = getopt_long(argc, argv, "a:b:hi:j:kl:o:p:q:r:t:u:wy:A:B:D:E:FGI:J:K:L:M:N:", long_options, &option_index); if (key == -1) { break; } switch (key) { case 'a': logLevelString = stString_copy(optarg); st_setLogLevelFromString(logLevelString); break; case 'b': cactusDiskDatabaseString = stString_copy(optarg); break; case 'h': usage(); return 0; case 'i': i = sscanf(optarg, "%" PRIi64 "", &spanningTrees); (void) i; assert(i == 1); assert(spanningTrees >= 0); break; case 'j': i = sscanf(optarg, "%" PRIi64 "", &maximumLength); assert(i == 1); assert(maximumLength >= 0); break; case 'k': useBanding = !useBanding; break; case 'l': i = sscanf(optarg, "%f", &pairwiseAlignmentBandingParameters->gapGamma); assert(i == 1); assert(pairwiseAlignmentBandingParameters->gapGamma >= 0.0); break; case 'L': i = sscanf(optarg, "%f", &matchGamma); assert(i == 1); assert(matchGamma >= 0.0); break; case 'o': i = sscanf(optarg, "%" PRIi64 "", &k); assert(i == 1); assert(k >= 0); pairwiseAlignmentBandingParameters->splitMatrixBiggerThanThis = (int64_t) k * k; break; case 'p': i = sscanf(optarg, "%" PRIi64 "", &k); assert(i == 1); assert(k >= 0); pairwiseAlignmentBandingParameters->anchorMatrixBiggerThanThis = (int64_t) k * k; break; case 'q': i = sscanf(optarg, "%" PRIi64 "", &k); assert(i == 1); assert(k >= 0); pairwiseAlignmentBandingParameters->repeatMaskMatrixBiggerThanThis = (int64_t) k * k; break; case 'r': i = sscanf(optarg, "%" PRIi64 "", &pairwiseAlignmentBandingParameters->diagonalExpansion); assert(i == 1); assert(pairwiseAlignmentBandingParameters->diagonalExpansion >= 0); assert(pairwiseAlignmentBandingParameters->diagonalExpansion % 2 == 0); break; case 't': i = sscanf(optarg, "%" PRIi64 "", &pairwiseAlignmentBandingParameters->constraintDiagonalTrim); assert(i == 1); assert(pairwiseAlignmentBandingParameters->constraintDiagonalTrim >= 0); break; case 'u': i = sscanf(optarg, "%" PRIi64 "", &minimumDegree); assert(i == 1); break; case 'w': pairwiseAlignmentBandingParameters->alignAmbiguityCharacters = 1; break; case 'y': pruneOutStubAlignments = 1; break; case 'A': i = sscanf(optarg, "%" PRIi64 "", &minimumIngroupDegree); assert(i == 1); break; case 'B': i = sscanf(optarg, "%" PRIi64 "", &minimumOutgroupDegree); assert(i == 1); break; case 'D': listOfEndAlignmentFiles = stString_split(optarg); break; case 'E': endAlignmentsToPrecomputeOutputFile = stString_copy(optarg); break; case 'F': useProgressiveMerging = 1; break; case 'G': calculateWhichEndsToComputeSeparately = 1; break; case 'I': i = sscanf(optarg, "%" PRIi64 "", &largeEndSize); assert(i == 1); break; case 'J': ingroupCoverageFilePath = stString_copy(optarg); break; case 'K': i = sscanf(optarg, "%" PRIi64, &minimumSizeToRescue); assert(i == 1); break; case 'M': i = sscanf(optarg, "%lf", &minimumCoverageToRescue); assert(i == 1); break; case 'N': i = sscanf(optarg, "%" PRIi64, &minimumNumberOfSpecies); if (i != 1) { st_errAbort("Error parsing minimumNumberOfSpecies parameter"); } break; default: usage(); return 1; } } st_setLogLevelFromString(logLevelString); /* * Load the flowerdisk */ stKVDatabaseConf *kvDatabaseConf = stKVDatabaseConf_constructFromString(cactusDiskDatabaseString); CactusDisk *cactusDisk = cactusDisk_construct(kvDatabaseConf, 0); //We precache the sequences st_logInfo("Set up the flower disk\n"); /* * Load the hmm */ StateMachine *sM = stateMachine5_construct(fiveState); /* * For each flower. */ if (calculateWhichEndsToComputeSeparately) { stList *flowers = flowerWriter_parseFlowersFromStdin(cactusDisk); if (stList_length(flowers) != 1) { st_errAbort("We are breaking up a flower's end alignments for precomputation but we have %" PRIi64 " flowers.\n", stList_length(flowers)); } stSortedSet *endsToAlignSeparately = getEndsToAlignSeparately(stList_get(flowers, 0), maximumLength, largeEndSize); assert(stSortedSet_size(endsToAlignSeparately) != 1); stSortedSetIterator *it = stSortedSet_getIterator(endsToAlignSeparately); End *end; while ((end = stSortedSet_getNext(it)) != NULL) { fprintf(stdout, "%s\t%" PRIi64 "\t%" PRIi64 "\n", cactusMisc_nameToStringStatic(end_getName(end)), end_getInstanceNumber(end), getTotalAdjacencyLength(end)); } return 0; //avoid cleanup costs stSortedSet_destructIterator(it); stSortedSet_destruct(endsToAlignSeparately); } else if (endAlignmentsToPrecomputeOutputFile != NULL) { /* * In this case we will align a set of end and save the alignments in a file. */ stList *names = flowerWriter_parseNames(stdin); Flower *flower = cactusDisk_getFlower(cactusDisk, *((Name *)stList_get(names, 0))); FILE *fileHandle = fopen(endAlignmentsToPrecomputeOutputFile, "w"); for(int64_t i=1; i<stList_length(names); i++) { End *end = flower_getEnd(flower, *((Name *)stList_get(names, i))); if (end == NULL) { st_errAbort("The end %" PRIi64 " was not found in the flower\n", *((Name *)stList_get(names, i))); } stSortedSet *endAlignment = makeEndAlignment(sM, end, spanningTrees, maximumLength, useProgressiveMerging, matchGamma, pairwiseAlignmentBandingParameters); writeEndAlignmentToDisk(end, endAlignment, fileHandle); stSortedSet_destruct(endAlignment); } fclose(fileHandle); return 0; //avoid cleanup costs stList_destruct(names); st_logInfo("Finished precomputing end alignments\n"); } else { /* * Compute complete flower alignments, possibly loading some precomputed alignments. */ bedRegion *bedRegions = NULL; size_t numBeds = 0; if (ingroupCoverageFilePath != NULL) { // Pre-load the mmap for the coverage file. FILE *coverageFile = fopen(ingroupCoverageFilePath, "rb"); if (coverageFile == NULL) { st_errnoAbort("Opening coverage file %s failed", ingroupCoverageFilePath); } fseek(coverageFile, 0, SEEK_END); int64_t coverageFileLen = ftell(coverageFile); assert(coverageFileLen >= 0); assert(coverageFileLen % sizeof(bedRegion) == 0); if (coverageFileLen == 0) { // mmap doesn't like length-0 mappings, for obvious // reasons. Pretend that the coverage file doesn't // exist in this case, since it contains no data. ingroupCoverageFilePath = NULL; } else { // Establish a memory mapping for the file. bedRegions = mmap(NULL, coverageFileLen, PROT_READ, MAP_SHARED, fileno(coverageFile), 0); if (bedRegions == MAP_FAILED) { st_errnoAbort("Failure mapping coverage file"); } numBeds = coverageFileLen / sizeof(bedRegion); } fclose(coverageFile); } stList *flowers = flowerWriter_parseFlowersFromStdin(cactusDisk); if (listOfEndAlignmentFiles != NULL && stList_length(flowers) != 1) { st_errAbort("We have precomputed alignments but %" PRIi64 " flowers to align.\n", stList_length(flowers)); } cactusDisk_preCacheStrings(cactusDisk, flowers); for (j = 0; j < stList_length(flowers); j++) { flower = stList_get(flowers, j); st_logInfo("Processing a flower\n"); stSortedSet *alignedPairs = makeFlowerAlignment3(sM, flower, listOfEndAlignmentFiles, spanningTrees, maximumLength, useProgressiveMerging, matchGamma, pairwiseAlignmentBandingParameters, pruneOutStubAlignments); st_logInfo("Created the alignment: %" PRIi64 " pairs\n", stSortedSet_size(alignedPairs)); stPinchIterator *pinchIterator = stPinchIterator_constructFromAlignedPairs(alignedPairs, getNextAlignedPairAlignment); /* * Run the cactus caf functions to build cactus. */ stPinchThreadSet *threadSet = stCaf_setup(flower); stCaf_anneal(threadSet, pinchIterator, NULL); if (minimumDegree < 2) { stCaf_makeDegreeOneBlocks(threadSet); } if (minimumIngroupDegree > 0 || minimumOutgroupDegree > 0 || minimumDegree > 1) { stCaf_melt(flower, threadSet, blockFilterFn, 0, 0, 0, INT64_MAX); } if (ingroupCoverageFilePath != NULL) { // Rescue any sequence that is covered by outgroups // but currently unaligned into single-degree blocks. stPinchThreadSetIt pinchIt = stPinchThreadSet_getIt(threadSet); stPinchThread *thread; while ((thread = stPinchThreadSetIt_getNext(&pinchIt)) != NULL) { Cap *cap = flower_getCap(flower, stPinchThread_getName(thread)); assert(cap != NULL); Sequence *sequence = cap_getSequence(cap); assert(sequence != NULL); rescueCoveredRegions(thread, bedRegions, numBeds, sequence_getName(sequence), minimumSizeToRescue, minimumCoverageToRescue); } stCaf_joinTrivialBoundaries(threadSet); } stCaf_finish(flower, threadSet, chainLengthForBigFlower, longChain, INT64_MAX, INT64_MAX); //Flower now destroyed. stPinchThreadSet_destruct(threadSet); st_logInfo("Ran the cactus core script.\n"); /* * Cleanup */ //Clean up the sorted set after cleaning up the iterator stPinchIterator_destruct(pinchIterator); stSortedSet_destruct(alignedPairs); st_logInfo("Finished filling in the alignments for the flower\n"); } stList_destruct(flowers); //st_errAbort("Done\n"); /* * Write and close the cactusdisk. */ cactusDisk_write(cactusDisk); return 0; //Exit without clean up is quicker, enable cleanup when doing memory leak detection. if (bedRegions != NULL) { // Clean up our mapping. munmap(bedRegions, numBeds * sizeof(bedRegion)); } } /////////////////////////////////////////////////////////////////////////// // Cleanup /////////////////////////////////////////////////////////////////////////// stateMachine_destruct(sM); cactusDisk_destruct(cactusDisk); stKVDatabaseConf_destruct(kvDatabaseConf); //destructCactusCoreInputParameters(cCIP); free(cactusDiskDatabaseString); if (listOfEndAlignmentFiles != NULL) { stList_destruct(listOfEndAlignmentFiles); } if (logLevelString != NULL) { free(logLevelString); } st_logInfo("Finished with the flower disk for this flower.\n"); //while(1); return 0; }