static void snapSoftToCloseHard(struct rbTree *vertexTree, struct rbTree *edgeTree, int maxSnapSize, int maxUncheckedSnapSize, struct nibTwoCache *seqCache, char *chromName) /* Snap hard vertices to nearby soft vertices of same type. */ { struct lm *lm = lmInit(0); addWaysInAndOut(vertexTree, edgeTree, lm); struct dlList *vList = sortedListFromTree(vertexTree); struct dlNode *node; int snapCount = 0; for (node = vList->head; !dlEnd(node); node = node->next) { if (snapVertex(node, maxSnapSize, maxUncheckedSnapSize, seqCache, chromName)) { rbTreeRemove(vertexTree, node->val); ++snapCount; } } /* Clean up ways in and out since have removed some nodes. */ for (node = vList->head; !dlEnd(node); node = node->next) { struct vertex *v = node->val; v->waysIn = v->waysOut = NULL; } if (snapCount > 0) { verbose(3, "Snapped %d close edges, now have %d vertices\n", snapCount, vertexTree->n); updateForwardedEdges(edgeTree); } dlListFree(&vList); lmCleanup(&lm); }
void listJobs() /* Report jobs running and recently finished. */ { struct job *job; struct dlNode *node; pmClear(&pmIn); pmPrintf(&pmIn, "%d running", dlCount(jobsRunning)); if (!pmSend(&pmIn, mainRudp)) return; for (node = jobsRunning->head; !dlEnd(node); node=node->next) { job = node->val; pmClear(&pmIn); pmPrintf(&pmIn, "%s", job->startMessage); if (!pmSend(&pmIn, mainRudp)) return; } pmClear(&pmIn); pmPrintf(&pmIn, "%d recent", dlCount(jobsFinished)); if (!pmSend(&pmIn, mainRudp)) return; for (node = jobsFinished->head; !dlEnd(node); node=node->next) { job = node->val; pmClear(&pmIn); pmPrintf(&pmIn, "%s", job->startMessage); if (!pmSend(&pmIn, mainRudp)) return; pmClear(&pmIn); pmPrintf(&pmIn, "%s", job->doneMessage); if (!pmSend(&pmIn, mainRudp)) return; } }
void optBranch(struct dlList *iList) /* Optimize branch instructions. */ { struct hash *labelHash = hashLabels(iList); struct dlNode *node, *next; for (node = iList->head; !dlEnd(node); node = next) { struct isx *isx = node->val; next = node->next; switch (isx->opType) { case poBeq: case poBne: case poBlt: case poBle: case poBgt: case poBge: case poBz: case poBnz: { if (!dlEnd(next)) { struct isx *nextIsx = next->val; if (nextIsx->opType == poJump) { struct dlNode *nn = next->next; if (!anyRealBefore(nn, isx->dest)) { // uglyf("Consolidating cond/jump into !cond\n"); isx->opType = invOp(isx->opType); isx->dest = nextIsx->dest; dlRemove(next); next = nn; } } } isx->dest = makeDirect(isx, labelHash); break; } case poJump: { isx->dest = makeDirect(isx, labelHash); removeNonJumps(node); break; } } } hashFree(&labelHash); }
static boolean snapVertex(struct dlNode *oldNode, int maxSnapSize, int maxUncheckedSnapSize, struct nibTwoCache *seqCache, char *chromName) /* Snap vertex to nearby previous hard vertex. */ { /* Figure out hard type we want to snap to, and return if not * soft to begin with. */ struct vertex *vOld = oldNode->val; int oldPos = vOld->position; enum ggVertexType currentType = vOld->type; /* If no seqCache, then set up things so not bothering to check it. */ if (seqCache == NULL) maxUncheckedSnapSize = maxSnapSize; /* Search backwards */ if (currentType == ggSoftEnd) { struct dlNode *node; for (node = oldNode->prev; !dlStart(node); node = node->prev) { struct vertex *v = node->val; int newPos = v->position; int dif = oldPos - newPos; if (dif > maxSnapSize) break; if (v->type == ggHardEnd) { if (checkSnapOk(vOld, v, FALSE, dif, maxUncheckedSnapSize, seqCache, chromName)) { vOld->movedTo = v; verbose(3, "snapping vertex %d to %d\n", oldPos, newPos); return TRUE; } } } } /* Search forward */ else if (currentType == ggSoftStart) { struct dlNode *node; for (node = oldNode->next; !dlEnd(node); node = node->next) { struct vertex *v = node->val; int newPos = v->position; int dif = newPos - oldPos; if (dif > maxSnapSize) break; if (v->type == ggHardStart) { if (checkSnapOk(vOld, v, TRUE, dif, maxUncheckedSnapSize, seqCache, chromName)) { vOld->movedTo = v; return TRUE; } } } } return FALSE; }
static struct isxAddress *makeDirect(struct isx *isx, struct hash *labelHash) /* Isx is a conditional or unconditional jump instruction. * If it's target is an unconditional jump, then eliminate * the middle man or men, and jump straight to final destination */ { struct isxAddress *dest = isx->dest; char *name = dest->name; struct dlNode *node = hashMustFindVal(labelHash, name); for (; !dlEnd(node); node = node->next) { struct isx *isx = node->val; switch (isx->opType) { case poLabel: case poLoopStart: case poLoopEnd: case poCondCase: case poCondStart: case poCondEnd: break; case poJump: // uglyf("Short circuiting %s\n", name); return makeDirect(node->val, labelHash); default: return dest; } } return dest; }
struct dlNode *dlValInList(struct dlList *list, void *val) /* Return node on list if any that has associated val. */ { struct dlNode *node; for (node = list->head; !dlEnd(node); node = node->next) if (node->val == val) return node; return NULL; }
FILE *openFromCache(struct dlList *cache, struct seqFilePos *sfp) /* Return open file handle via cache. The simple logic here * depends on not more than N files being returned at once. */ { static int maxCacheSize=16; int cacheSize = 0; struct dlNode *node; struct cachedFile *cf; int size; /* First loop through trying to find it in cache, counting * cache size as we go. */ for (node = cache->head; !dlEnd(node); node = node->next) { ++cacheSize; cf = node->val; if (sameString(sfp->file, cf->name)) { dlRemove(node); dlAddHead(cache, node); return cf->f; } } /* If cache has reached max size free least recently used. */ if (cacheSize >= maxCacheSize) { node = dlPopTail(cache); cf = node->val; carefulClose(&cf->f); freeMem(cf->name); freeMem(cf); freeMem(node); } /* Cache new file. */ AllocVar(cf); cf->name = cloneString(sfp->file); if (sfp->isTwoBit) { cf->f = (FILE *)twoBitOpen(sfp->file); } else if (sfp->isNib) { nibOpenVerify(sfp->file, &cf->f, &size); if (cf->f == NULL) errAbort("can't open nibfile %s\n",sfp->file); sfp->pos = size; } else cf->f = mustOpen(sfp->file, "rb"); dlAddValHead(cache, cf); return cf->f; }
void addChainToTree(struct wordTree *wt, struct dlList *chain) /* Add chain of words to tree. */ { struct dlNode *node; wt->useCount += 1; for (node = chain->head; !dlEnd(node); node = node->next) { struct wordInfo *info = node->val; verbose(2, " adding %s\n", info->word); wt = wordTreeAddFollowing(wt, info); } }
void addToTrie(struct trie *trie, struct dlList *ll) /* Add chain of letters to trie. */ { struct dlNode *node; struct trie *next; trie->useCount += 1; for (node = ll->head; !dlEnd(node); node = node->next) { UBYTE *s = node->val; UBYTE c = *s; if (trie->branchCount) { if (trie->singleFollowing) { next = trie->singleFollowing; if (next->letter != c) { /* Move from single to multiple followers. */ trie->singleFollowing = NULL; AllocArray(trie->multipleFollowing, 256); trie->multipleFollowing[next->letter] = next; /* Allocate and initialize new follower. */ AllocVar(next); next->letter = c; trie->multipleFollowing[c] = next; trie->branchCount += 1; } } else { next = trie->multipleFollowing[c]; if (next == NULL) { AllocVar(next); next->letter = c; trie->multipleFollowing[c] = next; trie->branchCount += 1; } } } else { AllocVar(next); next->letter = c; trie->singleFollowing = next; trie->branchCount = 1; } next->useCount += 1; trie = next; } }
struct job *findJobOnList(struct dlList *list, int jobId) /* Return job if it's on list, otherwise NULL */ { struct dlNode *node; struct job *job; for (node = list->head; !dlEnd(node); node = node->next) { job = node->val; if (job->jobId == jobId) return job; } return NULL; }
void addChainToTree(struct wordTree *wt, struct dlList *chain, struct lm *lm, struct rbTreeNode **stack) /* Add chain of words to tree. */ { struct dlNode *node; wt->useCount += 1; for (node = chain->head; !dlEnd(node); node = node->next) { char *word = node->val; verbose(2, " %s\n", word); wt = wordTreeAddFollowing(wt, word, lm, stack); } }
void doResurrect(char *line, struct sockaddr_in *hubIp) /* Send back I'm alive message */ { struct paraMessage pm; struct dlNode *node; int jobsReported = 0; pmInit(&pm, ntohl(hubIp->sin_addr.s_addr), paraHubPort); pmPrintf(&pm, "alive %s", hostName); for (node = jobsRunning->head; !dlEnd(node); node = node->next) { struct job *job = node->val; pmPrintf(&pm, " %d", job->jobId); ++jobsReported; } for (node = jobsFinished->head; !dlEnd(node); node = node->next) { struct job *job = node->val; if (jobsReported >= maxProcs) break; pmPrintf(&pm, " %d", job->jobId); ++jobsReported; } pmSend(&pm, mainRudp); }
struct wordTree *predictFromWordTree(struct wordTree *wt, struct dlNode *recent) /* Predict next word given tree and recently used word list. Will use all words in * recent list if can, but if there is not data in tree, will back off, and use * progressively less previous words until ultimately it just picks a random * word. */ { struct dlNode *node; for (node = recent; !dlEnd(node); node = node->next) { struct wordTree *result = predictNextFromAllPredecessors(wt, node); if (result != NULL) return result; } return NULL; }
static void wordTreeGenerateFile(struct wordStore *store, int maxSize, struct wordTree *firstWord, int maxOutputWords, char *fileName) /* Create file containing words base on tree probabilities. The wordTreeGenerateList does * most of work. */ { struct dlList *ll = wordTreeGenerateList(store, maxSize, firstWord, maxOutputWords); FILE *f = mustOpen(fileName, "w"); struct dlNode *node; for (node = ll->head; !dlEnd(node); node = node->next) { struct wordInfo *info = node->val; fprintf(f, "%s\n", info->word); } carefulClose(&f); dlListFree(&ll); }
char *dlListFragWords(struct dlNode *head) /* Return string containing all words in list pointed to by head. */ { struct dyString *dy = dyStringNew(0); dyStringAppendC(dy, '{'); struct dlNode *node; for (node = head; !dlEnd(node); node = node->next) { if (node != head) dyStringAppendC(dy, ' '); struct wordInfo *info = node->val; dyStringAppend(dy, info->word); } dyStringAppendC(dy, '}'); return dyStringCannibalize(&dy); }
void doStatus() /* Report status. */ { pmClear(&pmIn); pmPrintf(&pmIn, "%d of %d CPUs busy", busyProcs, maxProcs); if (busyProcs > 0) { struct dlNode *node; pmPrintf(&pmIn, ". Jobs:"); for (node = jobsRunning->head; !dlEnd(node); node = node->next) { struct job *job = node->val; pmPrintf(&pmIn, " %d", job->jobId); } } pmSend(&pmIn, mainRudp); }
char *predictNext(struct wordTree *wt, struct dlList *recent) /* Predict next word given list of recent words and wordTree. */ { struct dlNode *node; for (node = recent->head; !dlEnd(node); node = node->next) { char *word = node->val; struct wordTree key; key.word = word; wt = rbTreeFind(wt->following, &key); if (wt == NULL) errAbort("%s isn't a follower of %s\n", word, wt->word); } char *result = NULL; if (wt->following != NULL) result = pickRandomWord(wt->following); return result; }
struct wordTree *predictNextFromAllPredecessors(struct wordTree *wt, struct dlNode *list) /* Predict next word given tree and recently used word list. If tree doesn't * have statistics for what comes next given the words in list, then it returns * NULL. */ { verbose(2, " predictNextFromAllPredecessors(%s, %s)\n", wordTreeString(wt), dlListFragWords(list)); struct dlNode *node; for (node = list; !dlEnd(node); node = node->next) { struct wordInfo *info = node->val; wt = wordTreeFindInList(wt->children, info); verbose(2, " wordTreeFindInList(%s) = %p %s\n", info->word, wt, wordTreeString(wt)); if (wt == NULL || wt->children == NULL) break; } struct wordTree *result = NULL; if (wt != NULL && wt->children != NULL) result = pickRandom(wt->children); return result; }
struct cachedSeqFile *openNibFromCache(struct dlList *cache, char *dirName, char *seqName) /* Return open file handle via cache. */ { static int maxCacheSize=32; int cacheSize = 0; struct dlNode *node; struct cachedSeqFile *cn; char fileName[512]; /* First loop through trying to find it in cache, counting * cache size as we go. */ for (node = cache->head; !dlEnd(node); node = node->next) { ++cacheSize; cn = node->val; if (sameString(seqName, cn->name)) { dlRemove(node); dlAddHead(cache, node); return cn; } } /* If cache has reached max size free least recently used. */ if (cacheSize >= maxCacheSize) { node = dlPopTail(cache); cn = node->val; cachedSeqFileFree(&cn); freeMem(node); } /* Cache new file. */ AllocVar(cn); cn->name = cloneString(seqName); snprintf(fileName, sizeof(fileName), "%s/%s.nib", dirName, seqName); cn->fileName = cloneString(fileName); nibOpenVerify(fileName, &cn->f, &cn->size); dlAddValHead(cache, cn); return cn; }
FILE *openFromCache(struct dlList *cache, char *fileName) /* Return open file handle via cache. The simple logic here * depends on not more than N files being returned at once. */ { static int maxCacheSize=32; int cacheSize = 0; struct dlNode *node; struct cachedFile *cf; /* First loop through trying to find it in cache, counting * cache size as we go. */ for (node = cache->head; !dlEnd(node); node = node->next) { ++cacheSize; cf = node->val; if (sameString(fileName, cf->name)) { dlRemove(node); dlAddHead(cache, node); return cf->f; } } /* If cache has reached max size free least recently used. */ if (cacheSize >= maxCacheSize) { node = dlPopTail(cache); cf = node->val; carefulClose(&cf->f); freeMem(cf->name); freeMem(cf); freeMem(node); } /* Cache new file. */ AllocVar(cf); cf->name = cloneString(fileName); cf->f = mustOpen(fileName, "rb"); dlAddValHead(cache, cf); return cf->f; }
static struct hash *hashLabels(struct dlList *iList) /* Return hash of all labels. */ { struct dlNode *node; struct hash *hash = hashNew(0); for (node = iList->head; !dlEnd(node); node = node->next) { struct isx *isx = node->val; switch (isx->opType) { case poLabel: case poLoopStart: case poLoopEnd: case poCondCase: case poCondEnd: hashAdd(hash, isx->dest->name, node); break; } } return hash; }
struct wordTree *predictFromPreviousTypes(struct wordStore *store, struct dlList *past) /* Predict next based on general pattern of monomer order. */ { int maxToLookBack = 3; { /* Debugging block */ struct dlNode *node = nodesFromTail(past, maxToLookBack); verbose(3, "predictFromPreviousTypes("); for (; !dlEnd(node); node = node->next) { struct wordInfo *info = node->val; verbose(3, "%s, ", info->word); } verbose(3, ")\n"); } int pastDepth; struct dlNode *node; for (pastDepth = 1, node = past->tail; pastDepth <= maxToLookBack; ++pastDepth) { if (dlStart(node)) { verbose(2, "predictFromPreviousTypes with empty past\n"); return NULL; } struct wordInfo *info = node->val; struct wordType *prevType = hashFindVal(store->typeHash, info->word); if (prevType != NULL) { struct wordType *curType = advanceTypeWithWrap(store->typeList, prevType, pastDepth); struct wordInfo *curInfo = pickRandomFromType(curType); struct wordTree *curTree = wordTreeFindInList(store->markovChains->children, curInfo); verbose(3, "predictFromPreviousType pastDepth %d, curInfo %s, curTree %p %s\n", pastDepth, curInfo->word, curTree, curTree->info->word); return curTree; } node = node->prev; } verbose(2, "predictFromPreviousTypes past all unknown types\n"); return NULL; }
static boolean anyRealBefore(struct dlNode *node, struct isxAddress *label) /* Return TRUE if get any real instruction before label */ { for ( ; !dlEnd(node); node = node->next) { struct isx *isx = node->val; switch (isx->opType) { case poLabel: case poLoopStart: case poLoopEnd: case poCondCase: case poCondStart: case poCondEnd: if (isx->dest == label) return FALSE; break; default: return TRUE; } } return TRUE; }