static void snapSoftToCloseHard(struct rbTree *vertexTree, struct rbTree *edgeTree, int maxSnapSize,
int maxUncheckedSnapSize, struct nibTwoCache *seqCache, char *chromName)
/* Snap hard vertices to nearby soft vertices of same type. */
{
struct lm *lm = lmInit(0);
addWaysInAndOut(vertexTree, edgeTree, lm);
struct dlList *vList = sortedListFromTree(vertexTree);
struct dlNode *node;
int snapCount = 0;
for (node = vList->head; !dlEnd(node); node = node->next)
{
if (snapVertex(node, maxSnapSize, maxUncheckedSnapSize, seqCache, chromName))
{
rbTreeRemove(vertexTree, node->val);
++snapCount;
}
}
/* Clean up ways in and out since have removed some nodes. */
for (node = vList->head; !dlEnd(node); node = node->next)
{
struct vertex *v = node->val;
v->waysIn = v->waysOut = NULL;
}
if (snapCount > 0)
{
verbose(3, "Snapped %d close edges, now have %d vertices\n", snapCount, vertexTree->n);
updateForwardedEdges(edgeTree);
}
dlListFree(&vList);
lmCleanup(&lm);
}
void listJobs()
/* Report jobs running and recently finished. */
{
struct job *job;
struct dlNode *node;
pmClear(&pmIn);
pmPrintf(&pmIn, "%d running", dlCount(jobsRunning));
if (!pmSend(&pmIn, mainRudp))
return;
for (node = jobsRunning->head; !dlEnd(node); node=node->next)
{
job = node->val;
pmClear(&pmIn);
pmPrintf(&pmIn, "%s", job->startMessage);
if (!pmSend(&pmIn, mainRudp))
return;
}
pmClear(&pmIn);
pmPrintf(&pmIn, "%d recent", dlCount(jobsFinished));
if (!pmSend(&pmIn, mainRudp))
return;
for (node = jobsFinished->head; !dlEnd(node); node=node->next)
{
job = node->val;
pmClear(&pmIn);
pmPrintf(&pmIn, "%s", job->startMessage);
if (!pmSend(&pmIn, mainRudp))
return;
pmClear(&pmIn);
pmPrintf(&pmIn, "%s", job->doneMessage);
if (!pmSend(&pmIn, mainRudp))
return;
}
}
void optBranch(struct dlList *iList)
/* Optimize branch instructions. */
{
struct hash *labelHash = hashLabels(iList);
struct dlNode *node, *next;
for (node = iList->head; !dlEnd(node); node = next)
{
struct isx *isx = node->val;
next = node->next;
switch (isx->opType)
{
case poBeq:
case poBne:
case poBlt:
case poBle:
case poBgt:
case poBge:
case poBz:
case poBnz:
{
if (!dlEnd(next))
{
struct isx *nextIsx = next->val;
if (nextIsx->opType == poJump)
{
struct dlNode *nn = next->next;
if (!anyRealBefore(nn, isx->dest))
{
// uglyf("Consolidating cond/jump into !cond\n");
isx->opType = invOp(isx->opType);
isx->dest = nextIsx->dest;
dlRemove(next);
next = nn;
}
}
}
isx->dest = makeDirect(isx, labelHash);
break;
}
case poJump:
{
isx->dest = makeDirect(isx, labelHash);
removeNonJumps(node);
break;
}
}
}
hashFree(&labelHash);
}
static boolean snapVertex(struct dlNode *oldNode, int maxSnapSize, int maxUncheckedSnapSize,
struct nibTwoCache *seqCache, char *chromName)
/* Snap vertex to nearby previous hard vertex. */
{
/* Figure out hard type we want to snap to, and return if not
* soft to begin with. */
struct vertex *vOld = oldNode->val;
int oldPos = vOld->position;
enum ggVertexType currentType = vOld->type;
/* If no seqCache, then set up things so not bothering to check it. */
if (seqCache == NULL)
maxUncheckedSnapSize = maxSnapSize;
/* Search backwards */
if (currentType == ggSoftEnd)
{
struct dlNode *node;
for (node = oldNode->prev; !dlStart(node); node = node->prev)
{
struct vertex *v = node->val;
int newPos = v->position;
int dif = oldPos - newPos;
if (dif > maxSnapSize)
break;
if (v->type == ggHardEnd)
{
if (checkSnapOk(vOld, v, FALSE, dif, maxUncheckedSnapSize, seqCache, chromName))
{
vOld->movedTo = v;
verbose(3, "snapping vertex %d to %d\n", oldPos, newPos);
return TRUE;
}
}
}
}
/* Search forward */
else if (currentType == ggSoftStart)
{
struct dlNode *node;
for (node = oldNode->next; !dlEnd(node); node = node->next)
{
struct vertex *v = node->val;
int newPos = v->position;
int dif = newPos - oldPos;
if (dif > maxSnapSize)
break;
if (v->type == ggHardStart)
{
if (checkSnapOk(vOld, v, TRUE, dif, maxUncheckedSnapSize, seqCache, chromName))
{
vOld->movedTo = v;
return TRUE;
}
}
}
}
return FALSE;
}
static struct isxAddress *makeDirect(struct isx *isx, struct hash *labelHash)
/* Isx is a conditional or unconditional jump instruction.
* If it's target is an unconditional jump, then eliminate
* the middle man or men, and jump straight to final destination */
{
struct isxAddress *dest = isx->dest;
char *name = dest->name;
struct dlNode *node = hashMustFindVal(labelHash, name);
for (; !dlEnd(node); node = node->next)
{
struct isx *isx = node->val;
switch (isx->opType)
{
case poLabel:
case poLoopStart:
case poLoopEnd:
case poCondCase:
case poCondStart:
case poCondEnd:
break;
case poJump:
// uglyf("Short circuiting %s\n", name);
return makeDirect(node->val, labelHash);
default:
return dest;
}
}
return dest;
}
struct dlNode *dlValInList(struct dlList *list, void *val)
/* Return node on list if any that has associated val. */
{
struct dlNode *node;
for (node = list->head; !dlEnd(node); node = node->next)
if (node->val == val)
return node;
return NULL;
}
FILE *openFromCache(struct dlList *cache, struct seqFilePos *sfp)
/* Return open file handle via cache. The simple logic here
* depends on not more than N files being returned at once. */
{
static int maxCacheSize=16;
int cacheSize = 0;
struct dlNode *node;
struct cachedFile *cf;
int size;
/* First loop through trying to find it in cache, counting
* cache size as we go. */
for (node = cache->head; !dlEnd(node); node = node->next)
{
++cacheSize;
cf = node->val;
if (sameString(sfp->file, cf->name))
{
dlRemove(node);
dlAddHead(cache, node);
return cf->f;
}
}
/* If cache has reached max size free least recently used. */
if (cacheSize >= maxCacheSize)
{
node = dlPopTail(cache);
cf = node->val;
carefulClose(&cf->f);
freeMem(cf->name);
freeMem(cf);
freeMem(node);
}
/* Cache new file. */
AllocVar(cf);
cf->name = cloneString(sfp->file);
if (sfp->isTwoBit)
{
cf->f = (FILE *)twoBitOpen(sfp->file);
}
else if (sfp->isNib)
{
nibOpenVerify(sfp->file, &cf->f, &size);
if (cf->f == NULL)
errAbort("can't open nibfile %s\n",sfp->file);
sfp->pos = size;
}
else
cf->f = mustOpen(sfp->file, "rb");
dlAddValHead(cache, cf);
return cf->f;
}
void addChainToTree(struct wordTree *wt, struct dlList *chain)
/* Add chain of words to tree. */
{
struct dlNode *node;
wt->useCount += 1;
for (node = chain->head; !dlEnd(node); node = node->next)
{
struct wordInfo *info = node->val;
verbose(2, " adding %s\n", info->word);
wt = wordTreeAddFollowing(wt, info);
}
}
void addToTrie(struct trie *trie, struct dlList *ll)
/* Add chain of letters to trie. */
{
struct dlNode *node;
struct trie *next;
trie->useCount += 1;
for (node = ll->head; !dlEnd(node); node = node->next)
{
UBYTE *s = node->val;
UBYTE c = *s;
if (trie->branchCount)
{
if (trie->singleFollowing)
{
next = trie->singleFollowing;
if (next->letter != c)
{
/* Move from single to multiple followers. */
trie->singleFollowing = NULL;
AllocArray(trie->multipleFollowing, 256);
trie->multipleFollowing[next->letter] = next;
/* Allocate and initialize new follower. */
AllocVar(next);
next->letter = c;
trie->multipleFollowing[c] = next;
trie->branchCount += 1;
}
}
else
{
next = trie->multipleFollowing[c];
if (next == NULL)
{
AllocVar(next);
next->letter = c;
trie->multipleFollowing[c] = next;
trie->branchCount += 1;
}
}
}
else
{
AllocVar(next);
next->letter = c;
trie->singleFollowing = next;
trie->branchCount = 1;
}
next->useCount += 1;
trie = next;
}
}
struct job *findJobOnList(struct dlList *list, int jobId)
/* Return job if it's on list, otherwise NULL */
{
struct dlNode *node;
struct job *job;
for (node = list->head; !dlEnd(node); node = node->next)
{
job = node->val;
if (job->jobId == jobId)
return job;
}
return NULL;
}
void addChainToTree(struct wordTree *wt, struct dlList *chain,
struct lm *lm, struct rbTreeNode **stack)
/* Add chain of words to tree. */
{
struct dlNode *node;
wt->useCount += 1;
for (node = chain->head; !dlEnd(node); node = node->next)
{
char *word = node->val;
verbose(2, " %s\n", word);
wt = wordTreeAddFollowing(wt, word, lm, stack);
}
}
void doResurrect(char *line, struct sockaddr_in *hubIp)
/* Send back I'm alive message */
{
struct paraMessage pm;
struct dlNode *node;
int jobsReported = 0;
pmInit(&pm, ntohl(hubIp->sin_addr.s_addr), paraHubPort);
pmPrintf(&pm, "alive %s", hostName);
for (node = jobsRunning->head; !dlEnd(node); node = node->next)
{
struct job *job = node->val;
pmPrintf(&pm, " %d", job->jobId);
++jobsReported;
}
for (node = jobsFinished->head; !dlEnd(node); node = node->next)
{
struct job *job = node->val;
if (jobsReported >= maxProcs)
break;
pmPrintf(&pm, " %d", job->jobId);
++jobsReported;
}
pmSend(&pm, mainRudp);
}
struct wordTree *predictFromWordTree(struct wordTree *wt, struct dlNode *recent)
/* Predict next word given tree and recently used word list. Will use all words in
* recent list if can, but if there is not data in tree, will back off, and use
* progressively less previous words until ultimately it just picks a random
* word. */
{
struct dlNode *node;
for (node = recent; !dlEnd(node); node = node->next)
{
struct wordTree *result = predictNextFromAllPredecessors(wt, node);
if (result != NULL)
return result;
}
return NULL;
}
static void wordTreeGenerateFile(struct wordStore *store, int maxSize, struct wordTree *firstWord,
int maxOutputWords, char *fileName)
/* Create file containing words base on tree probabilities. The wordTreeGenerateList does
* most of work. */
{
struct dlList *ll = wordTreeGenerateList(store, maxSize, firstWord, maxOutputWords);
FILE *f = mustOpen(fileName, "w");
struct dlNode *node;
for (node = ll->head; !dlEnd(node); node = node->next)
{
struct wordInfo *info = node->val;
fprintf(f, "%s\n", info->word);
}
carefulClose(&f);
dlListFree(&ll);
}
char *dlListFragWords(struct dlNode *head)
/* Return string containing all words in list pointed to by head. */
{
struct dyString *dy = dyStringNew(0);
dyStringAppendC(dy, '{');
struct dlNode *node;
for (node = head; !dlEnd(node); node = node->next)
{
if (node != head)
dyStringAppendC(dy, ' ');
struct wordInfo *info = node->val;
dyStringAppend(dy, info->word);
}
dyStringAppendC(dy, '}');
return dyStringCannibalize(&dy);
}
void doStatus()
/* Report status. */
{
pmClear(&pmIn);
pmPrintf(&pmIn, "%d of %d CPUs busy", busyProcs, maxProcs);
if (busyProcs > 0)
{
struct dlNode *node;
pmPrintf(&pmIn, ". Jobs:");
for (node = jobsRunning->head; !dlEnd(node); node = node->next)
{
struct job *job = node->val;
pmPrintf(&pmIn, " %d", job->jobId);
}
}
pmSend(&pmIn, mainRudp);
}
char *predictNext(struct wordTree *wt, struct dlList *recent)
/* Predict next word given list of recent words and wordTree. */
{
struct dlNode *node;
for (node = recent->head; !dlEnd(node); node = node->next)
{
char *word = node->val;
struct wordTree key;
key.word = word;
wt = rbTreeFind(wt->following, &key);
if (wt == NULL)
errAbort("%s isn't a follower of %s\n", word, wt->word);
}
char *result = NULL;
if (wt->following != NULL)
result = pickRandomWord(wt->following);
return result;
}
struct wordTree *predictNextFromAllPredecessors(struct wordTree *wt, struct dlNode *list)
/* Predict next word given tree and recently used word list. If tree doesn't
* have statistics for what comes next given the words in list, then it returns
* NULL. */
{
verbose(2, " predictNextFromAllPredecessors(%s, %s)\n", wordTreeString(wt), dlListFragWords(list));
struct dlNode *node;
for (node = list; !dlEnd(node); node = node->next)
{
struct wordInfo *info = node->val;
wt = wordTreeFindInList(wt->children, info);
verbose(2, " wordTreeFindInList(%s) = %p %s\n", info->word, wt, wordTreeString(wt));
if (wt == NULL || wt->children == NULL)
break;
}
struct wordTree *result = NULL;
if (wt != NULL && wt->children != NULL)
result = pickRandom(wt->children);
return result;
}
struct cachedSeqFile *openNibFromCache(struct dlList *cache, char *dirName, char *seqName)
/* Return open file handle via cache. */
{
static int maxCacheSize=32;
int cacheSize = 0;
struct dlNode *node;
struct cachedSeqFile *cn;
char fileName[512];
/* First loop through trying to find it in cache, counting
* cache size as we go. */
for (node = cache->head; !dlEnd(node); node = node->next)
{
++cacheSize;
cn = node->val;
if (sameString(seqName, cn->name))
{
dlRemove(node);
dlAddHead(cache, node);
return cn;
}
}
/* If cache has reached max size free least recently used. */
if (cacheSize >= maxCacheSize)
{
node = dlPopTail(cache);
cn = node->val;
cachedSeqFileFree(&cn);
freeMem(node);
}
/* Cache new file. */
AllocVar(cn);
cn->name = cloneString(seqName);
snprintf(fileName, sizeof(fileName), "%s/%s.nib", dirName, seqName);
cn->fileName = cloneString(fileName);
nibOpenVerify(fileName, &cn->f, &cn->size);
dlAddValHead(cache, cn);
return cn;
}
FILE *openFromCache(struct dlList *cache, char *fileName)
/* Return open file handle via cache. The simple logic here
* depends on not more than N files being returned at once. */
{
static int maxCacheSize=32;
int cacheSize = 0;
struct dlNode *node;
struct cachedFile *cf;
/* First loop through trying to find it in cache, counting
* cache size as we go. */
for (node = cache->head; !dlEnd(node); node = node->next)
{
++cacheSize;
cf = node->val;
if (sameString(fileName, cf->name))
{
dlRemove(node);
dlAddHead(cache, node);
return cf->f;
}
}
/* If cache has reached max size free least recently used. */
if (cacheSize >= maxCacheSize)
{
node = dlPopTail(cache);
cf = node->val;
carefulClose(&cf->f);
freeMem(cf->name);
freeMem(cf);
freeMem(node);
}
/* Cache new file. */
AllocVar(cf);
cf->name = cloneString(fileName);
cf->f = mustOpen(fileName, "rb");
dlAddValHead(cache, cf);
return cf->f;
}
static struct hash *hashLabels(struct dlList *iList)
/* Return hash of all labels. */
{
struct dlNode *node;
struct hash *hash = hashNew(0);
for (node = iList->head; !dlEnd(node); node = node->next)
{
struct isx *isx = node->val;
switch (isx->opType)
{
case poLabel:
case poLoopStart:
case poLoopEnd:
case poCondCase:
case poCondEnd:
hashAdd(hash, isx->dest->name, node);
break;
}
}
return hash;
}
struct wordTree *predictFromPreviousTypes(struct wordStore *store, struct dlList *past)
/* Predict next based on general pattern of monomer order. */
{
int maxToLookBack = 3;
{ /* Debugging block */
struct dlNode *node = nodesFromTail(past, maxToLookBack);
verbose(3, "predictFromPreviousTypes(");
for (; !dlEnd(node); node = node->next)
{
struct wordInfo *info = node->val;
verbose(3, "%s, ", info->word);
}
verbose(3, ")\n");
}
int pastDepth;
struct dlNode *node;
for (pastDepth = 1, node = past->tail; pastDepth <= maxToLookBack; ++pastDepth)
{
if (dlStart(node))
{
verbose(2, "predictFromPreviousTypes with empty past\n");
return NULL;
}
struct wordInfo *info = node->val;
struct wordType *prevType = hashFindVal(store->typeHash, info->word);
if (prevType != NULL)
{
struct wordType *curType = advanceTypeWithWrap(store->typeList, prevType, pastDepth);
struct wordInfo *curInfo = pickRandomFromType(curType);
struct wordTree *curTree = wordTreeFindInList(store->markovChains->children, curInfo);
verbose(3, "predictFromPreviousType pastDepth %d, curInfo %s, curTree %p %s\n",
pastDepth, curInfo->word, curTree, curTree->info->word);
return curTree;
}
node = node->prev;
}
verbose(2, "predictFromPreviousTypes past all unknown types\n");
return NULL;
}
static boolean anyRealBefore(struct dlNode *node, struct isxAddress *label)
/* Return TRUE if get any real instruction before label */
{
for ( ; !dlEnd(node); node = node->next)
{
struct isx *isx = node->val;
switch (isx->opType)
{
case poLabel:
case poLoopStart:
case poLoopEnd:
case poCondCase:
case poCondStart:
case poCondEnd:
if (isx->dest == label)
return FALSE;
break;
default:
return TRUE;
}
}
return TRUE;
}
