void jobDone(char *line) /* Handle job-done message - forward it to managing host. */ { char *managingHost = nextWord(&line); char *jobIdString = nextWord(&line); // clearZombies(); if (jobIdString != NULL && line != NULL && line[0] != 0) { /* Remove job from list running list and put on recently finished list. */ struct job *job = findRunningJob(atoi(jobIdString)); if (job != NULL) { int status, err; err = waitpid(job->pid, &status, 0); if (err == -1 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { logDebug("paraNode sheparding %s pid %d status %d err %d errno %d", jobIdString, job->pid, status, err, errno); } job->doneMessage = cloneString(line); dlRemove(job->node); if (dlCount(jobsFinished) >= 4*maxProcs) { struct dlNode *node = dlPopTail(jobsFinished); struct job *oldJob = node->val; jobFree(&oldJob); } dlAddHead(jobsFinished, job->node); --busyProcs; } tellManagerJobIsDone(managingHost, jobIdString, line); } }
struct entity *mergeEntities(struct entity *entityList) /* Merge two entities. */ { struct entity *root = entityList, *ent; struct dlNode *node; if (root == NULL || root->next == NULL) return root; for (ent = root->next; ent != NULL; ent = ent->next) { node = ent->node; if (node != NULL) { dlRemove(node); ent->node = NULL; } root->exonList = slCat(root->exonList, ent->exonList); root->intronList = slCat(root->intronList, ent->intronList); root->cdaRefList = slCat(root->cdaRefList, ent->cdaRefList); if (root->start > ent->start) root->start = ent->start; if (root->end < ent->end) root->end = ent->end; } slSort(&root->exonList, cmpExons); root->exonList = weedDupeExons(root->exonList); slSort(&root->intronList, cmpIntrons); root->intronList = weedDupeIntrons(root->intronList); return root; }
static void memTrackerFree(void *vpt) /* Check cookies and free. */ { struct dlNode *node = vpt; node -= 1; dlRemove(node); memTracker->parent->free(node); }
struct dlNode *dlPopHead(struct dlList *list) /* Remove first node from list and return it. */ { struct dlNode *node = list->head; if (node->next == NULL) return NULL; dlRemove(node); return node; }
struct dlNode *dlPopTail(struct dlList *list) /* Remove last node from list and return it. */ { struct dlNode *node = list->tail; if (node->prev == NULL) return NULL; dlRemove(node); return node; }
void dlDelete(struct dlNode **nodePtr) /* Removes a node from list and frees it. */ { struct dlNode *node = *nodePtr; if (node != NULL) { dlRemove(node); freeMem(node); } }
FILE *openFromCache(struct dlList *cache, struct seqFilePos *sfp) /* Return open file handle via cache. The simple logic here * depends on not more than N files being returned at once. */ { static int maxCacheSize=16; int cacheSize = 0; struct dlNode *node; struct cachedFile *cf; int size; /* First loop through trying to find it in cache, counting * cache size as we go. */ for (node = cache->head; !dlEnd(node); node = node->next) { ++cacheSize; cf = node->val; if (sameString(sfp->file, cf->name)) { dlRemove(node); dlAddHead(cache, node); return cf->f; } } /* If cache has reached max size free least recently used. */ if (cacheSize >= maxCacheSize) { node = dlPopTail(cache); cf = node->val; carefulClose(&cf->f); freeMem(cf->name); freeMem(cf); freeMem(node); } /* Cache new file. */ AllocVar(cf); cf->name = cloneString(sfp->file); if (sfp->isTwoBit) { cf->f = (FILE *)twoBitOpen(sfp->file); } else if (sfp->isNib) { nibOpenVerify(sfp->file, &cf->f, &size); if (cf->f == NULL) errAbort("can't open nibfile %s\n",sfp->file); sfp->pos = size; } else cf->f = mustOpen(sfp->file, "rb"); dlAddValHead(cache, cf); return cf->f; }
void separateEntities(struct dlList *easyList, struct dlList *hardList) /* Separate out hard (overlapping) entities onto hard list. */ { struct dlNode *node, *next, *listNode; int sepCount = 0; dlSort(easyList, cmpEntities); for (node = easyList->head; node->next != NULL; node = next) { struct entity *ent = node->val; int eStart = ent->start, eEnd = ent->end; next = node->next; for (listNode = easyList->head; listNode->next != NULL; listNode = listNode->next) { struct entity *listEnt = listNode->val; int lStart = listEnt->start, lEnd = listEnt->end; int start = max(eStart, lStart); int end = min(eEnd, lEnd); int overlap = end - start; if (listEnt != ent && overlap > 0) { int eRefCount = slCount(ent->cdaRefList); int lRefCount = slCount(listEnt->cdaRefList); /* Move the one with less cDNA to the hard list. */ if (eRefCount > lRefCount) { dlRemove(listNode); dlAddTail(hardList, listNode); if (listNode == next) next = node; } else { dlRemove(node); dlAddTail(hardList, node); } ++sepCount; break; } } } }
boolean matchMaker(struct dlList *finList, struct dlList *toDoList, struct dlNode **retSourceNode, struct dlNode **retDestNode) /* Find a copy to make from the finished list to the toDo list. */ { struct dlNode *sourceNode, *destNode = NULL; struct machine *source, *dest; bool gotIt = FALSE; for (sourceNode = finList->head; sourceNode->next != NULL; sourceNode = sourceNode->next) { source = sourceNode->val; for (destNode = toDoList->head; destNode->next != NULL; destNode = destNode->next) { dest = destNode->val; if (dest->netSwitch == source->netSwitch) { gotIt = TRUE; break; } if (crossSwitchCount < crossSwitchMax) { ++crossSwitchCount; gotIt = TRUE; break; } } if (gotIt) break; } if (!gotIt) return FALSE; dlRemove(sourceNode); dlRemove(destNode); *retSourceNode = sourceNode; *retDestNode = destNode; //uglyf("Matched (%s %s) and (%s %s) cross %d\n", // source->name, source->netSwitch->name, // dest->name, dest->netSwitch->name, crossSwitchCount); return TRUE; }
static void removeNonJumps(struct dlNode *node) /* Node is a jump. Remove it if it just is a jump to the * next real (non-label) instruction */ { struct dlNode *n; struct isx *isx = node->val; struct isxAddress *dest = isx->dest; if (!anyRealBefore(node->next, dest)) { // uglyf("removing non-jump to %s\n", dest->name); dlRemove(node); } }
void optBranch(struct dlList *iList) /* Optimize branch instructions. */ { struct hash *labelHash = hashLabels(iList); struct dlNode *node, *next; for (node = iList->head; !dlEnd(node); node = next) { struct isx *isx = node->val; next = node->next; switch (isx->opType) { case poBeq: case poBne: case poBlt: case poBle: case poBgt: case poBge: case poBz: case poBnz: { if (!dlEnd(next)) { struct isx *nextIsx = next->val; if (nextIsx->opType == poJump) { struct dlNode *nn = next->next; if (!anyRealBefore(nn, isx->dest)) { // uglyf("Consolidating cond/jump into !cond\n"); isx->opType = invOp(isx->opType); isx->dest = nextIsx->dest; dlRemove(next); next = nn; } } } isx->dest = makeDirect(isx, labelHash); break; } case poJump: { isx->dest = makeDirect(isx, labelHash); removeNonJumps(node); break; } } } hashFree(&labelHash); }
void dgDisconnect(struct diGraph *dg, struct dgNode *a, struct dgNode *b) /* Disconnect nodes a and b. */ { struct dlNode *edgeInList; struct dgEdge *edge; dgRemoveFromConList(&a->nextList, b, NULL); edgeInList = dgRemoveFromConList(&b->prevList, a, NULL); if (edgeInList != NULL) { edge = edgeInList->val; dlRemove(edgeInList); freeMem(edgeInList); freeMem(edge); } }
static void *memTrackerRealloc(void *vpt, size_t size) /* Resize a memory block from memTrackerAlloc. */ { if (vpt == NULL) return memTrackerAlloc(size); else { struct dlNode *node = ((struct dlNode *)vpt)-1; size += sizeof(*node); dlRemove(node); node = memTracker->parent->realloc(node, size); if (node == NULL) return node; dlAddTail(memTracker->list, node); return (void*)(node+1); } }
struct dgNodeRef *dgConstrainedPriorityOrder(struct diGraph *dg) /* Return traversal of graph in priority order subject to * constraint that all parents must be output before * their children regardless of node priority. * Graph must be cycle free. */ { struct dlList *sortedList = newDlList(); struct dgNode *graphNode; struct dlNode *listNode; struct dgNodeRef *refList = NULL, *ref; if (dgHasCycles(dg)) errAbort("Call to dgConstrainedPriorityOrder on graph with cycles."); /* Make up list sorted by priority. */ for (graphNode = dg->nodeList; graphNode != NULL; graphNode = graphNode->next) { dlAddValTail(sortedList, graphNode); graphNode->visited = FALSE; } dlSort(sortedList, cmpPriority); /* Loop taking first member of list with no untraversed parents. */ while (!dlEmpty(sortedList)) { for (listNode = sortedList->head; listNode->next != NULL; listNode = listNode->next) { graphNode = listNode->val; if (dgParentsAllVisited(graphNode)) { dlRemove(listNode); freeMem(listNode); AllocVar(ref); ref->node = graphNode; slAddHead(&refList, ref); graphNode->visited = TRUE; break; } } } freeDlList(&sortedList); slReverse(&refList); return refList; }
static void mergeClusters(struct cluster *a, struct cluster *b) /* Merge cluster b into cluster a, and free remnants of b. */ { struct box *box; /* Merging with yourself is always problematic. */ if (a == b) return; /* Point all boxes in b to a. */ for (box = b->boxList; box != NULL; box = box->next) box->cluster = a; /* Add b's boxList to end of a's. */ a->boxList = slCat(a->boxList, b->boxList); /* Remove b from master cluster list. */ dlRemove(b->node); }
static void carefulFree(void *vpt) /* Check cookies and free. */ { struct carefulMemBlock *cmb = ((struct carefulMemBlock *)vpt)-1; size_t size = cmb->size; char *pEndCookie; carefulAlloced -= size; pEndCookie = (((char *)(cmb+1)) + size); if (cmb->startCookie != cmbStartCookie) errAbort("Bad start cookie %x freeing %llx\n", cmb->startCookie, ptrToLL(vpt)); if (memcmp(pEndCookie, cmbEndCookie, sizeof(cmbEndCookie)) != 0) errAbort("Bad end cookie %x%x%x%x freeing %llx\n", pEndCookie[0], pEndCookie[1], pEndCookie[2], pEndCookie[3], ptrToLL(vpt)); dlRemove((struct dlNode *)cmb); carefulParent->free(cmb); }
struct cachedSeqFile *openNibFromCache(struct dlList *cache, char *dirName, char *seqName) /* Return open file handle via cache. */ { static int maxCacheSize=32; int cacheSize = 0; struct dlNode *node; struct cachedSeqFile *cn; char fileName[512]; /* First loop through trying to find it in cache, counting * cache size as we go. */ for (node = cache->head; !dlEnd(node); node = node->next) { ++cacheSize; cn = node->val; if (sameString(seqName, cn->name)) { dlRemove(node); dlAddHead(cache, node); return cn; } } /* If cache has reached max size free least recently used. */ if (cacheSize >= maxCacheSize) { node = dlPopTail(cache); cn = node->val; cachedSeqFileFree(&cn); freeMem(node); } /* Cache new file. */ AllocVar(cn); cn->name = cloneString(seqName); snprintf(fileName, sizeof(fileName), "%s/%s.nib", dirName, seqName); cn->fileName = cloneString(fileName); nibOpenVerify(fileName, &cn->f, &cn->size); dlAddValHead(cache, cn); return cn; }
FILE *openFromCache(struct dlList *cache, char *fileName) /* Return open file handle via cache. The simple logic here * depends on not more than N files being returned at once. */ { static int maxCacheSize=32; int cacheSize = 0; struct dlNode *node; struct cachedFile *cf; /* First loop through trying to find it in cache, counting * cache size as we go. */ for (node = cache->head; !dlEnd(node); node = node->next) { ++cacheSize; cf = node->val; if (sameString(fileName, cf->name)) { dlRemove(node); dlAddHead(cache, node); return cf->f; } } /* If cache has reached max size free least recently used. */ if (cacheSize >= maxCacheSize) { node = dlPopTail(cache); cf = node->val; carefulClose(&cf->f); freeMem(cf->name); freeMem(cf); freeMem(node); } /* Cache new file. */ AllocVar(cf); cf->name = cloneString(fileName); cf->f = mustOpen(fileName, "rb"); dlAddValHead(cache, cf); return cf->f; }
void doKill(char *line) /* Kill a specific job. */ { char *jobIdString = nextWord(&line); int jobId = atoi(jobIdString); if (jobId != 0) { struct job *job = findRunningJob(jobId); if (job != NULL) { kill(job->pid, SIGTERM); dlRemove(job->node); jobFree(&job); --busyProcs; } } else { warn("Nothing to kill\n"); } }
void figureSynteny(char *inName, FILE *out) /* Figure out synteny stats - how much in a row aligns. */ { FILE *in; char line[512]; int lineCount = 0; char *words[64]; int wordCount; char *firstWord; char *queryName = ""; struct contig *contig; struct dlNode *contigNode; struct dlList *contigList = newDlList(); int lineState = 0; /* Keeps track of groups of four lines. */ struct slName *queryNameList = NULL; int maxSymCount = 64*1024; char *qSymBuf = needMem(maxSymCount+1); char *tSymBuf = needMem(maxSymCount+1); char *hSymBuf = needMem(maxSymCount+1); int symCount = 0; int qSymLen, tSymLen, hSymLen; int bestSegScore; int lastQoff = -1; int i; in = mustOpen(inName, "r"); while (fgets(line, sizeof(line), in)) { ++lineCount; if ((lineCount%100000) == 0) { printf("Processing line %d of %s\n", lineCount, inName); } if (++lineState == 5) lineState = 0; wordCount = chopLine(line, words); if (wordCount <= 0) continue; firstWord = words[0]; if (sameString(firstWord, "Aligning")) { char *queryString; char *targetString; char queryStrand, targetStrand; char *parts[8]; int partCount; /* Do some preliminary checking of this line. */ if (wordCount < 6) errAbort("Short line %d of %s", lineCount, inName); queryString = words[1]; queryStrand = words[2][0]; targetString = words[4]; targetStrand = words[5][0]; /* Extract the name of the query sequence. If it's new, * then write out contigs on previous query we've accumulated * so far and start a new list. */ partCount = chopString(queryString, ":-", parts, ArraySize(parts)); if (!sameString(parts[0], queryName)) { /* Allocate new name and keep track of it. */ struct slName *newName = newSlName(parts[0]); slAddHead(&queryNameList, newName); /* Set last Segment for this clone to impossible val. */ bestSegScore = -0x3fffffff; lastQoff = -1; /* Write out old contigs and empty out contig list. */ syntenyOnClone(queryName, contigList, out); freeContigList(&contigList); contigList = newDlList(); queryName = newName->name; } /* Make up a new contig, and fill it in with the data we * have so far about query. */ AllocVar(contig); contig->query = queryName; contig->qOffset = atoi(parts[1]); contig->qEndOffset = atoi(parts[2]); contig->qStrand = queryStrand; if (lastQoff != contig->qOffset) { lastQoff = contig->qOffset; bestSegScore = -0x3fffffff; } /* Parse target string and fill in contig with it's info. */ chopString(targetString, ":-", parts, ArraySize(parts)); contig->target = cloneString(parts[0]); contig->tOffset = atoi(parts[1]); contig->tEndOffset = atoi(parts[2]); contig->tStrand = targetStrand; /* We don't know start and end yet - set them to values * that will get easily replace by max/min. */ contig->qStart = contig->tStart = 0x3fffffff; lineState = -1; symCount = 0; } else if (sameString(firstWord, "best")) { if (wordCount < 3) errAbort("Short line %d of %s", lineCount, inName); contig->score = atoi(words[2]); if (contig->score > bestSegScore && contig->score >= minScore) { struct dlNode *tailNode; struct contig *tailContig; bestSegScore = contig->score; contig->isComplete = TRUE; contig->qSym = cloneStringZ(qSymBuf, symCount); contig->tSym = cloneStringZ(tSymBuf, symCount); contig->hSym = cloneStringZ(hSymBuf, symCount); contig->symCount = symCount; contig->qEnd = contig->qStart + countNonGap(qSymBuf, symCount); contig->tEnd = contig->tStart + countNonGap(tSymBuf, symCount); tailNode = contigList->tail; if (tailNode != NULL) { tailContig = tailNode->val; if (tailContig->qOffset == contig->qOffset) { freeContig(&tailContig); dlRemove(tailNode); freeMem(tailNode); } } contigNode = dlAddValTail(contigList, contig); } } else if (wordCount > 1 && isdigit(firstWord[0]) || firstWord[0] == '-') { int start, end; char *sym = words[1]; int symLen = strlen(sym); char firstChar = firstWord[0]; if (lineState != 0 && lineState != 2) errAbort("Bummer - phasing mismatch on lineState line %d of %s!\n", lineCount, inName); assert(lineState == 0 || lineState == 2); start = atoi(firstWord); end = start + symLen; if (symCount + symLen > maxSymCount) { errAbort("Single contig too long line %d of %s, can only handle up to %d symbols\n", lineCount, inName, maxSymCount); } if (lineState == 0) /* query symbols */ { qSymLen = symLen; if (isdigit(firstChar)) { start += contig->qOffset; end += contig->qOffset; contig->qStart = min(contig->qStart, start); contig->qEnd = max(contig->qEnd, end); } memcpy(qSymBuf+symCount, sym, symLen); } else /* target symbols */ { tSymLen = symLen; if (tSymLen != qSymLen) { errAbort("Target symbol size not same as query line %d of %s", lineCount, inName); } if (isdigit(firstChar)) { start += contig->tOffset; end += contig->tOffset; contig->tStart = min(contig->tStart, start); } memcpy(tSymBuf+symCount, sym, symLen); } } else if (firstWord[0] == '(') { lineState = -1; } else { assert(lineState == 1 || lineState == 3); if (lineState == 3) /* Hidden symbols. */ { char *sym = firstWord; int symLen = strlen(sym); hSymLen = symLen; if (hSymLen != qSymLen) { errAbort("Hidden symbol size not same as query line %d of %s", lineCount, inName); } memcpy(hSymBuf+symCount, sym, symLen); symCount += symLen; } } } syntenyOnClone(queryName, contigList, out); freeContigList(&contigList); fclose(in); slFreeList(&queryNameList); freeMem(qSymBuf); freeMem(tSymBuf); freeMem(hSymBuf); fprintf(out, "CloneSegCounts[] = \n"); for (i=0; i<ArraySize(cloneSegCounts); ++i) fprintf(out, "%d %d\n", i, cloneSegCounts[i]); fprintf(out, "\n"); fprintf(out, "kCounts[] = \n"); for (i=0; i<ArraySize(kCounts); ++i) fprintf(out, "%d %d\n", i, kCounts[i]); segAverageSize = round((double)segTotalSize/segCount); fprintf(out, "\n%d Segments, average size %d\n", segCount, segAverageSize); }
void ccCp(char *source, char *dest, char *hostList) /* Copy source to dest on all files in hostList. */ { time_t startTime = time(NULL); time_t curTime, lastTime = 0; struct machine *machineList = NULL; struct netSwitch *nsList; struct machine *m, *m2; struct dlList *toDoList = newDlList(); /* We haven't done these. */ struct dlList *finishedList = newDlList(); /* All done here. */ struct dlList *sourceList = newDlList(); /* These are sources for copies. */ struct dlList *workingList = newDlList(); /* These are copying data to themselves. */ struct dlList *errList = newDlList(); /* These are messed up 3x or more. */ bool firstOk = FALSE; struct dlNode *finNode, *node, *sourceNode, *destNode; struct dyString *cmd = newDyString(256); int machineCount; int machinesFinished = 0; char *thisHost = getenv("HOST"); off_t size; int goodMachines; double grandTotal; /* Get host and switch info. */ readHosts(hostList, &machineList, &nsList); machineCount = slCount(machineList); /* Make sure file exists.... */ if (!fileExists(source)) errAbort("%s doesn't exist\n", source); size = fileSize(source); printf("Copying %s (%lld bytes) to %d machines\n", source, (unsigned long long)size, machineCount); /* Add everything to the to-do list. */ for (m = machineList; m != NULL; m = m->next) { dlAddValTail(toDoList, m); } /* Loop through to-do list trying to do first copy. */ for (node = toDoList->head; node->next != NULL; node = node->next) { m = node->val; dyStringClear(cmd); m = node->val; if (sameString(thisHost, m->name)) { if (sameString(source, dest)) { /* Hey, this is too easy. */ firstOk = TRUE; ++machinesFinished; break; } else { dyStringPrintf(cmd, "cp %s %s", source, dest); } } else { dyStringPrintf(cmd, "rcp %s %s:%s", source, m->name, dest); } if (system(cmd->string) == 0) { dlRemove(node); dlAddTail(finishedList, node); firstOk = TRUE; ++machinesFinished; break; } else /* some error in rcp */ { warn("Problem with %s\n", cmd->string); m->errCount += 1; } } /* Loop around launching child processes to copy and * wait for them to finish. */ while (machinesFinished < machineCount) { int pid; int status; /* Start all possible copies. */ while (matchMaker(finishedList, toDoList, &sourceNode, &destNode)) { dlAddTail(sourceList, sourceNode); dlAddTail(workingList, destNode); m = destNode->val; m->sourceNode = sourceNode; startCopy(sourceNode->val, destNode->val, dest, thisHost, cmd); } curTime = time(NULL); if (curTime - lastTime >= 3) { printf("%d finished in %d seconds, %d in progress, %d to start, %d errors, %d total\n", dlCount(finishedList) + dlCount(sourceList), (int)(curTime - startTime), dlCount(workingList), dlCount(toDoList), dlCount(errList), machineCount); lastTime = curTime; } /* Wait for a child to finish. Figure out which machine it is. */ pid = wait(&status); finNode = NULL; for (node = workingList->head; node->next != NULL; node = node->next) { m = node->val; if (m->pid == pid) { finNode = node; break; } } if (finNode == NULL) { errAbort("Returned from wait on unknown child %d\n", pid); continue; } m = finNode->val; m->pid = 0; dlRemove(finNode); dlRemove(m->sourceNode); m2 = m->sourceNode->val; if (m->netSwitch != m2->netSwitch) --crossSwitchCount; dlAddTail(finishedList, m->sourceNode); if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { /* Good return - move self and source node to finished list. */ ++machinesFinished; dlAddTail(finishedList, finNode); } else { /* Bad return. Increment error count, and maybe move it to * error list. */ if (++m->errCount >= maxErrCount) { ++machinesFinished; dlAddTail(errList, finNode); fprintf(stderr, "Gave up on %s\n", m->name); } else { dlAddMiddle(toDoList, finNode); fprintf(stderr, "Retry %d on %s\n", m->errCount, m->name); } } } if (!dlEmpty(errList)) { fprintf(stderr, "errors in:"); for (node = errList->head; node->next != NULL; node = node->next) { m = node->val; fprintf(stderr, " %s", m->name); } fprintf(stderr, "\n"); } goodMachines = dlCount(finishedList); grandTotal = (double)goodMachines * (double)size; printf("Copied to %d of %d machines (grand total %e bytes) in %d seconds\n", goodMachines, machineCount, grandTotal, (int)(time(NULL) - startTime)); }
void dlRemoveTail(struct dlList *list) /* Remove tail from list. Node is not freed. */ { dlRemove(list->tail); }
void dlRemoveHead(struct dlList *list) /* Removes head from list. Node is not freed. */ { dlRemove(list->head); }