void listJobs()
/* Report jobs running and recently finished. */
{
struct job *job;
struct dlNode *node;
pmClear(&pmIn);
pmPrintf(&pmIn, "%d running", dlCount(jobsRunning));
if (!pmSend(&pmIn, mainRudp))
return;
for (node = jobsRunning->head; !dlEnd(node); node=node->next)
{
job = node->val;
pmClear(&pmIn);
pmPrintf(&pmIn, "%s", job->startMessage);
if (!pmSend(&pmIn, mainRudp))
return;
}
pmClear(&pmIn);
pmPrintf(&pmIn, "%d recent", dlCount(jobsFinished));
if (!pmSend(&pmIn, mainRudp))
return;
for (node = jobsFinished->head; !dlEnd(node); node=node->next)
{
job = node->val;
pmClear(&pmIn);
pmPrintf(&pmIn, "%s", job->startMessage);
if (!pmSend(&pmIn, mainRudp))
return;
pmClear(&pmIn);
pmPrintf(&pmIn, "%s", job->doneMessage);
if (!pmSend(&pmIn, mainRudp))
return;
}
}
int main(int argc, char *argv[])
{
char *outDir;
struct cdaAli *cdaList;
struct clonePair *pairList;
struct dlList **goodEntities, **badEntities; /* Array of lists, one for each chromosome. */
int i;
if (argc != 2)
{
errAbort("genieCon - generates constraint GFF files for Genie from cDNA alignments\n"
"usage\n"
" genieCon outputDir\n"
"genieCon will create one file of form conXX.gff for each chromosome, where\n"
"the XX is replaced by the chromosome name.");
}
outDir = argv[1];
dnaUtilOpen();
anyChromNames(&chromNames, &chromCount);
goodEntities = needMem(chromCount * sizeof(goodEntities[0]));
badEntities = needMem(chromCount * sizeof(badEntities[0]));
for (i=0; i<chromCount; ++i)
{
goodEntities[i] = newDlList();
badEntities[i] = newDlList();
}
cdaList = readAllCda();
printf("Read in %d alignments\n", slCount(cdaList));
cdaCoalesceBlocks(cdaList);
printf("Coalesced blocks\n");
pairList = pairClones(cdaList);
printf("Before weeding genomic had %d clones\n", slCount(pairList));
pairList = weedGenomic(pairList);
printf("after weeding genomic had %d clones\n", slCount(pairList));
makeEntities(pairList, goodEntities);
for (i=0; i<chromCount; ++i)
{
printf("Made %d gene-like entities on chromosome %s\n",
dlCount(goodEntities[i]), chromNames[i]);
}
for (i=0; i<chromCount; ++i)
{
if (dlCount(goodEntities[i]) > 0)
{
separateEntities(goodEntities[i], badEntities[i]);
printf("%d good %d bad entities on chromosome %s\n",
dlCount(goodEntities[i]), dlCount(badEntities[i]), chromNames[i]);
saveEntities(goodEntities[i], outDir, "ez", chromNames[i]);
saveEntities(badEntities[i], outDir, "odd", chromNames[i]);
}
}
return 0;
}
void dlSort(struct dlList *list,
int (*compare )(const void *elem1, const void *elem2))
/* Sort a singly linked list with Qsort and a temporary array.
* The arguments to the compare function in real, non-void, life
* are pointers to pointers of the type that is in the val field of
* the nodes of the list. */
{
int len = dlCount(list);
if (len > 1)
{
/* Move val's onto an array, sort, and then put back into list. */
struct dlSorter *sorter = needLargeMem(len * sizeof(sorter[0])), *s;
struct dlNode *node;
int i;
for (i=0, node = list->head; i<len; ++i, node = node->next)
{
s = &sorter[i];
s->node = node;
}
compareFunc = compare;
qsort(sorter, len, sizeof(sorter[0]), dlNodeCmp);
dlListInit(list);
for (i=0; i<len; ++i)
dlAddTail(list, sorter[i].node);
freeMem(sorter);
}
}
void jobDone(char *line)
/* Handle job-done message - forward it to managing host. */
{
char *managingHost = nextWord(&line);
char *jobIdString = nextWord(&line);
// clearZombies();
if (jobIdString != NULL && line != NULL && line[0] != 0)
{
/* Remove job from list running list and put on recently finished list. */
struct job *job = findRunningJob(atoi(jobIdString));
if (job != NULL)
{
int status, err;
err = waitpid(job->pid, &status, 0);
if (err == -1 || !WIFEXITED(status) || WEXITSTATUS(status) != 0)
{
logDebug("paraNode sheparding %s pid %d status %d err %d errno %d",
jobIdString, job->pid, status, err, errno);
}
job->doneMessage = cloneString(line);
dlRemove(job->node);
if (dlCount(jobsFinished) >= 4*maxProcs)
{
struct dlNode *node = dlPopTail(jobsFinished);
struct job *oldJob = node->val;
jobFree(&oldJob);
}
dlAddHead(jobsFinished, job->node);
--busyProcs;
}
tellManagerJobIsDone(managingHost, jobIdString, line);
}
}
int synQueueSize(struct synQueue *sq)
/* Return number of messages currently on queue. */
{
int size;
pthreadMutexLock(&sq->mutex);
size = dlCount(sq->queue);
pthreadMutexUnlock(&sq->mutex);
return size;
}
void dlAddMiddle(struct dlList *list, struct dlNode *node)
/* Add node to approximate middle of list. */
{
int listSize = dlCount(list);
int midPos = listSize/2;
if (midPos <= 0)
dlAddTail(list, node);
else
{
struct dlNode *ln = list->head;
int i;
for (i=0; i<midPos; ++i)
ln = ln->next;
dlAddAfter(ln, node);
}
}
void syntenyOnClone(char *queryName, struct dlList *contigList, FILE *out)
/* Figure out synteny for one query sequence thats in contigList. */
{
struct dlNode *node;
struct contig *lastContig = NULL, *contig = NULL;
int tStart, tEnd, qStart, qEnd, qSize;
boolean inRange;
int maxSynSkip = 50000;
int contigCount = dlCount(contigList);
int cloneSegCount = 0;
int kIx;
if (contigCount < 1)
return;
for (node = contigList->head; node->next != NULL; node = node->next)
{
contig = node->val;
if (contig->score < minScore)
continue;
inRange = TRUE;
if (lastContig != NULL && contig != NULL)
{
inRange = (sameString(contig->target, lastContig->target) &&
contig->tOffset - maxSynSkip < lastContig->tOffset && contig->tOffset + maxSynSkip > lastContig->tOffset
&& contig->tEndOffset - maxSynSkip < lastContig->tEndOffset && contig->tEndOffset + maxSynSkip > lastContig->tEndOffset);
}
if (!inRange || contig == NULL)
{
tEnd = lastContig->tEndOffset;
qEnd = lastContig->qEnd;
qSize = qEnd - qStart;
fprintf(out, "%s:%d-%d (%d) %s:%d-%d %c\n", queryName, qStart, qEnd, qSize,
lastContig->target, tStart, tEnd, lastContig->qStrand);
++cloneSegCount;
kIx = qSize/1000;
if (kIx >= ArraySize(kCounts))
kIx = ArraySize(kCounts)-1;
++kCounts[kIx];
++segCount;
segTotalSize += qSize;
}
if (!inRange || lastContig == NULL)
{
tStart = contig->tOffset;
qStart = contig->qStart;
}
lastContig = contig;
}
if (lastContig != NULL)
{
tEnd = lastContig->tEndOffset;
qEnd = lastContig->qEnd;
qSize = qEnd - qStart;
fprintf(out, "%s:%d-%d (%d) %s:%d-%d %c\n", queryName, qStart, qEnd, qSize,
lastContig->target, tStart, tEnd, lastContig->qStrand);
++cloneSegCount;
kIx = qSize/1000;
if (kIx >= ArraySize(kCounts))
kIx = ArraySize(kCounts)-1;
++kCounts[kIx];
++segCount;
segTotalSize += qSize;
}
if (cloneSegCount > ArraySize(cloneSegCounts))
cloneSegCount = ArraySize(cloneSegCounts)-1;
++cloneSegCounts[cloneSegCount];
fprintf(out, "\n");
}
void ccCp(char *source, char *dest, char *hostList)
/* Copy source to dest on all files in hostList. */
{
time_t startTime = time(NULL);
time_t curTime, lastTime = 0;
struct machine *machineList = NULL;
struct netSwitch *nsList;
struct machine *m, *m2;
struct dlList *toDoList = newDlList(); /* We haven't done these. */
struct dlList *finishedList = newDlList(); /* All done here. */
struct dlList *sourceList = newDlList(); /* These are sources for copies. */
struct dlList *workingList = newDlList(); /* These are copying data to themselves. */
struct dlList *errList = newDlList(); /* These are messed up 3x or more. */
bool firstOk = FALSE;
struct dlNode *finNode, *node, *sourceNode, *destNode;
struct dyString *cmd = newDyString(256);
int machineCount;
int machinesFinished = 0;
char *thisHost = getenv("HOST");
off_t size;
int goodMachines;
double grandTotal;
/* Get host and switch info. */
readHosts(hostList, &machineList, &nsList);
machineCount = slCount(machineList);
/* Make sure file exists.... */
if (!fileExists(source))
errAbort("%s doesn't exist\n", source);
size = fileSize(source);
printf("Copying %s (%lld bytes) to %d machines\n", source, (unsigned long long)size, machineCount);
/* Add everything to the to-do list. */
for (m = machineList; m != NULL; m = m->next)
{
dlAddValTail(toDoList, m);
}
/* Loop through to-do list trying to do first copy. */
for (node = toDoList->head; node->next != NULL; node = node->next)
{
m = node->val;
dyStringClear(cmd);
m = node->val;
if (sameString(thisHost, m->name))
{
if (sameString(source, dest))
{
/* Hey, this is too easy. */
firstOk = TRUE;
++machinesFinished;
break;
}
else
{
dyStringPrintf(cmd, "cp %s %s", source, dest);
}
}
else
{
dyStringPrintf(cmd, "rcp %s %s:%s", source, m->name, dest);
}
if (system(cmd->string) == 0)
{
dlRemove(node);
dlAddTail(finishedList, node);
firstOk = TRUE;
++machinesFinished;
break;
}
else /* some error in rcp */
{
warn("Problem with %s\n", cmd->string);
m->errCount += 1;
}
}
/* Loop around launching child processes to copy and
* wait for them to finish. */
while (machinesFinished < machineCount)
{
int pid;
int status;
/* Start all possible copies. */
while (matchMaker(finishedList, toDoList, &sourceNode, &destNode))
{
dlAddTail(sourceList, sourceNode);
dlAddTail(workingList, destNode);
m = destNode->val;
m->sourceNode = sourceNode;
startCopy(sourceNode->val, destNode->val, dest, thisHost, cmd);
}
curTime = time(NULL);
if (curTime - lastTime >= 3)
{
printf("%d finished in %d seconds, %d in progress, %d to start, %d errors, %d total\n",
dlCount(finishedList) + dlCount(sourceList), (int)(curTime - startTime),
dlCount(workingList), dlCount(toDoList), dlCount(errList), machineCount);
lastTime = curTime;
}
/* Wait for a child to finish. Figure out which machine it is. */
pid = wait(&status);
finNode = NULL;
for (node = workingList->head; node->next != NULL; node = node->next)
{
m = node->val;
if (m->pid == pid)
{
finNode = node;
break;
}
}
if (finNode == NULL)
{
errAbort("Returned from wait on unknown child %d\n", pid);
continue;
}
m = finNode->val;
m->pid = 0;
dlRemove(finNode);
dlRemove(m->sourceNode);
m2 = m->sourceNode->val;
if (m->netSwitch != m2->netSwitch)
--crossSwitchCount;
dlAddTail(finishedList, m->sourceNode);
if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
{
/* Good return - move self and source node to finished list. */
++machinesFinished;
dlAddTail(finishedList, finNode);
}
else
{
/* Bad return. Increment error count, and maybe move it to
* error list. */
if (++m->errCount >= maxErrCount)
{
++machinesFinished;
dlAddTail(errList, finNode);
fprintf(stderr, "Gave up on %s\n", m->name);
}
else
{
dlAddMiddle(toDoList, finNode);
fprintf(stderr, "Retry %d on %s\n", m->errCount, m->name);
}
}
}
if (!dlEmpty(errList))
{
fprintf(stderr, "errors in:");
for (node = errList->head; node->next != NULL; node = node->next)
{
m = node->val;
fprintf(stderr, " %s", m->name);
}
fprintf(stderr, "\n");
}
goodMachines = dlCount(finishedList);
grandTotal = (double)goodMachines * (double)size;
printf("Copied to %d of %d machines (grand total %e bytes) in %d seconds\n",
goodMachines, machineCount, grandTotal, (int)(time(NULL) - startTime));
}
static struct bfGraph *bfGraphFromRangeGraph(struct diGraph *rangeGraph,
boolean (*findEdgeRange)(struct dgEdge *edge, int *retMin, int *retMax),
struct dgNode *a, struct dgNode *b, int abMin, int abMax)
/* Construct a directed graph with two edges for each
* edge of the range graph. Where the range graph
* has the following info:
* sourceNode destNode minDistance maxDistance
* -------------------------------------------
* A B 2 5
* The bfGraph would have:
* sourceNode destNode edgeVal meaning
* --------------------------------------------
* A B 5 D(A) - D(B) <= 5
* B A -2 D(B) - D(A) <= -2
* Furthermore the bfGraph introduces a new node,
* "zero", which has a zero-valued connection to
* all other nodes.
*
* This graph can then be processed via the Bellman-
* Ford algorithm (see p. 532 of Cormen,Leiserson and
* Rivest's Introduction to Algorithms, MIT Press 1990)
* to see if the bfGraph, and by extension the range
* graph, is consistent.
*/
{
struct bfGraph *bfGraph;
struct bfVertex *vertices, *freeVertex, *newSource, *newDest, *zeroVertex;
struct bfEdge *edges, *newEdge, *freeEdge;
struct dgNode *oldSource, *oldDest, *oldNode;
int oldEdgeCount = dlCount(rangeGraph->edgeList);
int oldVertexCount = 0;
/* Coopt topoOrder field to store unique ID for each
* node, starting at 1. (We'll reserve 0 for the
* node we insert - the Belman Ford "zero" node.)*/
for (oldNode = rangeGraph->nodeList; oldNode != NULL; oldNode = oldNode->next)
oldNode->topoOrder = ++oldVertexCount;
/* Allocate space for new graph. */
AllocVar(bfGraph);
bfGraph->vertexCount = oldVertexCount + 1;
bfGraph->vertices = freeVertex = AllocArray(vertices, bfGraph->vertexCount);
bfGraph->edgeCount = 2*oldEdgeCount + oldVertexCount;
if (a != NULL)
bfGraph->edgeCount += 2;
bfGraph->edges = freeEdge = AllocArray(edges, bfGraph->edgeCount);
/* Get zero vertex. */
zeroVertex = freeVertex++;
/* Scan through old graph and add vertices to new graph. */
for (oldSource = rangeGraph->nodeList; oldSource != NULL;
oldSource = oldSource->next)
{
struct dgConnection *dgConn;
int rangeMin, rangeMax;
/* Allocate new source vertex. */
newSource = freeVertex++;
/* Make connection from zero vertex to this one. */
newEdge = freeEdge++;
newEdge->distance = 0;
newEdge->in = zeroVertex;
newEdge->out = newSource;
slAddHead(&zeroVertex->waysOut, newEdge);
/* Loop through all ways out of source. */
for (dgConn = oldSource->nextList; dgConn != NULL; dgConn = dgConn->next)
{
/* Find destination vertex and size range of edge. */
if ((*findEdgeRange)(dgConn->edgeOnList->val, &rangeMin, &rangeMax))
{
oldDest = dgConn->node;
newDest = &vertices[oldDest->topoOrder];
/* Make two new edges corresponding to old edge. */
newEdge = freeEdge++;
newEdge->distance = rangeMax;
newEdge->in = newSource;
newEdge->out = newDest;
slAddHead(&newSource->waysOut, newEdge);
newEdge = freeEdge++;
newEdge->distance = -rangeMin;
newEdge->in = newDest;
newEdge->out = newSource;
slAddHead(&newDest->waysOut, newEdge);
}
}
}
/* Add additional pair of edges for extra range if applicable. */
if (a != NULL)
{
newSource = &vertices[a->topoOrder];
newDest = &vertices[b->topoOrder];
newEdge = freeEdge++;
newEdge->distance = abMax;
newEdge->in = newSource;
newEdge->out = newDest;
slAddHead(&newSource->waysOut, newEdge);
newEdge = freeEdge++;
newEdge->distance = -abMin;
newEdge->in = newDest;
newEdge->out = newSource;
slAddHead(&newDest->waysOut, newEdge);
}
bfGraph->edgeCount = freeEdge - bfGraph->edges;
return bfGraph;
}
int carefulCountBlocksAllocated()
/* How many memory items are allocated? */
{
return dlCount(cmbAllocedList);
}