void lavToAxt(char *inLav, char *tNibDir, char *qNibDir, char *outAxt)
/* lavToAxt - Convert blastz lav file to an axt file (which includes sequence). */
{
FILE *f = mustOpen(outAxt, "w");
struct dlList *qCache = newDlList(), *tCache = newDlList();
parseIntoAxt(inLav, f, tNibDir, tCache, qNibDir, qCache);
carefulClose(&f);
}
int main(int argc, char *argv[])
{
char *outDir;
struct cdaAli *cdaList;
struct clonePair *pairList;
struct dlList **goodEntities, **badEntities; /* Array of lists, one for each chromosome. */
int i;
if (argc != 2)
{
errAbort("genieCon - generates constraint GFF files for Genie from cDNA alignments\n"
"usage\n"
" genieCon outputDir\n"
"genieCon will create one file of form conXX.gff for each chromosome, where\n"
"the XX is replaced by the chromosome name.");
}
outDir = argv[1];
dnaUtilOpen();
anyChromNames(&chromNames, &chromCount);
goodEntities = needMem(chromCount * sizeof(goodEntities[0]));
badEntities = needMem(chromCount * sizeof(badEntities[0]));
for (i=0; i<chromCount; ++i)
{
goodEntities[i] = newDlList();
badEntities[i] = newDlList();
}
cdaList = readAllCda();
printf("Read in %d alignments\n", slCount(cdaList));
cdaCoalesceBlocks(cdaList);
printf("Coalesced blocks\n");
pairList = pairClones(cdaList);
printf("Before weeding genomic had %d clones\n", slCount(pairList));
pairList = weedGenomic(pairList);
printf("after weeding genomic had %d clones\n", slCount(pairList));
makeEntities(pairList, goodEntities);
for (i=0; i<chromCount; ++i)
{
printf("Made %d gene-like entities on chromosome %s\n",
dlCount(goodEntities[i]), chromNames[i]);
}
for (i=0; i<chromCount; ++i)
{
if (dlCount(goodEntities[i]) > 0)
{
separateEntities(goodEntities[i], badEntities[i]);
printf("%d good %d bad entities on chromosome %s\n",
dlCount(goodEntities[i]), dlCount(badEntities[i]), chromNames[i]);
saveEntities(goodEntities[i], outDir, "ez", chromNames[i]);
saveEntities(badEntities[i], outDir, "odd", chromNames[i]);
}
}
return 0;
}
void chainToPsl(char *inName, char *tSizeFile, char *qSizeFile, char *targetList, char *queryList, char *outName)
/* chainToPsl - Convert chain file to psl format. */
{
struct hash *tSizeHash = readSizes(tSizeFile);
struct hash *qSizeHash = readSizes(qSizeFile);
struct lineFile *lf = lineFileOpen(inName, TRUE);
FILE *f = mustOpen(outName, "w");
struct hash *fileHash = newHash(0); /* No value. */
struct hash *tHash = newHash(20); /* seqFilePos value. */
struct hash *qHash = newHash(20); /* seqFilePos value. */
struct dlList *fileCache = newDlList();
struct chain *chain;
int q,t;
verbose(1, "Scanning %s\n", targetList);
hashFileList(targetList, fileHash, tHash);
verbose(1, "Scanning %s\n", queryList);
hashFileList(queryList, fileHash, qHash);
verbose(1, "Converting %s\n", inName);
while ((chain = chainRead(lf)) != NULL)
{
//uglyf("chain %s %s \n",chain->tName,chain->qName);
q = findSize(qSizeHash, chain->qName);
t = findSize(tSizeHash, chain->tName);
aliStringToPsl(lf, chain->qName, chain->tName, chain->qSize, chain->tSize,
min(chain->tEnd-chain->tStart, chain->qEnd-chain->qStart), chain->qStart, chain->qEnd, chain->tStart, chain->tEnd,
chain->qStrand, f, chain, tHash, qHash, fileCache);
chainFree(&chain);
}
lineFileClose(&lf);
carefulClose(&f);
}
int main(int argc, char *argv[])
/* Process command line. */
{
optionHash(&argc, argv);
if (argc != 9)
usage();
fileCache = newDlList();
maxGap = optionInt("maxGap", maxGap);
verboseSetLogFile("stdout");
ss = axtScoreSchemeDefault();
verbose(1,"Reading alignments from %s\n",argv[3]);
mrnaHash = readPslToBinKeeper(argv[2], argv[3]);
twoBitFile = twoBitOpen(argv[5]);
//verbose(1,"Reading alignments from %s\n",argv[]);
//pseudoHash = readPslToBinKeeper(argv[3], argv[]);
//verbose(1,"Reading mRNA sequences from %s\n",argv[5]);
//mrnaList = faReadAllMixed(argv[5]);
//if (mrnaList == NULL)
//errAbort("could not open %s\n",argv[5]);
//faHash = newHash(0);
//for (el = mrnaList; el != NULL ; el = el->next)
//hashAdd(faHash, el->name, el);
verbose(1,"Reading chains from %s\n",argv[6]);
chainHash = readChainToBinKeeper(argv[2], argv[6]);
outFile = fopen(argv[8],"w");
verbose(1,"Scoring %s\n",argv[1]);
checkExp(argv[1], argv[7], argv[4]);
fclose(outFile);
freeDnaSeqList(&mrnaList);
return(0);
}
struct diGraph *dgNew()
/* Return a new directed graph object. */
{
struct diGraph *dg;
AllocVar(dg);
dg->nodeHash = newHash(0);
dg->edgeList = newDlList();
return dg;
}
void pslPretty(char *pslName, char *targetList, char *queryList,
char *prettyName, boolean axt, char *checkFileName)
/* pslPretty - Convert PSL to human readable output. */
{
struct hash *fileHash = newHash(0); /* No value. */
struct hash *tHash = newHash(20); /* seqFilePos value. */
struct hash *qHash = newHash(20); /* seqFilePos value. */
struct dlList *fileCache = newDlList();
struct lineFile *lf = pslFileOpen(pslName);
FILE *f = mustOpen(prettyName, "w");
FILE *checkFile = NULL;
struct psl *psl;
int dotMod = dot;
if (checkFileName != NULL)
checkFile = mustOpen(checkFileName, "w");
/* fprintf(stderr,"Scanning %s\n", targetList); */
hashFileList(targetList, fileHash, tHash);
/* fprintf(stderr,"Scanning %s\n", queryList); */
hashFileList(queryList, fileHash, qHash);
/* fprintf(stderr,"Converting %s\n", pslName); */
while ((psl = pslNext(lf)) != NULL)
{
if (dot > 0)
{
if (--dotMod <= 0)
{
fprintf(stderr,"."); /* stderr flushes itself */
dotMod = dot;
}
}
prettyOne(psl, qHash, tHash, fileCache, f, axt, checkFile);
pslFree(&psl);
}
if (dot > 0)
fprintf(stderr,"\n");
if (checkFile != NULL)
{
fprintf(checkFile,"missLargeStart: %d\n", total_missLargeStart);
fprintf(checkFile,"missSmallStart: %d\n", total_missSmallStart);
fprintf(checkFile,"missLargeEnd: %d\n", total_missLargeEnd);
fprintf(checkFile,"missSmallEnd: %d\n", total_missSmallEnd);
fprintf(checkFile,"missLargeMiddle: %d\n", total_missLargeMiddle);
fprintf(checkFile,"missSmallMiddle: %d\n", total_missSmallMiddle);
fprintf(checkFile,"weirdSplice: %d\n", total_weirdSplice);
fprintf(checkFile,"doubleGap: %d\n", total_doubleGap);
fprintf(checkFile,"jumpBack: %d\n", total_jumpBack);
fprintf(checkFile,"perfect: %d\n", total_rnaPerfect);
fprintf(checkFile,"total: %d\n", total_rnaCount);
}
lineFileClose(&lf);
carefulClose(&f);
carefulClose(&checkFile);
}
static void carefulMemInit(size_t maxToAlloc)
/* Initialize careful memory system */
{
carefulMaxToAlloc = maxToAlloc;
cmbAllocedList = newDlList();
carefulAlignSize = sizeof(double);
if (sizeof(void *) > carefulAlignSize)
carefulAlignSize = sizeof(void *);
if (sizeof(long) > carefulAlignSize)
carefulAlignSize = sizeof(long);
if (sizeof(off_t) > carefulAlignSize)
carefulAlignSize = sizeof(off_t);
if (sizeof(long long) > carefulAlignSize)
carefulAlignSize = sizeof(long long);
carefulAlignAdd = carefulAlignSize-1;
carefulAlignMask = ~carefulAlignAdd;
}
struct dgNodeRef *dgConstrainedPriorityOrder(struct diGraph *dg)
/* Return traversal of graph in priority order subject to
* constraint that all parents must be output before
* their children regardless of node priority.
* Graph must be cycle free. */
{
struct dlList *sortedList = newDlList();
struct dgNode *graphNode;
struct dlNode *listNode;
struct dgNodeRef *refList = NULL, *ref;
if (dgHasCycles(dg))
errAbort("Call to dgConstrainedPriorityOrder on graph with cycles.");
/* Make up list sorted by priority. */
for (graphNode = dg->nodeList; graphNode != NULL; graphNode = graphNode->next)
{
dlAddValTail(sortedList, graphNode);
graphNode->visited = FALSE;
}
dlSort(sortedList, cmpPriority);
/* Loop taking first member of list with no untraversed parents. */
while (!dlEmpty(sortedList))
{
for (listNode = sortedList->head; listNode->next != NULL; listNode = listNode->next)
{
graphNode = listNode->val;
if (dgParentsAllVisited(graphNode))
{
dlRemove(listNode);
freeMem(listNode);
AllocVar(ref);
ref->node = graphNode;
slAddHead(&refList, ref);
graphNode->visited = TRUE;
break;
}
}
}
freeDlList(&sortedList);
slReverse(&refList);
return refList;
}
void figureSynteny(char *inName, FILE *out)
/* Figure out synteny stats - how much in a row aligns. */
{
FILE *in;
char line[512];
int lineCount = 0;
char *words[64];
int wordCount;
char *firstWord;
char *queryName = "";
struct contig *contig;
struct dlNode *contigNode;
struct dlList *contigList = newDlList();
int lineState = 0; /* Keeps track of groups of four lines. */
struct slName *queryNameList = NULL;
int maxSymCount = 64*1024;
char *qSymBuf = needMem(maxSymCount+1);
char *tSymBuf = needMem(maxSymCount+1);
char *hSymBuf = needMem(maxSymCount+1);
int symCount = 0;
int qSymLen, tSymLen, hSymLen;
int bestSegScore;
int lastQoff = -1;
int i;
in = mustOpen(inName, "r");
while (fgets(line, sizeof(line), in))
{
++lineCount;
if ((lineCount%100000) == 0)
{
printf("Processing line %d of %s\n", lineCount, inName);
}
if (++lineState == 5)
lineState = 0;
wordCount = chopLine(line, words);
if (wordCount <= 0)
continue;
firstWord = words[0];
if (sameString(firstWord, "Aligning"))
{
char *queryString;
char *targetString;
char queryStrand, targetStrand;
char *parts[8];
int partCount;
/* Do some preliminary checking of this line. */
if (wordCount < 6)
errAbort("Short line %d of %s", lineCount, inName);
queryString = words[1];
queryStrand = words[2][0];
targetString = words[4];
targetStrand = words[5][0];
/* Extract the name of the query sequence. If it's new,
* then write out contigs on previous query we've accumulated
* so far and start a new list. */
partCount = chopString(queryString, ":-", parts, ArraySize(parts));
if (!sameString(parts[0], queryName))
{
/* Allocate new name and keep track of it. */
struct slName *newName = newSlName(parts[0]);
slAddHead(&queryNameList, newName);
/* Set last Segment for this clone to impossible val. */
bestSegScore = -0x3fffffff;
lastQoff = -1;
/* Write out old contigs and empty out contig list. */
syntenyOnClone(queryName, contigList, out);
freeContigList(&contigList);
contigList = newDlList();
queryName = newName->name;
}
/* Make up a new contig, and fill it in with the data we
* have so far about query. */
AllocVar(contig);
contig->query = queryName;
contig->qOffset = atoi(parts[1]);
contig->qEndOffset = atoi(parts[2]);
contig->qStrand = queryStrand;
if (lastQoff != contig->qOffset)
{
lastQoff = contig->qOffset;
bestSegScore = -0x3fffffff;
}
/* Parse target string and fill in contig with it's info. */
chopString(targetString, ":-", parts, ArraySize(parts));
contig->target = cloneString(parts[0]);
contig->tOffset = atoi(parts[1]);
contig->tEndOffset = atoi(parts[2]);
contig->tStrand = targetStrand;
/* We don't know start and end yet - set them to values
* that will get easily replace by max/min. */
contig->qStart = contig->tStart = 0x3fffffff;
lineState = -1;
symCount = 0;
}
else if (sameString(firstWord, "best"))
{
if (wordCount < 3)
errAbort("Short line %d of %s", lineCount, inName);
contig->score = atoi(words[2]);
if (contig->score > bestSegScore && contig->score >= minScore)
{
struct dlNode *tailNode;
struct contig *tailContig;
bestSegScore = contig->score;
contig->isComplete = TRUE;
contig->qSym = cloneStringZ(qSymBuf, symCount);
contig->tSym = cloneStringZ(tSymBuf, symCount);
contig->hSym = cloneStringZ(hSymBuf, symCount);
contig->symCount = symCount;
contig->qEnd = contig->qStart + countNonGap(qSymBuf, symCount);
contig->tEnd = contig->tStart + countNonGap(tSymBuf, symCount);
tailNode = contigList->tail;
if (tailNode != NULL)
{
tailContig = tailNode->val;
if (tailContig->qOffset == contig->qOffset)
{
freeContig(&tailContig);
dlRemove(tailNode);
freeMem(tailNode);
}
}
contigNode = dlAddValTail(contigList, contig);
}
}
else if (wordCount > 1 && isdigit(firstWord[0]) || firstWord[0] == '-')
{
int start, end;
char *sym = words[1];
int symLen = strlen(sym);
char firstChar = firstWord[0];
if (lineState != 0 && lineState != 2)
errAbort("Bummer - phasing mismatch on lineState line %d of %s!\n", lineCount, inName);
assert(lineState == 0 || lineState == 2);
start = atoi(firstWord);
end = start + symLen;
if (symCount + symLen > maxSymCount)
{
errAbort("Single contig too long line %d of %s, can only handle up to %d symbols\n",
lineCount, inName, maxSymCount);
}
if (lineState == 0) /* query symbols */
{
qSymLen = symLen;
if (isdigit(firstChar))
{
start += contig->qOffset;
end += contig->qOffset;
contig->qStart = min(contig->qStart, start);
contig->qEnd = max(contig->qEnd, end);
}
memcpy(qSymBuf+symCount, sym, symLen);
}
else /* target symbols */
{
tSymLen = symLen;
if (tSymLen != qSymLen)
{
errAbort("Target symbol size not same as query line %d of %s",
lineCount, inName);
}
if (isdigit(firstChar))
{
start += contig->tOffset;
end += contig->tOffset;
contig->tStart = min(contig->tStart, start);
}
memcpy(tSymBuf+symCount, sym, symLen);
}
}
else if (firstWord[0] == '(')
{
lineState = -1;
}
else
{
assert(lineState == 1 || lineState == 3);
if (lineState == 3) /* Hidden symbols. */
{
char *sym = firstWord;
int symLen = strlen(sym);
hSymLen = symLen;
if (hSymLen != qSymLen)
{
errAbort("Hidden symbol size not same as query line %d of %s",
lineCount, inName);
}
memcpy(hSymBuf+symCount, sym, symLen);
symCount += symLen;
}
}
}
syntenyOnClone(queryName, contigList, out);
freeContigList(&contigList);
fclose(in);
slFreeList(&queryNameList);
freeMem(qSymBuf);
freeMem(tSymBuf);
freeMem(hSymBuf);
fprintf(out, "CloneSegCounts[] = \n");
for (i=0; i<ArraySize(cloneSegCounts); ++i)
fprintf(out, "%d %d\n", i, cloneSegCounts[i]);
fprintf(out, "\n");
fprintf(out, "kCounts[] = \n");
for (i=0; i<ArraySize(kCounts); ++i)
fprintf(out, "%d %d\n", i, kCounts[i]);
segAverageSize = round((double)segTotalSize/segCount);
fprintf(out, "\n%d Segments, average size %d\n", segCount, segAverageSize);
}
struct dgNodeRef *dgFindPath(struct diGraph *dg, struct dgNode *a, struct dgNode *b)
/* Find shortest path from a to b. Return NULL if can't be found. */
{
struct dgNodeRef *refList = NULL, *ref;
struct dgConnection *con;
struct dgNode *node, *nNode;
struct dlList *fifo;
struct dlNode *ffNode;
struct dgNode endNode;
int fifoSize = 1;
/* Do some quick and easy tests first to return if have no way out
* of node A, or if B directly follows A. */
if (a->nextList == NULL)
return NULL;
if (a == b)
{
AllocVar(ref);
ref->node = a;
return ref;
}
if ((con = dgFindNodeInConList(a->nextList, b)) != NULL)
{
AllocVar(refList);
refList->node = a;
node = con->node;
AllocVar(ref);
ref->node = node;
slAddTail(&refList, ref);
return refList;
}
/* Set up for breadth first traversal. Will use a doubly linked
* list as a fifo. */
for (node = dg->nodeList; node != NULL; node = node->next)
node->tempEntry = NULL;
fifo = newDlList();
dlAddValTail(fifo, a);
a->tempEntry = &endNode;
while ((ffNode = dlPopHead(fifo)) != NULL)
{
--fifoSize;
node = ffNode->val;
freeMem(ffNode);
for (con = node->nextList; con != NULL; con = con->next)
{
nNode = con->node;
if (nNode->tempEntry == NULL)
{
nNode->tempEntry = node;
if (nNode == b)
{
while (nNode != &endNode && nNode != NULL)
{
AllocVar(ref);
ref->node = nNode;
slAddHead(&refList, ref);
nNode = nNode->tempEntry;
}
break;
}
else
{
dlAddValTail(fifo, nNode);
++fifoSize;
if (fifoSize > 100000)
errAbort("Internal error in dgFindPath");
}
}
}
}
freeDlList(&fifo);
return refList;
}
void paraNode()
/* paraNode - a net server. */
{
char *line;
char *command;
struct sockaddr_in sai;
/* We have to know who we are... */
hostName = getMachine();
initRandom();
getTicksToHundreths();
/* log init */
if (optionExists("log"))
logOpenFile("paraNode", optionVal("log", NULL));
else
logOpenSyslog("paraNode", optionVal("logFacility", NULL));
logSetMinPriority(optionVal("logMinPriority", "info"));
logInfo("starting paraNode on %s", hostName);
/* Make job lists. */
jobsRunning = newDlList();
jobsFinished = newDlList();
/* Set up socket and self to listen to it. */
ZeroVar(&sai);
sai.sin_family = AF_INET;
sai.sin_port = htons(paraNodePort);
sai.sin_addr.s_addr = INADDR_ANY;
mainRudp = rudpMustOpenBound(&sai);
mainRudp->maxRetries = 12;
/* Event loop. */
findNow();
for (;;)
{
/* Get next incoming message and optionally check to make
* sure that it's from a host we trust, and check signature
* on first bit of incoming data. */
if (pmReceive(&pmIn, mainRudp))
{
findNow();
if (hubName == NULL || ntohl(pmIn.ipAddress.sin_addr.s_addr) == hubIp
|| ntohl(pmIn.ipAddress.sin_addr.s_addr) == localIp)
{
/* Host and signature look ok, read a string and
* parse out first word as command. */
line = pmIn.data;
logDebug("message from %s: \"%s\"",
paraFormatIp(ntohl(pmIn.ipAddress.sin_addr.s_addr)),
line);
command = nextWord(&line);
if (command != NULL)
{
if (sameString("quit", command))
break;
else if (sameString("run", command))
doRun(line, &pmIn.ipAddress);
else if (sameString("jobDone", command))
jobDone(line);
else if (sameString("status", command))
doStatus();
else if (sameString("kill", command))
doKill(line);
else if (sameString("check", command))
doCheck(line, &pmIn.ipAddress);
else if (sameString("resurrect", command))
doResurrect(line, &pmIn.ipAddress);
else if (sameString("listJobs", command))
listJobs();
else if (sameString("fetch", command))
doFetch(line);
else
logWarn("invalid command: \"%s\"", command);
}
logDebug("done command");
}
else
{
logWarn("command from unauthorized host %s",
paraFormatIp(ntohl(pmIn.ipAddress.sin_addr.s_addr)));
}
}
}
rudpClose(&mainRudp);
}
void ccCp(char *source, char *dest, char *hostList)
/* Copy source to dest on all files in hostList. */
{
time_t startTime = time(NULL);
time_t curTime, lastTime = 0;
struct machine *machineList = NULL;
struct netSwitch *nsList;
struct machine *m, *m2;
struct dlList *toDoList = newDlList(); /* We haven't done these. */
struct dlList *finishedList = newDlList(); /* All done here. */
struct dlList *sourceList = newDlList(); /* These are sources for copies. */
struct dlList *workingList = newDlList(); /* These are copying data to themselves. */
struct dlList *errList = newDlList(); /* These are messed up 3x or more. */
bool firstOk = FALSE;
struct dlNode *finNode, *node, *sourceNode, *destNode;
struct dyString *cmd = newDyString(256);
int machineCount;
int machinesFinished = 0;
char *thisHost = getenv("HOST");
off_t size;
int goodMachines;
double grandTotal;
/* Get host and switch info. */
readHosts(hostList, &machineList, &nsList);
machineCount = slCount(machineList);
/* Make sure file exists.... */
if (!fileExists(source))
errAbort("%s doesn't exist\n", source);
size = fileSize(source);
printf("Copying %s (%lld bytes) to %d machines\n", source, (unsigned long long)size, machineCount);
/* Add everything to the to-do list. */
for (m = machineList; m != NULL; m = m->next)
{
dlAddValTail(toDoList, m);
}
/* Loop through to-do list trying to do first copy. */
for (node = toDoList->head; node->next != NULL; node = node->next)
{
m = node->val;
dyStringClear(cmd);
m = node->val;
if (sameString(thisHost, m->name))
{
if (sameString(source, dest))
{
/* Hey, this is too easy. */
firstOk = TRUE;
++machinesFinished;
break;
}
else
{
dyStringPrintf(cmd, "cp %s %s", source, dest);
}
}
else
{
dyStringPrintf(cmd, "rcp %s %s:%s", source, m->name, dest);
}
if (system(cmd->string) == 0)
{
dlRemove(node);
dlAddTail(finishedList, node);
firstOk = TRUE;
++machinesFinished;
break;
}
else /* some error in rcp */
{
warn("Problem with %s\n", cmd->string);
m->errCount += 1;
}
}
/* Loop around launching child processes to copy and
* wait for them to finish. */
while (machinesFinished < machineCount)
{
int pid;
int status;
/* Start all possible copies. */
while (matchMaker(finishedList, toDoList, &sourceNode, &destNode))
{
dlAddTail(sourceList, sourceNode);
dlAddTail(workingList, destNode);
m = destNode->val;
m->sourceNode = sourceNode;
startCopy(sourceNode->val, destNode->val, dest, thisHost, cmd);
}
curTime = time(NULL);
if (curTime - lastTime >= 3)
{
printf("%d finished in %d seconds, %d in progress, %d to start, %d errors, %d total\n",
dlCount(finishedList) + dlCount(sourceList), (int)(curTime - startTime),
dlCount(workingList), dlCount(toDoList), dlCount(errList), machineCount);
lastTime = curTime;
}
/* Wait for a child to finish. Figure out which machine it is. */
pid = wait(&status);
finNode = NULL;
for (node = workingList->head; node->next != NULL; node = node->next)
{
m = node->val;
if (m->pid == pid)
{
finNode = node;
break;
}
}
if (finNode == NULL)
{
errAbort("Returned from wait on unknown child %d\n", pid);
continue;
}
m = finNode->val;
m->pid = 0;
dlRemove(finNode);
dlRemove(m->sourceNode);
m2 = m->sourceNode->val;
if (m->netSwitch != m2->netSwitch)
--crossSwitchCount;
dlAddTail(finishedList, m->sourceNode);
if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
{
/* Good return - move self and source node to finished list. */
++machinesFinished;
dlAddTail(finishedList, finNode);
}
else
{
/* Bad return. Increment error count, and maybe move it to
* error list. */
if (++m->errCount >= maxErrCount)
{
++machinesFinished;
dlAddTail(errList, finNode);
fprintf(stderr, "Gave up on %s\n", m->name);
}
else
{
dlAddMiddle(toDoList, finNode);
fprintf(stderr, "Retry %d on %s\n", m->errCount, m->name);
}
}
}
if (!dlEmpty(errList))
{
fprintf(stderr, "errors in:");
for (node = errList->head; node->next != NULL; node = node->next)
{
m = node->val;
fprintf(stderr, " %s", m->name);
}
fprintf(stderr, "\n");
}
goodMachines = dlCount(finishedList);
grandTotal = (double)goodMachines * (double)size;
printf("Copied to %d of %d machines (grand total %e bytes) in %d seconds\n",
goodMachines, machineCount, grandTotal, (int)(time(NULL) - startTime));
}
int rudpReceiveTimeOut(struct rudp *ru, void *messageBuf, int bufSize,
struct sockaddr_in *retFrom, int timeOut)
/* Read message into buffer of given size. Returns actual size read on
* success. On failure prints a warning, sets errno, and returns -1.
* Also returns ip address of message source. If timeOut is nonzero,
* it represents the timeout in milliseconds. It will set errno to
* ETIMEDOUT in this case.*/
{
char inBuf[udpEthMaxSize];
struct rudpHeader *head = (struct rudpHeader *)inBuf;
struct rudpHeader ackHead;
struct sockaddr_in sai;
socklen_t saiSize = sizeof(sai);
int readSize, err;
assert(bufSize <= rudpMaxSize);
ru->receiveCount += 1;
for (;;)
{
if (timeOut != 0)
{
if (!readReadyWait(ru->socket, timeOut))
{
warn("rudpReceive timed out\n");
errno = ETIMEDOUT;
return -1;
}
}
readSize = recvfrom(ru->socket, inBuf, sizeof(inBuf), 0,
(struct sockaddr*)&sai, &saiSize);
if (retFrom != NULL)
*retFrom = sai;
if (readSize < 0)
{
if (errno == EINTR)
continue;
warn("recvfrom error: %s", strerror(errno));
ru->failCount += 1;
return readSize;
}
if (readSize < sizeof(*head))
{
warn("rudpRecieve truncated message");
continue;
}
if (head->type != rudpData)
{
if (head->type != rudpAck)
warn("skipping non-data message %d in rudpReceive", head->type);
continue;
}
ackHead = *head;
ackHead.type = rudpAck;
err = sendto(ru->socket, &ackHead, sizeof(ackHead), 0,
(struct sockaddr *)&sai, sizeof(sai));
if (err < 0)
{
warn("problem sending ack in rudpRecieve: %s", strerror(errno));
}
readSize -= sizeof(*head);
if (readSize > bufSize)
{
warn("read more bytes than have room for in rudpReceive");
readSize = bufSize;
}
memcpy(messageBuf, head+1, readSize);
/* check for duplicate packet */
if (!ru->recvHash)
{ /* take advantage of new auto-expanding hashes */
ru->recvHash = newHashExt(4, FALSE); /* do not use local mem in hash */
ru->recvList = newDlList();
ru->recvCount = 0;
}
char hashKey[64];
char saiDottedQuad[17];
internetIpToDottedQuad(ntohl(sai.sin_addr.s_addr), saiDottedQuad);
safef(hashKey, sizeof(hashKey), "%s-%d-%d-%d"
, saiDottedQuad
, head->pid
, head->connId
, head->id
);
if (hashLookup(ru->recvHash, hashKey))
{
warn("duplicate packet filtered out: %s", hashKey);
continue;
}
long now = time(NULL);
struct packetSeen *p;
AllocVar(p);
AllocVar(p->node);
p->node->val = p;
p->recvHashKey = hashStoreName(ru->recvHash, hashKey);
p->lastChecked = now;
dlAddTail(ru->recvList, p->node);
++ru->recvCount;
sweepOutOldPacketsSeen(ru, now);
ru->lastIdReceived = head->id;
break;
}
return readSize;
}
