static void processMrnaFa(struct sqlConnection *conn, int taxon, char *type, char *db)
/* process isPcr results */
{
struct dyString *dy = dyStringNew(0);
struct lineFile *lf = lineFileOpen("mrna.fa", TRUE);
int lineSize;
char *line;
char *name;
char *dna;
boolean more = lineFileNext(lf, &line, &lineSize);
while(more)
{
if (line[0] != '>')
errAbort("unexpected error out of phase\n");
name = cloneString(line+1);
verbose(2,"name=%s\n",name);
dyStringClear(dy);
while((more=lineFileNext(lf, &line, &lineSize)))
{
if (line[0] == '>')
{
break;
}
dyStringAppend(dy,line);
}
dna = cloneString(dy->string);
while(1)
{
int oldProbe = 0;
dyStringClear(dy);
dyStringPrintf(dy, "select id from vgPrb "
"where taxon=%d and type='%s' and tName='%s' and state='new'",taxon,type,name);
oldProbe = sqlQuickNum(conn,dy->string);
if (oldProbe==0)
break; /* no more records match */
/* record exists and hasn't already been updated */
int vgPrb = findVgPrbBySeq(conn,dna,taxon);
if (vgPrb == 0)
{
dyStringClear(dy);
dyStringAppend(dy, "update vgPrb set");
dyStringAppend(dy, " seq = '");
dyStringAppend(dy, dna);
dyStringAppend(dy, "',\n");
dyStringPrintf(dy, " db = '%s',\n", db);
dyStringAppend(dy, " state = 'seq'\n");
dyStringPrintf(dy, " where id=%d\n", oldProbe);
dyStringPrintf(dy, " and state='%s'\n", "new");
verbose(2, "%s\n", dy->string);
sqlUpdate(conn, dy->string);
}
else /* probe seq already exists */
{
/* just re-map the probe table recs to it */
dyStringClear(dy);
dyStringPrintf(dy, "update vgPrbMap set vgPrb=%d where vgPrb=%d",vgPrb,oldProbe);
sqlUpdate(conn, dy->string);
/* and delete it from vgPrb */
dyStringClear(dy);
dyStringPrintf(dy, "delete from vgPrb where id=%d",oldProbe);
sqlUpdate(conn, dy->string);
}
}
freez(&name);
freez(&dna);
}
lineFileClose(&lf);
dyStringFree(&dy);
}
void autoUpgradeTableAddSessionKey(struct sqlConnection *conn, char *tableName)
/* Try to upgrade the table by adding sessionKey field
* in a safe way handling success failures and retries
* with multiple CGIs running. */
{
boolean testAgain = checkAutoUpgradeTableResultTimeIsOld(tableName);
if (testAgain)
{
// Get the advisory lock for this table
// This prevents multiple CGI processes from trying to upgrade simultaneously
char lockName[256];
safef(lockName, sizeof lockName, "AUTO_UPGRADE_%s", tableName);
sqlGetLock(conn, lockName);
// Make sure that the table has not been already upgraded by some earlier process.
// We do not want to upgrade more than once.
if (sqlFieldIndex(conn, tableName, "sessionKey") == -1)
{
char result[4096];
// Put a catch around the table upgrade attempt,
// both to allow us to continue if it fails,
// and to catch the error message and save it in the results file
struct errCatch *errCatch = errCatchNew();
if (errCatchStart(errCatch))
{
char query[256];
sqlSafef(query, sizeof query, "alter table %s add column sessionKey varchar(255) NOT NULL default ''", tableName);
sqlUpdate(conn, query);
}
errCatchEnd(errCatch);
if (errCatch->gotError)
{
safef(result, sizeof result, "AUTOUPGRADE FAILED\n%s", errCatch->message->string);
}
else
{
safef(result, sizeof result, "OK\n");
}
errCatchFree(&errCatch);
writeAutoUpgradeTableResult(tableName, result);
}
// Release the advisory lock for this table
sqlReleaseLock(conn, lockName);
}
else
{ // in the interests of speed, just use the old result
char *oldResult = readAutoUpgradeTableResult(tableName);
if (oldResult)
{
if (startsWith("AUTOUPGRADE FAILED", oldResult))
{
// cannot do this here since it is too early and the warn handler does not work right
// it ends up writing html and javascript before the cookie and response header have even been finished.
// warn("%s", oldResult);
// instead, save the error message for access later from select places like hgGateway
if (!dyUpgradeError)
dyUpgradeError = dyStringNew(256);
else
dyStringPrintf(dyUpgradeError,"\n");
dyStringPrintf(dyUpgradeError,"%s", oldResult);
}
}
}
}
static void populateMissingVgPrb(struct sqlConnection *conn)
/* populate vgPrb where missing, usually after new records added to visiGene */
{
struct sqlResult *sr;
char **row;
struct dyString *dy = dyStringNew(0);
struct sqlConnection *conn2 = sqlConnect(database);
struct sqlConnection *conn3 = sqlConnect(database);
int probeCount=0, vgPrbCount=0;
dyStringAppend(dy,
"select p.id,p.gene,antibody,probeType,fPrimer,rPrimer,p.seq,bac,g.taxon"
" from probe p join gene g"
" left join vgPrbMap m on m.probe = p.id"
" where g.id = p.gene"
" and m.probe is NULL");
sr = sqlGetResult(conn, dy->string);
while ((row = sqlNextRow(sr)) != NULL)
{
int id = sqlUnsigned(row[0]);
/* int gene = sqlUnsigned(row[1]); */
/* int antibody = sqlUnsigned(row[2]); */
/* int probeType = sqlUnsigned(row[3]); */
char *fPrimer = row[4];
char *rPrimer = row[5];
char *seq = row[6];
int bac = sqlUnsigned(row[7]);
int taxon = sqlUnsigned(row[8]);
char *peType = "none";
int peProbe = id;
char *peSeq = seq;
char *tName = "";
int tStart = 0;
int tEnd = 0;
char *tStrand = " ";
/*
char *peGene = "";
int bacInfo = 0;
int seqid = 0;
int pslid = 0;
*/
char *state = "new";
char *db = "";
int vgPrb = 0;
if (isNotEmpty(seq))
{
peType = "probe";
state = "seq";
}
else if (isNotEmpty(fPrimer) && isNotEmpty(rPrimer))
{
peType = "primersMrna";
}
else if (isNotEmpty(fPrimer) && isEmpty(rPrimer))
{ /* only have fPrimer, it's probably a comment, not dna seq */
peType = "refSeq"; /* use accession or gene */
}
else if (bac > 0)
{
peType = "bac"; /* use bacEndPairs */
}
else
{
peType = "refSeq"; /* use accession or gene */
}
if (!sameString(peSeq,""))
{
vgPrb = findVgPrbBySeq(conn3,peSeq,taxon);
}
if (vgPrb == 0)
{
dyStringClear(dy);
dyStringAppend(dy, "insert into vgPrb set");
dyStringPrintf(dy, " id=default,\n");
dyStringPrintf(dy, " type='%s',\n", peType);
dyStringAppend(dy, " seq='");
dyStringAppend(dy, peSeq);
dyStringAppend(dy, "',\n");
dyStringPrintf(dy, " tName='%s',\n", tName);
dyStringPrintf(dy, " tStart=%d,\n", tStart);
dyStringPrintf(dy, " tEnd=%d,\n", tEnd);
dyStringPrintf(dy, " tStrand='%s',\n", tStrand);
dyStringPrintf(dy, " db='%s',\n", db);
dyStringPrintf(dy, " taxon='%d',\n", taxon);
dyStringPrintf(dy, " state='%s'\n", state);
verbose(2, "%s\n", dy->string);
sqlUpdate(conn2, dy->string);
vgPrb = sqlLastAutoId(conn2);
vgPrbCount++;
}
dyStringClear(dy);
dyStringAppend(dy, "insert into vgPrbMap set");
dyStringPrintf(dy, " probe=%d,\n", peProbe);
dyStringPrintf(dy, " vgPrb=%d \n", vgPrb);
verbose(2, "%s\n", dy->string);
sqlUpdate(conn2, dy->string);
probeCount++;
}
verbose(1, "# new probe records found = %d, # new vgPrb records added = %d\n", probeCount, vgPrbCount);
dyStringFree(&dy);
sqlFreeResult(&sr);
sqlDisconnect(&conn3);
sqlDisconnect(&conn2);
}
static int doBacs(struct sqlConnection *conn, int taxon, char *db)
/* fetch available sequence for bacEndPairs */
{
struct dyString *dy = dyStringNew(0);
struct dnaSeq *chromSeq = NULL;
struct bac *bacs = bacRead(conn, taxon, db);
struct bac *bac = NULL;
char *chrom = cloneString("");
int count = 0;
verbose(1,"bac list read done.\n");
for(bac=bacs;bac;bac=bac->next)
{
if (differentWord(chrom,bac->chrom))
{
verbose(1,"switching to chrom %s\n",bac->chrom);
dnaSeqFree(&chromSeq);
chromSeq = hLoadChrom(bac->chrom,db);
freez(&chrom);
chrom = cloneString(bac->chrom);
}
char *dna = checkAndFetchBacDna(chromSeq, bac);
if (sameString(bac->strand,"-"))
{
reverseComplement(dna,strlen(dna));
}
dyStringClear(dy);
dyStringPrintf(dy, "select count(*) from vgPrb where id=%d and state='new'",bac->probe);
if (sqlQuickNum(conn,dy->string)>0)
{
/* record exists and hasn't already been updated */
int vgPrb = findVgPrbBySeq(conn,dna,taxon);
if (vgPrb == 0)
{
dyStringClear(dy);
dyStringAppend(dy, "update vgPrb set");
dyStringAppend(dy, " seq='");
dyStringAppend(dy, dna);
dyStringAppend(dy, "',\n");
dyStringPrintf(dy, " tName='%s',\n", bac->chrom);
dyStringPrintf(dy, " tStart=%d,\n", bac->chromStart);
dyStringPrintf(dy, " tEnd=%d,\n", bac->chromEnd);
dyStringPrintf(dy, " tStrand='%s',\n", bac->strand);
dyStringPrintf(dy, " db='%s',\n", db);
dyStringPrintf(dy, " state='%s'\n", "seq");
dyStringPrintf(dy, " where id=%d\n", bac->probe);
dyStringPrintf(dy, " and state='%s'\n", "new");
//verbose(2, "%s\n", dy->string); // the sql string could be quite large
sqlUpdate(conn, dy->string);
}
else /* probe seq already exists */
{
/* just re-map the probe table recs to it */
dyStringClear(dy);
dyStringPrintf(dy, "update vgPrbMap set vgPrb=%d where vgPrb=%d",vgPrb,bac->probe);
sqlUpdate(conn, dy->string);
/* and delete it from vgPrb */
dyStringClear(dy);
dyStringPrintf(dy, "delete from vgPrb where id=%d",bac->probe);
sqlUpdate(conn, dy->string);
}
++count;
verbose(2,"%d finished bac for probe id %d size %d\n",
count, bac->probe, bac->chromEnd - bac->chromStart);
}
freez(&dna);
}
freez(&chrom);
dnaSeqFree(&chromSeq);
bacFreeList(&bacs);
dyStringFree(&dy);
return count;
}
void condenseValues()
/* combine values for single snp */
{
char query[512];
struct sqlConnection *conn = hAllocConn();
struct sqlResult *sr;
char **row;
FILE *f;
struct dyString *ssList = newDyString(255);
struct dyString *buildList = newDyString(255);
char *currentSnpString = NULL;
int currentSnpNum = 0;
int count = 0;
char firstBuild[32];
char lastBuild[32];
f = hgCreateTabFile(".", "SNPSubSNPLinkCondense");
sqlSafef(query, sizeof(query), "select snp_id, subsnp_id, build_id from SNPSubSNPLink");
sr = sqlGetResult(conn, query);
while ((row = sqlNextRow(sr)) != NULL)
{
if (currentSnpString == NULL)
{
currentSnpString = cloneString(row[0]);
currentSnpNum = sqlUnsigned(row[0]);
dyStringPrintf(ssList, "%s", row[1]);
dyStringPrintf(buildList, "%s", row[2]);
safef(firstBuild, sizeof(firstBuild), row[2]);
safef(lastBuild, sizeof(firstBuild), row[2]);
}
else if (!sameString(row[0], currentSnpString))
{
fprintf(f, "%s\t%s\t%s\t%s\t%s\t%d\n",
currentSnpString, ssList->string, buildList->string, firstBuild, lastBuild, count);
if (currentSnpNum > sqlUnsigned(row[0]))
errAbort("snps out of order: %d before %s\n", currentSnpNum, row[0]);
currentSnpString = cloneString(row[0]);
currentSnpNum = sqlUnsigned(row[0]);
dyStringClear(ssList);
dyStringPrintf(ssList, "%s", row[1]);
dyStringClear(buildList);
dyStringPrintf(buildList, "%s", row[2]);
safef(firstBuild, sizeof(firstBuild), row[2]);
safef(lastBuild, sizeof(lastBuild), row[2]);
count = 1;
}
else
{
count++;
dyStringAppend(ssList, ",");
dyStringAppend(ssList, row[1]);
dyStringAppend(buildList, ",");
dyStringAppend(buildList, row[2]);
safef(lastBuild, sizeof(lastBuild), row[2]);
}
}
fprintf(f, "%s\t%s\t%s\t%s\t%s\t%d\n", currentSnpString, ssList->string, buildList->string,
firstBuild, lastBuild, count);
sqlFreeResult(&sr);
hFreeConn(&conn);
carefulClose(&f);
}
static void doBlat(struct sqlConnection *conn, int taxon, char *db)
/* place probe seq from non-BAC with blat that have no alignments yet */
{
int rc = 0;
char *blatSpec=NULL;
char cmdLine[256];
char path1[256];
char path2[256];
struct dyString *dy = dyStringNew(0);
/* (non-BACs needing alignment) */
dyStringClear(dy);
dyStringPrintf(dy,
"select concat(\"vgPrb_\",e.id), e.seq"
" from vgPrb e, vgPrbAli a"
" where e.id = a.vgPrb"
" and a.db = '%s'"
" and a.status = 'new'"
" and e.taxon = %d"
" and e.type <> 'bac'"
" and e.seq <> ''"
" order by e.id"
, db, taxon);
//restore:
rc = sqlSaveQuery(conn, dy->string, "blat.fa", TRUE);
verbose(1,"rc = %d = count of sequences for blat, to get psls for taxon %d\n",rc,taxon);
if (rc == 0)
{
unlink("blat.fa");
system("rm -f blatNearBest.psl; touch blatNearBest.psl"); /* make empty file */
return;
}
/* make .ooc and blat on kolossus */
safef(path1,sizeof(path1),"/gbdb/%s/%s.2bit",db,db);
safef(path2,sizeof(path2),"%s/%s.2bit",getCurrentDir(),db);
//restore:
verbose(1,"copy: [%s] to [%s]\n",path1,path2); copyFile(path1,path2);
safef(cmdLine,sizeof(cmdLine),
"ssh kolossus 'cd %s; blat -makeOoc=11.ooc -tileSize=11"
" -repMatch=1024 %s.2bit /dev/null /dev/null'",
getCurrentDir(),db);
//restore:
system("date"); verbose(1,"cmdLine: [%s]\n",cmdLine); system(cmdLine); system("date");
safef(cmdLine,sizeof(cmdLine),
"ssh kolossus 'cd %s; blat %s.2bit blat.fa -ooc=11.ooc -noHead blat.psl'",
getCurrentDir(),db);
//restore:
system("date"); verbose(1,"cmdLine: [%s]\n",cmdLine); system(cmdLine); system("date");
/* using blat even with -fastMap was way too slow - took over a day,
* so instead I will make a procedure to write a fake psl for the BACs
* which you will see called below */
safef(path2,sizeof(path2),"%s.2bit",db);
verbose(1,"rm %s\n",path2); unlink(path2);
safef(path2,sizeof(path2),"11.ooc");
verbose(1,"rm %s\n",path2); unlink(path2);
/* skip psl header and sort on query name */
safef(cmdLine,sizeof(cmdLine), "sort -k 10,10 blat.psl > blatS.psl");
verbose(1,"cmdLine=[%s]\n",cmdLine);
system(cmdLine);
/* keep near best within 5% of the best */
safef(cmdLine,sizeof(cmdLine),
"pslCDnaFilter -globalNearBest=0.005 -minId=0.96 -minNonRepSize=20 -minCover=0.50"
" blatS.psl blatNearBest.psl");
verbose(1,"cmdLine=[%s]\n",cmdLine);
system(cmdLine);
unlink("blat.fa");
unlink("blat.psl");
unlink("blatS.psl");
freez(&blatSpec);
dyStringFree(&dy);
}
void bedSqlFieldsExceptForChrom(struct hTableInfo *hti,
int *retFieldCount, char **retFields)
/* Given tableInfo figure out what fields are needed to
* add to a database query to have information to create
* a bed. The chromosome is not one of these fields - we
* assume that is already known since we're processing one
* chromosome at a time. Return a comma separated (no last
* comma) list of fields that can be freeMem'd, and the count
* of fields (this *including* the chromosome). */
{
struct dyString *fields = dyStringNew(128);
int fieldCount = fieldCount = hTableInfoBedFieldCount(hti);
dyStringPrintf(fields, "%s,%s", hti->startField, hti->endField);
if (fieldCount >= 4)
{
if (hti->nameField[0] != 0)
dyStringPrintf(fields, ",%s", hti->nameField);
else /* Put in . as placeholder. */
dyStringPrintf(fields, ",'.'");
}
if (fieldCount >= 5)
{
if (hti->scoreField[0] != 0)
dyStringPrintf(fields, ",%s", hti->scoreField);
else
dyStringPrintf(fields, ",0", hti->startField);
}
if (fieldCount >= 6)
{
if (hti->strandField[0] != 0)
dyStringPrintf(fields, ",%s", hti->strandField);
else
dyStringPrintf(fields, ",'.'");
}
if (fieldCount >= 8)
{
if (hti->cdsStartField[0] != 0)
dyStringPrintf(fields, ",%s,%s", hti->cdsStartField, hti->cdsEndField);
else
dyStringPrintf(fields, ",%s,%s", hti->startField, hti->endField);
}
if (fieldCount >= 12)
{
dyStringPrintf(fields, ",%s,%s,%s", hti->countField,
hti->endsSizesField, hti->startsField);
}
if (htiIsPsl(hti))
{
/* For psl format we need to know chrom size as well
* to handle reverse strand case. */
dyStringPrintf(fields, ",tSize");
}
*retFieldCount = fieldCount;
*retFields = dyStringCannibalize(&fields);
}
struct mimePart *parseMultiParts(struct mimeBuf *b, char *altHeader)
/* This is a recursive function. It parses multipart MIME messages.
Data that are binary or too large will be saved in mimePart->filename
otherwise saved as a c-string in mimePart->data. If multipart,
then first child is mimePart->child, subsequent sibs are in child->next.
altHeader is a string of headers that can be fed in if the headers have
already been read off the stream by an earlier process, i.e. apache.
*/
{
struct mimePart *p=AllocA(*p);
char *parentboundary = NULL, *boundary = NULL;
char *ct = NULL;
boolean autoBoundary = FALSE;
//debug
//fprintf(stderr,"altHeader=[%s]\n",altHeader);
if (sameOk(altHeader, "autoBoundary"))
{ /* process things with no explicit header.
* look for *MIME* \n\n-- */
struct dyString *dy = dyStringNew(0);
char *prevPrevLine = NULL;
char *prevLine = NULL;
char *line = NULL;
boolean found = FALSE;
autoBoundary = TRUE;
while (TRUE)
{
if (b->i >= b->eoi && b->eoi < b->eom) /* at end of input */
break;
line = getLineMB(b);
if (line && startsWith("--",line) // &&
//sameString(prevLine,"") &&
//prevPrevLine &&
//stringIn("MULTI",prevPrevLine) &&
//stringIn("MIME",prevPrevLine)
)
{
found = TRUE;
break;
}
freez(&prevPrevLine);
prevPrevLine = prevLine;
prevLine = line;
if (prevPrevLine)
touppers(prevPrevLine);
}
if (!found)
errAbort("autoBoundary: No initial boundary found.");
dyStringPrintf(dy, "CONTENT-TYPE:multipart/form-data; boundary=%s%s%s",
line+2, getNewLineByType(), getNewLineByType() );
altHeader = dyStringCannibalize(&dy);
//debug
//fprintf(stderr,"autoBoundary altHeader = [%s]\n",altHeader);
//fflush(stderr);
freez(&prevPrevLine);
freez(&prevLine);
freez(&line);
}
//debug
//fprintf(stderr,"\n");
readPartHeaderMB(b,p,altHeader);
ct = hashFindVal(p->hdr,"content-type"); /* use lowercase key */
//debug
//fprintf(stderr,"ct from hash:%s\n",ct);
//fflush(stderr);
if (ct && startsWith("multipart/",ct))
{
char bound[MAXBOUNDARY];
char *bnd = NULL;
struct mimePart *child = NULL;
/* these 3 vars just for processing epilog chunk: */
char *bp=NULL;
int size=0;
boolean hasZeros=FALSE;
/* save */
parentboundary = b->boundary;
boundary = getMimeHeaderFieldVal(ct,"boundary");
if (strlen(boundary) >= MAXBOUNDARY)
errAbort("error: boundary= value too long in MIME header Content-type:%s",ct);
safef(bound, sizeof(bound), "--%s",boundary); /* do not prepend CRLF to boundary yet */
freez(&boundary);
boundary = cloneString(bound);
//debug
//fprintf(stderr,"initial boundary parsed:%s\n",boundary);
//fflush(stderr);
if (!autoBoundary)
{
/* skip any extra "prolog" before the initial boundary marker */
while (TRUE)
{
bnd = getLineMB(b);
if (sameString(bnd,boundary))
break;
freez(&bnd);
}
//debug
//fprintf(stderr,"initial boundary found:%s\n",bnd);
//fflush(stderr);
freez(&bnd);
}
/* include crlf in the boundary so bodies won't have trailing a CRLF
* this is done here so that in case there's no extra CRLF
* between the header and the boundary, it will still work,
* so we only prepend the CRLF to the boundary after initial found */
safef(bound,sizeof(bound),"%s%s", getNewLineByType(), boundary);
freez(&boundary);
boundary=cloneString(bound);
setBoundaryMB(b, boundary);
while(TRUE)
{
int i = 0;
char c1 = ' ', c2 = ' ';
child = parseMultiParts(b,NULL);
slAddHead(&p->multi,child);
//call getLine, compare to boundary
/* skip extra initial boundary marker - it's moot anyway */
freez(&bnd);
//debug
//fprintf(stderr,"post-parse pre-getLineMB dumpMB: ");
//dumpMB(b); //debug
for (i=0;i<strlen(boundary);++i)
bound[i] = getcMB(b);
bound[i] = 0;
if (!sameString(bound,boundary))
errAbort("expected boundary %s, but found %s in MIME",boundary,bound);
//debug
//fprintf(stderr,"\nfound boundary:%s\n",bound);
//fflush(stderr);
c1 = getcMB(b);
if (c1 == '-')
{
c2 = getcMB(b);
if (c2 == '-')
break; /* last boundary found */
else
errAbort("expected -- after boundary %s, but found %c%c in MIME",boundary,c1,c2);
}
if (nlType == nlt_dos)
c2 = getcMB(b);
switch (nlType)
{
case nlt_dos:
if (c1 == 0x0d && c2 == 0x0a)
break;
else
errAbort("expected CRLF after boundary %s, but found %c%c in MIME",boundary,c1,c2);
case nlt_unix:
if (c1 == 0x0a)
break;
else
errAbort("expected LF after boundary %s, but found %c in MIME",boundary,c1);
case nlt_mac:
if (c1 == 0x0d)
break;
else
errAbort("expected CR after boundary %s, but found %c in MIME",boundary,c1);
default:
errAbort("unexpected nlType %d after boundary %s",nlType,boundary);
}
setEopMB(b);
}
freez(&bnd);
slReverse(&p->multi);
/* restore */
freez(&boundary);
boundary = parentboundary;
//debug
//fprintf(stderr,"restoring parent boundary = %s\n",boundary);
setBoundaryMB(b, boundary);
/* dump any "epilog" that may be between the
* end of the child boundary and the parent boundary */
getChunkMB(b, &bp, &size, &hasZeros);
//debug
//fprintf(stderr,"epilog size=%d\n",size);
}
else
{
char *bp=NULL;
int size=0;
boolean hasZeros=FALSE;
boolean toobig=FALSE;
boolean asFile=FALSE;
boolean convert=FALSE;
FILE *f = NULL;
struct dyString *dy=newDyString(1024);
//debug
//fprintf(stderr,"starting new part (non-multi), dumpMB: \n");
//dumpMB(b); //debug
//debug
//ct = hashFindVal(p->hdr,"content-transfer-encoding"); /* use lowercase key */
//fprintf(stderr,"cte from hash:%s\n",ct);
while(TRUE)
{
// break if eop, eod, eoi
getChunkMB(b, &bp, &size, &hasZeros);
//debug
//fprintf(stderr,"bp=%lu size=%d, hasZeros=%d \n",
// (unsigned long) bp,
// size,
// hasZeros);
if (hasZeros)
{
p->binary=TRUE;
}
//if (hasZeros && !asFile)
// {
// convert=TRUE;
// }
if (!asFile && p->size+size > MAXPARTSIZE)
{
toobig = TRUE;
convert=TRUE;
}
if (convert)
{
struct tempName uploadedData;
convert=FALSE;
asFile = TRUE;
makeTempName(&uploadedData, "hgSs", ".cgi");
p->fileName=cloneString(uploadedData.forCgi);
f = mustOpen(p->fileName,"w");
mustWrite(f,dy->string,dy->stringSize);
freeDyString(&dy);
}
if (asFile)
{
mustWrite(f,bp,size);
}
else
{
dyStringAppendN(dy,bp,size);
}
p->size+=size;
if (p->size > MAXDATASIZE)
errAbort("max data size allowable for upload in MIME exceeded %llu",(unsigned long long)MAXDATASIZE);
if (b->eop && b->i == b->eop) /* end of part */
{
break;
}
if (b->i == b->eoi && b->eoi < b->eom) /* end of data */
{
break;
}
moreMimeBuf(b);
}
if (dy)
{
p->data=needLargeMem(dy->stringSize+1);
memcpy(p->data,dy->string,dy->stringSize);
p->data[dy->stringSize] = 0;
freeDyString(&dy);
}
if (f)
carefulClose(&f);
//debug
//fprintf(stderr,"p->fileName=%s p->data=[%s]\n",p->fileName,p->data);
}
return p;
}
static struct bed *parseRegionInput(char *inputString)
/* scan the user region definition, turn into a bed list */
{
int itemCount = 0;
struct bed *bedList = NULL;
struct bed *bedEl;
int wordCount;
char *words[5];
struct lineFile *lf;
lf = lineFileOnString("userData", TRUE, inputString);
while (0 != (wordCount = lineFileChopNext(lf, words, ArraySize(words))))
{
char *chromName = NULL;
int chromStart = 0;
int chromEnd = 0;
char *regionName = NULL;
/* might be something of the form: chrom:start-end optionalRegionName */
if (((1 == wordCount) || (2 == wordCount)) &&
hgParseChromRange(NULL, words[0], &chromName,
&chromStart, &chromEnd))
{
if (2 == wordCount)
regionName = cloneString(words[1]);
}
else if (!((3 == wordCount) || (4 == wordCount)))
{
int i;
struct dyString *errMessage = dyStringNew(0);
for (i = 0; i < wordCount; ++i)
dyStringPrintf(errMessage, "%s ", words[i]);
errAbort("line %d: '%s'<BR>\n"
"illegal bed size, expected 3 or 4 fields, found %d\n",
lf->lineIx, dyStringCannibalize(&errMessage), wordCount);
}
else
{
chromName = hgOfficialChromName(database, words[0]);
chromStart = sqlSigned(words[1]);
chromEnd = sqlSigned(words[2]);
if (wordCount > 3)
regionName = cloneString(words[3]);
}
++itemCount;
if (itemCount > 1000)
{
warn("limit 1000 region definitions reached at line %d<BR>\n",
lf->lineIx);
break;
}
AllocVar(bedEl);
bedEl->chrom = chromName;
if (NULL == bedEl->chrom)
errAbort("at line %d, chrom name '%s' %s %s not recognized in this assembly %d",
lf->lineIx, words[0], words[1], words[2], wordCount);
bedEl->chromStart = chromStart;
bedEl->chromEnd = chromEnd;
if (illegalCoordinate(bedEl->chrom, bedEl->chromStart, bedEl->chromEnd))
errAbort("illegal input at line %d: %s %d %d",
lf->lineIx, bedEl->chrom, bedEl->chromStart, bedEl->chromEnd);
if (wordCount > 3)
bedEl->name = regionName;
else
bedEl->name = NULL;
/* if we wanted to give artifical names to each item */
#ifdef NOT
{
char name[128];
safef(name, ArraySize(name), "item_%04d", itemCount);
bedEl->name = cloneString(name);
}
#endif
slAddHead(&bedList, bedEl);
}
lineFileClose(&lf);
// slSort(&bedList, bedCmp); /* this would do chrom,chromStart order */
slReverse(&bedList); /* with no sort, it is in order as user entered */
return (bedList);
}
int main(int argc, char *argv[])
{
long enteredMainTime = clock1000();
cgiSpoof(&argc, argv);
char *prefix = cgiOptionalString("prefix");
char *database = cgiOptionalString("db");
initGenbankTableNames(database);
int exact = cgiOptionalInt("exact", 0);
struct sqlConnection *conn;
char query[2048];
char **row;
struct sqlResult *sr;
int count = 0;
boolean hasKnownCanonical;
struct dyString *str = newDyString(10000);
char *table;
pushAbortHandler(htmlVaBadRequestAbort);
if(prefix == NULL || database == NULL)
errAbort("%s", "Missing prefix and/or db CGI parameter");
conn = hAllocConn(database);
table = connGeneSuggestTable(conn);
if(table == NULL)
errAbort("gene autosuggest is not supported for db '%s'", database);
popAbortHandler();
hasKnownCanonical = sameString(table, "knownCanonical");
puts("Content-Type:text/plain");
puts("\n");
dyStringPrintf(str, "[\n");
if(exact)
{
// NOTE that exact is no longer used by the UI as of v271, but there are still some robots using it so we still support it.
if(hasKnownCanonical)
sqlSafef(query, sizeof(query), "select x.geneSymbol, k.chrom, kg.txStart, kg.txEnd, x.kgID, x.description "
"from knownCanonical k, knownGene kg, kgXref x where k.transcript = x.kgID and k.transcript = kg.name "
"and x.geneSymbol = '%s' order by x.geneSymbol, k.chrom, kg.txEnd - kg.txStart desc", prefix);
else
sqlSafef(query, sizeof(query), "select r.name2, r.chrom, r.txStart, r.txEnd, r.name, d.name "
"from %s r, %s g, %s d where r.name2 = '%s' and g.acc = r.name "
"and g.description = d.id order by r.name2, r.chrom, r.txEnd - r.txStart desc", table, gbCdnaInfoTable, descriptionTable, prefix);
}
else
{
// We use a LIKE query b/c it uses the geneSymbol index (substr queries do not use indices in mysql).
// Also note that we take advantage of the fact that searches are case-insensitive in mysql.
// Unfortunately, knownCanonical sometimes has multiple entries for a given gene (e.g. 2 TTn's in mm9 knownCanonical;
// 3 POU5F1's in hg19); we return all of them (#5962).
if(hasKnownCanonical)
sqlSafef(query, sizeof(query), "select x.geneSymbol, k.chrom, kg.txStart, kg.txEnd, x.kgID, x.description "
"from knownCanonical k, knownGene kg, kgXref x where k.transcript = x.kgID and k.transcript = kg.name "
"and x.geneSymbol LIKE '%s%%' order by x.geneSymbol, k.chrom, kg.txStart", prefix);
else
sqlSafef(query, sizeof(query), "select r.name2, r.chrom, r.txStart, r.txEnd, r.name, d.name "
"from %s r, %s g, %s d where r.name2 LIKE '%s%%' and g.acc = r.name "
"and g.description = d.id order by r.name2, r.chrom, r.txStart", table, gbCdnaInfoTable, descriptionTable, prefix);
}
sr = sqlGetResult(conn, query);
while ((row = sqlNextRow(sr)) != NULL)
{
// ignore funny chroms (e.g. _hap chroms. See redmine #4257.
if(!strchr(row[1], '_'))
{
// We have some very long descriptions, e.g. 4277 chars for hg38 CLOCK, so truncate:
const int maxDesc = 120;
char *description = row[5];
if (strlen(description) > maxDesc + 4)
strcpy(description + maxDesc, "...");
count++;
dyStringPrintf(str, "%s{\"value\": \"%s (%s)\", "
"\"id\": \"%s:%d-%s\", "
"\"geneSymbol\": \"%s\", "
"\"internalId\": \"%s\"}",
count == 1 ? "" : ",\n", row[0], jsonStringEscape(description),
row[1], atoi(row[2])+1, row[3],
jsonStringEscape(row[0]),
jsonStringEscape(row[4]));
}
}
dyStringPrintf(str, "\n]\n");
puts(dyStringContents(str));
cgiExitTime("hgSuggest", enteredMainTime);
return 0;
}
char *scanSettingsForCT(char *userName, char *sessionName, char *contents,
int *pLiveCount, int *pExpiredCount)
/* Parse the CGI-encoded session contents into {var,val} pairs and search
* for custom tracks. If found, refresh the custom track. Parsing code
* taken from cartParseOverHash.
* If any nonexistent custom track files are found, return a SQL update
* command that will remove those from this session. We can't just do
* the update here because that messes up the caller's query. */
{
int contentLength = strlen(contents);
struct dyString *newContents = dyStringNew(contentLength+1);
struct dyString *oneSetting = dyStringNew(contentLength / 4);
char *updateIfAny = NULL;
char *contentsToChop = cloneString(contents);
char *namePt = contentsToChop;
verbose(3, "Scanning %s %s\n", userName, sessionName);
while (isNotEmpty(namePt))
{
char *dataPt = strchr(namePt, '=');
char *nextNamePt;
if (dataPt == NULL)
errAbort("Mangled session content string %s", namePt);
*dataPt++ = 0;
nextNamePt = strchr(dataPt, '&');
if (nextNamePt != NULL)
*nextNamePt++ = 0;
dyStringClear(oneSetting);
dyStringPrintf(oneSetting, "%s=%s%s",
namePt, dataPt, (nextNamePt ? "&" : ""));
if (startsWith(CT_FILE_VAR_PREFIX, namePt))
{
boolean thisGotLiveCT = FALSE, thisGotExpiredCT = FALSE;
cgiDecode(dataPt, dataPt, strlen(dataPt));
verbose(3, "Found variable %s = %s\n", namePt, dataPt);
/* If the file does not exist, omit this setting from newContents so
* it doesn't get copied from session to session. If it does exist,
* leave it up to customFactoryTestExistence to parse the file for
* possible customTrash table references, some of which may exist
* and some not. */
if (! fileExists(dataPt))
{
verbose(3, "Removing %s from %s %s\n", oneSetting->string,
userName, sessionName);
thisGotExpiredCT = TRUE;
}
else
{
char *db = namePt + strlen(CT_FILE_VAR_PREFIX);
dyStringAppend(newContents, oneSetting->string);
customFactoryTestExistence(db, dataPt,
&thisGotLiveCT, &thisGotExpiredCT);
}
if (thisGotLiveCT && pLiveCount != NULL)
(*pLiveCount)++;
if (thisGotExpiredCT && pExpiredCount != NULL)
(*pExpiredCount)++;
if (thisGotExpiredCT)
{
if (verboseLevel() >= 3)
verbose(3, "Found expired custom track in %s %s: %s\n",
userName, sessionName, dataPt);
else
verbose(2, "Found expired custom track: %s\n", dataPt);
}
if (thisGotLiveCT)
verbose(4, "Found live custom track: %s\n", dataPt);
}
else
dyStringAppend(newContents, oneSetting->string);
namePt = nextNamePt;
}
if (newContents->stringSize != contentLength)
{
struct dyString *update = dyStringNew(contentLength*2);
if (newContents->stringSize > contentLength)
errAbort("Uh, why is newContents (%d) longer than original (%d)??",
newContents->stringSize, contentLength);
dyStringPrintf(update, "UPDATE %s set contents='", savedSessionTable);
dyStringAppendN(update, newContents->string, newContents->stringSize);
dyStringPrintf(update, "', lastUse=now(), useCount=useCount+1 "
"where userName=\"%s\" and sessionName=\"%s\";",
userName, sessionName);
verbose(3, "Removing one or more dead CT file settings from %s %s "
"(original length %d, now %d)\n",
userName, sessionName,
contentLength, newContents->stringSize);
updateIfAny = dyStringCannibalize(&update);
}
dyStringFree(&oneSetting);
dyStringFree(&newContents);
freeMem(contentsToChop);
return updateIfAny;
}
void bioImageLoad(char *setRaFile, char *itemTabFile)
/* bioImageLoad - Load data into bioImage database. */
{
struct hash *raHash = raReadSingle(setRaFile);
struct hash *rowHash;
struct lineFile *lf = lineFileOpen(itemTabFile, TRUE);
char *line, *words[256];
struct sqlConnection *conn = sqlConnect(database);
int rowSize;
int submissionSetId;
struct hash *fullDirHash = newHash(0);
struct hash *screenDirHash = newHash(0);
struct hash *thumbDirHash = newHash(0);
struct hash *treatmentHash = newHash(0);
struct hash *bodyPartHash = newHash(0);
struct hash *sliceTypeHash = newHash(0);
struct hash *imageTypeHash = newHash(0);
struct hash *sectionSetHash = newHash(0);
struct dyString *dy = dyStringNew(0);
/* Read first line of tab file, and from it get all the field names. */
if (!lineFileNext(lf, &line, NULL))
errAbort("%s appears to be empty", lf->fileName);
if (line[0] != '#')
errAbort("First line of %s needs to start with #, and then contain field names",
lf->fileName);
rowHash = hashRowOffsets(line+1);
rowSize = rowHash->elCount;
if (rowSize >= ArraySize(words))
errAbort("Too many fields in %s", lf->fileName);
/* Check that have all required fields */
{
char *fieldName;
int i;
for (i=0; i<ArraySize(requiredSetFields); ++i)
{
fieldName = requiredSetFields[i];
if (!hashLookup(raHash, fieldName))
errAbort("Field %s is not in %s", fieldName, setRaFile);
}
for (i=0; i<ArraySize(requiredItemFields); ++i)
{
fieldName = requiredItemFields[i];
if (!hashLookup(rowHash, fieldName))
errAbort("Field %s is not in %s", fieldName, itemTabFile);
}
for (i=0; i<ArraySize(requiredFields); ++i)
{
fieldName = requiredFields[i];
if (!hashLookup(rowHash, fieldName) && !hashLookup(raHash, fieldName))
errAbort("Field %s is not in %s or %s", fieldName, setRaFile, itemTabFile);
}
}
/* Create/find submission record. */
submissionSetId = saveSubmissionSet(conn, raHash);
/* Process rest of tab file. */
while (lineFileNextRowTab(lf, words, rowSize))
{
int fullDir = cachedId(conn, "location", "name",
fullDirHash, "fullDir", raHash, rowHash, words);
int screenDir = cachedId(conn, "location", "name",
screenDirHash, "screenDir", raHash, rowHash, words);
int thumbDir = cachedId(conn, "location",
"name", thumbDirHash, "thumbDir", raHash, rowHash, words);
int bodyPart = cachedId(conn, "bodyPart",
"name", bodyPartHash, "bodyPart", raHash, rowHash, words);
int sliceType = cachedId(conn, "sliceType",
"name", sliceTypeHash, "sliceType", raHash, rowHash, words);
int imageType = cachedId(conn, "imageType",
"name", imageTypeHash, "imageType", raHash, rowHash, words);
int treatment = cachedId(conn, "treatment",
"conditions", treatmentHash, "treatment", raHash, rowHash, words);
char *fileName = getVal("fileName", raHash, rowHash, words, NULL);
char *submitId = getVal("submitId", raHash, rowHash, words, NULL);
char *taxon = getVal("taxon", raHash, rowHash, words, NULL);
char *isEmbryo = getVal("isEmbryo", raHash, rowHash, words, NULL);
char *age = getVal("age", raHash, rowHash, words, NULL);
char *sectionSet = getVal("sectionSet", raHash, rowHash, words, "");
char *sectionIx = getVal("sectionIx", raHash, rowHash, words, "0");
char *gene = getVal("gene", raHash, rowHash, words, "");
char *locusLink = getVal("locusLink", raHash, rowHash, words, "");
char *refSeq = getVal("refSeq", raHash, rowHash, words, "");
char *genbank = getVal("genbank", raHash, rowHash, words, "");
char *priority = getVal("priority", raHash, rowHash, words, "200");
int sectionId = 0;
int oldId;
// char *xzy = getVal("xzy", raHash, rowHash, words, xzy);
if (sectionSet[0] != 0 && !sameString(sectionSet, "0"))
{
struct hashEl *hel = hashLookup(sectionSetHash, sectionSet);
if (hel != NULL)
sectionId = ptToInt(hel->val);
else
{
sqlUpdate(conn, "NOSQLINJ insert into sectionSet values(default)");
sectionId = sqlLastAutoId(conn);
hashAdd(sectionSetHash, sectionSet, intToPt(sectionId));
}
}
dyStringClear(dy);
sqlDyStringAppend(dy, "select id from image ");
dyStringPrintf(dy, "where fileName = '%s' ", fileName);
dyStringPrintf(dy, "and fullLocation = %d", fullDir);
oldId = sqlQuickNum(conn, dy->string);
if (oldId != 0)
{
if (replace)
{
dyStringClear(dy);
sqlDyStringPrintf(dy, "delete from image where id = %d", oldId);
sqlUpdate(conn, dy->string);
}
else
errAbort("%s is already in database line %d of %s",
fileName, lf->lineIx, lf->fileName);
}
dyStringClear(dy);
sqlDyStringAppend(dy, "insert into image set\n");
dyStringPrintf(dy, " id = default,\n");
dyStringPrintf(dy, " fileName = '%s',\n", fileName);
dyStringPrintf(dy, " fullLocation = %d,\n", fullDir);
dyStringPrintf(dy, " screenLocation = %d,\n", screenDir);
dyStringPrintf(dy, " thumbLocation = %d,\n", thumbDir);
dyStringPrintf(dy, " submissionSet = %d,\n", submissionSetId);
dyStringPrintf(dy, " sectionSet = %d,\n", sectionId);
dyStringPrintf(dy, " sectionIx = %s,\n", sectionIx);
dyStringPrintf(dy, " submitId = '%s',\n", submitId);
dyStringPrintf(dy, " gene = '%s',\n", gene);
dyStringPrintf(dy, " locusLink = '%s',\n", locusLink);
dyStringPrintf(dy, " refSeq = '%s',\n", refSeq);
dyStringPrintf(dy, " genbank = '%s',\n", genbank);
dyStringPrintf(dy, " priority = %s,\n", priority);
dyStringPrintf(dy, " taxon = %s,\n", taxon);
dyStringPrintf(dy, " isEmbryo = %s,\n", isEmbryo);
dyStringPrintf(dy, " age = %s,\n", age);
dyStringPrintf(dy, " bodyPart = %d,\n", bodyPart);
dyStringPrintf(dy, " sliceType = %d,\n", sliceType);
dyStringPrintf(dy, " imageType = %d,\n", imageType);
dyStringPrintf(dy, " treatment = %d\n", treatment);
sqlUpdate(conn, dy->string);
}
}
void endHandler(struct xap *xap, char *name)
/* Called at end of a tag */
{
struct table *table = xap->stack->object;
struct table *parentTable = xap->stack[1].object;
struct field *field;
struct fieldRef *fieldRef;
struct assocRef *assocRef;
char *text = skipLeadingSpaces(xap->stack->text->string);
char *primaryKeyVal = NULL;
struct assoc *assoc;
static struct dyString *uniq = NULL;
if (table->promoted) /* Simple case - copy text to parent table. */
{
for (fieldRef = table->parentKeys; fieldRef != NULL; fieldRef = fieldRef->next)
{
field = fieldRef->field;
if (field->table == parentTable)
{
struct dyString **parentContent = contentStack + table->fieldCount;
struct dyString *dy = parentContent[field->tablePos];
if (!field->isString && text[0] == 0)
text = "0";
dyStringAppend(dy, text);
break;
}
}
}
else
{
if (text[0] != 0)
{
field = hashFindVal(table->fieldHash, textField);
if (field == NULL)
errAbort("No text for %s expected in dtd", table->name);
dyStringAppendEscapedForTabFile(contentStack[field->tablePos], text);
}
/* Construct uniq string from fields, etc. */
if (uniq == NULL)
uniq = dyStringNew(0);
else
dyStringClear(uniq);
for (field = table->fieldList; field != NULL; field = field->next)
{
if (!(field->isPrimaryKey && field->isMadeUpKey))
{
struct dyString *dy = contentStack[field->tablePos];
if (dy->stringSize == 0 && !field->isString)
dyStringAppendC(dy, '0');
dyStringAppendN(uniq, dy->string, dy->stringSize);
dyStringAppendC(uniq, '\t');
}
}
for (assoc = table->assocList; assoc != NULL; assoc = assoc->next)
{
dyStringPrintf(uniq, "%p\t%s\t", assoc->f, assoc->childKey);
}
primaryKeyVal = hashFindVal(table->uniqHash, uniq->string);
if (primaryKeyVal == NULL)
{
struct dyString *priDy = contentStack[table->primaryKey->tablePos];
if (table->madeUpPrimary)
{
table->lastId += 1;
dyStringPrintf(priDy, "%d", table->lastId);
}
primaryKeyVal = priDy->string;
for (field = table->fieldList; field != NULL; field = field->next)
{
struct dyString *dy = contentStack[field->tablePos];
fprintf(table->tabFile, "%s", dy->string);
if (field->next != NULL)
fprintf(table->tabFile, "\t");
}
fprintf(table->tabFile, "\n");
hashAdd(table->uniqHash, uniq->string, cloneString(primaryKeyVal));
}
for (fieldRef = table->parentKeys; fieldRef != NULL; fieldRef = fieldRef->next)
{
field = fieldRef->field;
if (field->table == parentTable)
{
struct dyString **parentContent = contentStack + table->fieldCount;
struct dyString *dy = parentContent[field->tablePos];
dyStringAppend(dy, primaryKeyVal);
break;
}
}
for (assocRef = table->parentAssocs; assocRef != NULL;
assocRef = assocRef->next)
{
if (assocRef->parent == parentTable)
{
assoc = assocNew(assocRef->assoc->tabFile,
primaryKeyVal);
slAddHead(&parentTable->assocList, assoc);
}
}
slReverse(&table->assocList);
for (assoc = table->assocList; assoc != NULL; assoc = assoc->next)
fprintf(assoc->f, "%s\t%s\n", primaryKeyVal, assoc->childKey);
assocFreeList(&table->assocList);
}
contentStack += table->fieldCount;
}
char *webTimeStampedLinkToResource(char *fileName, boolean wrapInHtml)
// If wrapInHtml
// returns versioned link embedded in style or script html (free after use).
// else
// returns full path of a versioned path to the requested resource file (js, or css).
// NOTE: png, jpg and gif should also be supported but are untested.
//
// In production sites we use a versioned softlink that includes the CGI version. This has the following benefits:
// a) flushes user's web browser cache when the user visits a GB site whose version has changed since their last visit;
// b) enforces the requirement that static files are the same version as the CGIs (something that often fails to happen in mirrors).
// (see notes in redmine #3170).
//
// In dev trees we use mtime to create a pseudo-version; this forces web browsers to reload css/js file when it changes,
// so we don't get odd behavior that can be caused by caching of mis-matched javascript and style files in dev trees.
//
// In either case, the actual file has to have been previously created by running make in the appropriate directory (kent/src/hg/js
// or kent/src/hg/htdocs/style).
{
char baseName[PATH_LEN];
char extension[FILEEXT_LEN];
splitPath(fileName, NULL, baseName, extension);
boolean js = sameString(".js",extension);
boolean style = !js && sameString(".css",extension);
boolean image = !js
&& !style
&& ( sameString(".png",extension)
|| sameString(".jpg",extension)
|| sameString(".gif",extension));
if (!js && !style) // && !image) NOTE: This code has not been tested on images but should work.
errAbort("webTimeStampedLinkToResource: unknown resource type for %s.\n", fileName);
// Build and verify directory
char *dirName = "";
if (js)
dirName = cfgOptionDefault("browser.javaScriptDir", "js");
else if (style)
dirName = cfgOptionDefault("browser.styleDir","style");
else if (image)
dirName = cfgOptionDefault("browser.styleImagesDir","style/images");
struct dyString *fullDirName = NULL;
char *docRoot = hDocumentRoot();
if (docRoot != NULL)
fullDirName = dyStringCreate("%s/%s", docRoot, dirName);
else
// tolerate missing docRoot (i.e. when running from command line)
fullDirName = dyStringCreate("%s", dirName);
if (!fileExists(dyStringContents(fullDirName)))
errAbort("webTimeStampedLinkToResource: dir: %s doesn't exist.\n",
dyStringContents(fullDirName));
// build and verify real path to file
struct dyString *realFileName = dyStringCreate("%s/%s", dyStringContents(fullDirName), fileName);
if (!fileExists(dyStringContents(realFileName)))
errAbort("webTimeStampedLinkToResource: file: %s doesn't exist.\n",
dyStringContents(realFileName));
// build and verify link path including timestamp in the form of dir/baseName + timeStamp or CGI Version + ext
long mtime = fileModTime(dyStringContents(realFileName));
struct dyString *linkWithTimestamp;
char *scriptName = cgiScriptName();
if (scriptName == NULL)
scriptName = cloneString("");
boolean nonVersionedLinks = FALSE;
if (endsWith(scriptName, "qaPushQ"))
nonVersionedLinks = TRUE;
if (nonVersionedLinks)
linkWithTimestamp = dyStringCreate("%s/%s%s", dyStringContents(fullDirName), baseName, extension);
else if ((cfgOption("versionStamped") == NULL) && (hIsPreviewHost() || hIsPrivateHost()))
linkWithTimestamp = dyStringCreate("%s/%s-%ld%s", dyStringContents(fullDirName), baseName, mtime, extension);
else
linkWithTimestamp = dyStringCreate("%s/%s-v%s%s", dyStringContents(fullDirName), baseName, CGI_VERSION, extension);
if (hIsBrowserbox() && !fileExists(dyStringContents(linkWithTimestamp)))
// on the browserbox, both alpha and beta binaries can run
{
linkWithTimestamp = dyStringCreate("%s/%s-%ld%s", dyStringContents(fullDirName),
baseName, mtime, extension);
}
if (!fileExists(dyStringContents(linkWithTimestamp)))
errAbort("Cannot find correct version of file '%s'; this is due to an installation "
"error\n\nError details: %s does not exist", fileName, dyStringContents(linkWithTimestamp));
// Free up all that extra memory
dyStringFree(&realFileName);
dyStringFree(&fullDirName);
char *linkFull = dyStringCannibalize(&linkWithTimestamp);
char *link = linkFull;
if (docRoot != NULL)
{
link = cloneString(linkFull + strlen(docRoot) + 1);
freeMem(linkFull);
}
if (wrapInHtml) // wrapped for christmas
{
struct dyString *wrapped = dyStringNew(0);
if (js)
dyStringPrintf(wrapped,"<script type='text/javascript' SRC='../%s'></script>\n", link);
else if (style)
dyStringPrintf(wrapped,"<LINK rel='STYLESHEET' href='../%s' TYPE='text/css' />\n", link);
else // Will be image, since these are the only three choices allowed
dyStringPrintf(wrapped,"<IMG src='../%s' />\n", link);
freeMem(link);
link = dyStringCannibalize(&wrapped);
}
return link;
}
static struct slName *geneProbeList(struct sqlConnection *conn, int id)
/* Get list of gene names with hyperlinks to probe info page. */
{
struct slName *returnList = NULL;
char query[256], **row;
struct sqlResult *sr;
struct dyString *dy = dyStringNew(0);
struct probeAndColor *pcList = NULL, *pc;
int probeCount = 0;
/* Make up a list of all probes in this image. */
safef(query, sizeof(query),
"select probe,probeColor from imageProbe where image=%d", id);
sr = sqlGetResult(conn, query);
while ((row = sqlNextRow(sr)) != NULL)
{
AllocVar(pc);
pc->probe = sqlUnsigned(row[0]);
pc->probeColor = sqlUnsigned(row[1]);
slAddHead(&pcList, pc);
++probeCount;
}
slReverse(&pcList);
for (pc = pcList; pc != NULL; pc = pc->next)
{
int geneId;
char *geneName;
int probe = pc->probe;
char *geneUrl = NULL;
/* Get gene ID and name. */
safef(query, sizeof(query),
"select gene from probe where id = %d", probe);
geneId = sqlQuickNum(conn, query);
geneName = vgGeneNameFromId(conn, geneId);
/* Get url for known genes page if any. */
geneUrl = getKnownGeneUrl(conn, geneId);
/* Print gene name, surrounded by hyperlink to known genes
* page if possible. */
dyStringClear(dy);
if (geneUrl != NULL)
dyStringPrintf(dy, "<A HREF=\"%s\" target=_parent>",
geneUrl);
dyStringPrintf(dy, "%s", geneName);
if (geneUrl != NULL)
dyStringAppend(dy, "</A>");
freez(&geneName);
/* Add color if there's more than one probe for this image. */
if (probeCount > 1)
{
char *color;
safef(query, sizeof(query),
"select probeColor.name from probeColor "
"where probeColor.id = %d"
, pc->probeColor);
color = sqlQuickString(conn, query);
if (color != NULL)
dyStringPrintf(dy, " (%s)", color);
freez(&color);
}
/* Add to return list. */
slNameAddTail(&returnList, dy->string);
}
slFreeList(&pcList);
slReverse(&returnList);
return returnList;
}
char *menuBar(struct cart *cart, char *db)
// Return HTML for the menu bar (read from a configuration file);
// we fixup internal CGI's to add hgsid's and include the appropriate js and css files.
//
// Note this function is also called by hgTracks which extends the menu bar
// with a View menu defined in hgTracks/menu.c
{
char *docRoot = hDocumentRoot();
char *menuStr, buf[4096], uiVars[128];
FILE *fd;
char *navBarFile = "inc/globalNavBar.inc";
struct stat statBuf;
char *scriptName = cgiScriptName();
if (cart)
safef(uiVars, sizeof(uiVars), "%s=%s", cartSessionVarName(), cartSessionId(cart));
else
uiVars[0] = 0;
if(docRoot == NULL)
// tolerate missing docRoot (i.e. don't bother with menu when running from command line)
return NULL;
jsIncludeFile("jquery.js", NULL);
jsIncludeFile("jquery.plugins.js", NULL);
webIncludeResourceFile("nice_menu.css");
// Read in menu bar html
safef(buf, sizeof(buf), "%s/%s", docRoot, navBarFile);
fd = mustOpen(buf, "r");
fstat(fileno(fd), &statBuf);
int len = statBuf.st_size;
menuStr = needMem(len + 1);
mustRead(fd, menuStr, statBuf.st_size);
menuStr[len] = 0;
carefulClose(&fd);
if (cart)
{
char *newMenuStr = menuBarAddUiVars(menuStr, "/cgi-bin/hg", uiVars);
freez(&menuStr);
menuStr = newMenuStr;
}
if(scriptName)
{
// Provide hgTables options for some CGIs.
char hgTablesOptions[1024] = "";
char *track = (cart == NULL ? NULL :
(endsWith(scriptName, "hgGene") ?
cartOptionalString(cart, "hgg_type") :
cartOptionalString(cart, "g")));
if (track && cart && db &&
(endsWith(scriptName, "hgc") || endsWith(scriptName, "hgTrackUi") ||
endsWith(scriptName, "hgGene")))
{
struct trackDb *tdb = hTrackDbForTrack(db, track);
if (tdb)
{
struct trackDb *topLevel = trackDbTopLevelSelfOrParent(tdb);
safef(hgTablesOptions, sizeof hgTablesOptions,
"../cgi-bin/hgTables?hgta_doMainPage=1&hgta_group=%s&hgta_track=%s&hgta_table=%s&",
topLevel->grp, topLevel->track, tdb->table);
menuStr = replaceChars(menuStr, "../cgi-bin/hgTables?", hgTablesOptions);
trackDbFree(&tdb);
}
}
}
if(!loginSystemEnabled())
stripRegEx(menuStr, "<\\!-- LOGIN_START -->.*<\\!-- LOGIN_END -->", REG_ICASE);
if(scriptName)
{ // Provide optional official mirror servers menu items
char *geoMenu = geoMirrorMenu();
char *pattern = "<!-- OPTIONAL_MIRROR_MENU -->";
char *newMenuStr = replaceChars(menuStr, pattern, geoMenu);
freez(&menuStr);
menuStr = newMenuStr;
}
if(scriptName)
{
// Provide view menu for some CGIs.
struct dyString *viewItems = dyStringCreate("");
boolean hasViewMenu = TRUE;
if (endsWith(scriptName, "hgGenome"))
{
safef(buf, sizeof(buf), "../cgi-bin/hgGenome?%s&hgGenome_doPsOutput=1", uiVars);
dyStringPrintf(viewItems, "<li><a href='%s' id='%s'>%s</a></li>\n", buf, "pdfLink", "PDF/PS");
}
else
{
hasViewMenu = FALSE;
}
if (hasViewMenu)
{
struct dyString *viewMenu = dyStringCreate("<li class='menuparent' id='view'><span>View</span>\n<ul style='display: none; visibility: hidden;'>\n");
dyStringAppend(viewMenu, viewItems->string);
dyStringAppend(viewMenu, "</ul>\n</li>\n");
menuStr = replaceChars(menuStr, "<!-- OPTIONAL_VIEW_MENU -->", viewMenu->string);
dyStringFree(&viewMenu);
}
else if (!endsWith(scriptName, "hgTracks"))
{
replaceChars(menuStr, "<!-- OPTIONAL_VIEW_MENU -->", "");
}
dyStringFree(&viewItems);
}
if(scriptName)
{
// Provide context sensitive help links for some CGIs.
char *link = NULL;
char *label = NULL;
if (endsWith(scriptName, "hgBlat"))
{
link = "../goldenPath/help/hgTracksHelp.html#BLATAlign";
label = "Help on Blat";
}
else if (endsWith(scriptName, "hgHubConnect"))
{
link = "../goldenPath/help/hgTrackHubHelp.html";
label = "Help on Track Hubs";
}
else if (endsWith(scriptName, "hgNear"))
{
link = "../goldenPath/help/hgNearHelp.html";
label = "Help on Gene Sorter";
}
else if (endsWith(scriptName, "hgTables"))
{
link = "../goldenPath/help/hgTablesHelp.html";
label = "Help on Table Browser";
}
else if (endsWith(scriptName, "hgGenome"))
{
link = "../goldenPath/help/hgGenomeHelp.html";
label = "Help on Genome Graphs";
}
else if (endsWith(scriptName, "hgSession"))
{
link = "../goldenPath/help/hgSessionHelp.html";
label = "Help on Sessions";
}
else if (endsWith(scriptName, "hgVisiGene"))
{
link = "../goldenPath/help/hgTracksHelp.html#VisiGeneHelp";
label = "Help on VisiGene";
}
else if (endsWith(scriptName, "hgCustom"))
{
link = "../goldenPath/help/customTrack.html";
label = "Help on Custom Tracks";
}
// Don't overwrite any previously set defaults
if(!contextSpecificHelpLink && link)
contextSpecificHelpLink = link;
if(!contextSpecificHelpLabel && label)
contextSpecificHelpLabel = label;
}
if(contextSpecificHelpLink)
{
char buf[1024];
safef(buf, sizeof(buf), "<li><a href='%s'>%s</a></li>", contextSpecificHelpLink, contextSpecificHelpLabel);
menuStr = replaceChars(menuStr, "<!-- CONTEXT_SPECIFIC_HELP -->", buf);
}
return menuStr;
}
static struct slName *getProbeList(struct sqlConnection *conn, int id)
/* Get list of probes with hyperlinks to probe info page. */
{
struct slName *returnList = NULL;
char query[256];
char *sidUrl = cartSidUrlString(cart);
struct dyString *dy = dyStringNew(0);
struct slInt *probeList = NULL, *probe;
int submissionSource = 0;
/* Make up a list of all probes in this image. */
safef(query, sizeof(query),
"select probe from imageProbe where image=%d", id);
probeList = sqlQuickNumList(conn, query);
safef(query, sizeof(query),
"select submissionSet.submissionSource from image, submissionSet"
" where image.submissionSet = submissionSet.id and image.id=%d", id);
submissionSource = sqlQuickNum(conn, query);
for (probe = probeList; probe != NULL; probe = probe->next)
{
char *type;
/* Create hyperlink to probe page around gene name. */
dyStringClear(dy);
dyStringPrintf(dy, "<A HREF=\"%s?%s&%s=%d&%s=%d\" target=_parent>",
hgVisiGeneCgiName(), sidUrl, hgpDoProbe, probe->val, hgpSs, submissionSource);
safef(query, sizeof(query),
"select probeType.name from probeType,probe where probe.id = %d "
"and probe.probeType = probeType.id",
probe->val);
type = sqlQuickString(conn, query);
dyStringPrintf(dy, "%s", naForEmpty(type));
if (sameWord(type, "antibody"))
{
char *abName;
safef(query, sizeof(query),
"select antibody.name from probe,antibody "
"where probe.id = %d and probe.antibody = antibody.id"
, probe->val);
abName = sqlQuickString(conn, query);
if (abName != NULL)
{
dyStringPrintf(dy, " %s", abName);
freeMem(abName);
}
}
else if (sameWord(type, "RNA"))
{
safef(query, sizeof(query),
"select length(seq) from probe where id=%d", probe->val);
if (sqlQuickNum(conn, query) > 0)
dyStringPrintf(dy, " sequenced");
else
{
safef(query, sizeof(query),
"select length(fPrimer) from probe where id=%d", probe->val);
if (sqlQuickNum(conn, query) > 0)
dyStringPrintf(dy, " from primers");
}
}
else if (sameWord(type, "BAC"))
{
char *name;
safef(query, sizeof(query),
"select bac.name from probe,bac "
"where probe.id = %d and probe.bac = bac.id"
, probe->val);
name = sqlQuickString(conn, query);
if (name != NULL)
{
dyStringPrintf(dy, " %s", name);
freeMem(name);
}
}
dyStringPrintf(dy, "</A>");
freez(&type);
/* Add to return list. */
slNameAddTail(&returnList, dy->string);
}
slFreeList(&probeList);
slReverse(&returnList);
return returnList;
}
int main(int argc, char *argv[])
{
long enteredMainTime = clock1000();
struct dyString *output = newDyString(10000);
setUdcCacheDir();
cgiSpoof(&argc, argv);
pushWarnHandler(htmlVaBadRequestAbort);
pushAbortHandler(htmlVaBadRequestAbort);
char *database = cgiString("db");
char *cmd = cgiString("cmd");
char *jsonp = cgiOptionalString("jsonp");
if (!hDbExists(database))
errAbort("Invalid database '%s'", database);
if (!strcmp(cmd, "defaultPos"))
{
dyStringPrintf(output, "{\"pos\": \"%s\"}", hDefaultPos(database));
}
else if (!strcmp(cmd, "metaDb"))
{
// Return list of values for given metaDb var
// e.g. http://genome.ucsc.edu/hgApi?db=hg18&cmd=metaDb&var=cell
struct sqlConnection *conn = hAllocConn(database);
boolean metaDbExists = sqlTableExists(conn, "metaDb");
if (metaDbExists)
{
char *var = cgiOptionalString("var");
if (!var)
errAbort("Missing var parameter");
boolean fileSearch = (cgiOptionalInt("fileSearch",0) == 1);
struct slPair *pairs = mdbValLabelSearch(conn, var, MDB_VAL_STD_TRUNCATION, FALSE,
!fileSearch, fileSearch);
struct slPair *pair;
dyStringPrintf(output, "[\n");
for (pair = pairs; pair != NULL; pair = pair->next)
{
if (pair != pairs)
dyStringPrintf(output, ",\n");
dyStringPrintf(output, "['%s','%s']", javaScriptLiteralEncode(mdbPairLabel(pair)),
javaScriptLiteralEncode(mdbPairVal(pair)));
}
dyStringPrintf(output, "\n]\n");
}
else
errAbort("Assembly does not support metaDb");
}
// TODO: move to lib since hgTracks and hgApi share
#define METADATA_VALUE_PREFIX "hgt_mdbVal"
else if (startsWith(METADATA_VALUE_PREFIX, cmd))
{
// Returns metaDb value control: drop down or free text, with or without help link.
// e.g. http://genome.ucsc.edu/hgApi?db=hg18&cmd=hgt_mdbVal3&var=cell
// TODO: Move guts to lib, so that hgTracks::searchTracks.c and hgApi.c can share
struct sqlConnection *conn = hAllocConn(database);
boolean metaDbExists = sqlTableExists(conn, "metaDb");
if (metaDbExists)
{
char *var = cgiOptionalString("var");
if (!var)
errAbort("Missing var parameter");
int ix = atoi(cmd+strlen(METADATA_VALUE_PREFIX)); // 1 based index
if (ix == 0) //
errAbort("Unsupported 'cmd' parameter");
enum cvSearchable searchBy = cvSearchMethod(var);
char name[128];
safef(name,sizeof name,"%s%i",METADATA_VALUE_PREFIX,ix);
if (searchBy == cvSearchBySingleSelect || searchBy == cvSearchByMultiSelect)
{
boolean fileSearch = (cgiOptionalInt("fileSearch",0) == 1);
struct slPair *pairs = mdbValLabelSearch(conn, var, MDB_VAL_STD_TRUNCATION, FALSE,
!fileSearch, fileSearch);
if (slCount(pairs) > 0)
{
char *dropDownHtml =
cgiMakeSelectDropList((searchBy == cvSearchByMultiSelect),
name, pairs, NULL, ANYLABEL, "mdbVal",
"style='min-width: 200px; font-size: .9em;' "
"onchange='findTracksMdbValChanged(this);'");
if (dropDownHtml)
{
dyStringAppend(output,dropDownHtml);
freeMem(dropDownHtml);
}
slPairFreeList(&pairs);
}
}
else if (searchBy == cvSearchByFreeText)
{
dyStringPrintf(output,"<input type='text' name='%s' value='' class='mdbVal freeText' "
"onchange='findTracksMdbValChanged(this);' style='max-width:310px; "
"width:310px; font-size:.9em;'>", name);
}
else if (searchBy == cvSearchByWildList)
{
dyStringPrintf(output,"<input type='text' name='%s' value='' class='mdbVal wildList' "
"title='enter comma separated list of values' "
"onchange='findTracksMdbValChanged(this);' style='max-width:310px; "
"width:310px; font-size:.9em;'>", name);
}
else if (searchBy == cvSearchByDateRange || searchBy == cvSearchByIntegerRange)
{
// TO BE IMPLEMENTED
}
else
errAbort("Metadata variable not searchable");
dyStringPrintf(output,"<span id='helpLink%i'> </span>",ix);
}
else
errAbort("Assembly does not support metaDb");
}
else if (!strcmp(cmd, "tableMetadata"))
{ // returns an html table with metadata for a given track
char *trackName = cgiOptionalString("track");
boolean showLonglabel = (NULL != cgiOptionalString("showLonglabel"));
boolean showShortLabel = (NULL != cgiOptionalString("showShortLabel"));
if (trackName != NULL)
{
// hTrackDbForTrackAndAncestors avoids overhead of getting whole track list!
struct trackDb *tdb = hTrackDbForTrackAndAncestors(database, trackName);
if (tdb != NULL)
{
char * html = metadataAsHtmlTable(database,tdb,showLonglabel,showShortLabel);
if (html)
{
dyStringAppend(output,html);
freeMem(html);
}
else
dyStringPrintf(output,"No metadata found for track %s.",trackName);
}
else
dyStringPrintf(output,"Track %s not found",trackName);
}
else
dyStringAppend(output,"No track variable found");
}
else if (sameString(cmd, "codonToPos") || sameString(cmd, "exonToPos"))
{
char query[256];
struct sqlResult *sr;
char **row;
struct genePred *gp;
char *name = cgiString("name");
char *table = cgiString("table");
int num = cgiInt("num");
struct sqlConnection *conn = hAllocConn(database);
sqlSafef(query, sizeof(query), "select name, chrom, strand, txStart, txEnd, cdsStart, cdsEnd, exonCount, exonStarts, exonEnds from %s where name = '%s'", table, name);
sr = sqlGetResult(conn, query);
if ((row = sqlNextRow(sr)) != NULL)
{
gp = genePredLoad(row);
boolean found;
int start, end;
if (sameString(cmd, "codonToPos"))
found = codonToPos(gp, num, &start, &end);
else
found = exonToPos(gp, num, &start, &end);
if (found)
dyStringPrintf(output, "{\"pos\": \"%s:%d-%d\"}", gp->chrom, start + 1, end);
else
dyStringPrintf(output, "{\"error\": \"%d is an invalid %s for this gene\"}", num, sameString(cmd, "codonToPos") ? "codon" : "exon");
}
else
dyStringPrintf(output, "{\"error\": \"Couldn't find item: %s\"}", name);
sqlFreeResult(&sr);
hFreeConn(&conn);
}
else
{
warn("unknown cmd: %s",cmd);
errAbort("Unsupported 'cmd' parameter");
}
apiOut(dyStringContents(output), jsonp);
cgiExitTime("hgApi", enteredMainTime);
return 0;
}
char *getKnownGeneUrl(struct sqlConnection *conn, int geneId)
/* Given gene ID, try and find known gene on browser in same
* species. */
{
char query[256];
int taxon;
char *url = NULL;
char *genomeDb = NULL;
/* Figure out taxon. */
safef(query, sizeof(query),
"select taxon from gene where id = %d", geneId);
taxon = sqlQuickNum(conn, query);
genomeDb = hDbForTaxon(conn, taxon);
if (genomeDb != NULL)
{
/* Make sure known genes track exists - we may need
* to tweak this at some point for model organisms. */
safef(query, sizeof(query), "%s.knownToVisiGene", genomeDb);
if (!sqlTableExists(conn, query))
genomeDb = NULL;
}
/* If no db for that organism revert to human. */
if (genomeDb == NULL)
genomeDb = hDefaultDb();
safef(query, sizeof(query), "%s.knownToVisiGene", genomeDb);
if (sqlTableExists(conn, query))
{
struct dyString *dy = dyStringNew(0);
char *knownGene = NULL;
if (sqlCountColumnsInTable(conn, query) == 3)
{
dyStringPrintf(dy,
"select name from %s.knownToVisiGene where geneId = %d", genomeDb, geneId);
}
else
{
struct slName *imageList, *image;
safef(query, sizeof(query),
"select imageProbe.image from probe,imageProbe "
"where probe.gene=%d and imageProbe.probe=probe.id", geneId);
imageList = sqlQuickList(conn, query);
if (imageList != NULL)
{
dyStringPrintf(dy,
"select name from %s.knownToVisiGene ", genomeDb);
dyStringAppend(dy,
"where value in(");
for (image = imageList; image != NULL; image = image->next)
{
dyStringPrintf(dy, "'%s'", image->name);
if (image->next != NULL)
dyStringAppendC(dy, ',');
}
dyStringAppend(dy, ")");
slFreeList(&imageList);
}
}
if (dy->stringSize > 0)
{
knownGene = sqlQuickString(conn, dy->string);
if (knownGene != NULL)
{
dyStringClear(dy);
dyStringPrintf(dy, "../cgi-bin/hgGene?db=%s&hgg_gene=%s&hgg_chrom=none",
genomeDb, knownGene);
url = dyStringCannibalize(&dy);
}
}
dyStringFree(&dy);
}
freez(&genomeDb);
return url;
}
static void loadDatabase(char *database, char *track, int bedSize, struct bedStub *bedList)
/* Load database from bedList. */
{
struct sqlConnection *conn;
struct dyString *dy = newDyString(1024);
char *tab = (char *)NULL;
int loadOptions = (optionExists("onServer") ? SQL_TAB_FILE_ON_SERVER : 0);
if ( ! noLoad )
conn = sqlConnect(database);
if ((char *)NULL != tmpDir)
tab = cloneString(rTempName(tmpDir,"loadBed",".tab"));
else
tab = cloneString("bed.tab");
if (bedDetail && sqlTable == NULL && !customTrackLoader)
errAbort("bedDetail format requires sqlTable option");
if (bedDetail && !strictTab)
errAbort("bedDetail format must be tab separated");
if (bedDetail && !noBin)
noBin = TRUE;
/* First make table definition. */
if (sqlTable != NULL && !oldTable)
{
/* Read from file. */
char *sql, *s;
readInGulp(sqlTable, &sql, NULL);
/* Chop off end-of-statement semicolon if need be. */
s = strchr(sql, ';');
if (s != NULL) *s = 0;
if ( !noLoad )
{
if (renameSqlTable)
{
char *pos = stringIn("CREATE TABLE ", sql);
if (pos == NULL)
errAbort("Can't find CREATE TABLE in %s\n", sqlTable);
char *oldSql = cloneString(sql);
nextWord(&pos); nextWord(&pos);
char *tableName = nextWord(&pos);
sql = replaceChars(oldSql, tableName, track);
}
verbose(1, "Creating table definition for %s from sql: %s\n", track, sqlTable);
// add NOSQLINJ tag
sqlDyStringPrintf(dy, "%-s", sql);
sqlRemakeTable(conn, track, dy->string);
if (!noBin)
addBinToEmptyTable(conn, track);
adjustSqlTableColumns(conn, track, bedSize);
}
freez(&sql);
}
else if (!oldTable)
{
int minLength;
if (noLoad)
minLength=6;
else if (maxChromNameLength)
minLength = maxChromNameLength;
else
minLength = hGetMinIndexLength(database);
verbose(2, "INDEX chrom length: %d\n", minLength);
/* Create definition statement. */
verbose(1, "Creating table definition for %s, bedSize: %d\n", track, bedSize);
sqlDyStringPrintf(dy, "CREATE TABLE %s (\n", track);
if (!noBin)
dyStringAppend(dy, " bin smallint unsigned not null,\n");
dyStringAppend(dy, " chrom varchar(255) not null,\n");
dyStringAppend(dy, " chromStart int unsigned not null,\n");
dyStringAppend(dy, " chromEnd int unsigned not null,\n");
if (bedSize >= 4)
maybeBedGraph(4, dy, " name varchar(255) not null,\n");
if (bedSize >= 5)
{
if (allowNegativeScores)
maybeBedGraph(5, dy, " score int not null,\n");
else
maybeBedGraph(5, dy, " score int unsigned not null,\n");
}
if (bedSize >= 6)
maybeBedGraph(6, dy, " strand char(1) not null,\n");
if (bedSize >= 7)
maybeBedGraph(7, dy, " thickStart int unsigned not null,\n");
if (bedSize >= 8)
maybeBedGraph(8, dy, " thickEnd int unsigned not null,\n");
/* As of 2004-11-22 the reserved field is used as itemRgb in code */
if (bedSize >= 9)
maybeBedGraph(9, dy, " reserved int unsigned not null,\n");
if (bedSize >= 10)
maybeBedGraph(10, dy, " blockCount int unsigned not null,\n");
if (bedSize >= 11)
maybeBedGraph(11, dy, " blockSizes longblob not null,\n");
if (bedSize >= 12)
maybeBedGraph(12, dy, " chromStarts longblob not null,\n");
if (bedSize >= 13)
maybeBedGraph(13, dy, " expCount int unsigned not null,\n");
if (bedSize >= 14)
maybeBedGraph(14, dy, " expIds longblob not null,\n");
if (bedSize >= 15)
maybeBedGraph(15, dy, " expScores longblob not null,\n");
dyStringAppend(dy, "#Indices\n");
if (nameIx && (bedSize >= 4) && (0 == bedGraph))
dyStringAppend(dy, " INDEX(name(16)),\n");
if (noBin)
{
dyStringPrintf(dy, " INDEX(chrom(%d),chromStart)\n", minLength);
}
else
{
dyStringPrintf(dy, " INDEX(chrom(%d),bin)\n", minLength);
}
dyStringAppend(dy, ")\n");
if (noLoad)
verbose(2,"%s", dy->string);
else
sqlRemakeTable(conn, track, dy->string);
}
verbose(1, "Saving %s\n", tab);
writeBedTab(tab, bedList);
if ( ! noLoad )
{
verbose(1, "Loading %s\n", database);
if (customTrackLoader)
sqlLoadTabFile(conn, tab, track, loadOptions|SQL_TAB_FILE_WARN_ON_WARN);
else
sqlLoadTabFile(conn, tab, track, loadOptions);
if (! noHistory)
hgHistoryComment(conn, "Add %d element(s) from bed list to %s table",
slCount(bedList), track);
if(fillInScoreColumn != NULL)
{
char query[500];
char buf[500];
struct sqlResult *sr;
sqlSafef(query, sizeof(query), "select sum(score) from %s", track);
if(sqlQuickQuery(conn, query, buf, sizeof(buf)))
{
unsigned sum = sqlUnsigned(buf);
if (!sum)
{
sqlSafef(query, sizeof(query), "select min(%s), max(%s) from %s", fillInScoreColumn, fillInScoreColumn, track);
if ((sr = sqlGetResult(conn, query)) != NULL)
{
char **row = sqlNextRow(sr);
if(row != NULL)
{
float min = sqlFloat(row[0]);
float max = sqlFloat(row[1]);
if ( !(max == -1 && min == -1)) // if score is -1 then ignore, as if it werent present
{
if (max == min || sameString(row[0],row[1])) // this will lead to 'inf' score value in SQL update causing an error
errAbort("Could not set score in table %s max(%s)=min(%s)=%s\n", track, fillInScoreColumn, fillInScoreColumn, row[0]);
sqlFreeResult(&sr);
// Calculate a, b s/t f(x) = ax + b maps min-max => minScore-1000
float a = (1000-minScore) / (max - min);
float b = 1000 - ((1000-minScore) * max) / (max - min);
sqlSafef(query, sizeof(query), "update %s set score = round((%f * %s) + %f)", track, a, fillInScoreColumn, b);
int changed = sqlUpdateRows(conn, query, NULL);
verbose(2, "update query: %s; changed: %d\n", query, changed);
}
else
{
sqlFreeResult(&sr);
verbose(2, "score not updated; all values for column %s are -1\n", fillInScoreColumn);
}
}
}
}
}
}
sqlDisconnect(&conn);
/* if temp dir specified, unlink file to make it disappear */
if ((char *)NULL != tmpDir)
unlink(tab);
}
else
verbose(1, "No load option selected, see file: %s\n", tab);
} /* static void loadDatabase() */
char *visiGeneHypertextGenotype(struct sqlConnection *conn, int id)
/* Return genotype of organism if any in nifty hypertext format. */
{
int genotypeId;
struct slName *geneIdList, *geneId;
char query[256];
struct dyString *html;
/* Look up genotype ID. */
safef(query, sizeof(query),
"select specimen.genotype from image,specimen "
"where image.id=%d and image.specimen = specimen.id", id);
genotypeId = sqlQuickNum(conn, query);
if (genotypeId == 0)
return NULL;
/* Get list of genes involved. */
safef(query, sizeof(query),
"select distinct allele.gene from genotypeAllele,allele "
"where genotypeAllele.genotype=%d "
"and genotypeAllele.allele = allele.id"
, genotypeId);
geneIdList = sqlQuickList(conn, query);
if (geneIdList == NULL)
return cloneString("wild type");
/* Loop through each gene adding information to html. */
html = dyStringNew(0);
for (geneId = geneIdList; geneId != NULL; geneId = geneId->next)
{
char *geneName;
struct slName *alleleList, *allele;
int alleleCount;
boolean needsSlash = FALSE;
/* Get gene name. */
safef(query, sizeof(query), "select name from gene where id=%s",
geneId->name);
geneName = sqlQuickString(conn, query);
if (geneName == NULL)
internalErr();
/* Process each allele of gene. */
safef(query, sizeof(query),
"select allele.name from genotypeAllele,allele "
"where genotypeAllele.genotype=%d "
"and genotypeAllele.allele = allele.id "
"and allele.gene=%s"
, genotypeId, geneId->name);
alleleList = sqlQuickList(conn, query);
alleleCount = slCount(alleleList);
for (allele = alleleList; allele != NULL; allele = allele->next)
{
char *simplifiedAllele = getSimplifiedAllele(geneName, allele->name);
int repCount = 1, rep;
if (alleleCount == 1)
repCount = 2;
for (rep = 0; rep < repCount; ++rep)
{
if (needsSlash)
dyStringAppendC(html, '/');
else
needsSlash = TRUE;
dyStringAppend(html, geneName);
dyStringPrintf(html, "<SUP>%s</SUP>", simplifiedAllele);
}
freeMem(simplifiedAllele);
}
if (geneId->next != NULL)
dyStringAppendC(html, ' ');
slFreeList(&alleleList);
freeMem(geneName);
}
slFreeList(&geneIdList);
return dyStringCannibalize(&html);
}
static void doSeqAndExtFile(struct sqlConnection *conn, char *db, char *table)
{
int rc = 0;
char cmd[256];
char path[256];
char bedPath[256];
char gbdbPath[256];
char *fname=NULL;
struct dyString *dy = dyStringNew(0);
dyStringClear(dy);
dyStringPrintf(dy,
"select distinct concat('vgPrb_',e.id), e.seq"
" from vgPrb e join %s.%s v"
" left join %s.seq s on s.acc = v.qName"
" where concat('vgPrb_',e.id) = v.qName"
" and s.acc is NULL"
" order by e.id"
, db, table, db);
rc = sqlSaveQuery(conn, dy->string, "vgPrbExt.fa", TRUE);
verbose(1,"rc = %d = count of sequences for vgPrbExt.fa, to use with %s track %s\n",rc,db,table);
if (rc > 0) /* can set any desired minimum */
{
safef(bedPath,sizeof(bedPath),"/cluster/data/%s/bed/visiGene/",db);
if (!fileExists(bedPath))
{
safef(cmd,sizeof(cmd),"mkdir %s",bedPath);
verbose(1,"%s\n",cmd); system(cmd);
}
safef(gbdbPath,sizeof(gbdbPath),"/gbdb/%s/visiGene/",db);
if (!fileExists(gbdbPath))
{
safef(cmd,sizeof(cmd),"mkdir %s",gbdbPath);
verbose(1,"%s\n",cmd); system(cmd);
}
while(1)
{
int i=0;
safef(path,sizeof(path),"%svgPrbExt_AAAAAA.fa",bedPath);
char *c = rStringIn("AAAAAA",path);
srand( (unsigned)time( NULL ) );
for(i=0;i<6;++i)
{
*c++ += (int) 26 * (rand() / (RAND_MAX + 1.0));
}
if (!fileExists(path))
break;
}
safef(cmd,sizeof(cmd),"cp vgPrbExt.fa %s",path);
verbose(1,"%s\n",cmd); system(cmd);
fname = rStringIn("/", path);
++fname;
safef(cmd,sizeof(cmd),"ln -s %s %s%s",path,gbdbPath,fname);
verbose(1,"%s\n",cmd); system(cmd);
safef(cmd,sizeof(cmd),"hgLoadSeq %s %s%s", db, gbdbPath,fname);
verbose(1,"%s\n",cmd); system(cmd);
}
dyStringFree(&dy);
}
void paraFlow(char *fileName, int pfArgc, char **pfArgv)
/* parse and dump. */
{
struct pfCompile *pfc;
struct pfParse *program, *module;
char baseDir[256], baseName[128], baseSuffix[64];
char defFile[PATH_LEN];
char *parseFile = "out.parse";
char *typeFile = "out.typed";
char *boundFile = "out.bound";
char *scopeFile = "out.scope";
char *foldedFile = "out.folded";
char *cFile = "out.c";
FILE *parseF = mustOpen(parseFile, "w");
FILE *typeF = mustOpen(typeFile, "w");
FILE *scopeF = mustOpen(scopeFile, "w");
FILE *boundF = mustOpen(boundFile, "w");
FILE *foldedF = mustOpen(foldedFile, "w");
if (endPhase < 0)
return;
verbose(2, "Phase 0 - initialization\n");
pfc = pfCompileNew();
getPaths(pfc);
splitPath(fileName, baseDir, baseName, baseSuffix);
pfc->baseDir = cloneString(baseDir);
safef(defFile, sizeof(defFile), "%s%s.pfh", baseDir, baseName);
if (endPhase < 1)
return ;
verbose(2, "Phase 1 - tokenizing\n");
pfTokenizeInto(pfc, baseDir, baseName);
if (endPhase < 2)
return;
verbose(2, "Phase 2 - parsing\n");
program = pfParseInto(pfc);
dumpParseTree(pfc, program, parseF);
carefulClose(&parseF);
if (endPhase < 3)
return;
verbose(2, "Phase 3 - binding names\n");
pfBindVars(pfc, program);
dumpParseTree(pfc, program, boundF);
carefulClose(&boundF);
if (endPhase < 4)
return;
verbose(2, "Phase 4 - type checking\n");
pfTypeCheck(pfc, &program);
dumpParseTree(pfc, program, typeF);
carefulClose(&typeF);
if (endPhase < 5)
return;
verbose(2, "Phase 5 - polymorphic, para, and flow checks\n");
checkPolymorphic(pfc, pfc->scopeRefList);
checkParaFlow(pfc, program);
printScopeInfo(scopeF, 0, program);
carefulClose(&scopeF);
if (endPhase < 6)
return;
verbose(2, "Phase 6 - constant folding\n");
pfConstFold(pfc, program);
dumpParseTree(pfc, program, foldedF);
if (optionExists("asm"))
{
struct dyString *gccFiles;
if (endPhase < 7)
return;
verbose(2, "Phase 7 - nothing\n");
if (endPhase < 8)
return;
verbose(2, "Phase 8 - Code generation\n");
pfc->backEnd = backEndFind("mac-pentium");
gccFiles = asmCoder(pfc, program, baseDir, baseName);
if (endPhase < 9)
return;
verbose(2, "Phase 9 - Assembling pentium code\n");
{
char *libName = hashMustFindVal(pfc->cfgHash,"runAsmLib");
struct dyString *dy = dyStringNew(0);
int err;
dyStringPrintf(dy, "gcc ");
dyStringPrintf(dy, "-I %s ", pfc->cIncludeDir);
dyStringPrintf(dy, "-o %s%s ", baseDir, baseName);
dyStringAppend(dy, gccFiles->string);
dyStringPrintf(dy, "%s ", libName);
dyStringPrintf(dy, " %s ", pfc->runtimeLib);
dyStringPrintf(dy, "%s ", pfc->jkwebLib);
verbose(2, "%s\n", dy->string);
err = system(dy->string);
if (err != 0)
errAbort("Couldn't assemble: %s", dy->string);
dyStringFree(&dy);
}
dyStringFree(&gccFiles);
}
else
{
verbose(2, "Phase 7 - nothing\n");
if (endPhase < 8)
return;
verbose(2, "Phase 8 - C code generation\n");
pfCodeC(pfc, program, baseDir, cFile);
verbose(2, "%d modules, %d tokens, %d parseNodes\n",
pfc->moduleHash->elCount, pfc->tkz->tokenCount, pfParseCount(program));
if (endPhase < 9)
return;
verbose(2, "Phase 9 - compiling C code\n");
/* Now run gcc on it. */
{
struct dyString *dy = dyStringNew(0);
int err;
for (module = program->children; module != NULL; module = module->next)
{
if (module->name[0] != '<' && module->type != pptModuleRef)
{
struct pfModule *mod = hashMustFindVal(pfc->moduleHash, module->name);
char *cName = replaceSuffix(mod->fileName, ".pf", ".c");
char *oName = replaceSuffix(mod->fileName, ".pf", ".o");
dyStringClear(dy);
dyStringAppend(dy, "gcc ");
dyStringAppend(dy, "-O ");
dyStringPrintf(dy, "-I %s ", pfc->cIncludeDir);
dyStringAppend(dy, "-c ");
dyStringAppend(dy, "-o ");
dyStringPrintf(dy, "%s ", oName);
dyStringPrintf(dy, "%s ", cName);
verbose(2, "%s\n", dy->string);
err = system(dy->string);
if (err != 0)
errAbort("Couldn't compile %s.c", module->name);
freeMem(oName);
freeMem(cName);
}
}
dyStringClear(dy);
dyStringAppend(dy, "gcc ");
dyStringAppend(dy, "-O ");
dyStringPrintf(dy, "-I %s ", pfc->cIncludeDir);
dyStringPrintf(dy, "-o %s%s ", baseDir, baseName);
dyStringPrintf(dy, "%s ", cFile);
for (module = program->children; module != NULL; module = module->next)
{
if (module->name[0] != '<')
{
struct pfModule *mod = hashMustFindVal(pfc->moduleHash, module->name);
char *suffix = (module->type == pptModuleRef ? ".pfh" : ".pf");
char *oName = replaceSuffix(mod->fileName, suffix, ".o");
dyStringPrintf(dy, "%s ", oName);
freeMem(oName);
}
}
dyStringPrintf(dy, " %s ", pfc->runtimeLib);
dyStringPrintf(dy, "%s ", pfc->jkwebLib);
dyStringAppend(dy, "-lpthread -lm");
verbose(2, "%s\n", dy->string);
err = system(dy->string);
if (err != 0)
errnoAbort("problem compiling:\n", dy->string);
dyStringFree(&dy);
}
}
if (endPhase < 10)
return;
verbose(2, "Phase 10 - execution\n");
/* Now go run program itself. */
{
struct dyString *dy = dyStringNew(0);
int err;
int i;
if (baseDir[0] == 0)
dyStringPrintf(dy, "./%s", baseName);
else
dyStringPrintf(dy, "%s%s", baseDir, baseName);
for (i=0; i<pfArgc; ++i)
{
dyStringAppendC(dy, ' ');
dyStringAppend(dy, pfArgv[i]);
}
err = system(dy->string);
if (err != 0)
errAbort("problem running %s", baseName);
dyStringFree(&dy);
}
}
static void processIsPcr(struct sqlConnection *conn, int taxon, char *db)
/* process isPcr results */
{
/* >NM_010919:371+1088 2 718bp CGCGGATCCAAGGACATCTTGGACCTTCCG CCCAAGCTTGCATGTGCTGCAGCGACTGCG */
struct dyString *dy = dyStringNew(0);
struct lineFile *lf = lineFileOpen("isPcr.fa", TRUE);
int lineSize;
char *line;
char *name;
char *dna;
char *word, *end;
char *tName;
int tStart;
int tEnd;
char *tStrand;
int probeid=0; /* really a vgPrb id */
boolean more = lineFileNext(lf, &line, &lineSize);
while(more)
{
if (line[0] != '>')
errAbort("unexpected error out of phase\n");
name = cloneString(line);
verbose(1,"name=%s\n",name);
dyStringClear(dy);
while((more=lineFileNext(lf, &line, &lineSize)))
{
if (line[0] == '>')
{
break;
}
dyStringAppend(dy,line);
}
dna = cloneString(dy->string);
word = name+1;
end = strchr(word,':');
tName = cloneStringZ(word,end-word);
word = end+1;
end = strchr(word,'+');
tStrand = "+";
if (!end)
{
end = strchr(word,'-');
tStrand = "-";
}
tStart = atoi(word);
word = end+1;
end = strchr(word,' ');
tEnd = atoi(word);
word = end+1;
end = strchr(word,' ');
probeid = atoi(word);
dyStringClear(dy);
dyStringPrintf(dy, "select count(*) from vgPrb where id=%d and state='new'",probeid);
if (sqlQuickNum(conn,dy->string)>0)
{
/* record exists and hasn't already been updated */
int vgPrb = findVgPrbBySeq(conn,dna,taxon);
if (vgPrb == 0)
{
dyStringClear(dy);
dyStringAppend(dy, "update vgPrb set");
dyStringAppend(dy, " seq='");
dyStringAppend(dy, dna);
dyStringAppend(dy, "',\n");
dyStringPrintf(dy, " tName='%s',\n", tName);
dyStringPrintf(dy, " tStart=%d,\n", tStart);
dyStringPrintf(dy, " tEnd=%d,\n", tEnd);
dyStringPrintf(dy, " tStrand='%s',\n", tStrand);
dyStringPrintf(dy, " db='%s',\n", db);
dyStringPrintf(dy, " state='%s'\n", "seq");
dyStringPrintf(dy, " where id=%d\n", probeid);
dyStringPrintf(dy, " and state='%s'\n", "new");
verbose(2, "%s\n", dy->string);
sqlUpdate(conn, dy->string);
}
else /* probe seq already exists */
{
/* just re-map the probe table recs to it */
dyStringClear(dy);
dyStringPrintf(dy, "update vgPrbMap set vgPrb=%d where vgPrb=%d",vgPrb,probeid);
sqlUpdate(conn, dy->string);
/* and delete it from vgPrb */
dyStringClear(dy);
dyStringPrintf(dy, "delete from vgPrb where id=%d",probeid);
sqlUpdate(conn, dy->string);
}
}
freez(&tName);
freez(&name);
freez(&dna);
}
lineFileClose(&lf);
dyStringFree(&dy);
}
int edwFileFetch(struct sqlConnection *conn, struct edwFile *ef, int fd,
char *submitFileName, unsigned submitId, unsigned submitDirId, unsigned hostId)
/* Fetch file and if successful update a bunch of the fields in ef with the result.
* Returns fileId. */
{
ef->id = makeNewEmptyFileRecord(conn, submitId, submitDirId, ef->submitFileName, ef->size);
/* Update edwSubmit with file in transit info */
char query[256];
sqlSafef(query, sizeof(query), "update edwSubmit set fileIdInTransit=%lld where id=%u",
(long long)ef->id, submitId);
sqlUpdate(conn, query);
sqlSafef(query, sizeof(query), "select paraFetchStreams from edwHost where id=%u", hostId);
int paraFetchStreams = sqlQuickNum(conn, query);
struct paraFetchInterruptContext interruptContext = {.conn=conn, .submitId=submitId};
/* Wrap getting the file, the actual data transfer, with an error catcher that
* will remove partly uploaded files. Perhaps some day we'll attempt to rescue
* ones that are just truncated by downloading the rest, but not now. */
struct errCatch *errCatch = errCatchNew();
char tempName[PATH_LEN] = "";
char edwFile[PATH_LEN] = "", edwPath[PATH_LEN];
if (errCatchStart(errCatch))
{
/* Now make temp file name and open temp file in an atomic operation */
char *tempDir = edwTempDir();
safef(tempName, PATH_LEN, "%sedwSubmitXXXXXX", tempDir);
int localFd = mustMkstemp(tempName);
/* Update file name in database with temp file name so web app can track us. */
char query[PATH_LEN+128];
sqlSafef(query, sizeof(query),
"update edwFile set edwFileName='%s' where id=%lld",
tempName + strlen(edwRootDir), (long long)ef->id);
sqlUpdate(conn, query);
/* Do actual upload tracking how long it takes. */
ef->startUploadTime = edwNow();
mustCloseFd(&localFd);
if (!parallelFetchInterruptable(submitFileName, tempName, paraFetchStreams, 4, FALSE, FALSE,
paraFetchInterruptFunction, &interruptContext))
{
if (interruptContext.isInterrupted)
errAbort("Submission stopped by user.");
else
errAbort("parallel fetch of %s failed", submitFileName);
}
ef->endUploadTime = edwNow();
/* Rename file both in file system and (via ef) database. */
edwMakeFileNameAndPath(ef->id, submitFileName, edwFile, edwPath);
mustRename(tempName, edwPath);
if (endsWith(edwPath, ".gz") && !encode3IsGzipped(edwPath))
errAbort("%s has .gz suffix, but is not gzipped", submitFileName);
ef->edwFileName = cloneString(edwFile);
}
errCatchEnd(errCatch);
if (errCatch->gotError)
{
/* Attempt to remove any partial file. */
if (tempName[0] != 0)
{
verbose(1, "Removing partial %s\n", tempName);
parallelFetchRemovePartial(tempName);
remove(tempName);
}
handleSubmitError(conn, submitId, errCatch->message->string); // Throws further
assert(FALSE); // We never get here
}
errCatchFree(&errCatch);
/* Now we got the file. We'll go ahead and save the file name and stuff. */
sqlSafef(query, sizeof(query),
"update edwFile set"
" edwFileName='%s', startUploadTime=%lld, endUploadTime=%lld"
" where id = %d"
, ef->edwFileName, ef->startUploadTime, ef->endUploadTime, ef->id);
sqlUpdate(conn, query);
/* Wrap the validations in an error catcher that will save error to file table in database */
errCatch = errCatchNew();
boolean success = FALSE;
if (errCatchStart(errCatch))
{
/* Check MD5 sum here. */
unsigned char md5bin[16];
md5ForFile(edwPath, md5bin);
char md5[33];
hexBinaryString(md5bin, sizeof(md5bin), md5, sizeof(md5));
if (!sameWord(md5, ef->md5))
errAbort("%s has md5 mismatch: %s != %s. File may be corrupted in upload, or file may have "
"been changed since validateManifest was run. Please check that md5 of file "
"before upload is really %s. If it is then try submitting again, otherwise "
"rerun validateManifest and then try submitting again. \n",
ef->submitFileName, ef->md5, md5, ef->md5);
/* Finish updating a bunch more of edwFile record. Note there is a requirement in
* the validFile section that ef->updateTime be updated last. A nonzero ef->updateTime
* is used as a sign of record complete. */
struct dyString *dy = dyStringNew(0); /* Includes tag so query may be long */
sqlDyStringPrintf(dy, "update edwFile set md5='%s',size=%lld,updateTime=%lld",
md5, ef->size, ef->updateTime);
dyStringAppend(dy, ", tags='");
dyStringAppend(dy, ef->tags);
dyStringPrintf(dy, "' where id=%d", ef->id);
sqlUpdate(conn, dy->string);
dyStringFree(&dy);
/* Update edwSubmit so file no longer shown as in transit */
sqlSafef(query, sizeof(query), "update edwSubmit set fileIdInTransit=0 where id=%u", submitId);
sqlUpdate(conn, query);
success = TRUE;
}
errCatchEnd(errCatch);
if (errCatch->gotError)
{
handleFileError(conn, submitId, ef->id, errCatch->message->string);
}
return ef->id;
}
static void doPrimers(struct sqlConnection *conn, int taxon, char *db)
/* get probe seq from primers */
{
int rc = 0;
struct dyString *dy = dyStringNew(0);
char cmdLine[256];
char path1[256];
char path2[256];
dyStringClear(dy);
dyStringAppend(dy, "select e.id, p.fPrimer, p.rPrimer from probe p, vgPrbMap m, vgPrb e, gene g");
dyStringPrintf(dy, " where p.id = m.probe and m.vgPrb = e.id and g.id = p.gene and g.taxon = %d",taxon);
dyStringAppend(dy, " and e.state = 'new' and e.type='primersMrna'");
rc = sqlSaveQuery(conn, dy->string, "primers.query", FALSE);
verbose(1,"rc = %d = count of primers for mrna search for taxon %d\n",rc,taxon);
if (rc > 0) /* something to do */
{
dyStringClear(dy);
dyStringPrintf(dy, "select qName from %s.all_mrna",db);
rc = 0;
rc = sqlSaveQuery(conn, dy->string, "accFile.txt", FALSE);
safef(cmdLine,sizeof(cmdLine),"getRna %s accFile.txt mrna.fa",db);
system("date"); verbose(1,"cmdLine: [%s]\n",cmdLine); system(cmdLine); system("date");
verbose(1,"rc = %d = count of mrna for %s\n",rc,db);
system("date"); system("isPcr mrna.fa primers.query isPcr.fa -out=fa"); system("date");
system("ls -l");
processIsPcr(conn,taxon,db);
unlink("mrna.fa"); unlink("accFile.txt"); unlink("isPcr.fa");
}
unlink("primers.query");
/* find any remaining type primersMrna that couldn't be resolved and demote
* them to type primersGenome
*/
dyStringClear(dy);
dyStringAppend(dy, "update vgPrb set type='primersGenome'");
dyStringPrintf(dy, " where taxon = %d",taxon);
dyStringAppend(dy, " and state = 'new' and type='primersMrna'");
sqlUpdate(conn, dy->string);
/* get primers for those probes that did not find mrna isPcr matches
* and then do them against the genome instead */
dyStringClear(dy);
dyStringAppend(dy, "select e.id, p.fPrimer, p.rPrimer from probe p, vgPrbMap m, vgPrb e, gene g");
dyStringPrintf(dy, " where p.id = m.probe and m.vgPrb = e.id and g.id = p.gene and g.taxon = %d",taxon);
dyStringAppend(dy, " and e.state = 'new' and e.type='primersGenome'");
rc = 0;
rc = sqlSaveQuery(conn, dy->string, "primers.query", FALSE);
verbose(1,"rc = %d = count of primers for genome search for taxon %d\n",rc,taxon);
if (rc > 0) /* something to do */
{
safef(path1,sizeof(path1),"/gbdb/%s/%s.2bit",db,db);
safef(path2,sizeof(path2),"%s/%s.2bit",getCurrentDir(),db);
verbose(1,"copy: [%s] to [%s]\n",path1,path2); copyFile(path1,path2);
safef(cmdLine,sizeof(cmdLine),
"ssh kolossus 'cd %s; isPcr %s.2bit primers.query isPcr.fa -out=fa'",
getCurrentDir(),db);
system("date"); verbose(1,"cmdLine: [%s]\n",cmdLine); system(cmdLine); system("date");
safef(path2,sizeof(path2),"%s/%s.2bit",getCurrentDir(),db);
verbose(1,"rm %s\n",path2); unlink(path2); system("ls -l");
processIsPcr(conn,taxon,db);
unlink("mrna.fa"); unlink("accFile.txt"); unlink("isPcr.fa");
}
unlink("primers.query");
/* find any remaining type primersGenome that couldn't be resolved and demote
* them to type refSeq
*/
dyStringClear(dy);
dyStringAppend(dy, "update vgPrb set type='refSeq'");
dyStringPrintf(dy, " where taxon = %d",taxon);
dyStringAppend(dy, " and state = 'new' and type='primersGenome'");
sqlUpdate(conn, dy->string);
dyStringFree(&dy);
}
void jsTrackedVarCarryOver(struct dyString *dy, char *cgiVar, char *jsVar)
/* Carry over tracked variable (radio button?) to hidden form. */
{
dyStringPrintf(dy, "document.hiddenForm.%s.value=%s; ", cgiVar, jsVar);
}
static void asdDoQueryChunking(struct annoStreamDb *self, char *minChrom, uint minEnd)
/* Return a sqlResult for a query on table items in position range.
* If doing a whole genome query, just select all rows from table. */
{
struct annoStreamer *sSelf = &(self->streamer);
boolean hasWhere = FALSE;
struct dyString *query = self->makeBaselineQuery(self, &hasWhere);
if (sSelf->chrom != NULL && self->rowBuf.size > 0 && !self->doNextChunk)
{
// We're doing a region query, we already got some rows, and don't need another chunk:
resetRowBuf(&self->rowBuf);
self->eof = TRUE;
}
if (self->useMaxOutRows)
{
self->maxOutRows -= self->rowBuf.size;
if (self->maxOutRows <= 0)
self->eof = TRUE;
}
if (self->eof)
return;
int queryMaxItems = ASD_CHUNK_SIZE;
if (self->useMaxOutRows && self->maxOutRows < queryMaxItems)
queryMaxItems = self->maxOutRows;
if (self->hasBin)
{
// Results will be in bin order, but we can restore chromStart order by
// accumulating initial coarse-bin items and merge-sorting them with
// subsequent finest-bin items which will be in chromStart order.
if (self->doNextChunk && self->mergeBins && !self->gotFinestBin)
errAbort("annoStreamDb %s: can't continue merge in chunking query; "
"increase ASD_CHUNK_SIZE", sSelf->name);
self->mergeBins = TRUE;
if (self->qLm == NULL)
self->qLm = lmInit(0);
}
if (self->endFieldIndexName != NULL)
// Don't let mysql use a (chrom, chromEnd) index because that messes up
// sorting by chromStart.
sqlDyStringPrintf(query, " IGNORE INDEX (%s) ", self->endFieldIndexName);
if (sSelf->chrom != NULL)
{
uint start = sSelf->regionStart;
if (minChrom)
{
if (differentString(minChrom, sSelf->chrom))
errAbort("annoStreamDb %s: nextRow minChrom='%s' but region chrom='%s'",
sSelf->name, minChrom, sSelf->chrom);
if (start < minEnd)
start = minEnd;
}
if (self->doNextChunk && start < self->nextChunkStart)
start = self->nextChunkStart;
sqlDyStringAppend(query, hasWhere ? " and " : " where ");
sqlDyStringPrintf(query, "%s = '%s' and ", self->chromField, sSelf->chrom);
if (self->hasBin)
{
if (self->doNextChunk && self->gotFinestBin)
// It would be way more elegant to make a hAddBinTopLevelOnly but this will do:
dyStringPrintf(query, "bin > %d and ", self->minFinestBin);
hAddBinToQuery(start, sSelf->regionEnd, query);
}
if (self->doNextChunk)
sqlDyStringPrintf(query, "%s >= %u and ", self->startField, self->nextChunkStart);
sqlDyStringPrintf(query, "%s < %u and %s > %u ", self->startField, sSelf->regionEnd,
self->endField, start);
if (self->notSorted)
sqlDyStringPrintf(query, "order by %s ", self->startField);
sqlDyStringPrintf(query, "limit %d", queryMaxItems);
bufferRowsFromSqlQuery(self, query->string, queryMaxItems);
if (self->rowBuf.size == 0)
self->eof = TRUE;
}
else
{
// Genome-wide query: break it into chrom-by-chrom queries.
if (self->queryChrom == NULL)
self->queryChrom = self->chromList;
else if (!self->doNextChunk)
{
self->queryChrom = self->queryChrom->next;
resetMergeState(self);
}
if (minChrom != NULL)
{
// Skip chroms that precede minChrom
while (self->queryChrom != NULL && strcmp(self->queryChrom->name, minChrom) < 0)
{
self->queryChrom = self->queryChrom->next;
self->doNextChunk = FALSE;
resetMergeState(self);
}
if (self->hasBin)
{
self->mergeBins = TRUE;
if (self->qLm == NULL)
self->qLm = lmInit(0);
}
}
if (self->queryChrom == NULL)
self->eof = TRUE;
else
{
char *chrom = self->queryChrom->name;
int start = 0;
if (minChrom != NULL && sameString(chrom, minChrom))
start = minEnd;
if (self->doNextChunk && start < self->nextChunkStart)
start = self->nextChunkStart;
uint end = annoAssemblySeqSize(self->streamer.assembly, self->queryChrom->name);
sqlDyStringAppend(query, hasWhere ? " and " : " where ");
sqlDyStringPrintf(query, "%s = '%s' ", self->chromField, chrom);
if (start > 0 || self->doNextChunk)
{
dyStringAppend(query, "and ");
if (self->hasBin)
{
if (self->doNextChunk && self->gotFinestBin)
// It would be way more elegant to make a hAddBinTopLevelOnly but this will do:
dyStringPrintf(query, "bin > %d and ", self->minFinestBin);
hAddBinToQuery(start, end, query);
}
if (self->doNextChunk)
sqlDyStringPrintf(query, "%s >= %u and ", self->startField, self->nextChunkStart);
// region end is chromSize, so no need to constrain startField here:
sqlDyStringPrintf(query, "%s > %u ", self->endField, start);
}
if (self->notSorted)
sqlDyStringPrintf(query, "order by %s ", self->startField);
dyStringPrintf(query, "limit %d", queryMaxItems);
bufferRowsFromSqlQuery(self, query->string, queryMaxItems);
// If there happens to be no items on chrom, try again with the next chrom:
if (! self->eof && self->rowBuf.size == 0)
asdDoQueryChunking(self, minChrom, minEnd);
}
}
dyStringFree(&query);
}
void makeGoldAndGap(struct sqlConnection *conn, char *chromDir)
/* Read in .agp files in chromDir and use them to create the
* gold and gap tables for the corresponding chromosome(s). */
{
struct dyString *ds = newDyString(2048);
struct fileInfo *fiList, *fi;
char dir[256], chrom[128], ext[64];
char goldName[128], gapName[128];
char *agpName;
char *ptr;
char goldFileName[128];
char gapFileName[128];
if (! noLoad)
{
safef(goldFileName, ArraySize(goldFileName), "%s", goldTabName);
safef(gapFileName, ArraySize(gapFileName), "%s", gapTabName);
}
fiList = listDirX(chromDir, "*.agp", TRUE);
for (fi = fiList; fi != NULL; fi = fi->next)
{
/* Get full path name of .agp file and process it
* into table names. */
agpName = fi->name;
printf("Processing %s\n", agpName);
splitPath(agpName, dir, chrom, ext);
while ((ptr = strchr(chrom, '.')) != NULL)
*ptr = '_';
sprintf(goldName, "%s_gold", chrom);
sprintf(gapName, "%s_gap", chrom);
if (noLoad)
{
safef(goldFileName, ArraySize(goldFileName), "%s_gold.tab", chrom);
safef(gapFileName, ArraySize(gapFileName), "%s_gap.tab", chrom);
}
/* Create gold & gap tab separated files. */
splitAgp(fi->name, goldFileName, gapFileName);
/* Create gold table and load it up. */
dyStringClear(ds);
dyStringPrintf(ds, createGold, goldName);
dyStringPrintf(ds, goldSplitIndex, maxFragNameSize);
verbose(2, "%s", ds->string);
if (! noLoad)
sqlRemakeTable(conn, goldName, ds->string);
dyStringClear(ds);
dyStringPrintf(ds, "LOAD data local infile '%s' into table %s",
goldFileName, goldName);
if (! noLoad)
{
sqlUpdate(conn, ds->string);
remove(goldFileName);
}
/* Create gap table and load it up. */
dyStringClear(ds);
dyStringPrintf(ds, createGap, gapName);
dyStringAppend(ds, gapSplitIndex);
verbose(2, "%s", ds->string);
if (! noLoad)
{
sqlRemakeTable(conn, gapName, ds->string);
sqlMaybeMakeTable(conn, gapName, ds->string);
}
dyStringClear(ds);
dyStringPrintf(ds, "LOAD data local infile '%s' into table %s",
gapFileName, gapName);
if (! noLoad)
{
sqlUpdate(conn, ds->string);
remove(gapFileName);
}
}
freeDyString(&ds);
}