void getBackgroundStatus(char *url)
/* fetch status as the latest complete html block available */
{
char *html = NULL;
if (fileSize(url)==0)
{
htmlOpen("Background Status");
errAbort("No output found. Expecting output in [%s].", url);
htmlClose();
return;
}
readInGulp(url, &html, NULL);
int numLines = chopString(html, "\n", NULL, 1000000);
char **lines = NULL;
AllocArray(lines, numLines);
chopString(html, "\n", lines, numLines);
int end;
for (end=numLines-1; end >= 0 && ! (endsWith(lines[end], "</html>") || endsWith(lines[end], "</HTML>")) ; --end)
/* do nothing */ ;
if (end < 0)
{
htmlOpen("Background Status");
errAbort("No complete html found");
htmlClose();
return;
}
int start;
for (start=end; start >= 0 && ! (startsWith("<html>", lines[start]) || startsWith("<HTML>", lines[start])) ; --start)
/* do nothing */ ;
if (start < 0)
{
htmlOpen("Background Status");
errAbort("No html start tag found");
htmlClose();
return;
}
puts("Content-Type: text/html\n");
int line;
boolean autoRefreshFound = FALSE;
boolean successfullyUploaded = FALSE;
for (line=start; line <= end; line++)
{
puts(lines[line]);
if (startsWith("setTimeout(\"location = location;", lines[line]))
autoRefreshFound = TRUE;
if (startsWith("Output has been successfully uploaded", lines[line]))
successfullyUploaded = TRUE;
}
// if it looks like the background is no longer running,
// include the .err stdout output for more informative problem message
char urlErr[512];
char *textErr = NULL;
safef(urlErr, sizeof urlErr, "%s.err", url);
if (!autoRefreshFound && !successfullyUploaded && (fileSize(urlErr) > 0))
{
readInGulp(urlErr, &textErr, NULL);
printf("%s", textErr);
}
}
static void parseCustomTracks(char *db, char *inFile, char *trashFile)
/* parse tracks from input file, and also from trashfile if not null */
{
char *text;
struct customTrack *ctList = NULL, *oldCts = NULL;
readInGulp(inFile, &text, NULL);
/* read new CT's from input */
ctList = customFactoryParse(db, text, FALSE, NULL);
verbose(3, "parsed %d tracks from %s\n", slCount(ctList), inFile);
if (trashFile)
{
/* read old CT's from trash file */
oldCts = customFactoryParse(db, trashFile, TRUE, NULL);
/* merge old and new */
ctList = customTrackAddToList(ctList, oldCts, NULL, TRUE);
}
/* save to new trash file */
static struct tempName tn;
makeTempName(&tn, "ctTest", ".bed");
customTracksSaveFile(db, ctList, tn.forCgi);
/* reload from new trash file */
ctList = NULL;
ctList = customFactoryParse(db, tn.forCgi, TRUE, NULL);
customTracksSaveFile(db, ctList, "stdout");
/* cleanup */
unlink(tn.forCgi);
}
void rsyncEdwExpDataType(char *url, char *userId, char *password, char *outTab)
/* rsyncEdwExpDataType - Get experiment and data types from ENCODED via json.. */
{
char *jsonText = NULL;
if (jsonIn)
readInGulp(jsonIn, &jsonText, NULL);
else
jsonText = getJsonViaHttps(url, userId, password);
if (jsonOut)
writeGulp(jsonOut, jsonText, strlen(jsonText));
struct jsonElement *jsonRoot = jsonParse(jsonText);
char *expListName = "@graph";
struct jsonElement *jsonExpList = jsonMustFindNamedField(jsonRoot, "", expListName);
verbose(1, "Got @graph %p\n", jsonExpList);
struct slRef *ref, *refList = jsonListVal(jsonExpList, expListName);
verbose(1, "Got %d experiments\n", slCount(refList));
FILE *f = mustOpen(outTab, "w");
int realExpCount = 0;
for (ref = refList; ref != NULL; ref = ref->next)
{
struct jsonElement *el = ref->val;
char *acc = jsonStringField(el, "accession");
char *dataType = jsonStringField(el, "assay_term_name");
if (dataType != NULL)
{
fprintf(f, "%s\t%s\t%s\t%s\t%s\n", acc, dataType,
jsonOptionalStringField(el, "lab.title", ""),
jsonOptionalStringField(el, "biosample_term_name", ""),
jsonOptionalStringField(el, "award.rfa", ""));
++realExpCount;
}
}
verbose(1, "Got %d experiments with dataType\n", realExpCount);
carefulClose(&f);
}
void mafOrtholog(char *database, char *track, char *genePredFile, char *outFile)
/* mafOrtholog - find orthlogs in other species based on maf alignment and reference genePred */
{
struct slName *orgList = NULL;
FILE *f = mustOpen(outFile, "w");
struct genePredReader *gpr = genePredReaderFile(genePredFile, NULL);
struct genePred *gpList = genePredReaderAll(gpr), *gp = NULL;
struct sqlConnection *conn = hAllocConn();
if (optionExists("nibDir"))
nibDir = optionVal("nibDir", NULL);
if (optionExists("orgs"))
{
char *orgFile = optionVal("orgs", NULL);
char *buf;
readInGulp(orgFile, &buf, NULL);
orgList = stringToSlNames(buf);
}
for (gp = gpList ; gp != NULL ; gp=gp->next)
{
struct mafAli *maf = NULL;
if (thickOnly)
maf = mafLoadInRegion(conn, track,
gp->chrom, gp->cdsStart, gp->cdsEnd);
else
maf = mafLoadInRegion(conn, track,
gp->chrom, gp->txStart, gp->txEnd);
if (meFirst)
moveMeToFirst(maf, database);
printOrthologs(f, maf, gp);
mafAliFree(&maf);
}
carefulClose(&f);
}
void bwtFind(char *bwtFile, char *string)
/* bwtFind - Figure out if a string is in bwt transformed text.. */
{
char *bwtText;
size_t size;
readInGulp(bwtFile, &bwtText, &size);
printf("%s occurs %d times in %s\n", string, bwtSearch((UBYTE*)bwtText, size, string), bwtFile);
}
static char * readAutoUpgradeTableResult(char *tableName)
/* Read table upgrade result */
{
char path[AUTOUPGRPATHSIZE];
char *result = NULL;
makeResultName(tableName, path);
if (!fileExists(path))
return NULL; // There is no result yet
readInGulp(path, &result, NULL);
return result;
}
struct htmlPage *htmlPageGet(char *url)
/* Get page from URL (may be a file). */
{
if (fileExists(url))
{
char *buf;
readInGulp(url, &buf, NULL);
return htmlPageParseNoHead(url, buf);
}
else
return htmlPageGetWithCookies(url, NULL);
}
static void mafFrags(char *database, char *track, char *bedFile, char *mafFile)
/* mafFrags - Collect MAFs from regions specified in a 6 column bed file. */
{
struct slName *orgList = NULL;
struct lineFile *lf = lineFileOpen(bedFile, TRUE);
FILE *f = mustOpen(mafFile, "w");
if (optionExists("orgs"))
{
char *orgFile = optionVal("orgs", NULL);
char *buf;
readInGulp(orgFile, &buf, NULL);
orgList = stringToSlNames(buf);
/* Ensure that org list starts with database. */
struct slName *me = slNameFind(orgList, database);
if (me == NULL)
errAbort("Need to have reference database '%s' in %s", database, orgFile);
if (me != orgList)
{
slRemoveEl(&orgList, me);
slAddHead(&orgList, me);
}
}
mafWriteStart(f, "zero");
if (bed12)
{
char *row[12];
while (lineFileRow(lf, row))
{
struct bed *bed = bedLoadN(row, ArraySize(row));
struct mafAli *maf = mafFromBed12(database, track, bed, orgList);
if (meFirst)
moveMeToFirst(maf, bed->name);
mafWrite(f, maf);
mafAliFree(&maf);
bedFree(&bed);
}
}
else
{
char *row[6];
while (lineFileRow(lf, row))
{
struct bed *bed = bedLoadN(row, ArraySize(row));
processBed6(database, track, f, bed, orgList);
bedFree(&bed);
}
}
mafWriteEnd(f);
carefulClose(&f);
}
static void initTable(struct sqlConnection *conn, char *table, boolean nuke)
/* build tables */
{
char *sql = NULL;
char path[256];
if (nuke)
sqlDropTable(conn, table);
if (!sqlTableExists(conn, table))
{
safef(path,sizeof(path),"%s/%s.sql",sqlPath,table);
readInGulp(path, &sql, NULL);
sqlUpdate(conn, sql);
}
}
void replaceTextBetween(char *start, char *end, char *outerFile, char *middleFile)
/* replaceTextBetween - Replaces a section of text in the middle of a file.. */
{
/* Read outer file into memory. */
char *outer;
size_t outerSize;
readInGulp(outerFile, &outer, &outerSize);
/* Figure out the boundaries of the region we want to replace. */
char *s = stringIn(start, outer);
if (s == NULL)
errAbort("Can't find '%s' in %s", start, outerFile);
char *e = stringIn(end, s);
if (e == NULL)
errAbort("Can't find '%s' in %s", end, outerFile);
if (withEnds)
{
e += strlen(end);
}
else
{
s += strlen(start);
}
/* Read middle file into memory. */
char *middle;
size_t middleSize;
readInGulp(middleFile, &middle, &middleSize);
/* Write out file in three parts. */
int startSize = s - outer;
mustWrite(stdout, outer, startSize);
mustWrite(stdout, middle, middleSize);
int endSize = outer + outerSize - e;
mustWrite(stdout, e, endSize);
}
struct slName *keyFileList(struct column *col)
/* Make up list from key file for this column.
* return NULL if no key file. */
{
char *fileName = keyFileName(col);
char *buf;
struct slName *list;
if (fileName == NULL)
return NULL;
readInGulp(fileName, &buf, NULL);
list = stringToSlNames(buf);
freez(&buf);
return list;
}
void copyCnfToDefaultsFile(char *path, char *defaultFileName, int fileNo)
/* write the contents of the path to the fileNo */
{
if (fileExists(path))
{
char *cnf = NULL;
size_t cnfSize = 0;
readInGulp(path, &cnf, &cnfSize);
if (write (fileNo, cnf, cnfSize) == -1)
{
freeMem(cnf);
errAbort("Writing %s to file %s failed with errno %d", path, defaultFileName, errno);
}
freeMem(cnf);
}
}
static char *readHtmlRecursive(char *fileName, char *database)
/* Slurp in an html file. Wherever it contains insertHtmlRegex, recursively slurp that in
* and replace insertHtmlRegex with the contents. */
{
char *html;
readInGulp(fileName, &html, NULL);
if (isEmpty(html))
return html;
regmatch_t substrs[4];
while (regexMatchSubstr(html, insertHtmlRegex, substrs, ArraySize(substrs)))
{
struct dyString *dy = dyStringNew(0);
// All text before the regex match:
dyStringAppendN(dy, html, substrs[0].rm_so);
// Is there an #if before the #insert ?
boolean doInsert = TRUE;
if (substrs[1].rm_so != -1 &&
(! sameStringN(database, html+substrs[2].rm_so, (substrs[2].rm_eo - substrs[2].rm_so))))
doInsert = FALSE;
if (doInsert)
{
// Recursively pull in inserted file contents from relative path, replacing regex match:
char dir[PATH_LEN];
splitPath(fileName, dir, NULL, NULL);
char insertFileName[PATH_LEN+FILENAME_LEN];
safecpy(insertFileName, sizeof(insertFileName), dir);
safencat(insertFileName, sizeof(insertFileName), html+substrs[3].rm_so,
(substrs[3].rm_eo - substrs[3].rm_so));
if (!fileExists(insertFileName))
errAbort("readHtmlRecursive: relative path '%s' (#insert'ed in %s) not found",
insertFileName, fileName);
char *insertedText = readHtmlRecursive(insertFileName, database);
dyStringAppend(dy, insertedText);
freez(&insertedText);
}
// All text after the regex match:
dyStringAppend(dy, html+substrs[0].rm_eo);
freez(&html);
html = dyStringCannibalize(&dy);
}
return html;
}
void readAllWords(char *fileName, char ***retWords, int *retWordCount, char **retBuf)
/* Read in whole file and break it into words. You need to freeMem both
* *retWordCount and *retBuf when done. */
{
int wordCount;
char *buf = NULL;
char **words = NULL;
size_t bufSize;
readInGulp(fileName, &buf, &bufSize);
wordCount = chopByWhite(buf, NULL, 0);
if (wordCount != 0)
{
words = needMem(wordCount * sizeof(words[0]));
chopByWhite(buf, words, wordCount);
}
*retWords = words;
*retWordCount = wordCount;
*retBuf = buf;
}
static char *hAssemblyDescription(char *db)
/* Return a string containing db's description.html, or NULL if not found. */
//#*** LIBIFY: Code lifted from hgFind.c's hgPositionsHelpHtml.
{
char *htmlPath = hHtmlPath(db);
char *htmlString = NULL;
if (htmlPath != NULL)
{
if (fileExists(htmlPath))
readInGulp(htmlPath, &htmlString, NULL);
else if (startsWith("http://" , htmlPath) ||
startsWith("https://", htmlPath) ||
startsWith("ftp://" , htmlPath))
{
struct lineFile *lf = udcWrapShortLineFile(htmlPath, NULL, 256*1024);
htmlString = lineFileReadAll(lf);
lineFileClose(&lf);
}
}
return htmlString;
}
int main(int argc, char *argv[])
/* Process command line. */
{
optionInit(&argc, argv, options);
blockSize = optionInt("blockSize", blockSize);
itemsPerSlot = optionInt("itemsPerSlot", itemsPerSlot);
asFile = optionVal("as", asFile);
doCompress = !optionExists("unc");
sizesIs2Bit = optionExists("sizesIs2Bit");
extraIndex = optionVal("extraIndex", NULL);
tabSep = optionExists("tab");
udcDir = optionVal("udcDir", udcDefaultDir());
if (argc != 4)
usage();
udcSetDefaultDir(udcDir);
if (optionExists("type"))
{
// parse type
char *btype = cloneString(optionVal("type", ""));
char *plus = strchr(btype, '+');
if (plus)
{
*plus++ = 0;
if (isdigit(*plus))
bedP = sqlUnsigned(plus);
}
if (!startsWith("bed", btype))
errAbort("type must begin with \"bed\"");
btype +=3;
bedN = sqlUnsigned(btype);
if (bedN < 3)
errAbort("Bed must be 3 or higher, found %d\n", bedN);
if (bedN > 15)
errAbort("Bed must be 15 or lower, found %d\n", bedN);
}
else
{
if (asText)
errAbort("If you specify the .as file, you must specify the -type as well so that\n"
"the number of standard BED columns is known.");
}
/* If the haven't set bedN and bedP from the type var in the command line, then we sniff it
* out from file. */
char *bedFileName = argv[1];
if (bedN == 0)
{
/* Just read in single line and count fields. */
struct lineFile *lf = lineFileOpen(bedFileName, TRUE);
char *line;
if (!lineFileNextReal(lf, &line))
errAbort("%s is empty", lf->fileName);
int fieldCount;
if (tabSep)
fieldCount = chopByChar(line, '\t', NULL, 0); // Do not use chopString, see GOTCHA
else
fieldCount = chopByWhite(line, NULL, 0);
if (fieldCount > 256)
errAbort("Too many columns in %s, you sure it's a bed file?", lf->fileName);
lineFileClose(&lf);
/* Set up so that it looks like we are straight up bed for that many fields,
* or if more than or maximum defined fields, then for bed15+ */
bedN = fieldCount;
if (bedN > bedKnownFields)
{
bedP = bedN - bedKnownFields;
bedN = bedKnownFields;
}
}
/* Make sure that fields are defined, from bed spec if nowhere else. */
if (asFile)
readInGulp(asFile, &asText, NULL);
else
asText = bedAsDef(bedN, bedN + bedP);
bedToBigBed(bedFileName, argv[2], argv[3]);
optionFree();
if (verboseLevel() > 1)
printVmPeak();
return 0;
}
void loadDatabase(char *database, char *tab, char *track)
/* Load database from tab file. */
{
struct sqlConnection *conn = sqlConnect(database);
struct dyString *dy = newDyString(1024);
/* First make table definition. */
if (sqlTable != NULL)
{
/* Read from file. */
char *sql, *s;
readInGulp(sqlTable, &sql, NULL);
/* Chop of end-of-statement semicolon if need be. */
s = strchr(sql, ';');
if (s != NULL) *s = 0;
sqlRemakeTable(conn, track, sql);
freez(&sql);
}
else if (!oldTable)
{
/* Create definition statement. */
verbose(1, "Creating table definition for %s\n", track);
sqlDyStringPrintf(dy, "CREATE TABLE %s (\n", track);
if (!noBin)
dyStringAppend(dy, " bin smallint unsigned not null,\n");
dyStringAppend(dy, " level int unsigned not null,\n");
dyStringAppend(dy, " tName varchar(255) not null,\n");
dyStringAppend(dy, " tStart int unsigned not null,\n");
dyStringAppend(dy, " tEnd int unsigned not null,\n");
dyStringAppend(dy, " strand char(1) not null,\n");
dyStringAppend(dy, " qName varchar(255) not null,\n");
dyStringAppend(dy, " qStart int unsigned not null,\n");
dyStringAppend(dy, " qEnd int unsigned not null,\n");
dyStringAppend(dy, " chainId int unsigned not null,\n");
dyStringAppend(dy, " ali int unsigned not null,\n");
dyStringAppend(dy, " score double not null,\n");
dyStringAppend(dy, " qOver int not null, \n");
dyStringAppend(dy, " qFar int not null, \n");
dyStringAppend(dy, " qDup int not null, \n");
dyStringAppend(dy, " type varchar(255) not null,\n");
dyStringAppend(dy, " tN int not null, \n");
dyStringAppend(dy, " qN int not null, \n");
dyStringAppend(dy, " tR int not null, \n");
dyStringAppend(dy, " qR int not null, \n");
dyStringAppend(dy, " tNewR int not null, \n");
dyStringAppend(dy, " qNewR int not null, \n");
dyStringAppend(dy, " tOldR int not null, \n");
dyStringAppend(dy, " qOldR int not null, \n");
dyStringAppend(dy, " tTrf int not null, \n");
dyStringAppend(dy, " qTrf int not null, \n");
dyStringAppend(dy, "#Indices\n");
if (!noBin)
dyStringAppend(dy, " INDEX(tName(16),bin),\n");
dyStringAppend(dy, " INDEX(tName(16),tStart)\n");
dyStringAppend(dy, ")\n");
sqlRemakeTable(conn, track, dy->string);
}
dyStringClear(dy);
sqlDyStringPrintf(dy, "load data local infile '%s' into table %s", tab, track);
verbose(1, "Loading %s into %s\n", track, database);
sqlUpdate(conn, dy->string);
/* add a comment to the history table and finish up connection */
hgHistoryComment(conn, "Loaded net table %s", track);
sqlDisconnect(&conn);
}
int main(int argc, char *argv[])
{
char *com;
char **raFiles = NULL;
int raCount = 0;
char *expFile;
char *expression;
size_t size;
struct keyExp *exp;
int i;
if (argc < 4)
usage();
expFile = argv[1];
com = argv[2];
if (sameWord(com, "count"))
{
command = ctCount;
out = stdout;
raCount = argc-3;
raFiles = argv+3;
}
else if (sameWord(com, "print"))
{
if (argc < 5)
usage();
command = ctPrint;
selectKey = argv[3];
out = stdout;
raCount = argc-4;
raFiles = argv+4;
}
else if (sameWord(com, "save"))
{
if (argc < 6)
usage();
command = ctPrint;
selectKey = argv[3];
out = mustOpen(argv[4], "w");
raCount = argc-5;
raFiles = argv+5;
}
else if (sameWord(com, "stats"))
{
if (argc < 6)
usage();
command = ctStats;
selectKey = argv[3];
out = mustOpen(argv[4], "w");
raCount = argc-5;
raFiles = argv+5;
statHash = newHash(20);
}
else if (sameWord(com, "hist"))
{
if (argc < 6)
usage();
command = ctHist;
selectKey = argv[3];
out = mustOpen(argv[4], "w");
raCount = argc-5;
raFiles = argv+5;
statHash = newHash(18);
}
if (sameWord(expFile, "all"))
exp = NULL;
else
{
readInGulp(expFile, &expression, &size);
exp = keyExpParse(expression);
}
kvt = newKvt(128);
for (i=0; i<raCount; ++i)
{
scanFile(exp, raFiles[i]);
}
printf("%d matched %s\n", matchCount, expFile);
if (command == ctStats)
{
struct useCount *u;
slSort(&useCounts, cmpUse);
for (u = useCounts; u != NULL; u = u->next)
{
fprintf(out, "%d %s\n", u->count, u->what);
}
printf("%d unique values for %s\n", slCount(useCounts), selectKey);
}
else if (command == ctHist)
{
struct useCount *u;
fillInVal(useCounts);
slSort(&useCounts, cmpVal);
fprintf(out, "value uses\n");
fprintf(out, "------------\n");
for (u=useCounts; u != NULL; u = u->next)
{
fprintf(out, "%5d %5d\n", u->val, u->count);
}
}
return 0;
}
static void loadDatabase(char *database, char *track, int bedSize, struct bedStub *bedList)
/* Load database from bedList. */
{
struct sqlConnection *conn;
struct dyString *dy = newDyString(1024);
char *tab = (char *)NULL;
int loadOptions = (optionExists("onServer") ? SQL_TAB_FILE_ON_SERVER : 0);
if ( ! noLoad )
conn = sqlConnect(database);
if ((char *)NULL != tmpDir)
tab = cloneString(rTempName(tmpDir,"loadBed",".tab"));
else
tab = cloneString("bed.tab");
if (bedDetail && sqlTable == NULL)
errAbort("bedDetail format requires sqlTable option");
if (bedDetail && !strictTab)
errAbort("bedDetail format must be tab separated");
if (bedDetail && !noBin)
noBin = TRUE;
/* First make table definition. */
if (sqlTable != NULL && !oldTable)
{
/* Read from file. */
char *sql, *s;
readInGulp(sqlTable, &sql, NULL);
/* Chop off end-of-statement semicolon if need be. */
s = strchr(sql, ';');
if (s != NULL) *s = 0;
if ( !noLoad )
{
if (renameSqlTable)
{
char *pos = stringIn("CREATE TABLE ", sql);
if (pos == NULL)
errAbort("Can't find CREATE TABLE in %s\n", sqlTable);
char *oldSql = cloneString(sql);
nextWord(&pos); nextWord(&pos);
char *tableName = nextWord(&pos);
sql = replaceChars(oldSql, tableName, track);
}
verbose(1, "Creating table definition for %s\n", track);
sqlRemakeTable(conn, track, sql);
if (!noBin)
addBinToEmptyTable(conn, track);
adjustSqlTableColumns(conn, track, bedSize);
}
freez(&sql);
}
else if (!oldTable)
{
int minLength;
if (noLoad)
minLength=6;
else if (maxChromNameLength)
minLength = maxChromNameLength;
else
minLength = hGetMinIndexLength(database);
verbose(2, "INDEX chrom length: %d\n", minLength);
/* Create definition statement. */
verbose(1, "Creating table definition for %s\n", track);
dyStringPrintf(dy, "CREATE TABLE %s (\n", track);
if (!noBin)
dyStringAppend(dy, " bin smallint unsigned not null,\n");
dyStringAppend(dy, " chrom varchar(255) not null,\n");
dyStringAppend(dy, " chromStart int unsigned not null,\n");
dyStringAppend(dy, " chromEnd int unsigned not null,\n");
if (bedSize >= 4)
maybeBedGraph(4, dy, " name varchar(255) not null,\n");
if (bedSize >= 5)
{
if (allowNegativeScores)
maybeBedGraph(5, dy, " score int not null,\n");
else
maybeBedGraph(5, dy, " score int unsigned not null,\n");
}
if (bedSize >= 6)
maybeBedGraph(6, dy, " strand char(1) not null,\n");
if (bedSize >= 7)
maybeBedGraph(7, dy, " thickStart int unsigned not null,\n");
if (bedSize >= 8)
maybeBedGraph(8, dy, " thickEnd int unsigned not null,\n");
/* As of 2004-11-22 the reserved field is used as itemRgb in code */
if (bedSize >= 9)
maybeBedGraph(9, dy, " reserved int unsigned not null,\n");
if (bedSize >= 10)
maybeBedGraph(10, dy, " blockCount int unsigned not null,\n");
if (bedSize >= 11)
maybeBedGraph(11, dy, " blockSizes longblob not null,\n");
if (bedSize >= 12)
maybeBedGraph(12, dy, " chromStarts longblob not null,\n");
if (bedSize >= 13)
maybeBedGraph(13, dy, " expCount int unsigned not null,\n");
if (bedSize >= 14)
maybeBedGraph(14, dy, " expIds longblob not null,\n");
if (bedSize >= 15)
maybeBedGraph(15, dy, " expScores longblob not null,\n");
dyStringAppend(dy, "#Indices\n");
if (nameIx && (bedSize >= 4) && (0 == bedGraph))
dyStringAppend(dy, " INDEX(name(16)),\n");
if (noBin)
{
dyStringPrintf(dy, " INDEX(chrom(%d),chromStart)\n", minLength);
}
else
{
dyStringPrintf(dy, " INDEX(chrom(%d),bin)\n", minLength);
}
dyStringAppend(dy, ")\n");
if (noLoad)
verbose(2,"%s", dy->string);
else
sqlRemakeTable(conn, track, dy->string);
}
verbose(1, "Saving %s\n", tab);
writeBedTab(tab, bedList, bedSize);
if ( ! noLoad )
{
verbose(1, "Loading %s\n", database);
if (customTrackLoader)
sqlLoadTabFile(conn, tab, track, loadOptions|SQL_TAB_FILE_WARN_ON_WARN);
else
sqlLoadTabFile(conn, tab, track, loadOptions);
if (! noHistory)
{
char comment[256];
/* add a comment to the history table and finish up connection */
safef(comment, sizeof(comment),
"Add %d element(s) from bed list to %s table",
slCount(bedList), track);
hgHistoryComment(conn, comment);
}
if(fillInScoreColumn != NULL)
{
char query[500];
char buf[500];
struct sqlResult *sr;
safef(query, sizeof(query), "select sum(score) from %s", track);
if(sqlQuickQuery(conn, query, buf, sizeof(buf)))
{
unsigned sum = sqlUnsigned(buf);
if (!sum)
{
safef(query, sizeof(query), "select min(%s), max(%s) from %s", fillInScoreColumn, fillInScoreColumn, track);
if ((sr = sqlGetResult(conn, query)) != NULL)
{
char **row = sqlNextRow(sr);
if(row != NULL)
{
float min = sqlFloat(row[0]);
float max = sqlFloat(row[1]);
if ( !(max == -1 && min == -1)) // if score is -1 then ignore, as if it werent present
{
if (max == min || sameString(row[0],row[1])) // this will lead to 'inf' score value in SQL update causing an error
errAbort("Could not set score in table %s max(%s)=min(%s)=%s\n", track, fillInScoreColumn, fillInScoreColumn, row[0]);
sqlFreeResult(&sr);
// Calculate a, b s/t f(x) = ax + b maps min-max => minScore-1000
float a = (1000-minScore) / (max - min);
float b = 1000 - ((1000-minScore) * max) / (max - min);
safef(query, sizeof(query), "update %s set score = round((%f * %s) + %f)", track, a, fillInScoreColumn, b);
int changed = sqlUpdateRows(conn, query, NULL);
verbose(2, "update query: %s; changed: %d\n", query, changed);
}
else
{
sqlFreeResult(&sr);
verbose(2, "score not updated; all values for column %s are -1\n", fillInScoreColumn);
}
}
}
}
}
}
sqlDisconnect(&conn);
/* if temp dir specified, unlink file to make it disappear */
if ((char *)NULL != tmpDir)
unlink(tab);
}
else
verbose(1, "No load option selected, see file: %s\n", tab);
} /* static void loadDatabase() */
void hgFindSpec(char *org, char *database, char *hgFindSpecName, char *sqlFile,
char *hgRoot, boolean strict)
/* hgFindSpec - Create hgFindSpec table from text files. */
{
struct hash *uniqHash = newHash(8);
struct hash *htmlHash = newHash(8);
struct hgFindSpec *hfsList = NULL, *hfs;
char rootDir[512], orgDir[512], asmDir[512];
char tab[512];
snprintf(tab, sizeof(tab), "%s.tab", hgFindSpecName);
/* Create track list from hgRoot and hgRoot/org and hgRoot/org/assembly
* ra format database. */
sprintf(rootDir, "%s", hgRoot);
sprintf(orgDir, "%s/%s", hgRoot, org);
sprintf(asmDir, "%s/%s/%s", hgRoot, org, database);
layerOn(strict, database, asmDir, uniqHash, htmlHash, FALSE, &hfsList);
layerOn(strict, database, orgDir, uniqHash, htmlHash, FALSE, &hfsList);
layerOn(strict, database, rootDir, uniqHash, htmlHash, TRUE, &hfsList);
slSort(&hfsList, hgFindSpecCmp);
if (verboseLevel() > 0)
printf("Loaded %d search specs total\n", slCount(hfsList));
/* Write to tab-separated file. */
{
FILE *f = mustOpen(tab, "w");
for (hfs = hfsList; hfs != NULL; hfs = hfs->next)
hgFindSpecTabOut(hfs, f);
carefulClose(&f);
}
/* Update database */
{
char *create, *end;
char query[256];
struct sqlConnection *conn = sqlConnect(database);
/* Load in table definition. */
readInGulp(sqlFile, &create, NULL);
create = trimSpaces(create);
create = subTrackName(create, hgFindSpecName);
end = create + strlen(create)-1;
if (*end == ';') *end = 0;
sqlRemakeTable(conn, hgFindSpecName, create);
/* Load in regular fields. */
sqlSafef(query, sizeof query, "load data local infile '%s' into table %s", tab,
hgFindSpecName);
sqlUpdate(conn, query);
/* Load in settings fields. */
for (hfs = hfsList; hfs != NULL; hfs = hfs->next)
{
if (hfs->settingsHash != NULL)
{
char *settings = settingsFromHash(hfs->settingsHash);
updateBigTextField(conn, hgFindSpecName, "searchName",
hfs->searchName,
"searchSettings", settings);
freeMem(settings);
}
}
sqlDisconnect(&conn);
if (verboseLevel() > 0)
printf("Loaded database %s\n", database);
}
}
void hgTrackDb(char *org, char *database, char *trackDbName, char *sqlFile, char *hgRoot,
boolean strict)
/* hgTrackDb - Create trackDb table from text files. */
{
struct trackDb *td;
char tab[PATH_LEN];
safef(tab, sizeof(tab), "%s.tab", trackDbName);
struct trackDb *tdbList = buildTrackDb(org, database, hgRoot, strict);
tdbList = flatten(tdbList);
slSort(&tdbList, trackDbCmp);
verbose(1, "Loaded %d track descriptions total\n", slCount(tdbList));
/* Write to tab-separated file; hold off on html, since it must be encoded */
{
verbose(2, "Starting write of tabs to %s\n", tab);
FILE *f = mustOpen(tab, "w");
for (td = tdbList; td != NULL; td = td->next)
{
hVarSubstTrackDb(td, database);
char *hold = td->html;
td->html = "";
subChar(td->type, '\t', ' '); /* Tabs confuse things. */
subChar(td->shortLabel, '\t', ' '); /* Tabs confuse things. */
subChar(td->longLabel, '\t', ' '); /* Tabs confuse things. */
trackDbTabOut(td, f);
td->html = hold;
}
carefulClose(&f);
verbose(2, "Wrote tab representation to %s\n", tab);
}
/* Update database */
{
char *create, *end;
char query[256];
struct sqlConnection *conn = sqlConnect(database);
/* Load in table definition. */
readInGulp(sqlFile, &create, NULL);
create = trimSpaces(create);
create = substituteTrackName(create, trackDbName);
end = create + strlen(create)-1;
if (*end == ';') *end = 0;
sqlRemakeTable(conn, trackDbName, create);
/* Load in regular fields. */
sqlSafef(query, sizeof(query), "load data local infile '%s' into table %s", tab, trackDbName);
verbose(2, "sending mysql \"%s\"\n", query);
sqlUpdate(conn, query);
verbose(2, "done tab file load");
/* Load in html and settings fields. */
for (td = tdbList; td != NULL; td = td->next)
{
if (isEmpty(td->html))
{
if (strict && !trackDbLocalSetting(td, "parent") && !trackDbLocalSetting(td, "superTrack") &&
!sameString(td->track,"cytoBandIdeo"))
{
fprintf(stderr, "Warning: html missing for %s %s %s '%s'\n",org, database, td->track, td->shortLabel);
}
}
else
{
updateBigTextField(conn, trackDbName, "tableName", td->track, "html", td->html);
}
if (td->settingsHash != NULL)
{
char *settings = settingsFromHash(td->settingsHash);
updateBigTextField(conn, trackDbName, "tableName", td->track,
"settings", settings);
if (showSettings)
{
verbose(1, "%s: type='%s';", td->track, td->type);
if (isNotEmpty(settings))
{
char *oneLine = replaceChars(settings, "\n", "; ");
eraseTrailingSpaces(oneLine);
verbose(1, " %s", oneLine);
freeMem(oneLine);
}
verbose(1, "\n");
}
freeMem(settings);
}
}
sqlDisconnect(&conn);
verbose(1, "Loaded database %s\n", database);
}
}
