boolean accessControlDenied(char *db, char *table)
/* Return TRUE if table access is restricted to some host(s) other than
* the one we're running on. */
{
static char *currentHost = NULL;
struct slName *enabledHosts = NULL;
struct slName *sln = NULL;
static struct hash *dbToAcHash = NULL;
if (dbToAcHash == NULL)
dbToAcHash = hashNew(0);
struct hash *acHash = hashFindVal(dbToAcHash, db);
if (acHash == NULL)
{
struct sqlConnection *conn = hAllocConn(db);
acHash = accessControlInit(conn);
hFreeConn(&conn);
hashAdd(dbToAcHash, db, acHash);
}
if (acHash == NULL)
return FALSE;
enabledHosts = (struct slName *)hashFindVal(acHash, table);
if (enabledHosts == NULL)
return FALSE;
if (currentHost == NULL)
{
currentHost = cloneString(cgiServerName());
if (currentHost == NULL)
{
warn("accessControl: unable to determine current host");
return FALSE;
}
else
chopAtFirstDot(currentHost);
}
for (sln = enabledHosts; sln != NULL; sln = sln->next)
{
if (sameString(currentHost, sln->name))
return FALSE;
}
return TRUE;
}
int nearCountUniqAccRows(struct htmlPage *page)
/* Count number of unique rows in table containing just hyperlinked
* accessions. */
{
char *s, *e, *row, *acc;
int count = 0;
struct hash *uniqHash = hashNew(0);
if (page == NULL)
return -1;
/* Set s to first row. */
s = stringIn(nearStartTablePat, page->htmlText);
if (s == NULL)
return -1;
s += strlen(nearStartTablePat);
for (;;)
{
e = stringIn(nearEndRowPat, s);
if (e == NULL)
break;
row = cloneStringZ(s, e-s);
acc = qaStringBetween(row, ">", "</a>");
if (acc == NULL)
{
warn("Can't find acc text between > and </a> while counting uniq row %s",
row);
freez(&row);
break;
}
if (!hashLookup(uniqHash, acc))
{
hashAdd(uniqHash, acc, NULL);
++count;
}
freez(&row);
freez(&acc);
s = e + strlen(nearEndRowPat);
}
hashFree(&uniqHash);
return count;
}
struct hash *tableToAliasHash(struct sqlConnection *conn, char *table,
char *query)
/* Create hash of true name keyed by alias */
{
struct sqlResult *sr;
char **row;
struct hash *hash = hashNew(19);
char buf[256];
sqlSafef(buf, sizeof(buf), query, table);
sr = sqlGetResult(conn, buf);
while ((row = sqlNextRow(sr)) != NULL)
{
touppers(row[0]);
touppers(row[1]);
hashAdd(hash, row[0], lmCloneString(hash->lm, row[1]));
}
sqlFreeResult(&sr);
return hash;
}
static struct hash *collectQueryStats(char *pslFile, char *querySizeFile)
/* collect per-query statistics */
{
struct hash *queryStatsTbl = (querySizeFile != NULL)
? sumStatsLoad(querySizeFile)
: hashNew(queryHashPowTwo);
struct lineFile *pslLf = pslFileOpen(pslFile);
struct psl* psl;
while ((psl = pslNext(pslLf)) != NULL)
{
struct sumStats *ss = sumStatsGetForQuery(queryStatsTbl, psl->qName, psl->qSize);
sumStatsAccumulateQuery(ss, psl);
pslFree(&psl);
}
lineFileClose(&pslLf);
return queryStatsTbl;
}
struct hash *ccdsStatusValLoad(struct sqlConnection *conn)
/* load values from the imported ccdsStatusVals table. Table hashes
* status name to uid. Names are loaded both as-is and lower-case */
{
struct hash *statusVals = hashNew(0);
struct sqlResult *sr = sqlGetResult(conn,
"NOSQLINJ SELECT ccds_status_val_uid, ccds_status FROM CcdsStatusVals");
char **row;
while ((row = sqlNextRow(sr)) != NULL)
{
int uid = sqlSigned(row[0]);
char *stat = row[1];
hashAddInt(statusVals, stat, uid);
tolowers(stat);
hashAddInt(statusVals, stat, uid);
}
sqlFreeResult(&sr);
return statusVals;
}
struct hash *refSeqVerInfoFromFile(struct sqlConnection *conn, char *accList)
/* load refSeqVerInfo table for all native refseqs specified in a file, then validate it against
* the database. */
{
struct hash *refSeqVerInfoTbl = hashNew(18);
struct lineFile *lf = lineFileOpen(accList, TRUE);
int errCnt = 0;
char *line;
while (lineFileNextReal(lf, &line))
{
char *acc = trimSpaces(line);
if (fromFileAdd(refSeqVerInfoTbl, refSeqVerInfoNewFile(acc), conn) == refSeqVerInfoError)
errCnt++;
}
lineFileClose(&lf);
if (errCnt > 0)
errAbort("%d errors detected loading RefSeq accessioned from %s", errCnt, accList);
return refSeqVerInfoTbl;
}
struct hash *raTagVals(char *fileName, char *tag)
/* Return a hash of all values of given tag seen in any stanza of ra file. */
{
struct hash *hash = hashNew(0);
struct lineFile *lf = lineFileOpen(fileName, TRUE);
char *line;
while (lineFileNextFullReal(lf, &line))
{
char *word = nextWord(&line);
if (sameString(word, tag))
{
char *val = trimSpaces(line);
if (!hashLookup(hash, val))
hashAdd(hash, val, NULL);
}
}
lineFileClose(&lf);
return hash;
}
struct hash *raReadAll(char *fileName, char *keyField)
/* Return hash that contains all ra records in file keyed
* by given field, which must exist. The values of the
* hash are themselves hashes. */
{
struct lineFile *lf = lineFileOpen(fileName, TRUE);
struct hash *bigHash = hashNew(0);
struct hash *hash;
while ((hash = raNextRecord(lf)) != NULL)
{
char *key = hashFindVal(hash, keyField);
if (key == NULL)
errAbort("Couldn't find key field %s line %d of %s",
keyField, lf->lineIx, lf->fileName);
hashAdd(bigHash, key, hash);
}
lineFileClose(&lf);
return bigHash;
}
static boolean fieldExists(struct hash *fieldHash,
struct joinerSet *js, struct joinerField *jf)
/* Make sure field exists in at least one database. */
{
struct slName *db;
boolean gotIt = FALSE;
for (db = jf->dbList; db != NULL && !gotIt; db = db->next)
{
if (hashLookup(allDbHash, db->name))
{
struct sqlConnection *conn = sqlConnect(db->name);
struct slName *table, *tableList = getTablesForField(conn,
jf->splitPrefix, jf->table, jf->splitSuffix);
char fieldName[512];
sqlDisconnect(&conn);
for (table = tableList; table != NULL; table = table->next)
{
safef(fieldName, sizeof(fieldName), "%s.%s.%s",
db->name, table->name, jf->field);
if (hashLookup(fieldHash, fieldName))
{
gotIt = TRUE;
break;
}
}
slFreeList(&tableList);
}
else
{
/* Warn that database doesn't exist. Just warn once per database. */
static struct hash *uniqHash;
if (uniqHash == NULL)
uniqHash = hashNew(8);
if (!hashLookup(uniqHash, db->name))
{
hashAdd(uniqHash, db->name, NULL);
warn("Database %s doesn't exist", db->name);
}
}
}
return gotIt;
}
void endHandler(struct xap *xap, char *name)
/* Called at end of a tag */
{
struct type *type = xap->stack->object;
char *text = skipLeadingSpaces(xap->stack->text->string);
struct element *el;
for (el = type->elements; el != NULL; el = el->next)
{
if (!el->seenThisRound)
el->isOptional = TRUE;
}
if (text[0] == 0)
{
if (type->textAttribute != NULL)
type->textAttribute->isOptional = TRUE;
}
else
{
int textLen = strlen(text);
struct attribute *att = type->textAttribute;
if (att == NULL)
{
type->textAttribute = AllocVar(att);
att->name = "<text>";
att->values = hashNew(16);
if (type->count != 0)
att->isOptional = TRUE;
}
if (att->maxLen < textLen)
att->maxLen = textLen;
hashStore(att->values, text);
att->count += 1;
if (!att->nonInt)
if (!isAllInt(text) || hasLeftPaddedZero(text))
att->nonInt = TRUE;
if (!att->nonFloat)
if (!isAllFloat(text))
att->nonFloat = TRUE;
}
type->count += 1;
topType = type;
}
void trySubmitUpload(struct sqlConnection *conn, char *rawText)
/* Called when they've submitted from uploads page */
{
struct lineFile *lf = lineFileOnString("uploaded data", TRUE, rawText);
struct customPp *cpp = customPpNew(lf);
struct hash *settings = hashNew(8);
addIfNonempty(settings, hggMinVal, "minVal");
addIfNonempty(settings, hggMaxVal, "maxVal");
addIfNonempty(settings, hggMaxGapToFill, "maxGapToFill");
addIfNonempty(settings, hggLabelVals, "linesAt");
struct customTrack *trackList = chromGraphParser(database, cpp,
cartUsualString(cart, hggFormatType, formatNames[0]),
cartUsualString(cart, hggMarkerType, cgfMarkerGenomic),
cartUsualString(cart, hggColumnLabels, colLabelNames[0]),
nullIfAllSpace(cartUsualString(cart, hggDataSetName, NULL)),
nullIfAllSpace(cartUsualString(cart, hggDataSetDescription, NULL)),
settings, TRUE);
updateCustomTracks(trackList);
}
struct hash *chromMinSizeHash(struct bed *bedList)
/* Return hash full of lower bounds on chromosome sizes, taken
* from chromEnds on bedList (which just needs to be bed 3).
* Typically this is used when you want to make bin-keepers on
* these chromosomes. */
{
struct bed *bed;
struct hash *sizeHash = hashNew(16);
for (bed = bedList; bed != NULL; bed = bed->next)
{
struct minChromSize *chrom = hashFindVal(sizeHash, bed->chrom);
if (chrom == NULL)
{
lmAllocVar(sizeHash->lm, chrom);
hashAddSaveName(sizeHash, bed->chrom, chrom, &chrom->name);
}
chrom->minSize = max(chrom->minSize, bed->chromEnd);
}
return sizeHash;
}
struct hash *makeStageHash(char *fileName)
/* Return hash with keys that are stage names (st12 and the like)
* and values are ascii strings describing age in days.
* The input is two columns - stage name, and age in hours. */
{
struct lineFile *lf = lineFileOpen(fileName, TRUE);
char *row[2];
struct hash *hash = hashNew(0);
while (lineFileRow(lf, row))
{
char *stage = row[0];
double hours = atof(row[1]);
char days[16];
safef(days, sizeof(days), "%f", hours/24);
hashAdd(hash, stage, cloneString(days));
}
lineFileClose(&lf);
return hash;
}
struct hash *netToBkHash(char *netFile)
/* Read net file into a hash full of binKeepers keyed by chromosome.
* The binKeepers are full of nets. */
{
struct hash *netHash = hashNew(0);
struct lineFile *lf = lineFileOpen(netFile, TRUE);
struct chainNet *net, *netList = chainNetRead(lf);
for (net = netList; net != NULL; net = net->next)
{
if (hashLookup(netHash, net->name))
errAbort("%s has multiple %s records", netFile, net->name);
struct binKeeper *bk = binKeeperNew(0, net->size);
hashAdd(netHash, net->name, bk);
struct cnFill *fill;
for(fill=net->fillList; fill != NULL; fill = fill->next)
binKeeperAdd(bk, fill->tStart, fill->tStart+fill->tSize, fill);
}
lineFileClose(&lf);
return netHash;
}
struct slName *pslSetsQueryNames(struct pslSets *ps)
/* get list of all query names in all tables. slFree when done */
{
struct slName *qNames = NULL;
struct hash *qNameHash = hashNew(21);
int iSet;
struct hashCookie hc;
struct hashEl *hel;
for (iSet = 0; iSet < ps->numSets; iSet++)
addQueryNames(qNameHash, ps->sets[iSet]);
/* copy to a list */
hc = hashFirst(qNameHash);
while ((hel = hashNext(&hc)) != NULL)
slSafeAddHead(&qNames, slNameNew(hel->name));
hashFree(&qNameHash);
slNameSort(&qNames);
return qNames;
}
struct hash *blastTabLoadBestQuery(char *fileName)
/* Load all blastTabs from file, discarding ones that are not best for a given
* query. */
{
struct blastTab *bt, *btList = blastTabLoadAll(fileName);
struct hash *hash = hashNew(18);
for (bt = btList; bt != NULL; bt = bt->next)
{
struct hashEl *hel = hashLookup(hash, bt->query);
if (hel == NULL)
hashAdd(hash, bt->query, bt);
else
{
struct blastTab *oldBt = hel->val;
if (bt->bitScore > oldBt->bitScore)
hel->val = bt;
}
}
return hash;
}
static struct hash *hashTracksByGroup(struct trackDb *trackList)
/* Hash group names to lists of tracks in those groups; sort each list by trackDb priority. */
{
struct hash *hash = hashNew(0);
struct trackDb *tdb;
for (tdb = trackList; tdb != NULL; tdb = tdb->next)
{
struct hashEl *hel = hashLookup(hash, tdb->grp);
struct slRef *slr = slRefNew(tdb);
if (hel)
slAddHead(&(hel->val), slr);
else
hashAdd(hash, tdb->grp, slr);
}
struct hashCookie cookie = hashFirst(hash);
struct hashEl *hel;
while ((hel = hashNext(&cookie)) != NULL)
slSort(&hel->val, trackDbRefCmp);
return hash;
}
static void checkCustomTracks(char *db, char *outFile, char *expectedFile)
/* compare track lines of output file with expected. Return error
* settings are not a proper subset */
{
struct hash *expHash = hashNew(0);
struct customTrack *ct = NULL, *expCt = NULL;
struct customTrack *newCts = customFactoryParse(db, outFile, TRUE, NULL);
struct customTrack *expCts = customFactoryParse(db, expectedFile, TRUE, NULL);
verbose(3, "found %d tracks in output file %s, %d tracks in expected file %s\n",
slCount(newCts), outFile, slCount(expCts), expectedFile);
for (ct = expCts; ct != NULL; ct = ct->next)
hashAdd(expHash, ct->tdb->track, ct);
for (ct = newCts; ct != NULL; ct = ct->next)
{
if ((expCt = hashFindVal(expHash, ct->tdb->track)) == NULL)
errAbort("ct %s not found in expected", ct->tdb->track);
verbose(3, "output settings: %s %s\n%s\n", ct->tdb->track,
ct->tdb->shortLabel, ct->tdb->settings);
verbose(3, "expected settings: %s %s\n%s\n", expCt->tdb->track,
expCt->tdb->shortLabel, expCt->tdb->settings);
struct hash *newSettings = trackDbHashSettings(ct->tdb);
struct hash *expSettings = trackDbHashSettings(expCt->tdb);
struct hashCookie hc = hashFirst(expSettings);
struct hashEl *hel;
while ((hel = hashNext(&hc)) != NULL)
{
char *setting = (char *)hel->name;
/* ignore DB table name -- it will always differ */
if (sameString(setting, "dbTableName"))
continue;
char *expVal = (char *)hel->val;
char *newVal = NULL;
if ((newVal = (char *)hashFindVal(newSettings, setting)) == NULL)
errAbort("ct %s setting %s not found in new",
ct->tdb->track, setting);
if (differentString(newVal, expVal))
errAbort("ct %s setting %s differs from expected: %s should be %s",
ct->tdb->track, setting, newVal, expVal);
}
}
}
static struct hash *getCoveredTables(struct joiner *joiner, char *db,
struct sqlConnection *conn)
/* Get list of tables covered in database. */
{
struct hash *hash = hashNew(0);
struct joinerIgnore *ig;
struct slName *spec;
struct joinerSet *js;
struct joinerField *jf;
/* First put in all the ignored tables. */
for (ig = joiner->tablesIgnored; ig != NULL; ig = ig->next)
{
if (slNameInList(ig->dbList, db))
{
for (spec = ig->tableList; spec != NULL; spec = spec->next)
{
verbose(3,"ignoreTable: '%s'\n", spec->name);
addTablesLike(hash, conn, spec->name);
}
}
}
/* Now put in tables that are in one of the identifiers. */
for (js = joiner->jsList; js != NULL; js = js->next)
{
for (jf = js->fieldList; jf != NULL; jf = jf->next)
{
if (slNameInList(jf->dbList, db))
{
char spec[512];
safef(spec, sizeof(spec), "%s%s%s",
emptyForNull(jf->splitPrefix), jf->table,
emptyForNull(jf->splitSuffix));
addTablesLike(hash, conn, spec);
verbose(4,"ident: '%s', table: '%s'\n", js->name, spec);
}
}
}
return hash;
}
struct hash *loadGeno(char *genoFile)
/* load genome sequences into a hash. This supports the multi-sequence
* specs of twoBitLoadAll */
{
struct dnaSeq *genos = NULL, *geno;
struct hash *genoHash = hashNew(0);
if (nibIsFile(genoFile))
genos = nibLoadAllMasked(NIB_MASK_MIXED|NIB_BASE_NAME, genoFile);
else if (twoBitIsSpec(genoFile))
genos = twoBitLoadAll(genoFile);
else
genos = faReadDna(genoFile);
while ((geno = slPopHead(&genos)) != NULL)
{
tolowers(geno->dna);
hashAdd(genoHash, geno->name, geno);
}
return genoHash;
}
struct mgcLibraryTbl *mgcLibraryTblLoad(char *fileName)
/* load a file of mgcLibrary objects, building a hash by library id */
{
struct mgcLibraryTbl *mlt;
char *row[MGCLIBRARY_NUM_COLS];
char key[64];
struct lineFile *lf;
AllocVar(mlt);
mlt->idHash = hashNew(12); /* 4096 */
lf = gzLineFileOpen(fileName);
while (lineFileNextRowTab(lf, row, MGCLIBRARY_NUM_COLS))
{
struct mgcLibrary *mgcLibrary = mgcLibraryLoad(row);
safef(key, sizeof(key), "%d", mgcLibrary->id_lib);
hashAdd(mlt->idHash, key, mgcLibrary);
}
gzLineFileClose(&lf);
return mlt;
}
struct hash *makeSuppress()
/* Make a hash full of fields to suppress. */
{
struct hash *suppress = hashNew(4);
hashAdd(suppress, "objType", NULL); // Inherent in hierarchy or ignored
hashAdd(suppress, "subId", NULL); // Submission ID not worth carrying forward
hashAdd(suppress, "tableName", NULL); // We aren't interested in tables, just files
hashAdd(suppress, "project", NULL); // Always wgEncode
hashAdd(suppress, "expId", NULL); // Redundant with dccAccession
hashAdd(suppress, "cell", NULL); // Completely redundant with cellType - I checked
hashAdd(suppress, "sex", NULL); // This should be implied in cellType
if (!maniFields)
{
hashAdd(suppress, "dccAccession", NULL); // Redundant with meta object name
hashAdd(suppress, "composite", NULL); // Inherent in hierarchy now
hashAdd(suppress, "view", NULL); // This is in maniest
hashAdd(suppress, "replicate", NULL); // This is in manifest
hashAdd(suppress, "md5sum", NULL); // Also in manifest
}
return suppress;
}
struct slName *varsInAnyNode(struct metaNode *nodeList)
/* Return list of variables that are used in any node in list. */
{
struct hash *varHash = hashNew(6);
struct slName *var, *varList = NULL;
struct metaNode *node;
for (node = nodeList; node != NULL; node = node->next)
{
struct mdbVar *v;
for (v = node->vars; v != NULL; v = v->next)
{
if (!hashLookup(varHash, v->var))
{
var = slNameAddHead(&varList, v->var);
hashAdd(varHash, var->name, var);
}
}
}
hashFree(&varHash);
return varList;
}
struct chainNet *chainNetRead(struct lineFile *lf)
/* Read next net from file. Return NULL at end of file.*/
{
char *line, *words[3];
struct chainNet *net;
int wordCount;
if (!lineFileNextReal(lf, &line))
return NULL;
if (!startsWith("net ", line))
errAbort("Expecting 'net' first word of line %d of %s",
lf->lineIx, lf->fileName);
AllocVar(net);
wordCount = chopLine(line, words);
lineFileExpectAtLeast(lf, 3, wordCount);
net->name = cloneString(words[1]);
net->size = lineFileNeedNum(lf, words, 2);
net->nameHash = hashNew(6);
net->fillList = cnFillRead(net, lf);
return net;
}
struct hash *bedsIntoHashOfKeepers(struct bed *bedList)
/* Return a hash full of binKeepers, keyed by chromosome (or contig)
* that contains the bedList */
{
struct hash *sizeHash = chromMinSizeHash(bedList);
struct hash *keeperHash = hashNew(16);
struct bed *bed;
for (bed = bedList; bed != NULL; bed = bed->next)
{
struct binKeeper *keeper = hashFindVal(keeperHash, bed->chrom);
if (keeper == NULL)
{
struct minChromSize *chrom = hashMustFindVal(sizeHash, bed->chrom);
keeper = binKeeperNew(0, chrom->minSize);
hashAdd(keeperHash, chrom->name, keeper);
}
binKeeperAdd(keeper, bed->chromStart, bed->chromEnd, bed);
}
hashFree(&sizeHash);
return keeperHash;
}
struct slName *tagsInAny(struct meta *metaList)
/* Return list of variables that are used in any node in list. */
{
struct hash *tagHash = hashNew(6);
struct slName *tag, *tagList = NULL;
struct meta *meta;
for (meta = metaList; meta != NULL; meta = meta->next)
{
struct metaTagVal *v;
for (v = meta->tagList; v != NULL; v = v->next)
{
if (!hashLookup(tagHash, v->tag))
{
tag = slNameAddHead(&tagList, v->tag);
hashAdd(tagHash, tag->name, tag);
}
}
}
hashFree(&tagHash);
return tagList;
}
void md5ItemLoadAll(char *fileName, struct md5Item **retList, struct hash **retHash)
/* Read in all items from file. */
{
struct hash *hash = hashNew(16);
struct lineFile *lf = lineFileOpen(fileName, TRUE);
char *row[2];
struct md5Item *item, *list = NULL;
while (lineFileRow(lf, row))
{
AllocVar(item);
item->name = cloneString(row[1]);
item->md5 = cloneString(row[0]);
slAddHead(&list, item);
if (hashLookup(hash, item->name))
errAbort("%s duplicated line %d of %s", item->name, lf->lineIx, lf->fileName);
hashAdd(hash, item->name, item);
}
slReverse(&list);
*retList = list;
*retHash = hash;
}
static struct hash *hashLabels(struct dlList *iList)
/* Return hash of all labels. */
{
struct dlNode *node;
struct hash *hash = hashNew(0);
for (node = iList->head; !dlEnd(node); node = node->next)
{
struct isx *isx = node->val;
switch (isx->opType)
{
case poLabel:
case poLoopStart:
case poLoopEnd:
case poCondCase:
case poCondEnd:
hashAdd(hash, isx->dest->name, node);
break;
}
}
return hash;
}
void getChroms(struct sqlConnection *conn, struct hash **retHash,
struct chrom **retList)
/* Get hash of chromosomes from database. */
{
struct sqlResult *sr;
char **row;
struct chrom *chromList = NULL, *chrom;
struct hash *hash = hashNew(8);
sr = sqlGetResult(conn, "NOSQLINJ select chrom,size from chromInfo");
while ((row = sqlNextRow(sr)) != NULL)
{
AllocVar(chrom);
hashAddSaveName(hash, row[0], chrom, &chrom->name);
chrom->size = atoi(row[1]);
slAddHead(&chromList, chrom);
}
sqlFreeResult(&sr);
slReverse(&chromList);
*retHash = hash;
*retList = chromList;
}
struct slName *valsForVar(char *varName, struct taggedFile *tfList)
/* Return all values for given variable. */
{
struct slName *list = NULL;
struct hash *uniqHash = hashNew(7);
struct taggedFile *tf;
for (tf = tfList; tf != NULL; tf = tf->next)
{
char *val = metaTagValFindVal(tf->tagList, varName);
if (val != NULL)
{
if (hashLookup(uniqHash, val) == NULL)
{
hashAdd(uniqHash, val, NULL);
slNameAddHead(&list, val);
}
}
}
hashFree(&uniqHash);
slNameSort(&list);
return list;
}
