void freen(char *chrom)
/* Test something */
{
uglyTime(NULL);
struct sqlConnection *conn = sqlConnect("hg19");
uglyTime("connect");
char query[512];
sqlSafef(query, sizeof(query), "select * from knownGene where chrom='%s'", chrom);
struct sqlResult *sr = sqlGetResult(conn, query);
uglyTime("get result");
char **row;
struct rbTree *rt = rangeTreeNew();
while ((row = sqlNextRow(sr)) != NULL)
{
struct genePred *gp = genePredLoad(row);
int i;
int exonCount = gp->exonCount;
for (i=0; i<exonCount; ++i)
rangeTreeAdd(rt, gp->exonStarts[i], gp->exonEnds[i]);
}
uglyTime("Add rows");
struct range *list = rangeTreeList(rt);
uglyTime("Did list");
uglyf("%d items in chrom %s\n", slCount(list), chrom);
}
void threadExp(char *asciiSize, char *asciiJobCount, char *asciiThreadCount)
/* threadExp - Some pthread experiments. */
{
int size = atoi(asciiSize);
int jobCount = atoi(asciiJobCount);
int threadCount = atoi(asciiThreadCount);
pthread_t *threads;
int i;
double *results;
double sumErr = 0;
struct calcJob *jobList, *job;
if (size <= 0)
usage();
AllocArray(results, size);
AllocArray(threads, threadCount);
toDo = cacheQueueAlloc();
done = cacheQueueAlloc();
jobList = makeJobs(results, size, jobCount);
uglyTime("To init and alloc %d\n", size);
for (i=0; i<threadCount; ++i)
pthreadCreate(&threads[i], NULL, workerBee, NULL);
uglyTime("Thread spwan time");
doJobs(jobList);
uglyTime("job time");
for (i=0; i<size; ++i)
sumErr += (results[i]-i);
uglyTime("summing err");
printf("Total error = %e\n", sumErr);
printf("Total incer = %d\n", incer);
}
void make_graph (const char* fn)
{
FILE *fw = mustOpen(fn, "w");
int i, j, cnt;
int rec_num = 0;
if (po->COL_WIDTH == 2) po->COL_WIDTH = MAX(cols/20, 2);
/* edge_ptr describe edges */
AllocArray(edge_list, HEAP_SIZE);
/* Allocating heap structure */
struct fibheap *heap;
heap = fh_makeheap();
fh_setcmp(heap, edge_cmpr);
/* Generating seed list and push into heap */
progress("Generating seed list (minimum weight %d)", po->COL_WIDTH);
Edge __cur_min = {0, 0, po->COL_WIDTH};
Edge *_cur_min = &__cur_min;
Edge **cur_min = & _cur_min;
/* iterate over all genes to retrieve all edges */
for (i = 0; i < rows; i++)
{
for (j = i+1; j < rows; j++)
{
cnt = str_intersect_r(arr_c[i], arr_c[j]);
if (cnt < (*cur_min)->score) continue;
AllocVar(edge_ptr);
edge_ptr -> gene_one = i;
edge_ptr -> gene_two = j;
edge_ptr -> score = cnt;
fh_insert_fixed(heap, edge_ptr, cur_min);
rec_num++;
}
}
/*rec_num = heap->fh_n;*/
if (rec_num == 0)
errAbort("Not enough overlap between genes");
/* sort the seeds */
uglyTime("%d seeds generated", rec_num);
ReAllocArray(edge_list, rec_num);
fh_dump(heap, edge_list);
/* bi-clustering */
int n_blocks = 0;
progress("Clustering started");
n_blocks = cluster(fw, edge_list, rec_num);
uglyTime("%d clusters are written to %s", n_blocks, fn);
/* clean up */
for (i=0; i<rec_num; i++)
free(edge_list[i]);
free(edge_list);
}
void test(int count)
/* test - Test rbTree and rbIntTree. */
{
uglyTime(NULL);
rbIntTest(count);
uglyTime("rbIntTree");
rbTest(count);
uglyTime("rbTree");
hashTest(count);
uglyTime("hash");
}
void i16Dump(char *input)
/* i16Dump - Dump out info on i16. Useful for debugging. */
{
struct i16 *i16 = NULL;
struct i16Header *header = NULL;
if (mainIndex || overflowIndex || chromInfo)
{
i16 = i16Read(input, mmap);
uglyTime("Read %s", input);
header = i16->header;
}
else
header = i16ReadHeaderOnly(input);
printf("%d chromosomes, %lld bases, %lld indexed, %lld overflow bases, %lld overflow slots\n",
header->chromCount, header->dnaDiskSize, header->basesInIndex,
header->overflowBaseCount, header->overflowSlotCount);
printf("header->size=%lld\n", header->size);
if (chromInfo)
dumpChromInfo(i16, chromInfo);
if (overflowIndex)
dumpOverflowIndex(i16, overflowIndex, maxSize, maxOverflow);
if (mainIndex)
dumpMainIndex(i16, mainIndex, maxSize);
}
void doJobs(struct calcJob *jobList)
{
struct calcJob *job;
for (job = jobList; job != NULL; job = job->next)
synQueuePut(toDo, job);
uglyTime("To load syn queue");
for (job = jobList; job != NULL; job = job->next)
synQueueGet(done);
}
void cleanupTable(char *host, char *user, char *password, char *database, char *table, int target)
/* Trim table to target most recent items. */
{
uglyTime(NULL);
struct sqlConnection *conn = sqlConnectRemote(host, user, password, database);
struct dyString *query = dyStringNew(0);
sqlDyStringPrintf(query, "select count(*) from %s", table);
int initialCount = sqlQuickNum(conn, query->string);
uglyTime("%d initial vs %d target", initialCount, target);
if (target < initialCount)
{
/* Query database for id's ordered by age */
dyStringClear(query);
sqlDyStringPrintf(query, "select id,now()-lastUse age from %s order by age", table);
struct sqlResult *sr = sqlGetResult(conn, query->string);
/* Build up new query that'll delete old things. */
dyStringClear(query);
sqlDyStringPrintf(query, "delete from %s where id in (", table);
int i=0;
boolean addComma = FALSE;
char **row;
while ((row = sqlNextRow(sr)) != NULL)
{
if (++i > target)
{
if (addComma)
dyStringAppendC(query, ',');
else
addComma = TRUE;
dyStringPrintf(query, "'%s'", row[0]);
}
}
dyStringPrintf(query, ")");
sqlFreeResult(&sr);
uglyTime("made delete query %d chars", query->stringSize);
/* Excute deletion */
sqlUpdate(conn, query->string);
uglyTime("deleted");
}
sqlDisconnect(&conn);
}
int main(int argc, char *argv[])
/* Process command line. */
{
uglyTime(NULL);
optionHash(&argc, argv);
if (argc != 4)
usage();
threadExp(argv[1], argv[2], argv[3]);
return 0;
}
int main(int argc, char* argv[])
{
/* Start the timer */
uglyTime(NULL);
printf("\nQUBIC %.1f: greedy biclustering (compiled "__DATE__" "__TIME__")\n\n", VER);
rows = cols = 0;
/* get the program options defined in get_options.c */
get_options(argc, argv);
/*get the size of input expression matrix*/
get_matrix_size(po->FP);
progress("File %s contains %d genes by %d conditions", po -> FN, rows, cols);
if (rows < 3 || cols < 3)
{
/*neither rows number nor cols number can be too small*/
errAbort("Not enough genes or conditions to make inference");
}
genes = alloc2c(rows, LABEL_LEN);
conds = alloc2c(cols, LABEL_LEN);
/* Read in the gene names and condition names */
read_labels(po -> FP);
/* Read in the expression data */
if (po->IS_DISCRETE)
read_discrete(po -> FP);
else
{
read_continuous(po -> FP);
/* formatting rules */
discretize(addSuffix(po->FN, ".rules"));
}
fclose(po->FP);
/*we can do expansion by activate po->IS_SWITCH*/
if (po->IS_SWITCH)
{
read_and_solve_blocks(po->FB, addSuffix(po->BN, ".expansion"));
}
else
{
/* formatted file */
write_imported(addSuffix(po->FN, ".chars"));
/* the file that stores all blocks */
make_graph(addSuffix(po->FN, ".blocks"));
}/* end of main else */
free(po);
return 0;
}
int main(int argc, char *argv[])
/* Process command line. */
{
uglyTime(NULL);
dnaUtilOpen();
optionInit(&argc, argv, options);
maxSize = optionInt("maxSize", maxSize);
maxOverflow = optionInt("maxOverflow", maxOverflow);
mainIndex = optionVal("mainIndex", mainIndex);
overflowIndex = optionVal("overflowIndex", overflowIndex);
chromInfo= optionVal("chromInfo", chromInfo);
mmap = optionExists("mmap");
if (argc != 2)
usage();
i16Dump(argv[1]);
return 0;
}
int main(int argc, char *argv[])
/* Process command line. */
{
long enteredMainTime = clock1000();
uglyTime(NULL);
cgiSpoof(&argc, argv);
setUdcCacheDir();
oldCart = hashNew(0);
if (cgiVarExists(hgpDoDownload)) /* use cgiVars -- do not commit to any cart method yet */
{
struct sqlConnection *conn = sqlConnect(visiDb);
cart = cartAndCookieNoContent(hUserCookie(), excludeVars, oldCart);
doDownload(conn);
cartCheckout(&cart);
}
else
{
cartEmptyShell(doMiddle, hUserCookie(), excludeVars, oldCart);
}
cgiExitTime("hgVisiGene", enteredMainTime);
return 0;
}
void intersectOnChrom(char *db, struct sqlConnection *conn, char *chrom,
char *track1, char *track2)
/* Do intersection on one chromosome. */
{
int chromSize = hChromSize(chrom);
struct lm *lm = lmInit(0);
struct bed *bedList1, *bedList2, *andBed;
struct featureBits *fb1, *fb2;
Bits *bit1, *bit2;
int fieldCount1, fieldCount2;
struct binKeeper *bk2;
uglyTime(NULL);
scanChromTable(conn, chrom, track1);
scanChromTable(conn, chrom, track2);
uglyTime("Scan tracks");
bedList1 = getChromAsBed(conn, db, track1, chrom, lm, &fieldCount1);
bedList2 = getChromAsBed(conn, db, track2, chrom, lm, &fieldCount2);
uglyTime("Tracks as bed");
uglyf("%d items with %d fields in %s, ", slCount(bedList1), fieldCount1, track1);
uglyf("%d items with %d fields in %s\n", slCount(bedList2), fieldCount2, track2);
bit1 = bitAlloc(chromSize+8);
bit2 = bitAlloc(chromSize+8);
uglyTime("bitAlloc");
fb1 = fbList(db, chrom, track1, bedList1, chromSize);
fb2 = fbList(db, chrom, track1, bedList1, chromSize);
uglyTime("bed to featureBits list");
fbOrBits(bit1, chromSize, fb1, 0);
fbOrBits(bit2, chromSize, fb2, 0);
uglyTime("or into bits");
bitAnd(bit1, bit2, chromSize);
uglyTime("Anding bitfields");
andBed = bitsToBed4List(bit1, chromSize, chrom, 0, 0, chromSize, lm);
uglyTime("Converting bitfield to bed 4");
bitCountAllOverlaps(bedList1, bit2, fieldCount2);
uglyTime("Counting overlaps in track1 with bitfield of track2");
bk2 = fbToBinKeeper(fb2, chromSize);
uglyTime("Adding featureBits list from track 2 into binKeeper.");
bkCountAllOverlaps(bedList1, bk2, fieldCount2);
uglyTime("Count overlaps in track1 with binKeeper of track2");
featureBitsFreeList(&fb1);
featureBitsFreeList(&fb2);
uglyTime("free featureBits");
bitFree(&bit1);
bitFree(&bit2);
uglyTime("bitFree");
}
static void doFileSearch(char *db,char *organism,struct cart *cart,struct trackDb *tdbList)
{
struct sqlConnection *conn = hAllocConn(db);
boolean metaDbExists = sqlTableExists(conn, "metaDb");
if (!sqlTableExists(conn, "metaDb"))
{
warn("Assembly %s %s does not support Downloadable Files search.", organism, hFreezeFromDb(db));
hFreeConn(&conn);
return;
}
char *nameSearch = cartOptionalString(cart, TRACK_SEARCH_ON_NAME);
char *descSearch=NULL;
char *fileTypeSearch = cartOptionalString(cart, FILE_SEARCH_ON_FILETYPE);
boolean doSearch = sameWord(cartUsualString(cart, FILE_SEARCH,"no"), "search");
#ifdef ONE_FUNC
struct hash *parents = newHash(4);
#endif///def ONE_FUNC
boolean searchTermsExist = FALSE; // FIXME: Why is this needed?
int cols;
#ifdef USE_TABS
enum searchTab selectedTab = simpleTab;
char *currentTab = cartUsualString(cart, FILE_SEARCH_CURRENT_TAB, "simpleTab");
if (sameString(currentTab, "simpleTab"))
{
selectedTab = simpleTab;
descSearch = cartOptionalString(cart, TRACK_SEARCH_SIMPLE);
freez(&nameSearch);
}
else if (sameString(currentTab, "filesTab"))
{
selectedTab = filesTab;
descSearch = cartOptionalString(cart, TRACK_SEARCH_ON_DESCR);
}
#else///ifndef USE_TABS
enum searchTab selectedTab = filesTab;
descSearch = cartOptionalString(cart, TRACK_SEARCH_ON_DESCR);
#endif///ndef USE_TABS
#ifdef USE_TABS
struct trix *trix;
char trixFile[HDB_MAX_PATH_STRING];
getSearchTrixFile(db, trixFile, sizeof(trixFile));
trix = trixOpen(trixFile);
#endif///def USE_TABS
printf("<div style='max-width:1080px;'>");
// FIXME: Do we need a form at all?
printf("<form action='../cgi-bin/hgFileSearch' name='%s' id='%s' method='get'>\n\n",
FILE_SEARCH_FORM,FILE_SEARCH_FORM);
cartSaveSession(cart); // Creates hidden var of hgsid to avoid bad voodoo
printf("<input type='hidden' name='db' value='%s'>\n", db);
printf("<input type='hidden' name='%s' value=''>\n",TRACK_SEARCH_DEL_ROW);
printf("<input type='hidden' name='%s' value=''>\n",TRACK_SEARCH_ADD_ROW);
#ifdef USE_TABS
printf("<input type='hidden' name='%s' id='currentTab' value='%s'>\n",
FILE_SEARCH_CURRENT_TAB, currentTab);
printf("<div id='tabs' style='display:none; %s'>\n"
"<ul>\n"
"<li><a href='#simpleTab'><B style='font-size:.9em;font-family: arial, Geneva, "
"Helvetica, san-serif;'>Search</B></a></li>\n"
"<li><a href='#filesTab'><B style='font-size:.9em;font-family: arial, Geneva, "
"Helvetica, san-serif;'>Files</B></a></li>\n"
"</ul>\n",cgiBrowser()==btIE?"width:1060px;":"max-width:inherit;");
// Files tab
printf("<div id='simpleTab' style='max-width:inherit;'>\n");
printf("<table id='simpleTable' style='width:100%%; font-size:.9em;'><tr><td colspan='2'>");
printf("<input type='text' name='%s' id='simpleSearch' class='submitOnEnter' value='%s' "
"style='max-width:1000px; width:100%%;' onkeyup='findTracks.searchButtonsEnable(true);'>\n",
TRACK_SEARCH_SIMPLE,descSearch == NULL ? "" : descSearch);
if (selectedTab==simpleTab && descSearch)
searchTermsExist = TRUE;
printf("</td></tr><td style='max-height:4px;'></td></tr></table>");
printf("<input type='submit' name='%s' id='searchSubmit' value='search' style='font-size:.8em;'>\n",
FILE_SEARCH);
printf("<input type='button' name='clear' value='clear' class='clear' style='font-size:.8em;' "
"onclick='findTracks.clear();'>\n");
printf("<input type='submit' name='submit' value='cancel' class='cancel' "
"style='font-size:.8em;'>\n");
printf("</div>\n");
#endif///def USE_TABS
// Files tab
printf("<div id='filesTab' style='width:inherit;'>\n"
"<table id='filesTable' cellSpacing=0 style='width:inherit; font-size:.9em;'>\n");
cols = 8;
// Track Name contains
printf("<tr><td colspan=3></td>");
printf("<td nowrap><b style='max-width:100px;'>Track Name:</b></td>");
printf("<td align='right'>contains</td>\n");
printf("<td colspan='%d'>", cols - 4);
printf("<input type='text' name='%s' id='nameSearch' class='submitOnEnter' value='%s' "
"onkeyup='findTracks.searchButtonsEnable(true);' style='min-width:326px; font-size:.9em;'>",
TRACK_SEARCH_ON_NAME, nameSearch == NULL ? "" : nameSearch);
printf("</td></tr>\n");
// Description contains
printf("<tr><td colspan=2></td><td align='right'>and </td>");
printf("<td><b style='max-width:100px;'>Description:</b></td>");
printf("<td align='right'>contains</td>\n");
printf("<td colspan='%d'>", cols - 4);
printf("<input type='text' name='%s' id='descSearch' value='%s' class='submitOnEnter' "
"onkeyup='findTracks.searchButtonsEnable(true);' style='max-width:536px; "
"width:536px; font-size:.9em;'>",
TRACK_SEARCH_ON_DESCR, descSearch == NULL ? "" : descSearch);
printf("</td></tr>\n");
if (selectedTab==filesTab && descSearch)
searchTermsExist = TRUE;
// Set up Group dropdown
struct grp *grps = hLoadGrps(db);
grps = groupsFilterForTdbList(&grps,tdbList);
int numGroups = slCount(grps) + 1; // Add Any
char **groups = needMem(sizeof(char *) * numGroups);
char **labels = needMem(sizeof(char *) * numGroups);
groups[0] = ANYLABEL;
labels[0] = ANYLABEL;
int ix=1;
struct grp *grp = grps;
for (; grp != NULL; grp = grp->next,ix++)
{
groups[ix] = cloneString(grp->name);
labels[ix] = cloneString(grp->label);
}
printf("<tr><td colspan=2></td><td align='right'>and </td>\n");
printf("<td><b style='max-width:100px;'>Group:</b></td>");
printf("<td align='right'>is</td>\n");
printf("<td colspan='%d'>", cols - 4);
char *groupSearch = cartOptionalString(cart, TRACK_SEARCH_ON_GROUP);
cgiMakeDropListFull(TRACK_SEARCH_ON_GROUP, labels, groups, numGroups, groupSearch,
"class='groupSearch' style='min-width:40%; font-size:.9em;'");
printf("</td></tr>\n");
if (selectedTab==filesTab && groupSearch)
searchTermsExist = TRUE;
// Track Type is (drop down)
printf("<tr><td colspan=2></td><td align='right'>and </td>\n");
printf("<td nowrap><b style='max-width:100px;'>Data Format:</b></td>");
printf("<td align='right'>is</td>\n");
printf("<td colspan='%d'>", cols - 4);
char *dropDownHtml = fileFormatSelectHtml(FILE_SEARCH_ON_FILETYPE,fileTypeSearch,
"style='min-width:40%; font-size:.9em;'");
if (dropDownHtml)
{
puts(dropDownHtml);
freeMem(dropDownHtml);
}
printf("</td></tr>\n");
if (selectedTab==filesTab && fileTypeSearch)
searchTermsExist = TRUE;
// mdb selects
struct slPair *mdbSelects = NULL;
if (metaDbExists)
{
struct slPair *mdbVars = mdbVarsSearchable(conn,FALSE,TRUE); // Not tables, just files
mdbSelects = mdbSelectPairs(cart, mdbVars);
char *output = mdbSelectsHtmlRows(conn,mdbSelects,mdbVars,cols,TRUE); // just for fileSearch
if (output)
{
puts(output);
freeMem(output);
}
slPairFreeList(&mdbVars);
}
printf("</table>\n");
printf("<input type='submit' name='%s' id='searchSubmit' value='search' style='font-size:.8em;'>\n",
FILE_SEARCH);
printf("<input type='button' name='clear' value='clear' class='clear' style='font-size:.8em;' "
"onclick='findTracks.clear();'>\n");
printf("<input type='submit' name='submit' value='cancel' class='cancel' "
"style='font-size:.8em;'>\n");
printf("</div>\n");
#ifdef USE_TABS
printf("</div>\n"); // End tabs div
#endif///def USE_TABS
if (nameSearch != NULL && !strlen(nameSearch))
nameSearch = NULL;
if (descSearch != NULL && !strlen(descSearch))
descSearch = NULL;
if (groupSearch != NULL && sameString(groupSearch, ANYLABEL))
groupSearch = NULL;
printf("</form>\n");
printf("</div>"); // Restricts to max-width:1000px;
cgiDown(0.8);
if (measureTiming)
uglyTime("Generated search controls");
#ifdef USE_TABS
if (doSearch && selectedTab==simpleTab && isEmpty(descSearch))
doSearch = FALSE;
#endif///def USE_TABS
if (doSearch)
{
// Now search
#ifdef USE_TABS
struct slRef *foundTdbs = NULL;
if (selectedTab==simpleTab)
{
foundTdbs = simpleSearchForTdbs(trix,descWords,descWordCount);
// What to do now?
if (measureTiming)
uglyTime("Searched for tracks");
// Sort and Print results
if (selectedTab!=filesTab)
{
enum sortBy sortBy = cartUsualInt(cart,TRACK_SEARCH_SORT,sbRelevance);
int tracksFound = slCount(foundTdbs);
if (tracksFound > 1)
findTracksSort(&tracks,sortBy);
displayFoundTracks(cart,tracks,tracksFound,sortBy);
if (measureTiming)
uglyTime("Displayed found files");
}
}
else if (selectedTab==filesTab && mdbPairs != NULL)
#endif ///def USE_TABS
{
if (nameSearch || descSearch || groupSearch)
{ // Use nameSearch, descSearch and groupSearch to narrow down the list of composites.
if (isNotEmpty(nameSearch) || isNotEmpty(descSearch) || isNotEmpty(groupSearch))
{
struct trackDb *tdbList = hTrackDb(db);
struct trackDb *tdbsMatch = tdbFilterBy(&tdbList, nameSearch, descSearch,
groupSearch);
// Now we have a list of tracks, so we need a unique list of composites to add
doSearch = mdbSelectsAddFoundComposites(&mdbSelects,tdbsMatch);
}
}
if (doSearch && mdbSelects != NULL && isNotEmpty(fileTypeSearch))
fileSearchResults(db, conn, cart, mdbSelects, fileTypeSearch);
else
printf("<DIV id='filesFound'><BR>No files found.<BR></DIV><BR>\n");
if (measureTiming)
uglyTime("Searched for files");
}
slPairFreeList(&mdbSelects);
}
hFreeConn(&conn);
webNewSection("About " FILE_SEARCH_NAME);
printf("Search for downloadable ENCODE files by entering search terms in "
"the Track name or Description fields and/or by making selections with "
"the group, data format, and/or ENCODE metadata drop-downs.");
printf("<BR><a target='_blank' href='../goldenPath/help/fileSearch.html'>more help</a>\n");
}
