void hgMaxExp(char *database, char *table)
/* Output maximum expScore in table */
{
struct sqlConnection *conn = sqlConnect(database);
struct sqlResult *sr;
char query[256];
char **row;
/* Get list of all items with expression values. */
char *fieldNames = "name, expCount, expScores";
sqlSafef(query, sizeof(query), "select %s from %s", fieldNames, table);
sr = sqlGetResult(conn, query);
float maxScore = 0.0;
float minScore = 100000.0;
while ((row = sqlNextRow(sr)) != NULL)
{
char *name = row[0];
int expCount = sqlUnsigned(row[1]);
int commaCount;
float *expScores = NULL;
sqlFloatDynamicArray(row[2], &expScores, &commaCount);
if (expCount != commaCount)
errAbort("expCount and expScores don't match on %s in %s", name, table);
int i;
for (i=0; i<expCount; i++)
{
maxScore = max(maxScore, expScores[i]);
minScore = min(minScore, expScores[i]);
}
}
sqlFreeResult(&sr);
conn = sqlConnect(database);
sqlDisconnect(&conn); /* Disconnect because next step is slow. */
printf("max: %0.2f min: %0.2f\n", maxScore, minScore);
}
struct autoTest *autoTestLoad(char **row)
/* Load a autoTest from row fetched with select * from autoTest
* from database. Dispose of this with autoTestFree(). */
{
struct autoTest *ret;
AllocVar(ret);
ret->ptCount = sqlSigned(row[5]);
ret->difCount = sqlSigned(row[7]);
ret->valCount = sqlSigned(row[10]);
ret->id = sqlUnsigned(row[0]);
safecpy(ret->shortName, sizeof(ret->shortName), row[1]);
ret->longName = cloneString(row[2]);
{
char *s = cloneString(row[3]);
sqlStringArray(s, ret->aliases, 3);
}
{
char *s = row[4];
if(s != NULL && differentString(s, ""))
ret->threeD = pointCommaIn(&s, NULL);
}
{
int sizeOne;
sqlShortDynamicArray(row[6], &ret->pts, &sizeOne);
assert(sizeOne == ret->ptCount);
}
{
int sizeOne;
sqlUbyteDynamicArray(row[8], &ret->difs, &sizeOne);
assert(sizeOne == ret->difCount);
}
sqlSignedArray(row[9], ret->xy, 2);
{
int sizeOne;
sqlStringDynamicArray(row[11], &ret->vals, &sizeOne);
assert(sizeOne == ret->valCount);
}
ret->dblVal = sqlDouble(row[12]);
ret->fltVal = sqlFloat(row[13]);
{
int sizeOne;
sqlDoubleDynamicArray(row[14], &ret->dblArray, &sizeOne);
assert(sizeOne == ret->valCount);
}
{
int sizeOne;
sqlFloatDynamicArray(row[15], &ret->fltArray, &sizeOne);
assert(sizeOne == ret->valCount);
}
return ret;
}
struct expData *expDataLoad(char **row)
/* Load a expData from row fetched with select * from expData
* from database. Dispose of this with expDataFree(). */
{
struct expData *ret;
int sizeOne;
AllocVar(ret);
ret->expCount = sqlUnsigned(row[1]);
ret->name = cloneString(row[0]);
sqlFloatDynamicArray(row[2], &ret->expScores, &sizeOne);
assert(sizeOne == ret->expCount);
return ret;
}
struct gtexGeneBed *gtexGeneBedLoad(char **row)
/* Load a gtexGeneBed from row fetched with select * from gtexGeneBed
* from database. Dispose of this with gtexGeneBedFree(). */
{
struct gtexGeneBed *ret;
AllocVar(ret);
ret->expCount = sqlUnsigned(row[8]);
ret->chrom = cloneString(row[0]);
ret->chromStart = sqlUnsigned(row[1]);
ret->chromEnd = sqlUnsigned(row[2]);
ret->name = cloneString(row[3]);
ret->score = sqlUnsigned(row[4]);
safecpy(ret->strand, sizeof(ret->strand), row[5]);
ret->geneId = cloneString(row[6]);
ret->geneType = cloneString(row[7]);
{
int sizeOne;
sqlFloatDynamicArray(row[9], &ret->expScores, &sizeOne);
assert(sizeOne == ret->expCount);
}
return ret;
}
struct exprBed *exprBedLoad(char **row)
/* Load a exprBed from row fetched with select * from exprBed
* from database. Dispose of this with exprBedFree(). */
{
struct exprBed *ret;
int sizeOne;
AllocVar(ret);
ret->numExp = sqlUnsigned(row[8]);
ret->chrom = cloneString(row[0]);
ret->chromStart = sqlUnsigned(row[1]);
ret->chromEnd = sqlUnsigned(row[2]);
ret->name = cloneString(row[3]);
ret->size = sqlUnsigned(row[4]);
ret->uniqueAlign = sqlUnsigned(row[5]);
ret->score = sqlUnsigned(row[6]);
ret->hname = cloneString(row[7]);
sqlStringDynamicArray(row[9], &ret->hybes, &sizeOne);
assert(sizeOne == ret->numExp);
sqlFloatDynamicArray(row[10], &ret->scores, &sizeOne);
assert(sizeOne == ret->numExp);
return ret;
}
void hgExpDistance(char *database, char *posTable, char *expTable, char *outTable)
/* hgExpDistance - Create table that measures expression distance between pairs. */
{
struct sqlConnection *conn = sqlConnect(database);
struct sqlResult *sr;
char query[256];
char **row;
struct hash *expHash = hashNew(16);
int realExpCount = -1;
struct microData *geneList = NULL, *curGene, *gene;
int geneIx, geneCount = 0;
struct microData **geneArray = NULL;
float *weights = NULL;
char *tempDir = ".";
FILE *f = hgCreateTabFile(tempDir, outTable);
/* Get list/hash of all items with expression values. */
sqlSafef(query, sizeof(query), "select name,expCount,expScores from %s", posTable);
sr = sqlGetResult(conn, query);
while ((row = sqlNextRow(sr)) != NULL)
{
char *name = row[0];
if (!hashLookup(expHash, name))
{
int expCount = sqlUnsigned(row[1]);
int commaCount;
float *expScores = NULL;
sqlFloatDynamicArray(row[2], &expScores, &commaCount);
if (expCount != commaCount)
errAbort("expCount and expScores don't match on %s in %s", name, posTable);
if (realExpCount == -1)
realExpCount = expCount;
if (expCount != realExpCount)
errAbort("In %s some rows have %d experiments others %d",
name, expCount, realExpCount);
AllocVar(gene);
gene->expCount = expCount;
gene->expScores = expScores;
hashAddSaveName(expHash, name, gene, &gene->name);
slAddHead(&geneList, gene);
}
}
sqlFreeResult(&sr);
conn = sqlConnect(database);
slReverse(&geneList);
geneCount = slCount(geneList);
printf("Have %d elements in %s\n", geneCount, posTable);
weights = getWeights(realExpCount);
if (optionExists("lookup"))
geneList = lookupGenes(conn, optionVal("lookup", NULL), geneList);
geneCount = slCount(geneList);
printf("Got %d unique elements in %s\n", geneCount, posTable);
sqlDisconnect(&conn); /* Disconnect because next step is slow. */
if (geneCount < 1)
errAbort("ERROR: unique gene count less than one ?");
/* Get an array for sorting. */
AllocArray(geneArray, geneCount);
for (gene = geneList,geneIx=0; gene != NULL; gene = gene->next, ++geneIx)
geneArray[geneIx] = gene;
/* Print out closest 1000 in tab file. */
for (curGene = geneList; curGene != NULL; curGene = curGene->next)
{
calcDistances(curGene, geneList, weights);
qsort(geneArray, geneCount, sizeof(geneArray[0]), cmpMicroDataDistance);
for (geneIx=0; geneIx < 1000 && geneIx < geneCount; ++geneIx)
{
gene = geneArray[geneIx];
fprintf(f, "%s\t%s\t%f\n", curGene->name, gene->name, gene->distance);
}
dotOut();
}
printf("Made %s.tab\n", outTable);
/* Create and load table. */
conn = sqlConnect(database);
distanceTableCreate(conn, outTable);
hgLoadTabFile(conn, tempDir, outTable, &f);
printf("Loaded %s\n", outTable);
/* Add indices. */
sqlSafef(query, sizeof(query), "alter table %s add index(query)", outTable);
sqlUpdate(conn, query);
printf("Made query index\n");
if (optionExists("targetIndex"))
{
sqlSafef(query, sizeof(query), "alter table %s add index(target)", outTable);
sqlUpdate(conn, query);
printf("Made target index\n");
}
hgRemoveTabFile(tempDir, outTable);
}
void hgExpDistance(char *database, char *posTable, char *expTable, char *outTable)
/* hgExpDistance - Create table that measures expression distance between pairs. */
{
struct sqlConnection *conn = sqlConnect(database);
struct sqlResult *sr;
char query[256];
char **row;
struct hash *expHash = hashNew(16);
int realExpCount = -1;
struct microData *gene;
int rc, t;
pthread_t *threads = NULL;
pthread_attr_t attr;
int *threadID = NULL;
void *status;
char *tempDir = ".";
int arrayNum;
struct microDataDistance *geneDistPtr = NULL;
struct microDataDistance *geneDistArray = NULL;
int geneIx;
FILE *f = NULL;
/* Get list/hash of all items with expression values. */
safef(query, sizeof(query), "select name,expCount,expScores from %s", posTable);
sr = sqlGetResult(conn, query);
while ((row = sqlNextRow(sr)) != NULL)
{
char *name = row[0];
if (!hashLookup(expHash, name))
{
int expCount = sqlUnsigned(row[1]);
int commaCount;
float *expScores = NULL;
sqlFloatDynamicArray(row[2], &expScores, &commaCount);
if (expCount != commaCount)
errAbort("expCount and expScores don't match on %s in %s", name, posTable);
if (realExpCount == -1)
realExpCount = expCount;
if (expCount != realExpCount)
errAbort("In %s some rows have %d experiments others %d",
name, expCount, realExpCount);
AllocVar(gene);
gene->expCount = expCount;
gene->expScores = expScores;
hashAddSaveName(expHash, name, gene, &gene->name);
slAddHead(&geneList, gene);
}
}
sqlFreeResult(&sr);
conn = sqlConnect(database);
slReverse(&geneList);
geneCount = slCount(geneList);
printf("Have %d elements in %s\n", geneCount, posTable);
weights = getWeights(realExpCount);
if (optionExists("lookup"))
geneList = lookupGenes(conn, optionVal("lookup", NULL), geneList);
geneCount = slCount(geneList);
printf("Got %d unique elements in %s\n", geneCount, posTable);
sqlDisconnect(&conn); /* Disconnect because next step is slow. */
if (geneCount < 1)
errAbort("ERROR: unique gene count less than one ?");
f = hgCreateTabFile(tempDir, outTable);
synQ = synQueueNew();
/* instantiate threads */
AllocArray( threadID, numThreads );
AllocArray( threads, numThreads );
pthread_attr_init( &attr );
pthread_mutex_init( &mutexDotOut, NULL );
pthread_attr_setdetachstate( &attr, PTHREAD_CREATE_JOINABLE );
for (t = 0; t < numThreads; t++) {
threadID[t] = t;
rc = pthread_create( &threads[t], &attr, computeDistance,
(void *) &threadID[t]);
if (rc)
errAbort("ERROR: in pthread_create() %d\n", rc );
}
/* this thread will write to the file from the queue */
for (arrayNum = 0; arrayNum < geneCount; arrayNum++) {
geneDistArray = (struct microDataDistance *)synQueueGet( synQ );
geneDistPtr = geneDistArray;
/* Print out closest GENEDISTS distances in tab file. */
for (geneIx=0; geneIx < GENEDISTS && geneIx < geneCount;
++geneIx, geneDistPtr++)
if (geneDistPtr != NULL)
fprintf(f, "%s\t%s\t%f\n", geneDistPtr->name1,
geneDistPtr->name2, geneDistPtr->distance);
else
errAbort("ERROR: writing distance %d to file\n",
geneIx);
freeMem( geneDistArray );
}
/* synchronize all threads */
for (t = 0; t < numThreads; t++) {
rc = pthread_join( threads[t], &status);
if (rc)
errAbort("ERROR: in pthread_join() %d\n", rc );
}
printf("Made %s.tab\n", outTable);
slFreeList( &geneList );
pthread_mutex_destroy( &mutexDotOut );
pthread_attr_destroy( &attr );
/* Create and load table. */
conn = sqlConnect(database);
distanceTableCreate(conn, outTable);
hgLoadTabFile(conn, tempDir, outTable, &f);
printf("Loaded %s\n", outTable);
/* Add indices. */
safef(query, sizeof(query), "alter table %s add index(query(12))", outTable);
sqlUpdate(conn, query);
printf("Made query index\n");
if (optionExists("targetIndex"))
{
safef(query, sizeof(query), "alter table %s add index(target(12))", outTable);
sqlUpdate(conn, query);
printf("Made target index\n");
}
hgRemoveTabFile(tempDir, outTable);
}
void hgExpDistance(char *database, char *posTable, char *expTable, char *outTable)
/* hgExpDistance - Create table that measures expression distance between pairs. */
{
struct sqlConnection *conn = sqlConnect(database);
struct sqlResult *sr;
char query[256];
char **row;
struct hash *expHash = hashNew(16);
int realExpCount = -1;
struct microData *gene;
int rc, t;
pthread_t *threads = NULL;
pthread_attr_t attr;
int *threadID = NULL;
void *status;
char *tempDir = ".";
long time1, time2;
time1 = clock1000();
/* Get list/hash of all items with expression values. */
sqlSafef(query, sizeof(query), "select name,expCount,expScores from %s", posTable);
sr = sqlGetResult(conn, query);
while ((row = sqlNextRow(sr)) != NULL)
{
char *name = row[0];
if (!hashLookup(expHash, name))
{
int expCount = sqlUnsigned(row[1]);
int commaCount;
float *expScores = NULL;
sqlFloatDynamicArray(row[2], &expScores, &commaCount);
if (expCount != commaCount)
errAbort("expCount and expScores don't match on %s in %s", name, posTable);
if (realExpCount == -1)
realExpCount = expCount;
if (expCount != realExpCount)
errAbort("In %s some rows have %d experiments others %d",
name, expCount, realExpCount);
AllocVar(gene);
gene->expCount = expCount;
gene->expScores = expScores;
hashAddSaveName(expHash, name, gene, &gene->name);
slAddHead(&geneList, gene);
}
}
sqlFreeResult(&sr);
conn = sqlConnect(database);
slReverse(&geneList);
geneCount = slCount(geneList);
printf("Have %d elements in %s\n", geneCount, posTable);
weights = getWeights(realExpCount);
if (optionExists("lookup"))
geneList = lookupGenes(conn, optionVal("lookup", NULL), geneList);
geneCount = slCount(geneList);
printf("Got %d unique elements in %s\n", geneCount, posTable);
sqlDisconnect(&conn); /* Disconnect because next step is slow. */
if (geneCount < 1)
errAbort("ERROR: unique gene count less than one ?");
time2 = clock1000();
verbose(2, "records read time: %.2f seconds\n", (time2 - time1) / 1000.0);
f = hgCreateTabFile(tempDir, outTable);
/* instantiate threads */
AllocArray( threadID, numThreads );
AllocArray( threads, numThreads );
pthread_attr_init( &attr );
pthread_mutex_init( &mutexfilehandle, NULL );
pthread_attr_setdetachstate( &attr, PTHREAD_CREATE_JOINABLE );
for (t = 0; t < numThreads; t++) {
threadID[t] = t;
rc = pthread_create( &threads[t], &attr, computeDistance,
(void *) &threadID[t]);
if (rc)
errAbort("ERROR: in pthread_create() %d\n", rc );
}
/* synchronize all threads */
for (t = 0; t < numThreads; t++) {
rc = pthread_join( threads[t], &status);
if (rc)
errAbort("ERROR: in pthread_join() %d\n", rc );
}
printf("Made %s.tab\n", outTable);
slFreeList( &geneList );
pthread_mutex_destroy( &mutexfilehandle );
pthread_attr_destroy( &attr );
time1 = time2;
time2 = clock1000();
verbose(2, "distance computation time: %.2f seconds\n", (time2 - time1) / 1000.0);
/* Create and load table. */
conn = sqlConnect(database);
distanceTableCreate(conn, outTable);
hgLoadTabFile(conn, tempDir, outTable, &f);
printf("Loaded %s\n", outTable);
/* Add indices. */
sqlSafef(query, sizeof(query), "alter table %s add index(query(12))", outTable);
sqlUpdate(conn, query);
printf("Made query index\n");
if (optionExists("targetIndex"))
{
sqlSafef(query, sizeof(query), "alter table %s add index(target(12))", outTable);
sqlUpdate(conn, query);
printf("Made target index\n");
}
hgRemoveTabFile(tempDir, outTable);
time1 = time2;
time2 = clock1000();
verbose(2, "table create/load/index time: %.2f seconds\n", (time2 - time1) / 1000.0);
}