struct slInt * seeded_random_indices(int max, double proportion, int seed)
{
int i;
int numToPick = floor(max*proportion);
struct slInt *curr = NULL, *list = NULL, *shuffledList = NULL;
for(i = 0; i < max; i++)
{
if(!list)
{
list = slIntNew(i);
curr= list;
}
else
{
curr->next = slIntNew(i);
curr = curr->next;
}
}
shuffledList = seeded_shuffle_indices(list, seed);
slFreeList(&list);
curr = shuffledList;
for(i = 0; i < (numToPick-1); i++)
curr = curr->next;
slFreeList(curr->next);
curr->next =NULL;
return shuffledList;
}
struct slInt * orderIndicesByDescription(struct hash * config, matrix * data, int split)
{
matrix * fd = foldsDescriptionFromConfig(config);
matrix * orderedFd = matchOnColLabels(fd, data);
int fold, folds = foldsCountFromFoldsDescription(config);
if(orderedFd->cols != data->cols)
errAbort("ERROR: Columns differ for data and foldDescription file. Unsupported behaviour.\n");
struct slInt *curr = NULL, *result = NULL;
for(fold = 1; fold <= folds; fold++)
{
int j;
for(j = 0; j < orderedFd->cols; j++)
{
if(orderedFd->graph[(split-1)][j] == NULL_FLAG)
errAbort("ERROR: fold description didn't include an entry for sample %s in split %d", orderedFd->colLabels[j], split);
if(orderedFd->graph[(split-1)][j] == fold)
{
if(!result)
{
result = slIntNew(j);
curr = result;
}
else
{
curr->next = slIntNew(j);
curr = curr->next;
}
}
}
}
return result;
}
struct slName *bigBedListExtraIndexes(struct bbiFile *bbi)
/* Return list of names of extra indexes beyond primary chrom:start-end one" */
{
struct udcFile *udc = bbi->udc;
boolean isSwapped = bbi->isSwapped;
/* See if we have any extra indexes, and if so seek to there. */
bits64 offset = bbi->extraIndexListOffset;
if (offset == 0)
return NULL;
udcSeek(udc, offset);
/* Construct list of field that are being indexed. List is list of
* field numbers within asObj. */
int i;
struct slInt *intList = NULL, *intEl;
for (i=0; i<bbi->extraIndexCount; ++i)
{
bits16 type,fieldCount;
type = udcReadBits16(udc, isSwapped);
fieldCount = udcReadBits16(udc, isSwapped);
udcSeekCur(udc, sizeof(bits64)); // skip over fileOffset
udcSeekCur(udc, 4); // skip over reserved bits
if (fieldCount == 1)
{
bits16 fieldId = udcReadBits16(udc, isSwapped);
udcSeekCur(udc, 2); // skip over reserved bits
intEl = slIntNew(fieldId);
slAddHead(&intList, intEl);
}
else
{
warn("Not yet understanding indexes on multiple fields at once.");
internalErr();
}
}
/* Now have to make an asObject to find out name that corresponds to this field. */
struct asObject *as = bigBedAsOrDefault(bbi);
/* Make list of field names out of list of field numbers */
struct slName *nameList = NULL;
for (intEl = intList; intEl != NULL; intEl = intEl->next)
{
struct asColumn *col = slElementFromIx(as->columnList, intEl->val);
if (col == NULL)
{
warn("Inconsistent bigBed file %s", bbi->fileName);
internalErr();
}
slNameAddHead(&nameList, col->name);
}
asObjectFree(&as);
return nameList;
}
struct slInt * list_indices(int n)
{
struct slInt * result = NULL, * curr = NULL;
int i;
for(i = 0; i < n; i++)
{
if(!result)
{
result = slIntNew(i);
curr=result;
}
else
{
curr->next = slIntNew(i);
curr = curr->next;
}
}
return result;
}
struct slInt * copy_list(struct slInt * list)
{
struct slInt * result = NULL;
struct slInt * curr = NULL;
struct slInt * listCurr = list;
while(listCurr)
{
if(!result)
{
result = slIntNew(listCurr->val);
curr = result;
}
else
{
curr->next = slIntNew(listCurr->val);
curr = curr->next;
}
listCurr = listCurr->next;
}
return result;
}
struct slInt * index_complement(int max, struct slInt * list)
{
struct slInt * listCurr = list;
struct slInt * result = NULL;
struct slInt * resultCurr = NULL;
int i = 0;
//init complementary array as all complementary
int complementary[max];
for(i = 0; i < max; i++)
complementary[i] = 1;
//set flags in complementary array where list contains the index
while(listCurr != NULL)
{
complementary[listCurr->val] = 0;
listCurr=listCurr->next;
}
//iterate over complementary array, creating list from items flagged as complements
for(i = 0; i < max; i++)
{
if(complementary[i])
{
if(!result)
{
result=slIntNew(i);
resultCurr = result;
}
else
{
resultCurr->next = slIntNew(i);
resultCurr = resultCurr->next;
}
}
}
return result;
}
void rowsToCols(char *in, char *out)
/* rowsToCols - Convert rows to columns and vice versa in a text file.. */
{
struct slName *lineList = readAllLines(in);
struct tabRow *row, *rowList;
FILE *f;
int i, origCols;
char *s;
if (fs != NULL)
rowList = tabRowByChar(lineList, fs[0], in, varCol);
else if (fixed)
rowList = tabRowByFixedGuess(lineList, in);
else if (offsets)
{
struct slName *nameList = commaSepToSlNames(offsets), *name;
struct slInt *offList=NULL, *off;
for (name = nameList; name != NULL; name = name->next)
{
off = slIntNew(atoi(name->name));
slAddTail(&offList, off);
}
rowList = tabRowByFixedOffsets(lineList, offList, in);
}
else
rowList = tabRowByWhite(lineList, in, varCol);
origCols = tabRowMaxColCount(rowList);
f = mustOpen(out, "w");
for (i=0; i<origCols; ++i)
{
for (row = rowList; row != NULL; row = row->next)
{
if (i < row->colCount)
s = row->columns[i];
else
s = "";
fprintf(f, "%s", s);
if (row->next == NULL)
fprintf(f, "\n");
else
fprintf(f, "\t");
}
}
carefulClose(&f);
}
void bwtool_find_max(struct hash *options, char *favorites, char *regions, double fill,
char *bigfile, char *tmp_dir, char *outputfile)
/* find max points in a range */
{
boolean med_base = (hashFindVal(options, "median-base") != NULL) ? TRUE : FALSE;
boolean with_max = (hashFindVal(options, "with-max") != NULL) ? TRUE : FALSE;
struct metaBig *mb = metaBigOpen_check(bigfile, tmp_dir, NULL);
FILE *out = mustOpen(outputfile, "w");
struct bed6 *sections6 = readBed6Soft(regions);
struct bed *sections = bed12FromBed6(§ions6);
struct bed *section;
for (section = sections; section != NULL; section = section->next)
{
struct perBaseWig *pbwList = perBaseWigLoadContinue(mb, section->chrom, section->chromStart,
section->chromEnd);
struct perBaseWig *pbw;
struct slInt *ii;
int i, size;
double max = -DBL_MAX;
struct slInt *list = NULL;
for (pbw = pbwList; pbw != NULL; pbw = pbw->next)
{
int pbw_off = pbw->chromStart - section->chromStart;
for (i = 0; i < pbw->len; i++)
{
if (pbw->data[i] > max)
{
slFreeList(&list);
struct slInt *new_int = slIntNew(i + pbw_off);
slAddHead(&list, new_int);
max = pbw->data[i];
}
else if (pbw->data[i] == max)
{
struct slInt *new_int = slIntNew(i + pbw_off);
slAddHead(&list, new_int);
}
}
}
slReverse(&list);
if (list)
{
size = slCount(list);
if (med_base)
{
section->blockCount = 1;
AllocArray(section->blockSizes, sizeof(int));
AllocArray(section->chromStarts, sizeof(int));
section->blockSizes[0] = 1;
section->chromStarts[0] = median_base_calc(&list);
}
else
{
section->blockCount = size;
AllocArray(section->blockSizes, sizeof(int) * size);
AllocArray(section->chromStarts, sizeof(int) * size);
for (i = 0, ii = list; (i < size) && (ii != NULL); i++, ii = ii->next)
{
section->blockSizes[i] = 1;
section->chromStarts[i] = ii->val;
}
}
if (!with_max)
bedTabOutN(section, 12, out);
else
{
bedOutputN(section, 12, out, '\t', '\t');
fprintf(out, "%f\n", max);
}
slFreeList(&list);
}
perBaseWigFree(&pbwList);
}
metaBigClose(&mb);
bedFreeList(§ions);
carefulClose(&out);
}
void cleanTableSection(char *table, int startId, int endId)
/* clean a specific table section */
{
struct sqlResult *sr;
char **row;
char query[256];
int rc = 0;
int dc = 0;
int delCount = 0;
int count = 0;
int maxId = startId - 1;
int useCount = 0;
boolean deleteThis = FALSE;
int delRobotCount = 0;
int oldRecCount = 0;
struct slInt *delList = NULL;
time_t cleanSectionStart = time(NULL);
struct dyString *dy = dyStringNew(0);
while(TRUE)
{
verbose(2, "maxId: %d count=%d delCount=%d dc=%d\n", maxId, count, delCount, dc);
sqlSafef(query,sizeof(query),
"select id, firstUse, lastUse, useCount from %s"
" where id > %d order by id limit %d"
, table
, maxId
, chunkSize
);
sr = sqlGetResult(conn, query);
rc = 0;
dc = 0;
dyStringClear(dy);
while ((row = sqlNextRow(sr)) != NULL)
{
++count;
++rc;
maxId = sqlUnsigned(row[0]);
useCount = sqlSigned(row[3]);
int daysAgoFirstUse = toDaysAgo(row[1], maxId);
int daysAgoLastUse = toDaysAgo(row[2], maxId);
verbose(3, "id: %d, firstUse: [%s] [%d days ago], lastUse: [%s] [%d days ago], useCount: %d\n"
, maxId
, row[1], daysAgoFirstUse
, row[2], daysAgoLastUse
, useCount
);
deleteThis = FALSE;
if (sameString(table, sessionDbTableName))
{
if (daysAgoLastUse >= 14)
{
deleteThis = TRUE;
++oldRecCount;
}
else if ((daysAgoFirstUse >= 2) && useCount <= 1) /* reasonable new addition */
{
deleteThis = TRUE;
++delRobotCount;
}
}
if (sameString(table, userDbTableName))
{
if ((daysAgoFirstUse >= 7) && useCount < 7)
{
deleteThis = TRUE;
++delRobotCount;
}
else if ((daysAgoFirstUse >= 2) && useCount <= 1)
{
deleteThis = TRUE;
++delRobotCount;
}
else if (daysAgoLastUse >= 365) /* reasonable new addition */
{
deleteThis = TRUE;
++oldRecCount;
}
}
if (deleteThis)
{
++dc;
verbose(3, "TO DELETE id: %d, "
"firstUse: [%s] [%d days ago], lastUse: [%s] [%d days ago], useCount: %d\n"
, maxId
, row[1], daysAgoFirstUse
, row[2], daysAgoLastUse
, useCount
);
slAddHead(&delList, slIntNew(maxId));
}
}
sqlFreeResult(&sr);
if (rc < 1)
break;
if (dc > 0)
{
struct slInt *i;
for (i=delList;i;i=i->next)
{
dyStringClear(dy);
sqlDyStringPrintf(dy, "delete from %s where id=%d", table, i->val);
sqlUpdate(conn,dy->string);
}
slFreeList(&delList);
}
delCount+=dc;
if (maxId >= endId)
{
break; // we have done enough
}
verbose(3, "sleeping %d seconds\n", chunkWait);fflush(stderr);
sleep(chunkWait);
verbose(3, "awake\n");fflush(stderr);
}
verbose(1, "old recs deleted %d, robot recs deleted %d\n", oldRecCount, delRobotCount);fflush(stderr);
time_t cleanEnd = time(NULL);
int minutes = difftime(cleanEnd, cleanSectionStart) / 60;
verbose(1, "%s\n", ctime(&cleanEnd));
verbose(1, "%d minutes\n\n", minutes);
}
void cdwJobCleanFailed(int submitId)
/* Check out the symlink to determine its type. */
{
struct sqlConnection *conn = sqlConnect("cdw");
struct dyString *query = dyStringNew(0);
sqlDyStringPrintf(query,
"select id, commandLine, startTime, endTime, returnCode, pid from cdwJob where submitId=%d "
"order by commandLine,CAST(returnCode AS unsigned)",
submitId);
// NOTE we need this CAST on returnCode since it can be -1. we want success 0 first.
// TODO DO we need to add any other conditions such as distinguishing
// between running, queued, and done?
/* Scan through result set finding redundant rows beyond success row. */
struct sqlResult *sr = sqlGetResult(conn, query->string);
char **row;
char *lastCommand = "";
boolean success = FALSE;
struct slInt *list = NULL;
struct slInt *e;
while ((row = sqlNextRow(sr)) != NULL)
{
unsigned int id = sqlUnsigned(row[0]);
char *commandLine = row[1];
unsigned long startTime = sqlUnsignedLong(row[2]);
unsigned long endTime = sqlUnsignedLong(row[3]);
int returnCode = sqlSigned(row[4]);
unsigned int pid = sqlUnsigned(row[5]);
verbose(2, "%u\t%s\t%lu\t%lu\t%d\t%u\t%u\n", id, commandLine, startTime, endTime, returnCode, pid, submitId);
if (sameString(lastCommand, commandLine))
{
if (success) // we already succeeded, the old failure is unwanted baggage.
{
e = slIntNew(id); // or add it to a list of rows whose ids should get removed
slAddHead(&list, e);
}
}
else
{
if (returnCode == 0)
success = TRUE;
else
success = FALSE;
}
// note fields pid and submitId are defined as signed integers in cdwJob table, probably should be unsigned.
lastCommand = cloneString(commandLine);
}
sqlFreeResult(&sr);
slReverse(&list);
for(e=list;e;e=e->next)
{
dyStringClear(query);
sqlDyStringPrintf(query, "delete from cdwJob where id=%u", (unsigned int) e->val);
//printf("%s\n", query->string);
sqlUpdate(conn, query->string);
}
/* Clean up and go home */
dyStringFree(&query);
sqlDisconnect(&conn);
}
void tryToDeprecate(struct sqlConnection *conn)
/* CGI variables are set - if possible deprecate, otherwise put up error message. */
{
pushWarnHandler(localWarn);
fileList = cgiString("fileList");
reason = cloneString(trimSpaces(cgiString("reason")));
if (isEmpty(reason))
{
warn("Please enter a reason for deprecation.");
getFileListAndReason(conn);
}
else
{
/* Go through list of accessions and make sure they are all well formed and correspond to files that exist. */
boolean ok = TRUE;
struct slName *accList = slNameListOfUniqueWords(cloneString(fileList), FALSE);
struct slName *acc;
struct slInt *idList = NULL, *idEl;
for (acc = accList; acc != NULL; acc = acc->next)
{
char *licensePlate = acc->name;
if (!startsWith(edwLicensePlatePrefix, licensePlate))
{
ok = FALSE;
warn("%s is not an accession, doesn't start with %s", licensePlate, edwLicensePlatePrefix);
break;
}
char query[256];
sqlSafef(query, sizeof(query), "select fileId from edwValidFile where licensePlate='%s'", licensePlate);
int id = sqlQuickNum(conn, query);
if (id == 0)
{
ok = FALSE;
warn("%s - no such accession. ", licensePlate);
break;
}
/* check to see is it ok tor deprecate this file */
if (!okToDeprecateThisFile(conn, id, userEmail))
{
ok = FALSE;
warn("You can not deprecate %s which was originally uploaded by %s.\n",
licensePlate, edwFindOwnerNameFromFileId(conn, id));
warn("Please click the check box below to override this rule.");
break;
}
idEl = slIntNew(id);
slAddTail(&idList, idEl);
}
if (accList == NULL)
{
warn("Please enter some file accessions");
ok = FALSE;
}
/* If a problem then put up page to try again, otherwise do deprecation. */
if (!ok)
getFileListAndReason(conn);
else
{
deprecateFileList(conn, idList, reason);
printf("Deprecated %d files<BR>\n", slCount(idList));
cgiMakeButton("submit", "Deprecate More Files");
printf(" ");
edwPrintLogOutButton();
}
}
}
struct slInt * listTopN(matrix * A_ptr, int n, int direction, int ix)
/*Make a list of the top features in a row or column of matrix. Direction: 1=rows, 2=cols*/
{
//make an empty list, and a pointer to it.
struct slInt *list = NULL, *curr = NULL;
int i, j;
if(direction == 1)
{
//start the list by adding the first element (non null) in matrix
j = 0;
while(A_ptr->graph[j][ix] == NULL_FLAG && j < A_ptr->rows)
j++;
list=slIntNew(j);
curr = list;
for(i = (j+1); i < A_ptr->rows; i++)
{
curr = list;
//if the new element is the largest seen yet, add it to the front of the list
if(A_ptr->graph[i][ix] != NULL_FLAG && fabs(A_ptr->graph[i][ix]) > fabs(A_ptr->graph[curr->val][ix]))
{
slAddHead(&list, slIntNew(i));
}
else
{
//otherwise iterate the list looking for if it fits in it.
while(curr->next != NULL && (A_ptr->graph[i][ix] != NULL_FLAG && fabs(A_ptr->graph[i][ix]) < fabs(A_ptr->graph[curr->next->val][ix])))
curr = curr->next;
struct slInt *tmp = curr->next;
curr->next = slIntNew(i);
curr->next->next = tmp;
}
//if the list has gotten too long, reduce it
if(count_indices(list) > n)
{
curr = list;
j = 0;
while(curr->next->next && j < n)
{
curr = curr->next;
j++;
}
slFreeList(&curr->next);
curr->next = NULL;
}
}
}
else if(direction == 2)
{
//start the list by adding the first element in matrix
j = 0;
while(A_ptr->graph[ix][j] == NULL_FLAG && j < A_ptr->rows)
j++;
curr->val = j;
for(i = (j+1); i < A_ptr->cols; i++)
{
curr = list;
//if the new element is the largest seen yet, add it to the front of the list
if(A_ptr->graph[ix][i] != NULL_FLAG && fabs(A_ptr->graph[ix][i]) > fabs(A_ptr->graph[ix][curr->val]))
{
slAddHead(&list, slIntNew(i));
}
else
{
//otherwise iterate the list looking for if it fits in it.
while(curr->next != NULL && (A_ptr->graph[ix][i] != NULL_FLAG && fabs(A_ptr->graph[ix][i]) < fabs(A_ptr->graph[ix][curr->next->val])))
curr = curr->next;
struct slInt *tmp = curr->next;
curr->next = slIntNew(i);
curr->next->next = tmp;
}
//if the list has gotten too long, reduce it
if(count_indices(list) > n)
{
curr = list;
j = 0;
while(curr->next->next && j < n)
{
curr = curr->next;
j++;
}
slFreeList(&curr->next);
curr->next = NULL;
}
}
}
else
errAbort("When listing top N an invalid direction was supplied.\n");
return list;
}
