struct slInt * seeded_random_indices(int max, double proportion, int seed) { int i; int numToPick = floor(max*proportion); struct slInt *curr = NULL, *list = NULL, *shuffledList = NULL; for(i = 0; i < max; i++) { if(!list) { list = slIntNew(i); curr= list; } else { curr->next = slIntNew(i); curr = curr->next; } } shuffledList = seeded_shuffle_indices(list, seed); slFreeList(&list); curr = shuffledList; for(i = 0; i < (numToPick-1); i++) curr = curr->next; slFreeList(curr->next); curr->next =NULL; return shuffledList; }
struct slInt * orderIndicesByDescription(struct hash * config, matrix * data, int split) { matrix * fd = foldsDescriptionFromConfig(config); matrix * orderedFd = matchOnColLabels(fd, data); int fold, folds = foldsCountFromFoldsDescription(config); if(orderedFd->cols != data->cols) errAbort("ERROR: Columns differ for data and foldDescription file. Unsupported behaviour.\n"); struct slInt *curr = NULL, *result = NULL; for(fold = 1; fold <= folds; fold++) { int j; for(j = 0; j < orderedFd->cols; j++) { if(orderedFd->graph[(split-1)][j] == NULL_FLAG) errAbort("ERROR: fold description didn't include an entry for sample %s in split %d", orderedFd->colLabels[j], split); if(orderedFd->graph[(split-1)][j] == fold) { if(!result) { result = slIntNew(j); curr = result; } else { curr->next = slIntNew(j); curr = curr->next; } } } } return result; }
struct slName *bigBedListExtraIndexes(struct bbiFile *bbi) /* Return list of names of extra indexes beyond primary chrom:start-end one" */ { struct udcFile *udc = bbi->udc; boolean isSwapped = bbi->isSwapped; /* See if we have any extra indexes, and if so seek to there. */ bits64 offset = bbi->extraIndexListOffset; if (offset == 0) return NULL; udcSeek(udc, offset); /* Construct list of field that are being indexed. List is list of * field numbers within asObj. */ int i; struct slInt *intList = NULL, *intEl; for (i=0; i<bbi->extraIndexCount; ++i) { bits16 type,fieldCount; type = udcReadBits16(udc, isSwapped); fieldCount = udcReadBits16(udc, isSwapped); udcSeekCur(udc, sizeof(bits64)); // skip over fileOffset udcSeekCur(udc, 4); // skip over reserved bits if (fieldCount == 1) { bits16 fieldId = udcReadBits16(udc, isSwapped); udcSeekCur(udc, 2); // skip over reserved bits intEl = slIntNew(fieldId); slAddHead(&intList, intEl); } else { warn("Not yet understanding indexes on multiple fields at once."); internalErr(); } } /* Now have to make an asObject to find out name that corresponds to this field. */ struct asObject *as = bigBedAsOrDefault(bbi); /* Make list of field names out of list of field numbers */ struct slName *nameList = NULL; for (intEl = intList; intEl != NULL; intEl = intEl->next) { struct asColumn *col = slElementFromIx(as->columnList, intEl->val); if (col == NULL) { warn("Inconsistent bigBed file %s", bbi->fileName); internalErr(); } slNameAddHead(&nameList, col->name); } asObjectFree(&as); return nameList; }
struct slInt * list_indices(int n) { struct slInt * result = NULL, * curr = NULL; int i; for(i = 0; i < n; i++) { if(!result) { result = slIntNew(i); curr=result; } else { curr->next = slIntNew(i); curr = curr->next; } } return result; }
struct slInt * copy_list(struct slInt * list) { struct slInt * result = NULL; struct slInt * curr = NULL; struct slInt * listCurr = list; while(listCurr) { if(!result) { result = slIntNew(listCurr->val); curr = result; } else { curr->next = slIntNew(listCurr->val); curr = curr->next; } listCurr = listCurr->next; } return result; }
struct slInt * index_complement(int max, struct slInt * list) { struct slInt * listCurr = list; struct slInt * result = NULL; struct slInt * resultCurr = NULL; int i = 0; //init complementary array as all complementary int complementary[max]; for(i = 0; i < max; i++) complementary[i] = 1; //set flags in complementary array where list contains the index while(listCurr != NULL) { complementary[listCurr->val] = 0; listCurr=listCurr->next; } //iterate over complementary array, creating list from items flagged as complements for(i = 0; i < max; i++) { if(complementary[i]) { if(!result) { result=slIntNew(i); resultCurr = result; } else { resultCurr->next = slIntNew(i); resultCurr = resultCurr->next; } } } return result; }
void rowsToCols(char *in, char *out) /* rowsToCols - Convert rows to columns and vice versa in a text file.. */ { struct slName *lineList = readAllLines(in); struct tabRow *row, *rowList; FILE *f; int i, origCols; char *s; if (fs != NULL) rowList = tabRowByChar(lineList, fs[0], in, varCol); else if (fixed) rowList = tabRowByFixedGuess(lineList, in); else if (offsets) { struct slName *nameList = commaSepToSlNames(offsets), *name; struct slInt *offList=NULL, *off; for (name = nameList; name != NULL; name = name->next) { off = slIntNew(atoi(name->name)); slAddTail(&offList, off); } rowList = tabRowByFixedOffsets(lineList, offList, in); } else rowList = tabRowByWhite(lineList, in, varCol); origCols = tabRowMaxColCount(rowList); f = mustOpen(out, "w"); for (i=0; i<origCols; ++i) { for (row = rowList; row != NULL; row = row->next) { if (i < row->colCount) s = row->columns[i]; else s = ""; fprintf(f, "%s", s); if (row->next == NULL) fprintf(f, "\n"); else fprintf(f, "\t"); } } carefulClose(&f); }
void bwtool_find_max(struct hash *options, char *favorites, char *regions, double fill, char *bigfile, char *tmp_dir, char *outputfile) /* find max points in a range */ { boolean med_base = (hashFindVal(options, "median-base") != NULL) ? TRUE : FALSE; boolean with_max = (hashFindVal(options, "with-max") != NULL) ? TRUE : FALSE; struct metaBig *mb = metaBigOpen_check(bigfile, tmp_dir, NULL); FILE *out = mustOpen(outputfile, "w"); struct bed6 *sections6 = readBed6Soft(regions); struct bed *sections = bed12FromBed6(§ions6); struct bed *section; for (section = sections; section != NULL; section = section->next) { struct perBaseWig *pbwList = perBaseWigLoadContinue(mb, section->chrom, section->chromStart, section->chromEnd); struct perBaseWig *pbw; struct slInt *ii; int i, size; double max = -DBL_MAX; struct slInt *list = NULL; for (pbw = pbwList; pbw != NULL; pbw = pbw->next) { int pbw_off = pbw->chromStart - section->chromStart; for (i = 0; i < pbw->len; i++) { if (pbw->data[i] > max) { slFreeList(&list); struct slInt *new_int = slIntNew(i + pbw_off); slAddHead(&list, new_int); max = pbw->data[i]; } else if (pbw->data[i] == max) { struct slInt *new_int = slIntNew(i + pbw_off); slAddHead(&list, new_int); } } } slReverse(&list); if (list) { size = slCount(list); if (med_base) { section->blockCount = 1; AllocArray(section->blockSizes, sizeof(int)); AllocArray(section->chromStarts, sizeof(int)); section->blockSizes[0] = 1; section->chromStarts[0] = median_base_calc(&list); } else { section->blockCount = size; AllocArray(section->blockSizes, sizeof(int) * size); AllocArray(section->chromStarts, sizeof(int) * size); for (i = 0, ii = list; (i < size) && (ii != NULL); i++, ii = ii->next) { section->blockSizes[i] = 1; section->chromStarts[i] = ii->val; } } if (!with_max) bedTabOutN(section, 12, out); else { bedOutputN(section, 12, out, '\t', '\t'); fprintf(out, "%f\n", max); } slFreeList(&list); } perBaseWigFree(&pbwList); } metaBigClose(&mb); bedFreeList(§ions); carefulClose(&out); }
void cleanTableSection(char *table, int startId, int endId) /* clean a specific table section */ { struct sqlResult *sr; char **row; char query[256]; int rc = 0; int dc = 0; int delCount = 0; int count = 0; int maxId = startId - 1; int useCount = 0; boolean deleteThis = FALSE; int delRobotCount = 0; int oldRecCount = 0; struct slInt *delList = NULL; time_t cleanSectionStart = time(NULL); struct dyString *dy = dyStringNew(0); while(TRUE) { verbose(2, "maxId: %d count=%d delCount=%d dc=%d\n", maxId, count, delCount, dc); sqlSafef(query,sizeof(query), "select id, firstUse, lastUse, useCount from %s" " where id > %d order by id limit %d" , table , maxId , chunkSize ); sr = sqlGetResult(conn, query); rc = 0; dc = 0; dyStringClear(dy); while ((row = sqlNextRow(sr)) != NULL) { ++count; ++rc; maxId = sqlUnsigned(row[0]); useCount = sqlSigned(row[3]); int daysAgoFirstUse = toDaysAgo(row[1], maxId); int daysAgoLastUse = toDaysAgo(row[2], maxId); verbose(3, "id: %d, firstUse: [%s] [%d days ago], lastUse: [%s] [%d days ago], useCount: %d\n" , maxId , row[1], daysAgoFirstUse , row[2], daysAgoLastUse , useCount ); deleteThis = FALSE; if (sameString(table, sessionDbTableName)) { if (daysAgoLastUse >= 14) { deleteThis = TRUE; ++oldRecCount; } else if ((daysAgoFirstUse >= 2) && useCount <= 1) /* reasonable new addition */ { deleteThis = TRUE; ++delRobotCount; } } if (sameString(table, userDbTableName)) { if ((daysAgoFirstUse >= 7) && useCount < 7) { deleteThis = TRUE; ++delRobotCount; } else if ((daysAgoFirstUse >= 2) && useCount <= 1) { deleteThis = TRUE; ++delRobotCount; } else if (daysAgoLastUse >= 365) /* reasonable new addition */ { deleteThis = TRUE; ++oldRecCount; } } if (deleteThis) { ++dc; verbose(3, "TO DELETE id: %d, " "firstUse: [%s] [%d days ago], lastUse: [%s] [%d days ago], useCount: %d\n" , maxId , row[1], daysAgoFirstUse , row[2], daysAgoLastUse , useCount ); slAddHead(&delList, slIntNew(maxId)); } } sqlFreeResult(&sr); if (rc < 1) break; if (dc > 0) { struct slInt *i; for (i=delList;i;i=i->next) { dyStringClear(dy); sqlDyStringPrintf(dy, "delete from %s where id=%d", table, i->val); sqlUpdate(conn,dy->string); } slFreeList(&delList); } delCount+=dc; if (maxId >= endId) { break; // we have done enough } verbose(3, "sleeping %d seconds\n", chunkWait);fflush(stderr); sleep(chunkWait); verbose(3, "awake\n");fflush(stderr); } verbose(1, "old recs deleted %d, robot recs deleted %d\n", oldRecCount, delRobotCount);fflush(stderr); time_t cleanEnd = time(NULL); int minutes = difftime(cleanEnd, cleanSectionStart) / 60; verbose(1, "%s\n", ctime(&cleanEnd)); verbose(1, "%d minutes\n\n", minutes); }
void cdwJobCleanFailed(int submitId) /* Check out the symlink to determine its type. */ { struct sqlConnection *conn = sqlConnect("cdw"); struct dyString *query = dyStringNew(0); sqlDyStringPrintf(query, "select id, commandLine, startTime, endTime, returnCode, pid from cdwJob where submitId=%d " "order by commandLine,CAST(returnCode AS unsigned)", submitId); // NOTE we need this CAST on returnCode since it can be -1. we want success 0 first. // TODO DO we need to add any other conditions such as distinguishing // between running, queued, and done? /* Scan through result set finding redundant rows beyond success row. */ struct sqlResult *sr = sqlGetResult(conn, query->string); char **row; char *lastCommand = ""; boolean success = FALSE; struct slInt *list = NULL; struct slInt *e; while ((row = sqlNextRow(sr)) != NULL) { unsigned int id = sqlUnsigned(row[0]); char *commandLine = row[1]; unsigned long startTime = sqlUnsignedLong(row[2]); unsigned long endTime = sqlUnsignedLong(row[3]); int returnCode = sqlSigned(row[4]); unsigned int pid = sqlUnsigned(row[5]); verbose(2, "%u\t%s\t%lu\t%lu\t%d\t%u\t%u\n", id, commandLine, startTime, endTime, returnCode, pid, submitId); if (sameString(lastCommand, commandLine)) { if (success) // we already succeeded, the old failure is unwanted baggage. { e = slIntNew(id); // or add it to a list of rows whose ids should get removed slAddHead(&list, e); } } else { if (returnCode == 0) success = TRUE; else success = FALSE; } // note fields pid and submitId are defined as signed integers in cdwJob table, probably should be unsigned. lastCommand = cloneString(commandLine); } sqlFreeResult(&sr); slReverse(&list); for(e=list;e;e=e->next) { dyStringClear(query); sqlDyStringPrintf(query, "delete from cdwJob where id=%u", (unsigned int) e->val); //printf("%s\n", query->string); sqlUpdate(conn, query->string); } /* Clean up and go home */ dyStringFree(&query); sqlDisconnect(&conn); }
void tryToDeprecate(struct sqlConnection *conn) /* CGI variables are set - if possible deprecate, otherwise put up error message. */ { pushWarnHandler(localWarn); fileList = cgiString("fileList"); reason = cloneString(trimSpaces(cgiString("reason"))); if (isEmpty(reason)) { warn("Please enter a reason for deprecation."); getFileListAndReason(conn); } else { /* Go through list of accessions and make sure they are all well formed and correspond to files that exist. */ boolean ok = TRUE; struct slName *accList = slNameListOfUniqueWords(cloneString(fileList), FALSE); struct slName *acc; struct slInt *idList = NULL, *idEl; for (acc = accList; acc != NULL; acc = acc->next) { char *licensePlate = acc->name; if (!startsWith(edwLicensePlatePrefix, licensePlate)) { ok = FALSE; warn("%s is not an accession, doesn't start with %s", licensePlate, edwLicensePlatePrefix); break; } char query[256]; sqlSafef(query, sizeof(query), "select fileId from edwValidFile where licensePlate='%s'", licensePlate); int id = sqlQuickNum(conn, query); if (id == 0) { ok = FALSE; warn("%s - no such accession. ", licensePlate); break; } /* check to see is it ok tor deprecate this file */ if (!okToDeprecateThisFile(conn, id, userEmail)) { ok = FALSE; warn("You can not deprecate %s which was originally uploaded by %s.\n", licensePlate, edwFindOwnerNameFromFileId(conn, id)); warn("Please click the check box below to override this rule."); break; } idEl = slIntNew(id); slAddTail(&idList, idEl); } if (accList == NULL) { warn("Please enter some file accessions"); ok = FALSE; } /* If a problem then put up page to try again, otherwise do deprecation. */ if (!ok) getFileListAndReason(conn); else { deprecateFileList(conn, idList, reason); printf("Deprecated %d files<BR>\n", slCount(idList)); cgiMakeButton("submit", "Deprecate More Files"); printf(" "); edwPrintLogOutButton(); } } }
struct slInt * listTopN(matrix * A_ptr, int n, int direction, int ix) /*Make a list of the top features in a row or column of matrix. Direction: 1=rows, 2=cols*/ { //make an empty list, and a pointer to it. struct slInt *list = NULL, *curr = NULL; int i, j; if(direction == 1) { //start the list by adding the first element (non null) in matrix j = 0; while(A_ptr->graph[j][ix] == NULL_FLAG && j < A_ptr->rows) j++; list=slIntNew(j); curr = list; for(i = (j+1); i < A_ptr->rows; i++) { curr = list; //if the new element is the largest seen yet, add it to the front of the list if(A_ptr->graph[i][ix] != NULL_FLAG && fabs(A_ptr->graph[i][ix]) > fabs(A_ptr->graph[curr->val][ix])) { slAddHead(&list, slIntNew(i)); } else { //otherwise iterate the list looking for if it fits in it. while(curr->next != NULL && (A_ptr->graph[i][ix] != NULL_FLAG && fabs(A_ptr->graph[i][ix]) < fabs(A_ptr->graph[curr->next->val][ix]))) curr = curr->next; struct slInt *tmp = curr->next; curr->next = slIntNew(i); curr->next->next = tmp; } //if the list has gotten too long, reduce it if(count_indices(list) > n) { curr = list; j = 0; while(curr->next->next && j < n) { curr = curr->next; j++; } slFreeList(&curr->next); curr->next = NULL; } } } else if(direction == 2) { //start the list by adding the first element in matrix j = 0; while(A_ptr->graph[ix][j] == NULL_FLAG && j < A_ptr->rows) j++; curr->val = j; for(i = (j+1); i < A_ptr->cols; i++) { curr = list; //if the new element is the largest seen yet, add it to the front of the list if(A_ptr->graph[ix][i] != NULL_FLAG && fabs(A_ptr->graph[ix][i]) > fabs(A_ptr->graph[ix][curr->val])) { slAddHead(&list, slIntNew(i)); } else { //otherwise iterate the list looking for if it fits in it. while(curr->next != NULL && (A_ptr->graph[ix][i] != NULL_FLAG && fabs(A_ptr->graph[ix][i]) < fabs(A_ptr->graph[ix][curr->next->val]))) curr = curr->next; struct slInt *tmp = curr->next; curr->next = slIntNew(i); curr->next->next = tmp; } //if the list has gotten too long, reduce it if(count_indices(list) > n) { curr = list; j = 0; while(curr->next->next && j < n) { curr = curr->next; j++; } slFreeList(&curr->next); curr->next = NULL; } } } else errAbort("When listing top N an invalid direction was supplied.\n"); return list; }