void cdwGroupUser(char *groupName, int userCount, char *userEmails[]) /* cdwGroupUser - Change user group settings.. */ { struct sqlConnection *conn = cdwConnectReadWrite(); struct cdwGroup *group = cdwNeedGroupFromName(conn, groupName); /* Build up array of all users, in the process aborting if user not * found */ struct cdwUser *users[userCount]; int i; for (i=0; i<userCount; ++i) users[i] = cdwMustGetUserFromEmail(conn, userEmails[i]); /* Now go through user by user adding things */ for (i=0; i<userCount; ++i) { struct cdwUser *user = users[i]; char query[256]; boolean inGroup = cdwUserInGroup(conn, user->id, group->id); if (!inGroup && !clRemove) { sqlSafef(query, sizeof(query), "insert into cdwGroupUser (userId,groupId) values (%u,%u)", user->id, group->id); sqlUpdate(conn, query); } else if (inGroup && clRemove) { sqlSafef(query, sizeof(query), "delete from cdwGroupUser where userId=%u and groupId=%u", user->id, group->id); sqlUpdate(conn, query); } /* Deal with primary group */ if (clRemove) { if (group->id == user->primaryGroup) { /* If possible revert to another group. Otherwise will end up primary group 0 which * is ok too. */ sqlSafef(query, sizeof(query), "select groupId from cdwGroupUser where userId=%u and groupId != %u", user->id, group->id); int newPrimary = sqlQuickNum(conn, query); sqlSafef(query, sizeof(query), "update cdwUser set primaryGroup=%d where id=%u", newPrimary, user->id); sqlUpdate(conn, query); } } else { if (clPrimary || user->primaryGroup == 0) { sqlSafef(query, sizeof(query), "update cdwUser set primaryGroup=%u where id=%u", group->id, user->id); sqlUpdate(conn, query); } } } }
void cdwGroupFile(char *groupName, char *where) /* cdwGroupFile - Associate a file with a group.. */ { /* Get group from database, error out if no good */ struct sqlConnection *conn = cdwConnectReadWrite(); struct cdwGroup *group = cdwNeedGroupFromName(conn, groupName); /* Get list of all stanzas matching query */ struct tagStorm *tags = cdwTagStorm(conn); struct dyString *rqlQuery = dyStringNew(0); dyStringPrintf(rqlQuery, "select accession from cdwFileTags where accession"); if (where != NULL) dyStringPrintf(rqlQuery, " and %s", where); struct slRef *ref, *matchRefList = tagStanzasMatchingQuery(tags, rqlQuery->string); /* Make one pass through mostly for early error reporting and building up * hash of cdwValidFiles keyed by accession */ struct hash *validHash = hashNew(0); for (ref = matchRefList; ref != NULL; ref = ref->next) { struct tagStanza *stanza = ref->val; char *acc = tagFindVal(stanza, "accession"); if (acc != NULL) { struct cdwValidFile *vf = cdwValidFileFromLicensePlate(conn, acc); if (vf == NULL) errAbort("%s not found in cdwValidFile", acc); hashAdd(validHash, acc, vf); } } /* Second pass through matching list we call routine that actually adds * the group/file relationship. */ for (ref = matchRefList; ref != NULL; ref = ref->next) { struct tagStanza *stanza = ref->val; char *acc = tagFindVal(stanza, "accession"); if (acc != NULL) { struct cdwValidFile *vf = hashFindVal(validHash, acc); if (vf != NULL) { addGroupToValidFile(conn, vf, group); } } } if (clDry) verbose(1, "Would have %s", (clRemove ? "removed" : "added")); else verbose(1, "%s", (clRemove ? "Removed" : "Added")); verbose(1, " group %s to %d files\n", group->name, validHash->elCount); }
void cdwMakeRepeatQa(int startFileId, int endFileId) /* cdwMakeRepeatQa - Figure out what proportion of things align to repeats.. */ { struct sqlConnection *conn = cdwConnectReadWrite(); struct cdwFile *ef, *efList = cdwFileAllIntactBetween(conn, startFileId, endFileId); for (ef = efList; ef != NULL; ef = ef->next) { struct cdwValidFile *vf = cdwValidFileFromFileId(conn, ef->id); if (vf != NULL) { if (sameString(vf->format, "fastq")) fastqRepeatQa(conn, ef, vf); } } sqlDisconnect(&conn); }
void cdwMakePairedEndQa(unsigned startId, unsigned endId) /* cdwMakePairedEndQa - Do alignments of paired-end fastq files and calculate distrubution of * insert size. */ { struct sqlConnection *conn = cdwConnectReadWrite(); struct cdwFile *ef, *efList = cdwFileAllIntactBetween(conn, startId, endId); for (ef = efList; ef != NULL; ef = ef->next) { struct cdwValidFile *vf = cdwValidFileFromFileId(conn, ef->id); if (vf != NULL) { if (sameString(vf->format, "fastq") && !isEmpty(vf->pairedEnd)) pairedEndQa(conn, ef, vf); } } sqlDisconnect(&conn); }
void cdwChangeAccess(char *chmodString, char *rqlWhere) /* cdwChangeAccess - Change access to files.. */ { char cWhere, cDir, cAccess; parseChmodString(chmodString, &cWhere, &cDir, &cAccess); /* Get list of all stanzas matching query */ struct sqlConnection *conn = cdwConnectReadWrite(); struct tagStorm *tags = cdwTagStorm(conn); struct dyString *rqlQuery = dyStringNew(0); dyStringPrintf(rqlQuery, "select accession from cdwFileTags where accession and %s", rqlWhere); struct slRef *ref, *matchRefList = tagStanzasMatchingQuery(tags, rqlQuery->string); /* Make one pass through mostly for early error reporting and building up * hash of cdwValidFiles keyed by accession */ struct hash *validHash = hashNew(0); for (ref = matchRefList; ref != NULL; ref = ref->next) { struct tagStanza *stanza = ref->val; char *acc = tagFindVal(stanza, "accession"); if (acc != NULL) { struct cdwValidFile *vf = cdwValidFileFromLicensePlate(conn, acc); if (vf == NULL) errAbort("%s not found in cdwValidFile", acc); hashAdd(validHash, acc, vf); } } /* Second pass through matching list we call routine that actually adds * the group/file relationship. */ for (ref = matchRefList; ref != NULL; ref = ref->next) { struct tagStanza *stanza = ref->val; char *acc = tagFindVal(stanza, "accession"); if (acc != NULL) { struct cdwValidFile *vf = hashFindVal(validHash, acc); if (vf != NULL) { changeAccess(conn, vf->fileId, cWhere, cDir, cAccess); } } } }
void cdwReallyRemoveFiles(char *email, char *submitUrl, int fileCount, char *fileIds[]) /* cdwReallyRemoveFiles - Remove files from data warehouse. Generally you want to depricate them * instead. */ { /* First convert all fileIds to binary. Do this first so bad command lines get caught. */ long long ids[fileCount]; int i; for (i = 0; i<fileCount; ++i) ids[i] = sqlLongLong(fileIds[i]); /* Get hash of all submissions by user from that URL. Hash is keyed by ascii version of * submitId. */ struct sqlConnection *conn = cdwConnectReadWrite(); struct cdwUser *user = cdwMustGetUserFromEmail(conn, email); char query[256]; sqlSafef(query, sizeof(query), " select cdwSubmit.id,cdwSubmitDir.id from cdwSubmit,cdwSubmitDir " " where cdwSubmit.submitDirId=cdwSubmitDir.id and userId=%d " " and cdwSubmitDir.url='%s' ", user->id, submitUrl); struct hash *submitHash = sqlQuickHash(conn, query); /* Make sure that files and submission really go together. */ for (i=0; i<fileCount; ++i) { long long fileId = ids[i]; char buf[64]; sqlSafef(query, sizeof(query), "select submitId from cdwFile where id=%lld", fileId); char *result = sqlQuickQuery(conn, query, buf, sizeof(buf)); if (result == NULL) errAbort("%lld is not a fileId in the warehouse", fileId); if (hashLookup(submitHash, result) == NULL) errAbort("File ID %lld does not belong to submission set based on %s", fileId, submitUrl); } /* OK - paranoid checking is done, now let's remove each file from the tables it is in. */ for (i=0; i<fileCount; ++i) { cdwReallyRemoveFile(conn, ids[i], really); } }
void cdwUndeprecate(char *fileName) /* cdwUndeprecate - Undeprecate a list of file accessions.. */ { char **words, *buf; int wordCount; readAllWords(fileName, &words, &wordCount, &buf); verbose(1, "Read %d accessions from %s\n", wordCount, fileName); struct sqlConnection *conn = cdwConnectReadWrite(); int i; for (i=0; i<wordCount; ++i) { char query[512]; sqlSafef(query, sizeof(query), "select * from cdwValidFile where licensePlate='%s'", words[i]); struct cdwValidFile *vf = cdwValidFileLoadByQuery(conn, query); if (vf == NULL) errAbort("%s doesn't exist in cdwValidFile table", words[i]); sqlSafef(query, sizeof(query), "update cdwFile set deprecated='' where id=%u", vf->fileId); sqlUpdate(conn, query); } verbose(1, "Undeprecated %d files\n", i); }
void pairedEndQa(struct sqlConnection *conn, struct cdwFile *ef, struct cdwValidFile *vf) /* Look for other end, do a pairwise alignment, and save results in database. */ { verbose(2, "pairedEndQa on %u %s %s\n", ef->id, ef->cdwFileName, ef->submitFileName); /* Get other end, return if not found. */ struct cdwValidFile *otherVf = cdwOppositePairedEnd(conn, ef, vf); if (otherVf == NULL) return; if (otherVf->fileId > vf->fileId) return; struct cdwValidFile *vf1, *vf2; struct cdwQaPairedEndFastq *pair = cdwQaPairedEndFastqFromVfs(conn, vf, otherVf, &vf1, &vf2); if (pair != NULL) { cdwValidFileFree(&otherVf); return; } /* Get target assembly and figure out path for BWA index. */ struct cdwAssembly *assembly = cdwAssemblyForUcscDb(conn, vf->ucscDb); assert(assembly != NULL); char genoFile[PATH_LEN]; safef(genoFile, sizeof(genoFile), "%s%s/bwaData/%s.fa", cdwValDataDir, assembly->ucscDb, assembly->ucscDb); verbose(1, "aligning subsamples on %u vs. %u paired reads\n", vf1->fileId, vf2->fileId); /* Make alignments of subsamples. */ char *sample1 = NULL, *sample2 = NULL, *sai1 = NULL, *sai2 = NULL; makeTmpSai(conn, vf1, genoFile, &sample1, &sai1); makeTmpSai(conn, vf2, genoFile, &sample2, &sai2); /* Make paired end alignment */ char *tmpSam = cloneString(rTempName(cdwTempDir(), "cdwPairSample", ".sam")); char command[6*PATH_LEN]; safef(command, sizeof(command), "bwa sampe -n 1 -N 1 -f %s %s %s %s %s %s" , tmpSam, genoFile, sai1, sai2, sample1, sample2); mustSystem(command); /* Make ra file with pairing statistics */ char *tmpRa = cloneString(rTempName(cdwTempDir(), "cdwPairSample", ".ra")); safef(command, sizeof(command), "edwSamPairedEndStats -maxInsert=%d %s %s", maxInsert, tmpSam, tmpRa); mustSystem(command); /* Read RA file into variables. */ struct cdwQaPairedEndFastq *pe = cdwQaPairedEndFastqOneFromRa(tmpRa); /* Update database with record. */ struct sqlConnection *freshConn = cdwConnectReadWrite(); char query[256]; sqlSafef(query, sizeof(query), "insert into cdwQaPairedEndFastq " "(fileId1,fileId2,concordance,distanceMean,distanceStd,distanceMin,distanceMax,recordComplete) " " values (%u,%u,%g,%g,%g,%g,%g,1)" , vf1->fileId, vf2->fileId, pe->concordance, pe->distanceMean , pe->distanceStd, pe->distanceMin, pe->distanceMax); sqlUpdate(conn, query); sqlDisconnect(&freshConn); /* Clean up and go home. */ cdwValidFileFree(&otherVf); remove(sample1); remove(sample2); remove(sai1); remove(sai2); remove(tmpSam); remove(tmpRa); #ifdef SOON #endif /* SOON */ freez(&sample1); freez(&sample2); freez(&sai1); freez(&sai2); freez(&tmpSam); freez(&tmpRa); cdwQaPairedEndFastqFree(&pe); cdwValidFileFree(&otherVf); }