void edwMakePlateFileNameAndPath(int edwFileId, char *submitFileName, char licensePlate[edwMaxPlateSize], char edwFile[PATH_LEN], char serverPath[PATH_LEN]) /* Convert file id to local file name, and full file path. Make any directories needed * along serverPath. */ { /* Preserve suffix. Give ourselves up to two suffixes. */ int nameSize = strlen(submitFileName); char *suffix = lastMatchCharExcept(submitFileName, submitFileName + nameSize, '.', '/'); if (suffix != NULL) { char *secondSuffix = lastMatchCharExcept(submitFileName, suffix, '.', '/'); if (secondSuffix != NULL) suffix = secondSuffix; } suffix = emptyForNull(suffix); /* Figure out edw file name, starting with license plate. */ edwMakeLicensePlate(edwLicensePlatePrefix, edwFileId, licensePlate, edwMaxPlateSize); /* Figure out directory and make any components not already there. */ char edwDir[PATH_LEN]; edwDirForTime(edwNow(), edwDir); char uploadDir[PATH_LEN]; safef(uploadDir, sizeof(uploadDir), "%s%s", edwRootDir, edwDir); makeDirsOnPath(uploadDir); /* Figure out full file names */ safef(edwFile, PATH_LEN, "%s%s%s", edwDir, licensePlate, suffix); safef(serverPath, PATH_LEN, "%s%s", edwRootDir, edwFile); }
void recordIntoHistory(struct sqlConnection *conn, unsigned id, char *table, boolean success) /* Record success/failure into uploadAttempts and historyBits fields of table. */ { /* Get historyBits and fold status into it. */ char quickResult[32]; char query[256]; sqlSafef(query, sizeof(query), "select historyBits from %s where id=%u", table, id); if (sqlQuickQuery(conn, query, quickResult, sizeof(quickResult)) == NULL) internalErr(); char *lastTimeField; char *openResultField; long long historyBits = sqlLongLong(quickResult); historyBits <<= 1; if (success) { historyBits |= 1; lastTimeField = "lastOkTime"; openResultField = "openSuccesses"; } else { lastTimeField = "lastNotOkTime"; openResultField = "openFails"; } sqlSafef(query, sizeof(query), "update %s set historyBits=%lld, %s=%s+1, %s=%lld " "where id=%lld", table, historyBits, openResultField, openResultField, lastTimeField, edwNow(), (long long)id); sqlUpdate(conn, query); }
int makeNewEmptySubmitRecord(struct sqlConnection *conn, char *submitUrl, unsigned userId) /* Create a submit record around URL and return it's id. */ { struct dyString *query = dyStringNew(0); sqlDyStringAppend(query, "insert edwSubmit (url, startUploadTime, userId) "); sqlDyStringPrintf(query, "VALUES('%s', %lld, %d)", submitUrl, edwNow(), userId); sqlUpdate(conn, query->string); dyStringFree(&query); return sqlLastAutoId(conn); }
static boolean paraFetchInterruptFunction(void *v) /* Return TRUE if we need to interrupt. */ { struct paraFetchInterruptContext *context = v; long long now = edwNow(); if (context->lastChecked != now) // Only do check every second { context->isInterrupted = edwSubmitShouldStop(context->conn, context->submitId); context->lastChecked = now; } return context->isInterrupted; }
static int getLocalHost(struct sqlConnection *conn) /* Make up record for local host if it is not there already. */ { char query[256]; sqlSafef(query, sizeof(query), "select id from edwHost where name = '%s'", localHostName); int hostId = sqlQuickNum(conn, query); if (hostId == 0) { sqlSafef(query, sizeof(query), "insert edwHost(name, firstAdded) values('%s', %lld)", localHostName, edwNow()); sqlUpdate(conn, query); hostId = sqlLastAutoId(conn); } return hostId; }
int edwGetHost(struct sqlConnection *conn, char *hostName) /* Look up host name in table and return associated ID. If not found * make up new table entry. */ { /* If it's already in table, just return ID. */ char query[512]; sqlSafef(query, sizeof(query), "select id from edwHost where name='%s'", hostName); int hostId = sqlQuickNum(conn, query); if (hostId > 0) return hostId; sqlSafef(query, sizeof(query), "insert edwHost (name, firstAdded, paraFetchStreams) values('%s', %lld, 10)", hostName, edwNow()); sqlUpdate(conn, query); return sqlLastAutoId(conn); }
int edwGetSubmitDir(struct sqlConnection *conn, int hostId, char *submitDir) /* Get submitDir from database, creating it if it doesn't already exist. */ { /* If it's already in table, just return ID. */ char query[512]; sqlSafef(query, sizeof(query), "select id from edwSubmitDir where url='%s'", submitDir); int dirId = sqlQuickNum(conn, query); if (dirId > 0) return dirId; sqlSafef(query, sizeof(query), "insert edwSubmitDir (url, firstAdded, hostId) values('%s', %lld, %d)", submitDir, edwNow(), hostId); sqlUpdate(conn, query); return sqlLastAutoId(conn); }
static int getLocalSubmit(struct sqlConnection *conn) /* Get the submission that covers all of our local additions. */ { int dirId = getLocalSubmitDir(conn); char query[256]; sqlSafef(query, sizeof(query), "select id from edwSubmit where submitDirId='%d'", dirId); int submitId = sqlQuickNum(conn, query); if (submitId == 0) { sqlSafef(query, sizeof(query), "insert edwSubmit (submitDirId,startUploadTime) values(%d,%lld)", dirId, edwNow()); sqlUpdate(conn, query); submitId = sqlLastAutoId(conn); } return submitId; }
static int getLocalSubmitDir(struct sqlConnection *conn) /* Get submit dir for local submissions, making it up if it does not exist. */ { int hostId = getLocalHost(conn); char query[256]; sqlSafef(query, sizeof(query), "select id from edwSubmitDir where url='%s' and hostId=%d", localHostDir, hostId); int dirId = sqlQuickNum(conn, query); if (dirId == 0) { sqlSafef(query, sizeof(query), "insert edwSubmitDir(url,hostId,firstAdded) values('%s',%d,%lld)", localHostDir, hostId, edwNow()); sqlUpdate(conn, query); dirId = sqlLastAutoId(conn); } return dirId; }
char *edwTempDirForToday(char dir[PATH_LEN]) /* Fills in dir with temp dir of the day, and returns a pointer to it. */ { char dayDir[PATH_LEN]; edwDirForTime(edwNow(), dayDir); safef(dir, PATH_LEN, "%s%stmp/", edwRootDir, dayDir); /* Bracket time consuming call to makeDirsOnPath with check that we didn't just do same * thing. */ static char lastDayDir[PATH_LEN] = ""; if (!sameString(dayDir, lastDayDir)) { strcpy(lastDayDir, dayDir); int len = strlen(dir); dir[len-1] = 0; makeDirsOnPath(dir); dir[len-1] = '/'; } return dir; }
void notOverlappingSelf(struct sqlConnection *conn, char *url) /* Ensure we are only submission going on for this URL, allowing for time out * and command line override. */ { if (doNow) // Allow command line override return; /* Fetch most recent submission from this URL. */ struct edwSubmit *old = edwMostRecentSubmission(conn, url); if (old == NULL) return; /* See if we have something in progress, meaning started but not ended. */ if (old->endUploadTime == 0 && isEmpty(old->errorMessage)) { /* Check submission last alive time against our usual time out. */ long long maxStartTime = edwSubmitMaxStartTime(old, conn); if (edwNow() - maxStartTime < edwSingleFileTimeout) errAbort("Submission of %s already is in progress. Please come back in an hour", url); } edwSubmitFree(&old); }
void edwMakeFileNameAndPath(int edwFileId, char *submitFileName, char edwFile[PATH_LEN], char serverPath[PATH_LEN]) /* Convert file id to local file name, and full file path. Make any directories needed * along serverPath. */ { /* Preserve suffix. Give ourselves up to two suffixes. */ char *suffix = edwFindDoubleFileSuffix(submitFileName); /* Figure out edw file name, starting with baseName. */ char baseName[32]; edwMakeBabyName(edwFileId, baseName, sizeof(baseName)); /* Figure out directory and make any components not already there. */ char edwDir[PATH_LEN]; edwDirForTime(edwNow(), edwDir); char uploadDir[PATH_LEN]; safef(uploadDir, sizeof(uploadDir), "%s%s", edwRootDir, edwDir); makeDirsOnPath(uploadDir); /* Figure out full file names */ safef(edwFile, PATH_LEN, "%s%s%s", edwDir, baseName, suffix); safef(serverPath, PATH_LEN, "%s%s", edwRootDir, edwFile); }
long long edwGettingFile(struct sqlConnection *conn, char *submitDir, char *submitFileName) /* See if we are in process of getting file. Return file record id if it exists even if * it's not complete. Return -1 if record does not exist. */ { /* First see if we have even got the directory. */ char query[PATH_LEN+512]; sqlSafef(query, sizeof(query), "select id from edwSubmitDir where url='%s'", submitDir); int submitDirId = sqlQuickNum(conn, query); if (submitDirId <= 0) return -1; /* Then see if we have file that matches submitDir and submitFileName. */ sqlSafef(query, sizeof(query), "select id from edwFile " "where submitFileName='%s' and submitDirId = %d and errorMessage = '' and deprecated=''" " and (endUploadTime > startUploadTime or startUploadTime < %lld) " "order by submitId desc limit 1" , submitFileName, submitDirId , (long long)edwNow() - edwSingleFileTimeout); long long id = sqlQuickLongLong(conn, query); if (id == 0) return -1; return id; }
struct edwFile *edwGetLocalFile(struct sqlConnection *conn, char *localAbsolutePath, char *symLinkMd5Sum) /* Get record of local file from database, adding it if it doesn't already exist. * Can make it a symLink rather than a copy in which case pass in valid MD5 sum * for symLinkM5dSum. */ { /* First do a reality check on the local absolute path. Is there a file there? */ if (localAbsolutePath[0] != '/') errAbort("Using relative path in edwAddLocalFile."); long long size = fileSize(localAbsolutePath); if (size == -1) errAbort("%s does not exist", localAbsolutePath); long long updateTime = fileModTime(localAbsolutePath); /* Get file if it's in database already. */ int submitDirId = getLocalSubmitDir(conn); int submitId = getLocalSubmit(conn); char query[256+PATH_LEN]; sqlSafef(query, sizeof(query), "select * from edwFile where submitId=%d and submitFileName='%s'", submitId, localAbsolutePath); struct edwFile *ef = edwFileLoadByQuery(conn, query); /* If we got something in database, check update time and size, and if it's no change just * return existing database id. */ if (ef != NULL && ef->updateTime == updateTime && ef->size == size) return ef; /* If we got here, then we need to make a new file record. Start with pretty empty record * that just has file ID, submitted file name and a few things*/ sqlSafef(query, sizeof(query), "insert edwFile (submitId,submitDirId,submitFileName,startUploadTime) " " values(%d, %d, '%s', %lld)" , submitId, submitDirId, localAbsolutePath, edwNow()); sqlUpdate(conn, query); long long fileId = sqlLastAutoId(conn); /* Create big data warehouse file/path name. */ char edwFile[PATH_LEN], edwPath[PATH_LEN]; edwMakeFileNameAndPath(fileId, localAbsolutePath, edwFile, edwPath); /* We're a little paranoid so md5 it */ char *md5; /* Do copy or symbolic linking of file into warehouse managed dir. */ if (symLinkMd5Sum) { md5 = symLinkMd5Sum; makeSymLink(localAbsolutePath, edwPath); } else { copyFile(localAbsolutePath, edwPath); md5 = md5HexForFile(localAbsolutePath); } /* Update file record. */ sqlSafef(query, sizeof(query), "update edwFile set edwFileName='%s', endUploadTime=%lld," "updateTime=%lld, size=%lld, md5='%s' where id=%lld" , edwFile, edwNow(), updateTime, size, md5, fileId); sqlUpdate(conn, query); /* Now, it's a bit of a time waste, but cheap in code, to just load it back from DB. */ sqlSafef(query, sizeof(query), "select * from edwFile where id=%lld", fileId); return edwFileLoadByQuery(conn, query); }
int edwFileFetch(struct sqlConnection *conn, struct edwFile *ef, int fd, char *submitFileName, unsigned submitId, unsigned submitDirId, unsigned hostId) /* Fetch file and if successful update a bunch of the fields in ef with the result. * Returns fileId. */ { ef->id = makeNewEmptyFileRecord(conn, submitId, submitDirId, ef->submitFileName, ef->size); /* Update edwSubmit with file in transit info */ char query[256]; sqlSafef(query, sizeof(query), "update edwSubmit set fileIdInTransit=%lld where id=%u", (long long)ef->id, submitId); sqlUpdate(conn, query); sqlSafef(query, sizeof(query), "select paraFetchStreams from edwHost where id=%u", hostId); int paraFetchStreams = sqlQuickNum(conn, query); struct paraFetchInterruptContext interruptContext = {.conn=conn, .submitId=submitId}; /* Wrap getting the file, the actual data transfer, with an error catcher that * will remove partly uploaded files. Perhaps some day we'll attempt to rescue * ones that are just truncated by downloading the rest, but not now. */ struct errCatch *errCatch = errCatchNew(); char tempName[PATH_LEN] = ""; char edwFile[PATH_LEN] = "", edwPath[PATH_LEN]; if (errCatchStart(errCatch)) { /* Now make temp file name and open temp file in an atomic operation */ char *tempDir = edwTempDir(); safef(tempName, PATH_LEN, "%sedwSubmitXXXXXX", tempDir); int localFd = mustMkstemp(tempName); /* Update file name in database with temp file name so web app can track us. */ char query[PATH_LEN+128]; sqlSafef(query, sizeof(query), "update edwFile set edwFileName='%s' where id=%lld", tempName + strlen(edwRootDir), (long long)ef->id); sqlUpdate(conn, query); /* Do actual upload tracking how long it takes. */ ef->startUploadTime = edwNow(); mustCloseFd(&localFd); if (!parallelFetchInterruptable(submitFileName, tempName, paraFetchStreams, 4, FALSE, FALSE, paraFetchInterruptFunction, &interruptContext)) { if (interruptContext.isInterrupted) errAbort("Submission stopped by user."); else errAbort("parallel fetch of %s failed", submitFileName); } ef->endUploadTime = edwNow(); /* Rename file both in file system and (via ef) database. */ edwMakeFileNameAndPath(ef->id, submitFileName, edwFile, edwPath); mustRename(tempName, edwPath); if (endsWith(edwPath, ".gz") && !encode3IsGzipped(edwPath)) errAbort("%s has .gz suffix, but is not gzipped", submitFileName); ef->edwFileName = cloneString(edwFile); } errCatchEnd(errCatch); if (errCatch->gotError) { /* Attempt to remove any partial file. */ if (tempName[0] != 0) { verbose(1, "Removing partial %s\n", tempName); parallelFetchRemovePartial(tempName); remove(tempName); } handleSubmitError(conn, submitId, errCatch->message->string); // Throws further assert(FALSE); // We never get here } errCatchFree(&errCatch); /* Now we got the file. We'll go ahead and save the file name and stuff. */ sqlSafef(query, sizeof(query), "update edwFile set" " edwFileName='%s', startUploadTime=%lld, endUploadTime=%lld" " where id = %d" , ef->edwFileName, ef->startUploadTime, ef->endUploadTime, ef->id); sqlUpdate(conn, query); /* Wrap the validations in an error catcher that will save error to file table in database */ errCatch = errCatchNew(); boolean success = FALSE; if (errCatchStart(errCatch)) { /* Check MD5 sum here. */ unsigned char md5bin[16]; md5ForFile(edwPath, md5bin); char md5[33]; hexBinaryString(md5bin, sizeof(md5bin), md5, sizeof(md5)); if (!sameWord(md5, ef->md5)) errAbort("%s has md5 mismatch: %s != %s. File may be corrupted in upload, or file may have " "been changed since validateManifest was run. Please check that md5 of file " "before upload is really %s. If it is then try submitting again, otherwise " "rerun validateManifest and then try submitting again. \n", ef->submitFileName, ef->md5, md5, ef->md5); /* Finish updating a bunch more of edwFile record. Note there is a requirement in * the validFile section that ef->updateTime be updated last. A nonzero ef->updateTime * is used as a sign of record complete. */ struct dyString *dy = dyStringNew(0); /* Includes tag so query may be long */ sqlDyStringPrintf(dy, "update edwFile set md5='%s',size=%lld,updateTime=%lld", md5, ef->size, ef->updateTime); dyStringAppend(dy, ", tags='"); dyStringAppend(dy, ef->tags); dyStringPrintf(dy, "' where id=%d", ef->id); sqlUpdate(conn, dy->string); dyStringFree(&dy); /* Update edwSubmit so file no longer shown as in transit */ sqlSafef(query, sizeof(query), "update edwSubmit set fileIdInTransit=0 where id=%u", submitId); sqlUpdate(conn, query); success = TRUE; } errCatchEnd(errCatch); if (errCatch->gotError) { handleFileError(conn, submitId, ef->id, errCatch->message->string); } return ef->id; }
void edwSubmit(char *submitUrl, char *email) /* edwSubmit - Submit URL with validated.txt to warehouse. */ { /* Parse out url a little into submitDir and submitFile */ char *lastSlash = strrchr(submitUrl, '/'); if (lastSlash == NULL) errAbort("%s is not a valid URL - it has no '/' in it.", submitUrl); char *submitFile = lastSlash+1; int submitDirSize = submitFile - submitUrl; char submitDir[submitDirSize+1]; memcpy(submitDir, submitUrl, submitDirSize); submitDir[submitDirSize] = 0; // Add trailing zero /* Make sure user has access. */ struct sqlConnection *conn = edwConnectReadWrite(); struct edwUser *user = edwMustGetUserFromEmail(conn, email); int userId = user->id; /* See if we are already running on same submission. If so council patience and quit. */ notOverlappingSelf(conn, submitUrl); /* Make a submit record. */ int submitId = makeNewEmptySubmitRecord(conn, submitUrl, userId); /* The next errCatch block will fill these in if all goes well. */ struct submitFileRow *sfrList = NULL, *oldList = NULL, *newList = NULL; int oldCount = 0; long long oldBytes = 0, newBytes = 0, byteCount = 0; /* Start catching errors from here and writing them in submitId. If we don't * throw we'll end up having a list of all files in the submit in sfrList. */ struct errCatch *errCatch = errCatchNew(); char query[1024]; if (errCatchStart(errCatch)) { /* Make sure they got a bit of space, enough for a reasonable submit file. * We do this here just because we can make error message more informative. */ long long diskFreeSpace = freeSpaceOnFileSystem(edwRootDir); if (diskFreeSpace < 4*1024*1024) errAbort("No space left in warehouse!"); /* Open remote submission file. This is most likely where we will fail. */ int hostId=0, submitDirId = 0; long long startUploadTime = edwNow(); int remoteFd = edwOpenAndRecordInDir(conn, submitDir, submitFile, submitUrl, &hostId, &submitDirId); /* Copy to local temp file. */ char tempSubmitFile[PATH_LEN]; fetchFdToTempFile(remoteFd, tempSubmitFile); mustCloseFd(&remoteFd); long long endUploadTime = edwNow(); /* Calculate MD5 sum, and see if we already have such a file. */ char *md5 = md5HexForFile(tempSubmitFile); int fileId = findFileGivenMd5AndSubmitDir(conn, md5, submitDirId); /* If we already have it, then delete temp file, otherwise put file in file table. */ char submitLocalPath[PATH_LEN]; if (fileId != 0) { remove(tempSubmitFile); char submitRelativePath[PATH_LEN]; sqlSafef(query, sizeof(query), "select edwFileName from edwFile where id=%d", fileId); sqlNeedQuickQuery(conn, query, submitRelativePath, sizeof(submitRelativePath)); safef(submitLocalPath, sizeof(submitLocalPath), "%s%s", edwRootDir, submitRelativePath); } else { /* Looks like it's the first time we've seen this submission file, so * save the file itself. We'll get to the records inside the file in a bit. */ fileId = makeNewEmptyFileRecord(conn, submitId, submitDirId, submitFile, 0); /* Get file/path names for submission file inside warehouse. */ char edwFile[PATH_LEN]; edwMakeFileNameAndPath(fileId, submitFile, edwFile, submitLocalPath); /* Move file to final resting place and get update time and size from local file system. */ mustRename(tempSubmitFile, submitLocalPath); time_t updateTime = fileModTime(submitLocalPath); off_t size = fileSize(submitLocalPath); /* Update file table which now should be complete including updateTime. */ sqlSafef(query, sizeof(query), "update edwFile set " " updateTime=%lld, size=%lld, md5='%s', edwFileName='%s'," " startUploadTime=%lld, endUploadTime=%lld" " where id=%u\n", (long long)updateTime, (long long)size, md5, edwFile, startUploadTime, endUploadTime, fileId); sqlUpdate(conn, query); } /* By now there is a submit file on the local file system. We parse it out. */ edwParseSubmitFile(conn, submitLocalPath, submitUrl, &sfrList); /* Save our progress so far to submit table. */ sqlSafef(query, sizeof(query), "update edwSubmit" " set submitFileId=%lld, submitDirId=%lld, fileCount=%d where id=%d", (long long)fileId, (long long)submitDirId, slCount(sfrList), submitId); sqlUpdate(conn, query); /* Weed out files we already have. */ struct submitFileRow *sfr, *sfrNext; for (sfr = sfrList; sfr != NULL; sfr = sfrNext) { sfrNext = sfr->next; struct edwFile *bf = sfr->file; long long fileId; if ((fileId = edwGotFile(conn, submitDir, bf->submitFileName, bf->md5, bf->size)) >= 0) { ++oldCount; oldBytes += bf->size; sfr->md5MatchFileId = fileId; slAddHead(&oldList, sfr); } else slAddHead(&newList, sfr); byteCount += bf->size; } sfrList = NULL; slReverse(&newList); slReverse(&oldList); /* Update database with oldFile count. */ sqlSafef(query, sizeof(query), "update edwSubmit set oldFiles=%d,oldBytes=%lld,byteCount=%lld where id=%u", oldCount, oldBytes, byteCount, submitId); sqlUpdate(conn, query); /* Deal with old files. This may throw an error. We do it before downloading new * files since we want to fail fast if we are going to fail. */ int updateCount = handleOldFileTags(conn, oldList, doUpdate); sqlSafef(query, sizeof(query), "update edwSubmit set metaChangeCount=%d where id=%u", updateCount, submitId); sqlUpdate(conn, query); } errCatchEnd(errCatch); if (errCatch->gotError) { handleSubmitError(conn, submitId, errCatch->message->string); /* The handleSubmitError will keep on throwing. */ } errCatchFree(&errCatch); /* Go through list attempting to load the files if we don't already have them. */ struct submitFileRow *sfr; for (sfr = newList; sfr != NULL; sfr = sfr->next) { if (edwSubmitShouldStop(conn, submitId)) break; struct edwFile *bf = sfr->file; int submitUrlSize = strlen(submitDir) + strlen(bf->submitFileName) + 1; char submitUrl[submitUrlSize]; safef(submitUrl, submitUrlSize, "%s%s", submitDir, bf->submitFileName); if (edwGotFile(conn, submitDir, bf->submitFileName, bf->md5, bf->size)<0) { /* We can't get a ID for this file. There's two possible reasons - * either somebody is in the middle of fetching it or nobody's started. * If somebody is in the middle of fetching it, assume they died * if they took more than an hour, and start up another fetch. * So here we fetch unless somebody else is fetching recently. */ if (edwGettingFile(conn, submitDir, bf->submitFileName) < 0) { verbose(1, "Fetching %s\n", bf->submitFileName); getSubmittedFile(conn, bf, submitDir, submitUrl, submitId); newBytes += bf->size; sqlSafef(query, sizeof(query), "update edwSubmit set newFiles=newFiles+1,newBytes=%lld where id=%d", newBytes, submitId); sqlUpdate(conn, query); } } else { verbose(2, "Already got %s\n", bf->submitFileName); sqlSafef(query, sizeof(query), "update edwSubmit set oldFiles=oldFiles+1 where id=%d", submitId); sqlUpdate(conn, query); } if (sfr->replacesFile != 0) { /* What happens when the replacement doesn't validate? */ verbose(2, "Replacing %s with %s\n", sfr->replaces, bf->submitFileName); sqlSafef(query, sizeof(query), "update edwFile set replacedBy=%u, deprecated='%s' where id=%u", bf->id, sfr->replaceReason, sfr->replacesFile); sqlUpdate(conn, query); } } /* If we made it here, update submit endUploadTime */ sqlSafef(query, sizeof(query), "update edwSubmit set endUploadTime=%lld where id=%d", edwNow(), submitId); sqlUpdate(conn, query); /* Get a real submission record and then set things up so mail user when all done. */ struct edwSubmit *submit = edwSubmitFromId(conn, submitId); sqlDisconnect(&conn); // We'll be waiting a while so free connection waitForValidationAndSendEmail(submit, email); }
void monitorSubmission(struct sqlConnection *conn) /* Write out information about submission. */ { char *url = trimSpaces(cgiString("url")); cgiMakeHiddenVar("url", url); struct edwSubmit *sub = edwMostRecentSubmission(conn, url); time_t startTime = 0, endTime = 0, endUploadTime = 0; if (sub == NULL) { int posInQueue = edwSubmitPositionInQueue(conn, url, NULL); if (posInQueue == 0) printf("%s is first in the submission queue, but upload has not started<BR>\n", url); else if (posInQueue > 0) printf("%s is in submission queue with %d submissions ahead of it<BR>\n", url, posInQueue); else { printf("%s status unknown.", url); } } else { startTime = sub->startUploadTime; endUploadTime = sub->endUploadTime; endTime = (endUploadTime ? endUploadTime : edwNow()); int timeSpan = endTime - startTime; long long thisUploadSize = sub->byteCount - sub->oldBytes; long long curSize = 0; // Amount of current file we know we've transferred. /* Print title letting them know if upload is done or in progress. */ printf("<B>Submission by %s is ", userEmail); if (!isEmpty(sub->errorMessage)) { if (endUploadTime == 0) printf("having problems..."); else printf("stopped by uploader request."); } else if (endUploadTime != 0) { printf("uploaded."); } else printf("in progress..."); printf("</B><BR>\n"); /* Print URL and how far along we are at the file level. */ if (!isEmpty(sub->errorMessage)) { printf("<B>error:</B> %s<BR>\n", sub->errorMessage); cgiMakeButton("getUrl", "try submission again"); printf("<BR>"); } printf("<B>url:</B> %s<BR>\n", sub->url); printf("<B>files count:</B> %d<BR>\n", sub->fileCount); if (sub->oldFiles > 0) printf("<B>files already in warehouse:</B> %u<BR>\n", sub->oldFiles); if (sub->metaChangeCount > 0) printf("<B>old files with new tags in this submission</B> %d<BR>", sub->metaChangeCount); if (sub->oldFiles != sub->fileCount) { printf("<B>files transferred:</B> %u<BR>\n", sub->newFiles); printf("<B>files remaining:</B> %u<BR>\n", sub->fileCount - sub->oldFiles - sub->newFiles); } /* Report validation status */ printf("<B>new files validated:</B> %u of %u<BR>\n", edwSubmitCountNewValid(sub, conn), sub->newFiles); /* Print error message, and in case of error skip file-in-transfer info. */ if (isEmpty(sub->errorMessage)) { /* If possible print information about file en route */ if (endUploadTime == 0) { struct edwFile *ef = edwFileInProgress(conn, sub->id); if (ef != NULL) { char path[PATH_LEN]; safef(path, sizeof(path), "%s%s", edwRootDir, ef->edwFileName); if (ef->endUploadTime > 0) curSize = ef->size; else curSize = paraFetchedSoFar(path); printf("<B>file in route:</B> %s", ef->submitFileName); printf(" (%d%% transferred)<BR>\n", (int)(100.0 * curSize / ef->size)); } } } /* Report bytes transferred */ long long transferredThisTime = curSize + sub->newBytes; printf("<B>total bytes transferred:</B> "); long long totalTransferred = transferredThisTime + sub->oldBytes; printLongWithCommas(stdout, totalTransferred); printf(" of "); printLongWithCommas(stdout, sub->byteCount); if (sub->byteCount != 0) printf(" (%d%%)<BR>\n", (int)(100.0 * totalTransferred / sub->byteCount)); else printf("<BR>\n"); /* Report transfer speed if possible */ if (isEmpty(sub->errorMessage)) { if (timeSpan > 0) { printf("<B>transfer speed:</B> "); printLongWithCommas(stdout, (curSize + sub->newBytes)/timeSpan); printf(" bytes/sec<BR>\n"); } /* Report start time and duration */ printf("<B>submission started:</B> %s<BR>\n", ctime(&startTime)); struct dyString *duration = edwFormatDuration(timeSpan); /* Try and give them an ETA if we aren't finished */ if (endUploadTime == 0 && timeSpan > 0) { printf("<B>time so far:</B> %s<BR>\n", duration->string); double bytesPerSecond = (double)transferredThisTime/timeSpan; long long bytesRemaining = thisUploadSize - curSize - sub->newBytes; if (bytesPerSecond > 0) { long long estimatedFinish = bytesRemaining/bytesPerSecond; struct dyString *eta = edwFormatDuration(estimatedFinish); printf("<B>estimated finish in:</B> %s<BR>\n", eta->string); } } else { printf("<B>submission time:</B> %s<BR>\n", duration->string); cgiMakeButton("getUrl", "submit another data set"); } } } cgiMakeButton("monitor", "refresh status"); if (endUploadTime == 0 && isEmpty(sub->errorMessage)) cgiMakeButton(stopButtonName, "stop upload"); printf(" <input type=\"button\" value=\"browse submissions\" " "onclick=\"window.location.href='edwWebBrowse';\">\n"); edwPrintLogOutButton(); }