void edwMakePlateFileNameAndPath(int edwFileId, char *submitFileName,
char licensePlate[edwMaxPlateSize], char edwFile[PATH_LEN], char serverPath[PATH_LEN])
/* Convert file id to local file name, and full file path. Make any directories needed
* along serverPath. */
{
/* Preserve suffix. Give ourselves up to two suffixes. */
int nameSize = strlen(submitFileName);
char *suffix = lastMatchCharExcept(submitFileName, submitFileName + nameSize, '.', '/');
if (suffix != NULL)
{
char *secondSuffix = lastMatchCharExcept(submitFileName, suffix, '.', '/');
if (secondSuffix != NULL)
suffix = secondSuffix;
}
suffix = emptyForNull(suffix);
/* Figure out edw file name, starting with license plate. */
edwMakeLicensePlate(edwLicensePlatePrefix, edwFileId, licensePlate, edwMaxPlateSize);
/* Figure out directory and make any components not already there. */
char edwDir[PATH_LEN];
edwDirForTime(edwNow(), edwDir);
char uploadDir[PATH_LEN];
safef(uploadDir, sizeof(uploadDir), "%s%s", edwRootDir, edwDir);
makeDirsOnPath(uploadDir);
/* Figure out full file names */
safef(edwFile, PATH_LEN, "%s%s%s", edwDir, licensePlate, suffix);
safef(serverPath, PATH_LEN, "%s%s", edwRootDir, edwFile);
}
void recordIntoHistory(struct sqlConnection *conn, unsigned id, char *table, boolean success)
/* Record success/failure into uploadAttempts and historyBits fields of table. */
{
/* Get historyBits and fold status into it. */
char quickResult[32];
char query[256];
sqlSafef(query, sizeof(query), "select historyBits from %s where id=%u", table, id);
if (sqlQuickQuery(conn, query, quickResult, sizeof(quickResult)) == NULL)
internalErr();
char *lastTimeField;
char *openResultField;
long long historyBits = sqlLongLong(quickResult);
historyBits <<= 1;
if (success)
{
historyBits |= 1;
lastTimeField = "lastOkTime";
openResultField = "openSuccesses";
}
else
{
lastTimeField = "lastNotOkTime";
openResultField = "openFails";
}
sqlSafef(query, sizeof(query),
"update %s set historyBits=%lld, %s=%s+1, %s=%lld "
"where id=%lld",
table, historyBits, openResultField, openResultField, lastTimeField, edwNow(),
(long long)id);
sqlUpdate(conn, query);
}
int makeNewEmptySubmitRecord(struct sqlConnection *conn, char *submitUrl, unsigned userId)
/* Create a submit record around URL and return it's id. */
{
struct dyString *query = dyStringNew(0);
sqlDyStringAppend(query, "insert edwSubmit (url, startUploadTime, userId) ");
sqlDyStringPrintf(query, "VALUES('%s', %lld, %d)", submitUrl, edwNow(), userId);
sqlUpdate(conn, query->string);
dyStringFree(&query);
return sqlLastAutoId(conn);
}
static boolean paraFetchInterruptFunction(void *v)
/* Return TRUE if we need to interrupt. */
{
struct paraFetchInterruptContext *context = v;
long long now = edwNow();
if (context->lastChecked != now) // Only do check every second
{
context->isInterrupted = edwSubmitShouldStop(context->conn, context->submitId);
context->lastChecked = now;
}
return context->isInterrupted;
}
static int getLocalHost(struct sqlConnection *conn)
/* Make up record for local host if it is not there already. */
{
char query[256];
sqlSafef(query, sizeof(query), "select id from edwHost where name = '%s'", localHostName);
int hostId = sqlQuickNum(conn, query);
if (hostId == 0)
{
sqlSafef(query, sizeof(query), "insert edwHost(name, firstAdded) values('%s', %lld)",
localHostName, edwNow());
sqlUpdate(conn, query);
hostId = sqlLastAutoId(conn);
}
return hostId;
}
int edwGetHost(struct sqlConnection *conn, char *hostName)
/* Look up host name in table and return associated ID. If not found
* make up new table entry. */
{
/* If it's already in table, just return ID. */
char query[512];
sqlSafef(query, sizeof(query), "select id from edwHost where name='%s'", hostName);
int hostId = sqlQuickNum(conn, query);
if (hostId > 0)
return hostId;
sqlSafef(query, sizeof(query), "insert edwHost (name, firstAdded, paraFetchStreams) values('%s', %lld, 10)",
hostName, edwNow());
sqlUpdate(conn, query);
return sqlLastAutoId(conn);
}
int edwGetSubmitDir(struct sqlConnection *conn, int hostId, char *submitDir)
/* Get submitDir from database, creating it if it doesn't already exist. */
{
/* If it's already in table, just return ID. */
char query[512];
sqlSafef(query, sizeof(query), "select id from edwSubmitDir where url='%s'", submitDir);
int dirId = sqlQuickNum(conn, query);
if (dirId > 0)
return dirId;
sqlSafef(query, sizeof(query),
"insert edwSubmitDir (url, firstAdded, hostId) values('%s', %lld, %d)",
submitDir, edwNow(), hostId);
sqlUpdate(conn, query);
return sqlLastAutoId(conn);
}
static int getLocalSubmit(struct sqlConnection *conn)
/* Get the submission that covers all of our local additions. */
{
int dirId = getLocalSubmitDir(conn);
char query[256];
sqlSafef(query, sizeof(query), "select id from edwSubmit where submitDirId='%d'", dirId);
int submitId = sqlQuickNum(conn, query);
if (submitId == 0)
{
sqlSafef(query, sizeof(query), "insert edwSubmit (submitDirId,startUploadTime) values(%d,%lld)",
dirId, edwNow());
sqlUpdate(conn, query);
submitId = sqlLastAutoId(conn);
}
return submitId;
}
static int getLocalSubmitDir(struct sqlConnection *conn)
/* Get submit dir for local submissions, making it up if it does not exist. */
{
int hostId = getLocalHost(conn);
char query[256];
sqlSafef(query, sizeof(query), "select id from edwSubmitDir where url='%s' and hostId=%d",
localHostDir, hostId);
int dirId = sqlQuickNum(conn, query);
if (dirId == 0)
{
sqlSafef(query, sizeof(query), "insert edwSubmitDir(url,hostId,firstAdded) values('%s',%d,%lld)",
localHostDir, hostId, edwNow());
sqlUpdate(conn, query);
dirId = sqlLastAutoId(conn);
}
return dirId;
}
char *edwTempDirForToday(char dir[PATH_LEN])
/* Fills in dir with temp dir of the day, and returns a pointer to it. */
{
char dayDir[PATH_LEN];
edwDirForTime(edwNow(), dayDir);
safef(dir, PATH_LEN, "%s%stmp/", edwRootDir, dayDir);
/* Bracket time consuming call to makeDirsOnPath with check that we didn't just do same
* thing. */
static char lastDayDir[PATH_LEN] = "";
if (!sameString(dayDir, lastDayDir))
{
strcpy(lastDayDir, dayDir);
int len = strlen(dir);
dir[len-1] = 0;
makeDirsOnPath(dir);
dir[len-1] = '/';
}
return dir;
}
void notOverlappingSelf(struct sqlConnection *conn, char *url)
/* Ensure we are only submission going on for this URL, allowing for time out
* and command line override. */
{
if (doNow) // Allow command line override
return;
/* Fetch most recent submission from this URL. */
struct edwSubmit *old = edwMostRecentSubmission(conn, url);
if (old == NULL)
return;
/* See if we have something in progress, meaning started but not ended. */
if (old->endUploadTime == 0 && isEmpty(old->errorMessage))
{
/* Check submission last alive time against our usual time out. */
long long maxStartTime = edwSubmitMaxStartTime(old, conn);
if (edwNow() - maxStartTime < edwSingleFileTimeout)
errAbort("Submission of %s already is in progress. Please come back in an hour", url);
}
edwSubmitFree(&old);
}
void edwMakeFileNameAndPath(int edwFileId, char *submitFileName, char edwFile[PATH_LEN], char serverPath[PATH_LEN])
/* Convert file id to local file name, and full file path. Make any directories needed
* along serverPath. */
{
/* Preserve suffix. Give ourselves up to two suffixes. */
char *suffix = edwFindDoubleFileSuffix(submitFileName);
/* Figure out edw file name, starting with baseName. */
char baseName[32];
edwMakeBabyName(edwFileId, baseName, sizeof(baseName));
/* Figure out directory and make any components not already there. */
char edwDir[PATH_LEN];
edwDirForTime(edwNow(), edwDir);
char uploadDir[PATH_LEN];
safef(uploadDir, sizeof(uploadDir), "%s%s", edwRootDir, edwDir);
makeDirsOnPath(uploadDir);
/* Figure out full file names */
safef(edwFile, PATH_LEN, "%s%s%s", edwDir, baseName, suffix);
safef(serverPath, PATH_LEN, "%s%s", edwRootDir, edwFile);
}
long long edwGettingFile(struct sqlConnection *conn, char *submitDir, char *submitFileName)
/* See if we are in process of getting file. Return file record id if it exists even if
* it's not complete. Return -1 if record does not exist. */
{
/* First see if we have even got the directory. */
char query[PATH_LEN+512];
sqlSafef(query, sizeof(query), "select id from edwSubmitDir where url='%s'", submitDir);
int submitDirId = sqlQuickNum(conn, query);
if (submitDirId <= 0)
return -1;
/* Then see if we have file that matches submitDir and submitFileName. */
sqlSafef(query, sizeof(query),
"select id from edwFile "
"where submitFileName='%s' and submitDirId = %d and errorMessage = '' and deprecated=''"
" and (endUploadTime > startUploadTime or startUploadTime < %lld) "
"order by submitId desc limit 1"
, submitFileName, submitDirId
, (long long)edwNow() - edwSingleFileTimeout);
long long id = sqlQuickLongLong(conn, query);
if (id == 0)
return -1;
return id;
}
struct edwFile *edwGetLocalFile(struct sqlConnection *conn, char *localAbsolutePath,
char *symLinkMd5Sum)
/* Get record of local file from database, adding it if it doesn't already exist.
* Can make it a symLink rather than a copy in which case pass in valid MD5 sum
* for symLinkM5dSum. */
{
/* First do a reality check on the local absolute path. Is there a file there? */
if (localAbsolutePath[0] != '/')
errAbort("Using relative path in edwAddLocalFile.");
long long size = fileSize(localAbsolutePath);
if (size == -1)
errAbort("%s does not exist", localAbsolutePath);
long long updateTime = fileModTime(localAbsolutePath);
/* Get file if it's in database already. */
int submitDirId = getLocalSubmitDir(conn);
int submitId = getLocalSubmit(conn);
char query[256+PATH_LEN];
sqlSafef(query, sizeof(query), "select * from edwFile where submitId=%d and submitFileName='%s'",
submitId, localAbsolutePath);
struct edwFile *ef = edwFileLoadByQuery(conn, query);
/* If we got something in database, check update time and size, and if it's no change just
* return existing database id. */
if (ef != NULL && ef->updateTime == updateTime && ef->size == size)
return ef;
/* If we got here, then we need to make a new file record. Start with pretty empty record
* that just has file ID, submitted file name and a few things*/
sqlSafef(query, sizeof(query),
"insert edwFile (submitId,submitDirId,submitFileName,startUploadTime) "
" values(%d, %d, '%s', %lld)"
, submitId, submitDirId, localAbsolutePath, edwNow());
sqlUpdate(conn, query);
long long fileId = sqlLastAutoId(conn);
/* Create big data warehouse file/path name. */
char edwFile[PATH_LEN], edwPath[PATH_LEN];
edwMakeFileNameAndPath(fileId, localAbsolutePath, edwFile, edwPath);
/* We're a little paranoid so md5 it */
char *md5;
/* Do copy or symbolic linking of file into warehouse managed dir. */
if (symLinkMd5Sum)
{
md5 = symLinkMd5Sum;
makeSymLink(localAbsolutePath, edwPath);
}
else
{
copyFile(localAbsolutePath, edwPath);
md5 = md5HexForFile(localAbsolutePath);
}
/* Update file record. */
sqlSafef(query, sizeof(query),
"update edwFile set edwFileName='%s', endUploadTime=%lld,"
"updateTime=%lld, size=%lld, md5='%s' where id=%lld"
, edwFile, edwNow(), updateTime, size, md5, fileId);
sqlUpdate(conn, query);
/* Now, it's a bit of a time waste, but cheap in code, to just load it back from DB. */
sqlSafef(query, sizeof(query), "select * from edwFile where id=%lld", fileId);
return edwFileLoadByQuery(conn, query);
}
int edwFileFetch(struct sqlConnection *conn, struct edwFile *ef, int fd,
char *submitFileName, unsigned submitId, unsigned submitDirId, unsigned hostId)
/* Fetch file and if successful update a bunch of the fields in ef with the result.
* Returns fileId. */
{
ef->id = makeNewEmptyFileRecord(conn, submitId, submitDirId, ef->submitFileName, ef->size);
/* Update edwSubmit with file in transit info */
char query[256];
sqlSafef(query, sizeof(query), "update edwSubmit set fileIdInTransit=%lld where id=%u",
(long long)ef->id, submitId);
sqlUpdate(conn, query);
sqlSafef(query, sizeof(query), "select paraFetchStreams from edwHost where id=%u", hostId);
int paraFetchStreams = sqlQuickNum(conn, query);
struct paraFetchInterruptContext interruptContext = {.conn=conn, .submitId=submitId};
/* Wrap getting the file, the actual data transfer, with an error catcher that
* will remove partly uploaded files. Perhaps some day we'll attempt to rescue
* ones that are just truncated by downloading the rest, but not now. */
struct errCatch *errCatch = errCatchNew();
char tempName[PATH_LEN] = "";
char edwFile[PATH_LEN] = "", edwPath[PATH_LEN];
if (errCatchStart(errCatch))
{
/* Now make temp file name and open temp file in an atomic operation */
char *tempDir = edwTempDir();
safef(tempName, PATH_LEN, "%sedwSubmitXXXXXX", tempDir);
int localFd = mustMkstemp(tempName);
/* Update file name in database with temp file name so web app can track us. */
char query[PATH_LEN+128];
sqlSafef(query, sizeof(query),
"update edwFile set edwFileName='%s' where id=%lld",
tempName + strlen(edwRootDir), (long long)ef->id);
sqlUpdate(conn, query);
/* Do actual upload tracking how long it takes. */
ef->startUploadTime = edwNow();
mustCloseFd(&localFd);
if (!parallelFetchInterruptable(submitFileName, tempName, paraFetchStreams, 4, FALSE, FALSE,
paraFetchInterruptFunction, &interruptContext))
{
if (interruptContext.isInterrupted)
errAbort("Submission stopped by user.");
else
errAbort("parallel fetch of %s failed", submitFileName);
}
ef->endUploadTime = edwNow();
/* Rename file both in file system and (via ef) database. */
edwMakeFileNameAndPath(ef->id, submitFileName, edwFile, edwPath);
mustRename(tempName, edwPath);
if (endsWith(edwPath, ".gz") && !encode3IsGzipped(edwPath))
errAbort("%s has .gz suffix, but is not gzipped", submitFileName);
ef->edwFileName = cloneString(edwFile);
}
errCatchEnd(errCatch);
if (errCatch->gotError)
{
/* Attempt to remove any partial file. */
if (tempName[0] != 0)
{
verbose(1, "Removing partial %s\n", tempName);
parallelFetchRemovePartial(tempName);
remove(tempName);
}
handleSubmitError(conn, submitId, errCatch->message->string); // Throws further
assert(FALSE); // We never get here
}
errCatchFree(&errCatch);
/* Now we got the file. We'll go ahead and save the file name and stuff. */
sqlSafef(query, sizeof(query),
"update edwFile set"
" edwFileName='%s', startUploadTime=%lld, endUploadTime=%lld"
" where id = %d"
, ef->edwFileName, ef->startUploadTime, ef->endUploadTime, ef->id);
sqlUpdate(conn, query);
/* Wrap the validations in an error catcher that will save error to file table in database */
errCatch = errCatchNew();
boolean success = FALSE;
if (errCatchStart(errCatch))
{
/* Check MD5 sum here. */
unsigned char md5bin[16];
md5ForFile(edwPath, md5bin);
char md5[33];
hexBinaryString(md5bin, sizeof(md5bin), md5, sizeof(md5));
if (!sameWord(md5, ef->md5))
errAbort("%s has md5 mismatch: %s != %s. File may be corrupted in upload, or file may have "
"been changed since validateManifest was run. Please check that md5 of file "
"before upload is really %s. If it is then try submitting again, otherwise "
"rerun validateManifest and then try submitting again. \n",
ef->submitFileName, ef->md5, md5, ef->md5);
/* Finish updating a bunch more of edwFile record. Note there is a requirement in
* the validFile section that ef->updateTime be updated last. A nonzero ef->updateTime
* is used as a sign of record complete. */
struct dyString *dy = dyStringNew(0); /* Includes tag so query may be long */
sqlDyStringPrintf(dy, "update edwFile set md5='%s',size=%lld,updateTime=%lld",
md5, ef->size, ef->updateTime);
dyStringAppend(dy, ", tags='");
dyStringAppend(dy, ef->tags);
dyStringPrintf(dy, "' where id=%d", ef->id);
sqlUpdate(conn, dy->string);
dyStringFree(&dy);
/* Update edwSubmit so file no longer shown as in transit */
sqlSafef(query, sizeof(query), "update edwSubmit set fileIdInTransit=0 where id=%u", submitId);
sqlUpdate(conn, query);
success = TRUE;
}
errCatchEnd(errCatch);
if (errCatch->gotError)
{
handleFileError(conn, submitId, ef->id, errCatch->message->string);
}
return ef->id;
}
void edwSubmit(char *submitUrl, char *email)
/* edwSubmit - Submit URL with validated.txt to warehouse. */
{
/* Parse out url a little into submitDir and submitFile */
char *lastSlash = strrchr(submitUrl, '/');
if (lastSlash == NULL)
errAbort("%s is not a valid URL - it has no '/' in it.", submitUrl);
char *submitFile = lastSlash+1;
int submitDirSize = submitFile - submitUrl;
char submitDir[submitDirSize+1];
memcpy(submitDir, submitUrl, submitDirSize);
submitDir[submitDirSize] = 0; // Add trailing zero
/* Make sure user has access. */
struct sqlConnection *conn = edwConnectReadWrite();
struct edwUser *user = edwMustGetUserFromEmail(conn, email);
int userId = user->id;
/* See if we are already running on same submission. If so council patience and quit. */
notOverlappingSelf(conn, submitUrl);
/* Make a submit record. */
int submitId = makeNewEmptySubmitRecord(conn, submitUrl, userId);
/* The next errCatch block will fill these in if all goes well. */
struct submitFileRow *sfrList = NULL, *oldList = NULL, *newList = NULL;
int oldCount = 0;
long long oldBytes = 0, newBytes = 0, byteCount = 0;
/* Start catching errors from here and writing them in submitId. If we don't
* throw we'll end up having a list of all files in the submit in sfrList. */
struct errCatch *errCatch = errCatchNew();
char query[1024];
if (errCatchStart(errCatch))
{
/* Make sure they got a bit of space, enough for a reasonable submit file.
* We do this here just because we can make error message more informative. */
long long diskFreeSpace = freeSpaceOnFileSystem(edwRootDir);
if (diskFreeSpace < 4*1024*1024)
errAbort("No space left in warehouse!");
/* Open remote submission file. This is most likely where we will fail. */
int hostId=0, submitDirId = 0;
long long startUploadTime = edwNow();
int remoteFd = edwOpenAndRecordInDir(conn, submitDir, submitFile, submitUrl,
&hostId, &submitDirId);
/* Copy to local temp file. */
char tempSubmitFile[PATH_LEN];
fetchFdToTempFile(remoteFd, tempSubmitFile);
mustCloseFd(&remoteFd);
long long endUploadTime = edwNow();
/* Calculate MD5 sum, and see if we already have such a file. */
char *md5 = md5HexForFile(tempSubmitFile);
int fileId = findFileGivenMd5AndSubmitDir(conn, md5, submitDirId);
/* If we already have it, then delete temp file, otherwise put file in file table. */
char submitLocalPath[PATH_LEN];
if (fileId != 0)
{
remove(tempSubmitFile);
char submitRelativePath[PATH_LEN];
sqlSafef(query, sizeof(query), "select edwFileName from edwFile where id=%d", fileId);
sqlNeedQuickQuery(conn, query, submitRelativePath, sizeof(submitRelativePath));
safef(submitLocalPath, sizeof(submitLocalPath), "%s%s", edwRootDir, submitRelativePath);
}
else
{
/* Looks like it's the first time we've seen this submission file, so
* save the file itself. We'll get to the records inside the file in a bit. */
fileId = makeNewEmptyFileRecord(conn, submitId, submitDirId, submitFile, 0);
/* Get file/path names for submission file inside warehouse. */
char edwFile[PATH_LEN];
edwMakeFileNameAndPath(fileId, submitFile, edwFile, submitLocalPath);
/* Move file to final resting place and get update time and size from local file system. */
mustRename(tempSubmitFile, submitLocalPath);
time_t updateTime = fileModTime(submitLocalPath);
off_t size = fileSize(submitLocalPath);
/* Update file table which now should be complete including updateTime. */
sqlSafef(query, sizeof(query),
"update edwFile set "
" updateTime=%lld, size=%lld, md5='%s', edwFileName='%s',"
" startUploadTime=%lld, endUploadTime=%lld"
" where id=%u\n",
(long long)updateTime, (long long)size, md5, edwFile,
startUploadTime, endUploadTime, fileId);
sqlUpdate(conn, query);
}
/* By now there is a submit file on the local file system. We parse it out. */
edwParseSubmitFile(conn, submitLocalPath, submitUrl, &sfrList);
/* Save our progress so far to submit table. */
sqlSafef(query, sizeof(query),
"update edwSubmit"
" set submitFileId=%lld, submitDirId=%lld, fileCount=%d where id=%d",
(long long)fileId, (long long)submitDirId, slCount(sfrList), submitId);
sqlUpdate(conn, query);
/* Weed out files we already have. */
struct submitFileRow *sfr, *sfrNext;
for (sfr = sfrList; sfr != NULL; sfr = sfrNext)
{
sfrNext = sfr->next;
struct edwFile *bf = sfr->file;
long long fileId;
if ((fileId = edwGotFile(conn, submitDir, bf->submitFileName, bf->md5, bf->size)) >= 0)
{
++oldCount;
oldBytes += bf->size;
sfr->md5MatchFileId = fileId;
slAddHead(&oldList, sfr);
}
else
slAddHead(&newList, sfr);
byteCount += bf->size;
}
sfrList = NULL;
slReverse(&newList);
slReverse(&oldList);
/* Update database with oldFile count. */
sqlSafef(query, sizeof(query),
"update edwSubmit set oldFiles=%d,oldBytes=%lld,byteCount=%lld where id=%u",
oldCount, oldBytes, byteCount, submitId);
sqlUpdate(conn, query);
/* Deal with old files. This may throw an error. We do it before downloading new
* files since we want to fail fast if we are going to fail. */
int updateCount = handleOldFileTags(conn, oldList, doUpdate);
sqlSafef(query, sizeof(query),
"update edwSubmit set metaChangeCount=%d where id=%u", updateCount, submitId);
sqlUpdate(conn, query);
}
errCatchEnd(errCatch);
if (errCatch->gotError)
{
handleSubmitError(conn, submitId, errCatch->message->string);
/* The handleSubmitError will keep on throwing. */
}
errCatchFree(&errCatch);
/* Go through list attempting to load the files if we don't already have them. */
struct submitFileRow *sfr;
for (sfr = newList; sfr != NULL; sfr = sfr->next)
{
if (edwSubmitShouldStop(conn, submitId))
break;
struct edwFile *bf = sfr->file;
int submitUrlSize = strlen(submitDir) + strlen(bf->submitFileName) + 1;
char submitUrl[submitUrlSize];
safef(submitUrl, submitUrlSize, "%s%s", submitDir, bf->submitFileName);
if (edwGotFile(conn, submitDir, bf->submitFileName, bf->md5, bf->size)<0)
{
/* We can't get a ID for this file. There's two possible reasons -
* either somebody is in the middle of fetching it or nobody's started.
* If somebody is in the middle of fetching it, assume they died
* if they took more than an hour, and start up another fetch.
* So here we fetch unless somebody else is fetching recently. */
if (edwGettingFile(conn, submitDir, bf->submitFileName) < 0)
{
verbose(1, "Fetching %s\n", bf->submitFileName);
getSubmittedFile(conn, bf, submitDir, submitUrl, submitId);
newBytes += bf->size;
sqlSafef(query, sizeof(query),
"update edwSubmit set newFiles=newFiles+1,newBytes=%lld where id=%d",
newBytes, submitId);
sqlUpdate(conn, query);
}
}
else
{
verbose(2, "Already got %s\n", bf->submitFileName);
sqlSafef(query, sizeof(query), "update edwSubmit set oldFiles=oldFiles+1 where id=%d",
submitId);
sqlUpdate(conn, query);
}
if (sfr->replacesFile != 0)
{
/* What happens when the replacement doesn't validate? */
verbose(2, "Replacing %s with %s\n", sfr->replaces, bf->submitFileName);
sqlSafef(query, sizeof(query),
"update edwFile set replacedBy=%u, deprecated='%s' where id=%u",
bf->id, sfr->replaceReason, sfr->replacesFile);
sqlUpdate(conn, query);
}
}
/* If we made it here, update submit endUploadTime */
sqlSafef(query, sizeof(query),
"update edwSubmit set endUploadTime=%lld where id=%d",
edwNow(), submitId);
sqlUpdate(conn, query);
/* Get a real submission record and then set things up so mail user when all done. */
struct edwSubmit *submit = edwSubmitFromId(conn, submitId);
sqlDisconnect(&conn); // We'll be waiting a while so free connection
waitForValidationAndSendEmail(submit, email);
}
void monitorSubmission(struct sqlConnection *conn)
/* Write out information about submission. */
{
char *url = trimSpaces(cgiString("url"));
cgiMakeHiddenVar("url", url);
struct edwSubmit *sub = edwMostRecentSubmission(conn, url);
time_t startTime = 0, endTime = 0, endUploadTime = 0;
if (sub == NULL)
{
int posInQueue = edwSubmitPositionInQueue(conn, url, NULL);
if (posInQueue == 0)
printf("%s is first in the submission queue, but upload has not started<BR>\n", url);
else if (posInQueue > 0)
printf("%s is in submission queue with %d submissions ahead of it<BR>\n", url, posInQueue);
else
{
printf("%s status unknown.", url);
}
}
else
{
startTime = sub->startUploadTime;
endUploadTime = sub->endUploadTime;
endTime = (endUploadTime ? endUploadTime : edwNow());
int timeSpan = endTime - startTime;
long long thisUploadSize = sub->byteCount - sub->oldBytes;
long long curSize = 0; // Amount of current file we know we've transferred.
/* Print title letting them know if upload is done or in progress. */
printf("<B>Submission by %s is ", userEmail);
if (!isEmpty(sub->errorMessage))
{
if (endUploadTime == 0)
printf("having problems...");
else
printf("stopped by uploader request.");
}
else if (endUploadTime != 0)
{
printf("uploaded.");
}
else
printf("in progress...");
printf("</B><BR>\n");
/* Print URL and how far along we are at the file level. */
if (!isEmpty(sub->errorMessage))
{
printf("<B>error:</B> %s<BR>\n", sub->errorMessage);
cgiMakeButton("getUrl", "try submission again");
printf("<BR>");
}
printf("<B>url:</B> %s<BR>\n", sub->url);
printf("<B>files count:</B> %d<BR>\n", sub->fileCount);
if (sub->oldFiles > 0)
printf("<B>files already in warehouse:</B> %u<BR>\n", sub->oldFiles);
if (sub->metaChangeCount > 0)
printf("<B>old files with new tags in this submission</B> %d<BR>", sub->metaChangeCount);
if (sub->oldFiles != sub->fileCount)
{
printf("<B>files transferred:</B> %u<BR>\n", sub->newFiles);
printf("<B>files remaining:</B> %u<BR>\n", sub->fileCount - sub->oldFiles - sub->newFiles);
}
/* Report validation status */
printf("<B>new files validated:</B> %u of %u<BR>\n", edwSubmitCountNewValid(sub, conn),
sub->newFiles);
/* Print error message, and in case of error skip file-in-transfer info. */
if (isEmpty(sub->errorMessage))
{
/* If possible print information about file en route */
if (endUploadTime == 0)
{
struct edwFile *ef = edwFileInProgress(conn, sub->id);
if (ef != NULL)
{
char path[PATH_LEN];
safef(path, sizeof(path), "%s%s", edwRootDir, ef->edwFileName);
if (ef->endUploadTime > 0)
curSize = ef->size;
else
curSize = paraFetchedSoFar(path);
printf("<B>file in route:</B> %s", ef->submitFileName);
printf(" (%d%% transferred)<BR>\n", (int)(100.0 * curSize / ef->size));
}
}
}
/* Report bytes transferred */
long long transferredThisTime = curSize + sub->newBytes;
printf("<B>total bytes transferred:</B> ");
long long totalTransferred = transferredThisTime + sub->oldBytes;
printLongWithCommas(stdout, totalTransferred);
printf(" of ");
printLongWithCommas(stdout, sub->byteCount);
if (sub->byteCount != 0)
printf(" (%d%%)<BR>\n", (int)(100.0 * totalTransferred / sub->byteCount));
else
printf("<BR>\n");
/* Report transfer speed if possible */
if (isEmpty(sub->errorMessage))
{
if (timeSpan > 0)
{
printf("<B>transfer speed:</B> ");
printLongWithCommas(stdout, (curSize + sub->newBytes)/timeSpan);
printf(" bytes/sec<BR>\n");
}
/* Report start time and duration */
printf("<B>submission started:</B> %s<BR>\n", ctime(&startTime));
struct dyString *duration = edwFormatDuration(timeSpan);
/* Try and give them an ETA if we aren't finished */
if (endUploadTime == 0 && timeSpan > 0)
{
printf("<B>time so far:</B> %s<BR>\n", duration->string);
double bytesPerSecond = (double)transferredThisTime/timeSpan;
long long bytesRemaining = thisUploadSize - curSize - sub->newBytes;
if (bytesPerSecond > 0)
{
long long estimatedFinish = bytesRemaining/bytesPerSecond;
struct dyString *eta = edwFormatDuration(estimatedFinish);
printf("<B>estimated finish in:</B> %s<BR>\n", eta->string);
}
}
else
{
printf("<B>submission time:</B> %s<BR>\n", duration->string);
cgiMakeButton("getUrl", "submit another data set");
}
}
}
cgiMakeButton("monitor", "refresh status");
if (endUploadTime == 0 && isEmpty(sub->errorMessage))
cgiMakeButton(stopButtonName, "stop upload");
printf(" <input type=\"button\" value=\"browse submissions\" "
"onclick=\"window.location.href='edwWebBrowse';\">\n");
edwPrintLogOutButton();
}
