void getUrl(struct sqlConnection *conn)
/* Put up URL. */
{
edwMustGetUserFromEmail(conn, userEmail);
printf("Please enter a URL for a validated manifest file:<BR>");
printf("URL ");
cgiMakeTextVar("url", emptyForNull(cgiOptionalString("url")), 80);
cgiMakeButton("submitUrl", "submit");
printf("<BR>\n");
cgiMakeCheckBox("update", FALSE);
printf(" Update information associated with files that have already been uploaded.");
printf("<BR>Submission by %s", userEmail);
edwPrintLogOutButton();
}
void getFileListAndReason(struct sqlConnection *conn)
/* Put up URL. */
{
edwMustGetUserFromEmail(conn, userEmail);
printf("Please enter in accessions (ENCFF identifiers) for files to deprecate:<BR>");
cgiMakeTextArea("fileList", fileList, 4, 60);
printf("<BR>");
printf("Please enter in reason for deprecating files:<BR>");
cgiMakeTextArea("reason", reason, 4, 60);
printf("<BR>");
printf("Allow me to deprecate files not uploaded by me: ");
cgiMakeCheckBox("allowBox", FALSE);
printf("<BR>");
cgiMakeButton("submit", "submit");
edwPrintLogOutButton();
}
void submitUrl(struct sqlConnection *conn)
/* Submit validated manifest if it is not already in process. Show
* progress once it is in progress. */
{
/* Parse email and URL out of CGI vars. Do a tiny bit of error checking. */
char *url = trimSpaces(cgiString("url"));
if (!stringIn("://", url))
errAbort("%s doesn't seem to be a valid URL, no '://'", url);
/* Do some reality checks that email and URL actually exist. */
edwMustGetUserFromEmail(conn, userEmail);
int sd = netUrlMustOpenPastHeader(url);
close(sd);
edwAddSubmitJob(conn, userEmail, url, cgiBoolean("update"));
/* Give the system a half second to react and then put up status info about submission */
sleep1000(1000);
monitorSubmission(conn);
}
void edwSubmit(char *submitUrl, char *email)
/* edwSubmit - Submit URL with validated.txt to warehouse. */
{
/* Parse out url a little into submitDir and submitFile */
char *lastSlash = strrchr(submitUrl, '/');
if (lastSlash == NULL)
errAbort("%s is not a valid URL - it has no '/' in it.", submitUrl);
char *submitFile = lastSlash+1;
int submitDirSize = submitFile - submitUrl;
char submitDir[submitDirSize+1];
memcpy(submitDir, submitUrl, submitDirSize);
submitDir[submitDirSize] = 0; // Add trailing zero
/* Make sure user has access. */
struct sqlConnection *conn = edwConnectReadWrite();
struct edwUser *user = edwMustGetUserFromEmail(conn, email);
int userId = user->id;
/* See if we are already running on same submission. If so council patience and quit. */
notOverlappingSelf(conn, submitUrl);
/* Make a submit record. */
int submitId = makeNewEmptySubmitRecord(conn, submitUrl, userId);
/* The next errCatch block will fill these in if all goes well. */
struct submitFileRow *sfrList = NULL, *oldList = NULL, *newList = NULL;
int oldCount = 0;
long long oldBytes = 0, newBytes = 0, byteCount = 0;
/* Start catching errors from here and writing them in submitId. If we don't
* throw we'll end up having a list of all files in the submit in sfrList. */
struct errCatch *errCatch = errCatchNew();
char query[1024];
if (errCatchStart(errCatch))
{
/* Make sure they got a bit of space, enough for a reasonable submit file.
* We do this here just because we can make error message more informative. */
long long diskFreeSpace = freeSpaceOnFileSystem(edwRootDir);
if (diskFreeSpace < 4*1024*1024)
errAbort("No space left in warehouse!");
/* Open remote submission file. This is most likely where we will fail. */
int hostId=0, submitDirId = 0;
long long startUploadTime = edwNow();
int remoteFd = edwOpenAndRecordInDir(conn, submitDir, submitFile, submitUrl,
&hostId, &submitDirId);
/* Copy to local temp file. */
char tempSubmitFile[PATH_LEN];
fetchFdToTempFile(remoteFd, tempSubmitFile);
mustCloseFd(&remoteFd);
long long endUploadTime = edwNow();
/* Calculate MD5 sum, and see if we already have such a file. */
char *md5 = md5HexForFile(tempSubmitFile);
int fileId = findFileGivenMd5AndSubmitDir(conn, md5, submitDirId);
/* If we already have it, then delete temp file, otherwise put file in file table. */
char submitLocalPath[PATH_LEN];
if (fileId != 0)
{
remove(tempSubmitFile);
char submitRelativePath[PATH_LEN];
sqlSafef(query, sizeof(query), "select edwFileName from edwFile where id=%d", fileId);
sqlNeedQuickQuery(conn, query, submitRelativePath, sizeof(submitRelativePath));
safef(submitLocalPath, sizeof(submitLocalPath), "%s%s", edwRootDir, submitRelativePath);
}
else
{
/* Looks like it's the first time we've seen this submission file, so
* save the file itself. We'll get to the records inside the file in a bit. */
fileId = makeNewEmptyFileRecord(conn, submitId, submitDirId, submitFile, 0);
/* Get file/path names for submission file inside warehouse. */
char edwFile[PATH_LEN];
edwMakeFileNameAndPath(fileId, submitFile, edwFile, submitLocalPath);
/* Move file to final resting place and get update time and size from local file system. */
mustRename(tempSubmitFile, submitLocalPath);
time_t updateTime = fileModTime(submitLocalPath);
off_t size = fileSize(submitLocalPath);
/* Update file table which now should be complete including updateTime. */
sqlSafef(query, sizeof(query),
"update edwFile set "
" updateTime=%lld, size=%lld, md5='%s', edwFileName='%s',"
" startUploadTime=%lld, endUploadTime=%lld"
" where id=%u\n",
(long long)updateTime, (long long)size, md5, edwFile,
startUploadTime, endUploadTime, fileId);
sqlUpdate(conn, query);
}
/* By now there is a submit file on the local file system. We parse it out. */
edwParseSubmitFile(conn, submitLocalPath, submitUrl, &sfrList);
/* Save our progress so far to submit table. */
sqlSafef(query, sizeof(query),
"update edwSubmit"
" set submitFileId=%lld, submitDirId=%lld, fileCount=%d where id=%d",
(long long)fileId, (long long)submitDirId, slCount(sfrList), submitId);
sqlUpdate(conn, query);
/* Weed out files we already have. */
struct submitFileRow *sfr, *sfrNext;
for (sfr = sfrList; sfr != NULL; sfr = sfrNext)
{
sfrNext = sfr->next;
struct edwFile *bf = sfr->file;
long long fileId;
if ((fileId = edwGotFile(conn, submitDir, bf->submitFileName, bf->md5, bf->size)) >= 0)
{
++oldCount;
oldBytes += bf->size;
sfr->md5MatchFileId = fileId;
slAddHead(&oldList, sfr);
}
else
slAddHead(&newList, sfr);
byteCount += bf->size;
}
sfrList = NULL;
slReverse(&newList);
slReverse(&oldList);
/* Update database with oldFile count. */
sqlSafef(query, sizeof(query),
"update edwSubmit set oldFiles=%d,oldBytes=%lld,byteCount=%lld where id=%u",
oldCount, oldBytes, byteCount, submitId);
sqlUpdate(conn, query);
/* Deal with old files. This may throw an error. We do it before downloading new
* files since we want to fail fast if we are going to fail. */
int updateCount = handleOldFileTags(conn, oldList, doUpdate);
sqlSafef(query, sizeof(query),
"update edwSubmit set metaChangeCount=%d where id=%u", updateCount, submitId);
sqlUpdate(conn, query);
}
errCatchEnd(errCatch);
if (errCatch->gotError)
{
handleSubmitError(conn, submitId, errCatch->message->string);
/* The handleSubmitError will keep on throwing. */
}
errCatchFree(&errCatch);
/* Go through list attempting to load the files if we don't already have them. */
struct submitFileRow *sfr;
for (sfr = newList; sfr != NULL; sfr = sfr->next)
{
if (edwSubmitShouldStop(conn, submitId))
break;
struct edwFile *bf = sfr->file;
int submitUrlSize = strlen(submitDir) + strlen(bf->submitFileName) + 1;
char submitUrl[submitUrlSize];
safef(submitUrl, submitUrlSize, "%s%s", submitDir, bf->submitFileName);
if (edwGotFile(conn, submitDir, bf->submitFileName, bf->md5, bf->size)<0)
{
/* We can't get a ID for this file. There's two possible reasons -
* either somebody is in the middle of fetching it or nobody's started.
* If somebody is in the middle of fetching it, assume they died
* if they took more than an hour, and start up another fetch.
* So here we fetch unless somebody else is fetching recently. */
if (edwGettingFile(conn, submitDir, bf->submitFileName) < 0)
{
verbose(1, "Fetching %s\n", bf->submitFileName);
getSubmittedFile(conn, bf, submitDir, submitUrl, submitId);
newBytes += bf->size;
sqlSafef(query, sizeof(query),
"update edwSubmit set newFiles=newFiles+1,newBytes=%lld where id=%d",
newBytes, submitId);
sqlUpdate(conn, query);
}
}
else
{
verbose(2, "Already got %s\n", bf->submitFileName);
sqlSafef(query, sizeof(query), "update edwSubmit set oldFiles=oldFiles+1 where id=%d",
submitId);
sqlUpdate(conn, query);
}
if (sfr->replacesFile != 0)
{
/* What happens when the replacement doesn't validate? */
verbose(2, "Replacing %s with %s\n", sfr->replaces, bf->submitFileName);
sqlSafef(query, sizeof(query),
"update edwFile set replacedBy=%u, deprecated='%s' where id=%u",
bf->id, sfr->replaceReason, sfr->replacesFile);
sqlUpdate(conn, query);
}
}
/* If we made it here, update submit endUploadTime */
sqlSafef(query, sizeof(query),
"update edwSubmit set endUploadTime=%lld where id=%d",
edwNow(), submitId);
sqlUpdate(conn, query);
/* Get a real submission record and then set things up so mail user when all done. */
struct edwSubmit *submit = edwSubmitFromId(conn, submitId);
sqlDisconnect(&conn); // We'll be waiting a while so free connection
waitForValidationAndSendEmail(submit, email);
}
