void updateSumAndAll(struct encode2Manifest *mi, char *md5, char *rootDir)
/* Update mi to reflect new md5. Since this means file has changed we'll go
* look for file and get new size and update time. We also update validation key. */
{
char path[PATH_LEN];
safef(path, sizeof(path), "%s/%s", rootDir, mi->fileName);
off_t size = fileSize(path);
if (size == -1)
errAbort("%s doesn't exist", path);
mi->size = size;
mi->modifiedTime = fileModTime(path);
mi->md5sum = md5;
mi->validKey = encode3CalcValidationKey(md5, size);
}
static boolean checkAutoUpgradeTableResultTimeIsOld(char *tableName)
/* Has enough time passed since the last upgrade check?
* The idea is to only check once every few minutes
* rather than each time the CGI runs. */
{
char path[AUTOUPGRPATHSIZE];
makeResultName(tableName, path);
if (!fileExists(path))
return TRUE; // If there is no result yet we should test and make one
struct timeval rawtime;
gettimeofday( &rawtime, NULL );
time_t now = rawtime.tv_sec;
time_t fMT = fileModTime(path);
double diff = difftime(now, fMT);
if (diff > (3 * 60)) // 3 minutes in seconds
return TRUE; // The result is old we should test it again in case situation changed
return FALSE;
}
static void rTokInto(struct pfCompile *pfc, char *baseDir, char *modName,
boolean lookForPfh)
/* Tokenize module, and recursively any thing it goes into. */
{
char *pfPath = findSourcePath(baseDir, pfc->paraLibPath, modName, ".pf");
char *pfhPath = replaceSuffix(pfPath, ".pf", ".pfh");
char *oPath = replaceSuffix(pfPath, ".pf", ".o");
char *cPath = replaceSuffix(pfPath, ".pf", ".c");
char *fileName = NULL;
struct pfModule *module;
boolean isPfh = FALSE;
struct pfToken *tok;
/* Look too see if can use just module header.
* We can if it exists and is newer than module. */
if (lookForPfh)
{
if (fileExists(pfhPath) && fileExists(cPath) && fileExists(oPath))
{
unsigned long pfTime = fileModTime(pfPath);
unsigned long pfhTime = fileModTime(pfhPath);
unsigned long cTime = fileModTime(cPath);
unsigned long oTime = fileModTime(oPath);
if (pfTime < pfhTime && pfhTime <= cTime && cTime <= oTime)
{
fileName = pfhPath;
isPfh = TRUE;
}
}
}
if (fileName == NULL)
fileName = pfPath;
/* Tokenize file and add module to hash. */
module = tokenizeFile(pfc->tkz, fileName, modName);
module->isPfh = isPfh;
hashAdd(pfc->moduleHash, modName, module);
/* Look and see if there are any 'include' or 'import' to follow. */
for (tok = module->tokList; tok != NULL; tok = tok->next)
{
if (tok->type == pftInclude || tok->type == pftImport)
{
struct pfToken *modTok = tok->next;
if (modTok->type != pftString)
expectingGot("module name in single quotes", modTok);
if (sameString(modTok->val.s, modName))
errAt(modTok, "A module can't include/import itself.");
if (!hashLookup(pfc->moduleHash, modTok->val.s))
rTokInto(pfc, baseDir, modTok->val.s, TRUE);
}
}
/* Add module to list */
slAddHead(&pfc->moduleList, module);
/* Clean up and go home. */
freeMem(pfPath);
freeMem(pfhPath);
freeMem(oPath);
freeMem(cPath);
}
char *webTimeStampedLinkToResource(char *fileName, boolean wrapInHtml)
// If wrapInHtml
// returns versioned link embedded in style or script html (free after use).
// else
// returns full path of a versioned path to the requested resource file (js, or css).
// NOTE: png, jpg and gif should also be supported but are untested.
//
// In production sites we use a versioned softlink that includes the CGI version. This has the following benefits:
// a) flushes user's web browser cache when the user visits a GB site whose version has changed since their last visit;
// b) enforces the requirement that static files are the same version as the CGIs (something that often fails to happen in mirrors).
// (see notes in redmine #3170).
//
// In dev trees we use mtime to create a pseudo-version; this forces web browsers to reload css/js file when it changes,
// so we don't get odd behavior that can be caused by caching of mis-matched javascript and style files in dev trees.
//
// In either case, the actual file has to have been previously created by running make in the appropriate directory (kent/src/hg/js
// or kent/src/hg/htdocs/style).
{
char baseName[PATH_LEN];
char extension[FILEEXT_LEN];
splitPath(fileName, NULL, baseName, extension);
boolean js = sameString(".js",extension);
boolean style = !js && sameString(".css",extension);
boolean image = !js
&& !style
&& ( sameString(".png",extension)
|| sameString(".jpg",extension)
|| sameString(".gif",extension));
if (!js && !style) // && !image) NOTE: This code has not been tested on images but should work.
errAbort("webTimeStampedLinkToResource: unknown resource type for %s.\n", fileName);
// Build and verify directory
char *dirName = "";
if (js)
dirName = cfgOptionDefault("browser.javaScriptDir", "js");
else if (style)
dirName = cfgOptionDefault("browser.styleDir","style");
else if (image)
dirName = cfgOptionDefault("browser.styleImagesDir","style/images");
struct dyString *fullDirName = NULL;
char *docRoot = hDocumentRoot();
if (docRoot != NULL)
fullDirName = dyStringCreate("%s/%s", docRoot, dirName);
else
// tolerate missing docRoot (i.e. when running from command line)
fullDirName = dyStringCreate("%s", dirName);
if (!fileExists(dyStringContents(fullDirName)))
errAbort("webTimeStampedLinkToResource: dir: %s doesn't exist.\n",
dyStringContents(fullDirName));
// build and verify real path to file
struct dyString *realFileName = dyStringCreate("%s/%s", dyStringContents(fullDirName), fileName);
if (!fileExists(dyStringContents(realFileName)))
errAbort("webTimeStampedLinkToResource: file: %s doesn't exist.\n",
dyStringContents(realFileName));
// build and verify link path including timestamp in the form of dir/baseName + timeStamp or CGI Version + ext
long mtime = fileModTime(dyStringContents(realFileName));
struct dyString *linkWithTimestamp;
char *scriptName = cgiScriptName();
if (scriptName == NULL)
scriptName = cloneString("");
boolean nonVersionedLinks = FALSE;
if (endsWith(scriptName, "qaPushQ"))
nonVersionedLinks = TRUE;
if (nonVersionedLinks)
linkWithTimestamp = dyStringCreate("%s/%s%s", dyStringContents(fullDirName), baseName, extension);
else if ((cfgOption("versionStamped") == NULL) && (hIsPreviewHost() || hIsPrivateHost()))
linkWithTimestamp = dyStringCreate("%s/%s-%ld%s", dyStringContents(fullDirName), baseName, mtime, extension);
else
linkWithTimestamp = dyStringCreate("%s/%s-v%s%s", dyStringContents(fullDirName), baseName, CGI_VERSION, extension);
if (!fileExists(dyStringContents(linkWithTimestamp)))
errAbort("Cannot find correct version of file '%s'; this is due to an installation error\n\nError details: %s does not exist",
fileName, dyStringContents(linkWithTimestamp));
// Free up all that extra memory
dyStringFree(&realFileName);
dyStringFree(&fullDirName);
char *linkFull = dyStringCannibalize(&linkWithTimestamp);
char *link = linkFull;
if (docRoot != NULL)
{
link = cloneString(linkFull + strlen(docRoot) + 1);
freeMem(linkFull);
}
if (wrapInHtml) // wrapped for christmas
{
struct dyString *wrapped = dyStringNew(0);
if (js)
dyStringPrintf(wrapped,"<script type='text/javascript' SRC='/%s'></script>\n", link);
else if (style)
dyStringPrintf(wrapped,"<LINK rel='STYLESHEET' href='/%s' TYPE='text/css' />\n", link);
else // Will be image, since these are the only three choices allowed
dyStringPrintf(wrapped,"<IMG src='/%s' />\n", link);
freeMem(link);
link = dyStringCannibalize(&wrapped);
}
return link;
}
struct edwFile *edwGetLocalFile(struct sqlConnection *conn, char *localAbsolutePath,
char *symLinkMd5Sum)
/* Get record of local file from database, adding it if it doesn't already exist.
* Can make it a symLink rather than a copy in which case pass in valid MD5 sum
* for symLinkM5dSum. */
{
/* First do a reality check on the local absolute path. Is there a file there? */
if (localAbsolutePath[0] != '/')
errAbort("Using relative path in edwAddLocalFile.");
long long size = fileSize(localAbsolutePath);
if (size == -1)
errAbort("%s does not exist", localAbsolutePath);
long long updateTime = fileModTime(localAbsolutePath);
/* Get file if it's in database already. */
int submitDirId = getLocalSubmitDir(conn);
int submitId = getLocalSubmit(conn);
char query[256+PATH_LEN];
sqlSafef(query, sizeof(query), "select * from edwFile where submitId=%d and submitFileName='%s'",
submitId, localAbsolutePath);
struct edwFile *ef = edwFileLoadByQuery(conn, query);
/* If we got something in database, check update time and size, and if it's no change just
* return existing database id. */
if (ef != NULL && ef->updateTime == updateTime && ef->size == size)
return ef;
/* If we got here, then we need to make a new file record. Start with pretty empty record
* that just has file ID, submitted file name and a few things*/
sqlSafef(query, sizeof(query),
"insert edwFile (submitId,submitDirId,submitFileName,startUploadTime) "
" values(%d, %d, '%s', %lld)"
, submitId, submitDirId, localAbsolutePath, edwNow());
sqlUpdate(conn, query);
long long fileId = sqlLastAutoId(conn);
/* Create big data warehouse file/path name. */
char edwFile[PATH_LEN], edwPath[PATH_LEN];
edwMakeFileNameAndPath(fileId, localAbsolutePath, edwFile, edwPath);
/* We're a little paranoid so md5 it */
char *md5;
/* Do copy or symbolic linking of file into warehouse managed dir. */
if (symLinkMd5Sum)
{
md5 = symLinkMd5Sum;
makeSymLink(localAbsolutePath, edwPath);
}
else
{
copyFile(localAbsolutePath, edwPath);
md5 = md5HexForFile(localAbsolutePath);
}
/* Update file record. */
sqlSafef(query, sizeof(query),
"update edwFile set edwFileName='%s', endUploadTime=%lld,"
"updateTime=%lld, size=%lld, md5='%s' where id=%lld"
, edwFile, edwNow(), updateTime, size, md5, fileId);
sqlUpdate(conn, query);
/* Now, it's a bit of a time waste, but cheap in code, to just load it back from DB. */
sqlSafef(query, sizeof(query), "select * from edwFile where id=%lld", fileId);
return edwFileLoadByQuery(conn, query);
}
void edwSubmit(char *submitUrl, char *email)
/* edwSubmit - Submit URL with validated.txt to warehouse. */
{
/* Parse out url a little into submitDir and submitFile */
char *lastSlash = strrchr(submitUrl, '/');
if (lastSlash == NULL)
errAbort("%s is not a valid URL - it has no '/' in it.", submitUrl);
char *submitFile = lastSlash+1;
int submitDirSize = submitFile - submitUrl;
char submitDir[submitDirSize+1];
memcpy(submitDir, submitUrl, submitDirSize);
submitDir[submitDirSize] = 0; // Add trailing zero
/* Make sure user has access. */
struct sqlConnection *conn = edwConnectReadWrite();
struct edwUser *user = edwMustGetUserFromEmail(conn, email);
int userId = user->id;
/* See if we are already running on same submission. If so council patience and quit. */
notOverlappingSelf(conn, submitUrl);
/* Make a submit record. */
int submitId = makeNewEmptySubmitRecord(conn, submitUrl, userId);
/* The next errCatch block will fill these in if all goes well. */
struct submitFileRow *sfrList = NULL, *oldList = NULL, *newList = NULL;
int oldCount = 0;
long long oldBytes = 0, newBytes = 0, byteCount = 0;
/* Start catching errors from here and writing them in submitId. If we don't
* throw we'll end up having a list of all files in the submit in sfrList. */
struct errCatch *errCatch = errCatchNew();
char query[1024];
if (errCatchStart(errCatch))
{
/* Make sure they got a bit of space, enough for a reasonable submit file.
* We do this here just because we can make error message more informative. */
long long diskFreeSpace = freeSpaceOnFileSystem(edwRootDir);
if (diskFreeSpace < 4*1024*1024)
errAbort("No space left in warehouse!");
/* Open remote submission file. This is most likely where we will fail. */
int hostId=0, submitDirId = 0;
long long startUploadTime = edwNow();
int remoteFd = edwOpenAndRecordInDir(conn, submitDir, submitFile, submitUrl,
&hostId, &submitDirId);
/* Copy to local temp file. */
char tempSubmitFile[PATH_LEN];
fetchFdToTempFile(remoteFd, tempSubmitFile);
mustCloseFd(&remoteFd);
long long endUploadTime = edwNow();
/* Calculate MD5 sum, and see if we already have such a file. */
char *md5 = md5HexForFile(tempSubmitFile);
int fileId = findFileGivenMd5AndSubmitDir(conn, md5, submitDirId);
/* If we already have it, then delete temp file, otherwise put file in file table. */
char submitLocalPath[PATH_LEN];
if (fileId != 0)
{
remove(tempSubmitFile);
char submitRelativePath[PATH_LEN];
sqlSafef(query, sizeof(query), "select edwFileName from edwFile where id=%d", fileId);
sqlNeedQuickQuery(conn, query, submitRelativePath, sizeof(submitRelativePath));
safef(submitLocalPath, sizeof(submitLocalPath), "%s%s", edwRootDir, submitRelativePath);
}
else
{
/* Looks like it's the first time we've seen this submission file, so
* save the file itself. We'll get to the records inside the file in a bit. */
fileId = makeNewEmptyFileRecord(conn, submitId, submitDirId, submitFile, 0);
/* Get file/path names for submission file inside warehouse. */
char edwFile[PATH_LEN];
edwMakeFileNameAndPath(fileId, submitFile, edwFile, submitLocalPath);
/* Move file to final resting place and get update time and size from local file system. */
mustRename(tempSubmitFile, submitLocalPath);
time_t updateTime = fileModTime(submitLocalPath);
off_t size = fileSize(submitLocalPath);
/* Update file table which now should be complete including updateTime. */
sqlSafef(query, sizeof(query),
"update edwFile set "
" updateTime=%lld, size=%lld, md5='%s', edwFileName='%s',"
" startUploadTime=%lld, endUploadTime=%lld"
" where id=%u\n",
(long long)updateTime, (long long)size, md5, edwFile,
startUploadTime, endUploadTime, fileId);
sqlUpdate(conn, query);
}
/* By now there is a submit file on the local file system. We parse it out. */
edwParseSubmitFile(conn, submitLocalPath, submitUrl, &sfrList);
/* Save our progress so far to submit table. */
sqlSafef(query, sizeof(query),
"update edwSubmit"
" set submitFileId=%lld, submitDirId=%lld, fileCount=%d where id=%d",
(long long)fileId, (long long)submitDirId, slCount(sfrList), submitId);
sqlUpdate(conn, query);
/* Weed out files we already have. */
struct submitFileRow *sfr, *sfrNext;
for (sfr = sfrList; sfr != NULL; sfr = sfrNext)
{
sfrNext = sfr->next;
struct edwFile *bf = sfr->file;
long long fileId;
if ((fileId = edwGotFile(conn, submitDir, bf->submitFileName, bf->md5, bf->size)) >= 0)
{
++oldCount;
oldBytes += bf->size;
sfr->md5MatchFileId = fileId;
slAddHead(&oldList, sfr);
}
else
slAddHead(&newList, sfr);
byteCount += bf->size;
}
sfrList = NULL;
slReverse(&newList);
slReverse(&oldList);
/* Update database with oldFile count. */
sqlSafef(query, sizeof(query),
"update edwSubmit set oldFiles=%d,oldBytes=%lld,byteCount=%lld where id=%u",
oldCount, oldBytes, byteCount, submitId);
sqlUpdate(conn, query);
/* Deal with old files. This may throw an error. We do it before downloading new
* files since we want to fail fast if we are going to fail. */
int updateCount = handleOldFileTags(conn, oldList, doUpdate);
sqlSafef(query, sizeof(query),
"update edwSubmit set metaChangeCount=%d where id=%u", updateCount, submitId);
sqlUpdate(conn, query);
}
errCatchEnd(errCatch);
if (errCatch->gotError)
{
handleSubmitError(conn, submitId, errCatch->message->string);
/* The handleSubmitError will keep on throwing. */
}
errCatchFree(&errCatch);
/* Go through list attempting to load the files if we don't already have them. */
struct submitFileRow *sfr;
for (sfr = newList; sfr != NULL; sfr = sfr->next)
{
if (edwSubmitShouldStop(conn, submitId))
break;
struct edwFile *bf = sfr->file;
int submitUrlSize = strlen(submitDir) + strlen(bf->submitFileName) + 1;
char submitUrl[submitUrlSize];
safef(submitUrl, submitUrlSize, "%s%s", submitDir, bf->submitFileName);
if (edwGotFile(conn, submitDir, bf->submitFileName, bf->md5, bf->size)<0)
{
/* We can't get a ID for this file. There's two possible reasons -
* either somebody is in the middle of fetching it or nobody's started.
* If somebody is in the middle of fetching it, assume they died
* if they took more than an hour, and start up another fetch.
* So here we fetch unless somebody else is fetching recently. */
if (edwGettingFile(conn, submitDir, bf->submitFileName) < 0)
{
verbose(1, "Fetching %s\n", bf->submitFileName);
getSubmittedFile(conn, bf, submitDir, submitUrl, submitId);
newBytes += bf->size;
sqlSafef(query, sizeof(query),
"update edwSubmit set newFiles=newFiles+1,newBytes=%lld where id=%d",
newBytes, submitId);
sqlUpdate(conn, query);
}
}
else
{
verbose(2, "Already got %s\n", bf->submitFileName);
sqlSafef(query, sizeof(query), "update edwSubmit set oldFiles=oldFiles+1 where id=%d",
submitId);
sqlUpdate(conn, query);
}
if (sfr->replacesFile != 0)
{
/* What happens when the replacement doesn't validate? */
verbose(2, "Replacing %s with %s\n", sfr->replaces, bf->submitFileName);
sqlSafef(query, sizeof(query),
"update edwFile set replacedBy=%u, deprecated='%s' where id=%u",
bf->id, sfr->replaceReason, sfr->replacesFile);
sqlUpdate(conn, query);
}
}
/* If we made it here, update submit endUploadTime */
sqlSafef(query, sizeof(query),
"update edwSubmit set endUploadTime=%lld where id=%d",
edwNow(), submitId);
sqlUpdate(conn, query);
/* Get a real submission record and then set things up so mail user when all done. */
struct edwSubmit *submit = edwSubmitFromId(conn, submitId);
sqlDisconnect(&conn); // We'll be waiting a while so free connection
waitForValidationAndSendEmail(submit, email);
}
