static void h1n1DownloadPdb(char *item, char *pdbUrl, struct tempName *tmpPdb)
/* uncompress PDB to trash */
{
int inFd = netOpenHttpExt(pdbUrl, "GET", NULL);
int inFdRedir = 0;
char *pdbUrlRedir = NULL;
if (!netSkipHttpHeaderLinesHandlingRedirect(inFd, pdbUrl, &inFdRedir, &pdbUrlRedir))
errAbort("Unable to access predicted 3D structure file: %s", pdbUrl);
if (pdbUrlRedir != NULL)
{
close(inFd);
inFd = inFdRedir;
freez(&pdbUrlRedir);
}
trashDirFile(tmpPdb, "hgct", item, ".pdb");
FILE *outFh = mustOpen(tmpPdb->forCgi, "w");
struct lineFile *inLf = lineFileDecompressFd(pdbUrl, TRUE, inFd);
char *line;
while (lineFileNext(inLf, &line, NULL))
{
fputs(line, outFh);
fputc('\n', outFh);
}
lineFileClose(&inLf);
carefulClose(&outFh);
}
struct htmlPage *htmlPageFromForm(struct htmlPage *origPage, struct htmlForm *form,
char *buttonName, char *buttonVal)
/* Return a new htmlPage based on response to pressing indicated button
* on indicated form in origPage. */
{
struct htmlPage *newPage = NULL;
struct dyString *dyUrl = dyStringNew(0);
struct dyString *dyHeader = dyStringNew(0);
struct dyString *dyText = NULL;
char *url = htmlExpandUrl(origPage->url, form->action);
char *cgiVars = NULL;
int contentLength = 0;
int sd = -1;
dyStringAppend(dyUrl, url);
cookieOutput(dyHeader, origPage->cookies);
if (sameWord(form->method, "GET"))
{
cgiVars = htmlFormCgiVars(origPage, form, buttonName, buttonVal, dyHeader);
dyStringAppend(dyUrl, "?");
dyStringAppend(dyUrl, cgiVars);
verbose(3, "GET %s\n", dyUrl->string);
sd = netOpenHttpExt(dyUrl->string, form->method, dyHeader->string);
}
else if (sameWord(form->method, "POST"))
{
cgiVars = htmlFormCgiVars(origPage, form, buttonName, buttonVal, dyHeader);
contentLength = strlen(cgiVars);
verbose(3, "POST %s\n", dyUrl->string);
dyStringPrintf(dyHeader, "Content-length: %d\r\n", contentLength);
sd = netOpenHttpExt(dyUrl->string, form->method, dyHeader->string);
mustWriteFd(sd, cgiVars, contentLength);
}
dyText = netSlurpFile(sd);
close(sd);
newPage = htmlPageParse(url, dyStringCannibalize(&dyText));
freez(&url);
dyStringFree(&dyUrl);
dyStringFree(&dyHeader);
freez(&cgiVars);
return newPage;
}
char *gsUploadUrl(char *gsToken, char *user, char *uploadFileName, off_t contentLength, char *base64Md5, char *contentType)
/* call uploadurl */
{
// UPLOADURLS
// TODO deal with creating parent dirs if uploadFileName contains a path? maybe not.
// old: "https://identity.genomespace.org/datamanager/uploadurls/users/"
// old "https://dm.genomespace.org/datamanager/v1.0/uploadurl/users/" // if this works, use default dir fetched earlier instead
char *dmSvr = getGenomeSpaceConfig("dmServer");
char uploadUrl[1024];
safef(uploadUrl, sizeof(uploadUrl),
"%s/v1.0/uploadurl/users/"
"%s/"
"%s"
"?Content-Length=%lld"
"&Content-MD5=%s"
"&Content-Type=%s"
, dmSvr
, user
, uploadFileName
, (long long) contentLength
, cgiEncode(base64Md5)
, contentType
);
struct dyString *reqExtra = newDyString(256);
dyStringPrintf(reqExtra, "Cookie: gs-token=%s\r\n", gsToken);
int sd = netOpenHttpExt(uploadUrl, "GET", reqExtra->string);
if (sd < 0)
errAbort("failed to open socket for [%s]", uploadUrl);
char *responseCode = NULL;
char *s3UploadUrl = parseResponse(sd, &responseCode);
if (sameString(responseCode, "404 Not Found"))
errAbort("GenomeSpace: %s, if a path was used in the output name, it may indicate the path does not exist in GenomeSpace.", responseCode);
if (!sameString(responseCode, "200 OK"))
errAbort("GenomeSpace: %s", responseCode);
dyStringFree(&reqExtra);
return s3UploadUrl;
}
static char *getGsPersonalDirectory(char *gsToken)
/* Get User's default directory from GenomeSpace DM
* Returns a url like [https://identity.genomespace.org/datamanager/files/users/<user>]
*/
{
// DEFAULT DIRECTORY
// old1 char *defaultDirectoryUrl = "https://identity.genomespace.org/datamanager/defaultdirectory";
// old2 char *defaultDirectoryUrl = "https://dmtest.genomespace.org:8444/datamanager/defaultdirectory";
// old3 char *defaultDirectoryUrl = "https://dm.genomespace.org/datamanager/v1.0/defaultdirectory";
// NOTE the defaultdirectory method got renamed to personaldirectory
// old4 char *personalDirectoryUrl = "https://dm.genomespace.org/datamanager/v1.0/personaldirectory";
char *dmSvr = getGenomeSpaceConfig("dmServer");
char personalDirectoryUrl[1024];
safef(personalDirectoryUrl, sizeof personalDirectoryUrl, "%s/v1.0/personaldirectory", dmSvr);
struct dyString *reqExtra = newDyString(256);
dyStringPrintf(reqExtra, "Cookie: gs-token=%s\r\n", gsToken);
int sd = netOpenHttpExt(personalDirectoryUrl, "GET", reqExtra->string);
if (sd < 0)
errAbort("failed to open socket for [%s]", personalDirectoryUrl);
struct dyString *dy = netSlurpFile(sd);
close(sd);
char *personalDirectory = NULL;
if (strstr(dy->string, "HTTP/1.1 303 See Other"))
{
char *valStart = strstr(dy->string, "Location: ");
if (valStart)
{
valStart += strlen("Location: ");
char *valEnd = strstr(valStart, "\r\n");
if (!valEnd)
errAbort("location not found in response headers");
personalDirectory = cloneStringZ(valStart, valEnd - valStart);
}
}
dyStringFree(&dy);
dyStringFree(&reqExtra);
return personalDirectory;
}
char *htmlSlurpWithCookies(char *url, struct htmlCookie *cookies)
/* Send get message to url with cookies, and return full response as
* a dyString. This is not parsed or validated, and includes http
* header lines. Typically you'd pass this to htmlPageParse() to
* get an actual page. */
{
struct dyString *dyHeader = dyStringNew(0);
struct dyString *dyText;
int sd;
cookieOutput(dyHeader, cookies);
sd = netOpenHttpExt(url, "GET", dyHeader->string);
dyText = netSlurpFile(sd);
close(sd);
dyStringFree(&dyHeader);
return dyStringCannibalize(&dyText);
}
void syncOneRecord(struct sqlConnection *conn, char *type, struct jsonWrite *json,
char *table, long long id)
/* Send over one record and save UUID result to row of table defined by id in idField. */
{
/* Construct dyString for URL */
struct dyString *dyUrl = dyStringNew(0);
dyStringPrintf(dyUrl, "http://%s:%s@%s/%s/", gUserId, gPassword, gHost, type);
verbose(2, "%s\n", dyUrl->string);
/* Construct dyString for http header */
struct dyString *dyHeader = dyStringNew(0);
dyStringPrintf(dyHeader, "Content-length: %d\r\n", json->dy->stringSize);
dyStringPrintf(dyHeader, "Content-type: text/javascript\r\n");
/* Send header and then JSON */
int sd = netOpenHttpExt(dyUrl->string, "POST", dyHeader->string);
mustWriteFd(sd, json->dy->string, json->dy->stringSize);
/* Grab response */
struct dyString *dyText = netSlurpFile(sd);
close(sd);
uglyf("%s\n", dyText->string);
/* Turn it into htmlPage structure - this will also parse out http header */
struct htmlPage *response = htmlPageParse(dyUrl->string, dyText->string);
uglyf("status %s %d\n", response->status->version, response->status->status);
/* If we got bad status abort with hopefully very informative message. */
int status = response->status->status;
if (status != 200 && status != 201) // HTTP codes
{
errAbort("ERROR - Metadatabase returns %d to our post request\n"
"POSTED JSON: %s\n"
"URL: %s\n"
"FULL RESPONSE: %s\n", status, json->dy->string, dyUrl->string, dyText->string);
}
/* Parse uuid out of json response. It should look something like
{
"status": "success",
"@graph": [
{
"description": "The macs2 peak calling software from Tao Liu.",
"name": "macs2", "title": "macs2", "url": "https://github.com/taoliu/MACS/",
"uuid": "9bda84fd-9872-49e3-9aa0-b71adbf9f31d",
"schema_version": "1",
"source_url": "https://github.com/taoliu/MACS/",
"references": [],
"@id": "/software/9bda84fd-9872-49e3-9aa0-b71adbf9f31d/",
"@type": ["software", "item"],
"aliases": []
}
],
"@type": ["result"]
}
*/
struct jsonElement *jsonRoot = jsonParse(response->htmlText);
struct jsonElement *graph = jsonMustFindNamedField(jsonRoot, "", "@graph");
struct slRef *ref = jsonListVal(graph, "@graph");
assert(slCount(ref) == 1);
struct jsonElement *graphEl = ref->val;
char *uuid = jsonStringField(graphEl, "uuid");
uglyf("Got uuid %s\n", uuid);
/* Save uuid to table */
char query[256];
sqlSafef(query, sizeof(query),
"update %s set metaUuid='%s' where id=%lld", table, uuid, id);
sqlUpdate(conn, query);
/* Clean up */
dyStringFree(&dyUrl);
dyStringFree(&dyHeader);
dyStringFree(&dyText);
response->fullText = NULL; // avoid double free of this
htmlPageFree(&response);
}
char *gsS3Upload(char *s3UploadUrl, char *inputFileName, off_t contentLength, char *base64Md5, char *hexMd5, char *contentType, boolean progress, char *fileName)
/* call s3 upload */
{
// S3 UPLOAD to Amazon Storage
struct dyString *reqExtra = newDyString(256);
dyStringPrintf(reqExtra, "Content-Length: %lld\r\n", (long long)contentLength);
dyStringPrintf(reqExtra, "Content-MD5: %s\r\n", base64Md5);
dyStringPrintf(reqExtra, "Content-Type: %s\r\n", contentType);
int sd = netOpenHttpExt(s3UploadUrl, "PUT", reqExtra->string);
if (sd < 0)
errAbort("failed to open socket for [%s]", s3UploadUrl);
unsigned char buffer[S3UPBUFSIZE];
int bufRead = 0;
FILE *f = mustOpen(inputFileName,"rb");
off_t totalUploaded = 0;
int lastPctUploaded = -1;
// upload the file contents
while ((bufRead = fread(&buffer, 1, S3UPBUFSIZE, f)) > 0)
{
int bufWrite = 0;
while (bufWrite < bufRead)
{
int socketWrite = write(sd, buffer + bufWrite, bufRead - bufWrite);
if (socketWrite == -1)
{
if (errno == 32) // broken pipe often happens when the ssh connection shuts down or has errors.
{
warn("broken pipe, S3 server closed the ssh connection.");
break;
}
errnoAbort("error writing to socket for GenomeSpace upload");
}
bufWrite += socketWrite;
}
if (errno == 32)
break;
totalUploaded += bufRead;
int pctUploaded = 100.0*totalUploaded/contentLength;
if (progress && (pctUploaded != lastPctUploaded))
{
char nicenumber[1024]="";
sprintWithGreekByte(nicenumber, sizeof(nicenumber), contentLength);
// Various global flags must be reset to draw a fresh html output page.
webHeadAlreadyOutputed = FALSE;
webInTextMode = FALSE;
includedResourceFiles = NULL;
htmlWarnBoxSetUpAlready=FALSE;
htmlOpen("Uploading Output to GenomeSpace");
printf("Name: %s<br>\n", fileName);
printf("Size: %s<br>\n", nicenumber);
printf("Progress: %0d%%<br>\n", pctUploaded);
printf("<br>\n");
printf("<FORM ACTION=\"/cgi-bin/hgTables\" METHOD=GET>\n"
"<INPUT TYPE=SUBMIT NAME=\"%s\" VALUE=\"Back\" >"
"<INPUT TYPE=SUBMIT NAME=\"Refresh\" VALUE=\"Refresh\" onclick='window.location=window.location;return false;' >"
"</FORM>\n"
, hgtaDoMainPage);
puts("<script type=\"text/javascript\">");
puts("<!--");
puts("setTimeout(\"location = location;\",5000);");
puts("-->");
puts("</script>");
htmlClose();
fflush(stdout);
lastPctUploaded = pctUploaded;
}
}
carefulClose(&f);
char *responseCode = NULL;
char *s3UploadResponse = parseResponse(sd, &responseCode);
if (!sameString(responseCode, "200 OK"))
errAbort("Amazon S3 Response: %s", responseCode);
dyStringFree(&reqExtra);
return s3UploadResponse;
}
