void eHTTPStream::haveData(void *vdata, int len)
{
__u8 *data=(__u8*)vdata;
while (len)
{
int valid=len;
if (!metadataleft)
{
// not in metadata mode.. process mp3 data (stream to input buffer)
// are we just at the beginning of metadata? (pointer)
if (metadatainterval && (metadatainterval == bytes))
{
// enable metadata mode
metadataleft=*data++*16;
metadatapointer=0;
len--;
bytes=0;
continue;
} else if (metadatainterval && (metadatainterval < bytes))
eFatal("metadatainterval < bytes");
// otherwise there's really data.
if (metadatainterval)
{
// is metadata in our buffer?
if ((valid + bytes) > metadatainterval)
valid=metadatainterval-bytes;
}
buffer.write(data, valid);
data+=valid;
len-=valid;
bytes+=valid;
} else
{
// metadata ... process it.
int meta=len;
if (meta > metadataleft)
meta=metadataleft;
memcpy(metadata+metadatapointer, data, meta);
metadatapointer+=meta;
data+=meta;
len-=meta;
metadataleft-=meta;
if (!metadataleft)
processMetaData();
}
}
dataAvailable();
}
static void getMetaData(struct sqlConnection *conn, struct refSeqVerInfo *refSeqVerInfoLst, char *outFile)
/* get request prot sequences from database */
{
struct sqlConnection *conn2 = sqlConnect(sqlGetDatabase(conn));
static char *hdr = "#mrnaAcc\t" "protAcc\t" "geneName\t" "ncbiGeneId\t" "cds\t" "product\t" "status\n";
FILE *fh = mustOpen(outFile, "w");
fputs(hdr, fh);
struct refSeqVerInfo *rsvi;
for (rsvi = refSeqVerInfoLst; rsvi != NULL; rsvi = rsvi->next)
processMetaData(fh, conn, conn2, rsvi);
carefulClose(&fh);
sqlDisconnect(&conn2);
}
static void getMetaData(struct sqlConnection *conn, struct hash *refSeqVerInfoTbl, char *outFile)
/* get request prot sequences from database */
{
struct sqlConnection *conn2 = sqlConnect(sqlGetDatabase(conn));
static char *hdr = "#mrnaAcc\t" "protAcc\t" "geneName\t" "ncbiGeneId\t" "cds\t" "product\n";
FILE *fh = mustOpen(outFile, "w");
fputs(hdr, fh);
struct hashCookie cookie = hashFirst(refSeqVerInfoTbl);
struct hashEl *hel;
while ((hel = hashNext(&cookie)) != NULL)
{
processMetaData(fh, conn, conn2, hel->val);
}
carefulClose(&fh);
sqlDisconnect(&conn2);
}
void databaseUpdate(struct gbSelect* select)
/* update the database from genbank state on disk */
{
struct sqlConnection *conn = hAllocConn(gDatabase);
struct gbStatusTbl* statusTbl;
boolean maxShrinkageExceeded;
char typePrefix[32], tmpDir[PATH_LEN];
gbVerbEnter(3, "update %s", gbSelectDesc(select));
/* Setup tmp dir for load, must be unique for each update due to
* initialLoad feature */
if (select->accPrefix != NULL)
safef(typePrefix, sizeof(typePrefix), "%s.%s", gbFmtSelect(select->type),
select->accPrefix);
else
safef(typePrefix, sizeof(typePrefix), "%s", gbFmtSelect(select->type));
safef(tmpDir, sizeof(tmpDir), "%s/%s/%s/%s",
gWorkDir, select->release->name, select->release->genome->database,
typePrefix);
if (!(gOptions.flags & DBLOAD_DRY_RUN))
gbMakeDirs(tmpDir);
/* Build list of entries that need processed. This also flags updates that
* have the change and new entries so we can limit the per-update processing.
*/
statusTbl = gbBuildState(conn, select, &gOptions, gMaxShrinkage, tmpDir,
gbVerbose, FALSE, &maxShrinkageExceeded);
if (maxShrinkageExceeded)
{
fprintf(stderr, "Warning: switching to dryRun mode due to maxShrinkage being exceeded\n");
gMaxShrinkageError = TRUE;
gOptions.flags |= DBLOAD_DRY_RUN;
}
if (gOptions.flags & DBLOAD_DRY_RUN)
{
gbVerbLeave(3, "dry run, skipping update %s", gbSelectDesc(select));
gbStatusTblFree(&statusTbl);
hFreeConn(&conn);
return;
}
checkForStop(); /* last safe place */
/* count global number of extFileChgs */
gExtFileChged += statusTbl->numExtChg;
/* first clean out old and changed */
deleteOutdated(conn, select, statusTbl, tmpDir);
/* meta data MUST be done first, it sets some gbStatus data */
processMetaData(conn, select, statusTbl, tmpDir);
processAligns(conn, select, statusTbl, tmpDir);
/* now it's safe to update the status table, delay commit for initialLoad */
if (gOptions.flags & DBLOAD_INITIAL)
slSafeAddHead(&gPendingStatusUpdates,
gbStatusTblUpdate(statusTbl, conn, FALSE));
else
gbStatusTblUpdate(statusTbl, conn, TRUE);
/* add this and partition to the loaded table, if not already there.
* set the extFile updated flag updates were done or this is the initial load */
updateLoadedTbl(select);
if (gOptions.flags & DBLOAD_INITIAL)
gbLoadedTblSetExtFileUpdated(gLoadedTbl, select);
if ((gOptions.flags & DBLOAD_INITIAL) == 0)
gbLoadedTblCommit(gLoadedTbl);
/* print before freeing memory */
gbVerbLeave(3, "update %s", gbSelectDesc(select));
gbStatusTblFree(&statusTbl);
hFreeConn(&conn);
}
