static int compressbuf(lsi_param_t *rec, lsi_module_t *pModule,
int isSend)
{
const char *pSendingStr, *pCompressStr, *pZipStr, *pModuleStr;
zbufinfo_t *pBufInfo;
z_stream *pStream;
int ret;
int finish = Z_NO_FLUSH;
zmoddata_t *myData = (zmoddata_t *)g_api->get_module_data(
rec->session, pModule, LSI_DATA_HTTP);
if (myData == NULL)
return LSI_ERROR;
pSendingStr = isSend ? SENDING_STR : RECVING_STR;
pBufInfo = isSend ? myData->send : myData->recv;
if (pBufInfo == NULL)
return LSI_ERROR;
pStream = &pBufInfo->zstream;
pCompressStr = pBufInfo->compresslevel ? COMPRESS_STR : DECOMPRESS_STR;
pZipStr = pBufInfo->compresslevel ? ZIP_STR : UNZIP_STR;
pModuleStr = g_api->get_module_name(pModule);
if (pBufInfo->zstate == Z_UNINITED)
{
if (initstream(pStream, pBufInfo->compresslevel) == LS_FAIL)
{
g_api->log(rec->session, LSI_LOG_ERROR,
"[%s%s] initZstream init method [%s], failed.\n",
pModuleStr, pCompressStr, pZipStr);
ret = LSI_HKPT_RECV_RESP_BODY;
g_api->enable_hook(rec->session, pModule, 0,
&ret, 1); // Disable
return g_api->stream_write_next(rec, (const char *)rec->ptr1,
rec->len1);
}
g_api->log(rec->session, LSI_LOG_DEBUG,
"[%s%s] initZstream init method [%s], succeeded.\n",
pModuleStr, pCompressStr, pZipStr);
pBufInfo->zstate = Z_INITED;
}
if (pBufInfo->zstate < Z_EOF)
{
pStream->avail_in = rec->len1;
pStream->next_in = (Byte *)rec->ptr1;
if (rec->flag_in & LSI_CBFI_EOF)
{
pBufInfo->zstate = Z_EOF;
rec->flag_in |= LSI_CBFI_FLUSH;
}
else if (rec->flag_in & LSI_CBFI_FLUSH)
finish = Z_PARTIAL_FLUSH;
}
rec->flag_in &= ~LSI_CBFI_EOF;
if (pBufInfo->zstate == Z_EOF)
finish = Z_FINISH;
return doCompression(rec, pBufInfo, finish, pModuleStr, pSendingStr,
pCompressStr);
}
void SMTWorker::doWork()
{
assert(m_pCurFile);
assert(m_phFhSink);
uint32 buffSize = BLOCKSIZE;
bool endFile = false;
if ((m_pCurFile->getSize() - m_uiTotRead) < buffSize)
{
buffSize = (uint32)(((uint64)m_pCurFile->getSize()) - m_uiTotRead);
endFile = true;
}
if (buffSize == 0)
{
if (m_pCurFile->isCompressed())
{
//need to make sure we finish the compression off
doCompression(nullptr, 0, true);
}
else
{
finishTask();
m_pCT->endTask(m_uiId);
}
}
else
{
UTIL::MISC::Buffer buff(buffSize);
m_hFhSource.read(buff, buffSize);
m_uiTotRead += buffSize;
m_uiTotFileRead += buffSize;
m_pMD5Norm->update(buff, buffSize);
if (!m_pCurFile->isCompressed())
writeFile(buff, buffSize, endFile);
else
doCompression(buff, buffSize, endFile);
}
}
void docIdTidy(char *database, char *docIdDir)
/* docIdTidy - tidy up the docId library by compressing and md5suming where appropriate. */
{
char query[10 * 1024];
struct sqlResult *sr;
char **row;
struct sqlConnection *conn = sqlConnect(database);
struct tempName tn;
trashDirFile(&tn, "docId", "meta", ".txt");
char *tempFile = tn.forCgi;
struct toDoList *toDoList = NULL;
safef(query, sizeof query, "select * from %s", docIdTable);
sr = sqlGetResult(conn, query);
while ((row = sqlNextRow(sr)) != NULL)
{
struct docIdSub *docIdSub = docIdSubLoad(row);
cgiDecode(docIdSub->metaData, docIdSub->metaData, strlen(docIdSub->metaData));
FILE *f = mustOpen(tempFile, "w");
fwrite(docIdSub->metaData, strlen(docIdSub->metaData), 1, f);
fclose(f);
boolean validated;
//printf("metadata is %s\n", docIdSub->metaData);
struct mdbObj *mdbObj = mdbObjsLoadFromFormattedFile(tempFile, &validated);
unlink(tempFile);
char *docIdType = mdbObjFindValue(mdbObj, "type");
char buffer[10 * 1024];
safef(buffer, sizeof buffer, "%d", docIdSub->ix);
char *path = docIdGetPath(buffer, docIdDir, docIdType, NULL);
//docIdDecorate(docIdSub->ix));
printf("path %s\n", path);
struct toDoList *toDoItem = NULL;
if (!fileIsCompressed(path))
{
if (toDoItem == NULL) AllocVar(toDoItem);
printf("foo\n");
toDoItem->needs |= NEEDS_COMPRESSION;
toDoItem->path = path;
toDoItem->docId = docIdSub->ix;
}
printf("docId %d md5sum %s valReport %s\n",docIdSub->ix, docIdSub->md5sum, docIdSub->valReport);
//if (docIdSub->md5sum == NULL)
if (sameString(docIdSub->md5sum, ""))
{
printf("mdsum\n");
if (toDoItem == NULL) AllocVar(toDoItem);
toDoItem->needs |= NEEDS_MD5SUM;
toDoItem->path = path;
toDoItem->docId = docIdSub->ix;
}
//if (docIdSub->valReport == NULL)
if (sameString(docIdSub->valReport,""))
{
printf("report\n");
if (toDoItem == NULL) AllocVar(toDoItem);
toDoItem->needs |= NEEDS_REPORT;
toDoItem->path = path;
toDoItem->docId = docIdSub->ix;
}
if (toDoItem)
slAddHead(&toDoList, toDoItem);
}
sqlFreeResult(&sr);
doCompression(toDoList);
doReports(toDoList);
doMd5Summing(toDoList);
}
