static int DoStore(DPS_AGENT *Agent, urlid_t rec_id, Byte *Doc, size_t DocSize, char *Client) {
z_stream zstream;
DPS_BASE_PARAM P;
int rc = DPS_OK;
Byte *CDoc = NULL;
size_t dbnum = ((size_t)rec_id) % ((Agent->flags & DPS_FLAG_UNOCON) ? Agent->Conf->dbl.nitems : Agent->dbl.nitems);
DPS_DB *db = (Agent->flags & DPS_FLAG_UNOCON) ? &Agent->Conf->dbl.db[dbnum] : &Agent->dbl.db[dbnum];
zstream.zalloc = Z_NULL;
zstream.zfree = Z_NULL;
zstream.opaque = Z_NULL;
zstream.next_in = Doc;
if (deflateInit2(&zstream, 9, Z_DEFLATED, 15, 9, Z_DEFAULT_STRATEGY) == Z_OK) {
zstream.avail_in = DocSize;
zstream.avail_out = 2 * DocSize;
CDoc = zstream.next_out = (Byte *) DpsMalloc(2 * DocSize + 1);
if (zstream.next_out == NULL) {
return DPS_ERROR;
}
deflate(&zstream, Z_FINISH);
deflateEnd(&zstream);
/* store operations */
bzero(&P, sizeof(P));
P.subdir = "store";
P.basename = "doc";
P.indname = "doc";
P.rec_id = rec_id;
P.mode = DPS_WRITE_LOCK;
P.NFiles = (db->StoredFiles) ? db->StoredFiles : DpsVarListFindInt(&Agent->Vars, "StoredFiles", 0x100);
P.vardir = (db->vardir) ? db->vardir : DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR);
P.A = Agent;
if (DpsBaseWrite(&P, CDoc, zstream.total_out) != DPS_OK) {
DpsLog(Agent, DPS_LOG_ERROR, "store/doc write error: %s", strerror(errno));
rc = DPS_ERROR;
}
DpsBaseClose(&P);
if (rc == DPS_OK) DpsLog(Agent, DPS_LOG_EXTRA, "[%s] Stored rec_id: %x Size: %d Ratio: %5.2f%%", Client,
rec_id, DocSize, 100.0 * zstream.total_out / DocSize);
if (Agent->Flags.OptimizeAtUpdate) {
DpsBaseOptimize(&P, ((int)rec_id) >> DPS_BASE_BITS);
}
extern __C_LINK int __DPSCALL DpsBaseRelocate(DPS_AGENT *Agent, int base_type) {
DPS_BASE_PARAM O, N;
DPS_BASE_PARAM *Old = &O, *New = &N;
size_t base, i, ndel, mdel = 128, data_len;
urlid_t *todel = (int*)DpsMalloc(128 * sizeof(urlid_t));
void *data;
bzero(Old, sizeof(O));
bzero(New, sizeof(N));
switch(base_type) {
case 0: /* stored */
Old->subdir = "store";
Old->basename = "doc";
Old->indname = "doc";
Old->mode = DPS_WRITE_LOCK;
Old->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "OldStoredFiles", 0x100);
Old->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR);
Old->A = Agent;
New->subdir = "store";
New->basename = "doc";
New->indname = "doc";
New->mode = DPS_WRITE_LOCK;
New->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "StoredFiles", 0x100);
New->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR);
New->A = Agent;
DpsLog(Agent, DPS_LOG_INFO, "Relocating stored database");
break;
case 1: /* URL data */
Old->subdir = DPS_URLDIR;
Old->basename = "info";
Old->indname = "info";
Old->mode = DPS_WRITE_LOCK;
Old->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "OldURLDataFiles", 0x300);
Old->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR);
Old->A = Agent;
#ifdef HAVE_ZLIB
O.zlib_method = Z_DEFLATED;
O.zlib_level = 9;
O.zlib_windowBits = DPS_BASE_INFO_WINDOWBITS;
O.zlib_memLevel = 9;
O.zlib_strategy = DPS_BASE_INFO_STRATEGY;
#endif
New->subdir = DPS_URLDIR;
New->basename = "info";
New->indname = "info";
New->mode = DPS_WRITE_LOCK;
New->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "URLDataFiles", 0x300);
New->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR);
New->A = Agent;
#ifdef HAVE_ZLIB
N.zlib_method = Z_DEFLATED;
N.zlib_level = 9;
N.zlib_windowBits = DPS_BASE_INFO_WINDOWBITS;
N.zlib_memLevel = 9;
N.zlib_strategy = DPS_BASE_INFO_STRATEGY;
#endif
DpsLog(Agent, DPS_LOG_INFO, "Relocating URLData database");
break;
case 2: /* tree wrd */
Old->subdir = DPS_TREEDIR;
Old->basename = "wrd";
Old->indname = "wrd";
Old->mode = DPS_WRITE_LOCK;
Old->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "OldWrdFiles", 0x300);
Old->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR);
Old->A = Agent;
#ifdef HAVE_ZLIB
O.zlib_method = Z_DEFLATED;
O.zlib_level = 9;
O.zlib_windowBits = DPS_BASE_WRD_WINDOWBITS;
O.zlib_memLevel = 9;
O.zlib_strategy = DPS_BASE_WRD_STRATEGY;
#endif
New->subdir = DPS_TREEDIR;
New->basename = "wrd";
New->indname = "wrd";
New->mode = DPS_WRITE_LOCK;
New->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "WrdFiles", 0x300);
New->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR);
New->A = Agent;
#ifdef HAVE_ZLIB
N.zlib_method = Z_DEFLATED;
N.zlib_level = 9;
N.zlib_windowBits = DPS_BASE_WRD_WINDOWBITS;
N.zlib_memLevel = 9;
N.zlib_strategy = DPS_BASE_WRD_STRATEGY;
#endif
DpsLog(Agent, DPS_LOG_INFO, "Relocating Wrd database");
break;
default:
DPS_FREE(todel);
return DPS_OK;
}
for (base = 0; base < O.NFiles; base++) {
ndel = 0;
if (have_sigterm || have_sigint || have_sigalrm) {
DpsLog(Agent, DPS_LOG_EXTRA, "%s signal received. Exiting chackup", (have_sigterm) ? "SIGTERM" :
(have_sigint) ? "SIGINT" : "SIGALRM");
DpsBaseClose(Old);
DpsBaseClose(New);
DPS_FREE(todel);
return DPS_OK;
}
Old->rec_id = (urlid_t)(base << DPS_BASE_BITS);
if (DpsBaseOpen(Old, DPS_READ_LOCK) != DPS_OK) {
DpsBaseClose(Old);
DpsBaseClose(New);
continue;
}
if (lseek(O.Ifd, (off_t)0, SEEK_SET) == (off_t)-1) {
DpsLog(Agent, DPS_LOG_ERROR, "Can't seeek for file %s", Old->Ifilename);
DpsBaseClose(Old);
DpsBaseClose(New);
DPS_FREE(todel);
return DPS_ERROR;
}
while (read(Old->Ifd, &Old->Item, sizeof(DPS_BASEITEM)) == sizeof(DPS_BASEITEM)) {
if (Old->Item.rec_id != 0) {
if (ndel >= mdel) {
mdel += 128;
todel = (urlid_t*)DpsRealloc(todel, mdel * sizeof(urlid_t));
if (todel == NULL) {
DpsBaseClose(Old);
DpsBaseClose(New);
DpsLog(Agent, DPS_LOG_ERROR, "Can't realloc %d bytes %s:%d", mdel * sizeof(urlid_t), __FILE__, __LINE__);
DPS_FREE(todel);
return DPS_ERROR;
}
}
todel[ndel++] = Old->Item.rec_id;
}
}
DpsBaseClose(Old);
for (i = 0; i < ndel; i++) {
Old->rec_id = todel[i];
data = DpsBaseARead(Old, &data_len);
if (data == NULL) continue;
DpsBaseDelete(Old);
DpsBaseClose(Old);
New->rec_id = todel[i];
DpsBaseWrite(New, data, data_len);
DpsBaseClose(New);
DPS_FREE(data);
}
DpsLog(Agent, DPS_LOG_EXTRA, "\tbase: %d [0x%x], %d records relocated", base, base, ndel);
}
DPS_FREE(todel);
for (base = N.NFiles; base < O.NFiles; base++) {
Old->rec_id = (urlid_t)(base << DPS_BASE_BITS);
if (DpsBaseOpen(Old, DPS_READ_LOCK) != DPS_OK) {
DpsBaseClose(Old);
continue;
}
unlink(O.Ifilename);
unlink(O.Sfilename);
DpsBaseClose(Old);
}
return DPS_OK;
}
