static int DoStore(DPS_AGENT *Agent, urlid_t rec_id, Byte *Doc, size_t DocSize, char *Client) { z_stream zstream; DPS_BASE_PARAM P; int rc = DPS_OK; Byte *CDoc = NULL; size_t dbnum = ((size_t)rec_id) % ((Agent->flags & DPS_FLAG_UNOCON) ? Agent->Conf->dbl.nitems : Agent->dbl.nitems); DPS_DB *db = (Agent->flags & DPS_FLAG_UNOCON) ? &Agent->Conf->dbl.db[dbnum] : &Agent->dbl.db[dbnum]; zstream.zalloc = Z_NULL; zstream.zfree = Z_NULL; zstream.opaque = Z_NULL; zstream.next_in = Doc; if (deflateInit2(&zstream, 9, Z_DEFLATED, 15, 9, Z_DEFAULT_STRATEGY) == Z_OK) { zstream.avail_in = DocSize; zstream.avail_out = 2 * DocSize; CDoc = zstream.next_out = (Byte *) DpsMalloc(2 * DocSize + 1); if (zstream.next_out == NULL) { return DPS_ERROR; } deflate(&zstream, Z_FINISH); deflateEnd(&zstream); /* store operations */ bzero(&P, sizeof(P)); P.subdir = "store"; P.basename = "doc"; P.indname = "doc"; P.rec_id = rec_id; P.mode = DPS_WRITE_LOCK; P.NFiles = (db->StoredFiles) ? db->StoredFiles : DpsVarListFindInt(&Agent->Vars, "StoredFiles", 0x100); P.vardir = (db->vardir) ? db->vardir : DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); P.A = Agent; if (DpsBaseWrite(&P, CDoc, zstream.total_out) != DPS_OK) { DpsLog(Agent, DPS_LOG_ERROR, "store/doc write error: %s", strerror(errno)); rc = DPS_ERROR; } DpsBaseClose(&P); if (rc == DPS_OK) DpsLog(Agent, DPS_LOG_EXTRA, "[%s] Stored rec_id: %x Size: %d Ratio: %5.2f%%", Client, rec_id, DocSize, 100.0 * zstream.total_out / DocSize); if (Agent->Flags.OptimizeAtUpdate) { DpsBaseOptimize(&P, ((int)rec_id) >> DPS_BASE_BITS); }
extern __C_LINK int __DPSCALL DpsBaseRelocate(DPS_AGENT *Agent, int base_type) { DPS_BASE_PARAM O, N; DPS_BASE_PARAM *Old = &O, *New = &N; size_t base, i, ndel, mdel = 128, data_len; urlid_t *todel = (int*)DpsMalloc(128 * sizeof(urlid_t)); void *data; bzero(Old, sizeof(O)); bzero(New, sizeof(N)); switch(base_type) { case 0: /* stored */ Old->subdir = "store"; Old->basename = "doc"; Old->indname = "doc"; Old->mode = DPS_WRITE_LOCK; Old->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "OldStoredFiles", 0x100); Old->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); Old->A = Agent; New->subdir = "store"; New->basename = "doc"; New->indname = "doc"; New->mode = DPS_WRITE_LOCK; New->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "StoredFiles", 0x100); New->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); New->A = Agent; DpsLog(Agent, DPS_LOG_INFO, "Relocating stored database"); break; case 1: /* URL data */ Old->subdir = DPS_URLDIR; Old->basename = "info"; Old->indname = "info"; Old->mode = DPS_WRITE_LOCK; Old->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "OldURLDataFiles", 0x300); Old->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); Old->A = Agent; #ifdef HAVE_ZLIB O.zlib_method = Z_DEFLATED; O.zlib_level = 9; O.zlib_windowBits = DPS_BASE_INFO_WINDOWBITS; O.zlib_memLevel = 9; O.zlib_strategy = DPS_BASE_INFO_STRATEGY; #endif New->subdir = DPS_URLDIR; New->basename = "info"; New->indname = "info"; New->mode = DPS_WRITE_LOCK; New->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "URLDataFiles", 0x300); New->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); New->A = Agent; #ifdef HAVE_ZLIB N.zlib_method = Z_DEFLATED; N.zlib_level = 9; N.zlib_windowBits = DPS_BASE_INFO_WINDOWBITS; N.zlib_memLevel = 9; N.zlib_strategy = DPS_BASE_INFO_STRATEGY; #endif DpsLog(Agent, DPS_LOG_INFO, "Relocating URLData database"); break; case 2: /* tree wrd */ Old->subdir = DPS_TREEDIR; Old->basename = "wrd"; Old->indname = "wrd"; Old->mode = DPS_WRITE_LOCK; Old->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "OldWrdFiles", 0x300); Old->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); Old->A = Agent; #ifdef HAVE_ZLIB O.zlib_method = Z_DEFLATED; O.zlib_level = 9; O.zlib_windowBits = DPS_BASE_WRD_WINDOWBITS; O.zlib_memLevel = 9; O.zlib_strategy = DPS_BASE_WRD_STRATEGY; #endif New->subdir = DPS_TREEDIR; New->basename = "wrd"; New->indname = "wrd"; New->mode = DPS_WRITE_LOCK; New->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "WrdFiles", 0x300); New->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); New->A = Agent; #ifdef HAVE_ZLIB N.zlib_method = Z_DEFLATED; N.zlib_level = 9; N.zlib_windowBits = DPS_BASE_WRD_WINDOWBITS; N.zlib_memLevel = 9; N.zlib_strategy = DPS_BASE_WRD_STRATEGY; #endif DpsLog(Agent, DPS_LOG_INFO, "Relocating Wrd database"); break; default: DPS_FREE(todel); return DPS_OK; } for (base = 0; base < O.NFiles; base++) { ndel = 0; if (have_sigterm || have_sigint || have_sigalrm) { DpsLog(Agent, DPS_LOG_EXTRA, "%s signal received. Exiting chackup", (have_sigterm) ? "SIGTERM" : (have_sigint) ? "SIGINT" : "SIGALRM"); DpsBaseClose(Old); DpsBaseClose(New); DPS_FREE(todel); return DPS_OK; } Old->rec_id = (urlid_t)(base << DPS_BASE_BITS); if (DpsBaseOpen(Old, DPS_READ_LOCK) != DPS_OK) { DpsBaseClose(Old); DpsBaseClose(New); continue; } if (lseek(O.Ifd, (off_t)0, SEEK_SET) == (off_t)-1) { DpsLog(Agent, DPS_LOG_ERROR, "Can't seeek for file %s", Old->Ifilename); DpsBaseClose(Old); DpsBaseClose(New); DPS_FREE(todel); return DPS_ERROR; } while (read(Old->Ifd, &Old->Item, sizeof(DPS_BASEITEM)) == sizeof(DPS_BASEITEM)) { if (Old->Item.rec_id != 0) { if (ndel >= mdel) { mdel += 128; todel = (urlid_t*)DpsRealloc(todel, mdel * sizeof(urlid_t)); if (todel == NULL) { DpsBaseClose(Old); DpsBaseClose(New); DpsLog(Agent, DPS_LOG_ERROR, "Can't realloc %d bytes %s:%d", mdel * sizeof(urlid_t), __FILE__, __LINE__); DPS_FREE(todel); return DPS_ERROR; } } todel[ndel++] = Old->Item.rec_id; } } DpsBaseClose(Old); for (i = 0; i < ndel; i++) { Old->rec_id = todel[i]; data = DpsBaseARead(Old, &data_len); if (data == NULL) continue; DpsBaseDelete(Old); DpsBaseClose(Old); New->rec_id = todel[i]; DpsBaseWrite(New, data, data_len); DpsBaseClose(New); DPS_FREE(data); } DpsLog(Agent, DPS_LOG_EXTRA, "\tbase: %d [0x%x], %d records relocated", base, base, ndel); } DPS_FREE(todel); for (base = N.NFiles; base < O.NFiles; base++) { Old->rec_id = (urlid_t)(base << DPS_BASE_BITS); if (DpsBaseOpen(Old, DPS_READ_LOCK) != DPS_OK) { DpsBaseClose(Old); continue; } unlink(O.Ifilename); unlink(O.Sfilename); DpsBaseClose(Old); } return DPS_OK; }