__C_LINK int __DPSCALL DpsCacheMakeIndexes(DPS_AGENT *Indexer, DPS_DB *db) {
DPS_UINT8URLIDLIST L8;
DPS_UINT4URLIDLIST L4;
DPS_VARLIST *v = &Indexer->Conf->Vars;
size_t i, r;
char *ind, *nm, *lfname;
bzero(&L4, sizeof(DPS_UINT4URLIDLIST));
bzero(&L8, sizeof(DPS_UINT8URLIDLIST));
r = (size_t) 'l';
for (i = 0; i < v->Root[r].nvars; i++) {
if (!strncasecmp("Limit-", v->Root[r].Var[i].name, 6)) {
ind = v->Root[r].Var[i].val;
lfname = v->Root[r].Var[i].name;
nm = lfname + 6;
if (!strcasecmp(ind, "category")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Category index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating category index");
if (DPS_OK == DpsLimit8(Indexer, &L8, "Category", DPS_IFIELD_TYPE_HEX8STR, db)) {
MakeNestedIndex(Indexer, &L8, DPS_LIMFNAME_CAT, db);
}
} else if (!strcasecmp(ind, "tag")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Tag index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating tag index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "Tag", DPS_IFIELD_TYPE_STRCRC32, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_TAG, db);
}
} else if (!strcasecmp(ind, "link")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Link index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating link index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "link", DPS_IFIELD_TYPE_INT, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_LINK, db);
}
} else if (!strcasecmp(ind, "time")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Time index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating time index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "last_mod_time", DPS_IFIELD_TYPE_HOUR, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_TIME, db);
}
} else if (!strcasecmp(ind, "hostname")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Hostname index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating hostname index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "url", DPS_IFIELD_TYPE_HOSTNAME, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_HOST, db);
}
} else if (!strcasecmp(ind, "language")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Language index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating language index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "Content-Language", DPS_IFIELD_TYPE_STR2CRC32, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_LANG, db);
}
} else if (!strcasecmp(ind, "content")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Content-Type index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating Content-Type index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "Content-Type", DPS_IFIELD_TYPE_STRCRC32, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_CTYPE, db);
}
} else if (!strcasecmp(ind, "siteid")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Site_id index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating Site_id index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "site_id", DPS_IFIELD_TYPE_INT, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_SITE, db);
}
} else {
char *buf, *req, *dbaddr;
DPS_DB ldb, *pdb = &ldb;
size_t buf_len = dps_strlen(nm) + 16;
if ((buf = (char*) DpsMalloc(buf_len * sizeof(char))) == NULL) {
DpsLog(Indexer, DPS_LOG_ERROR, "Can't alloc %d chars at %s:%d", buf_len, __FILE__, __LINE__);
return DPS_ERROR;
}
dps_setproctitle("[%d] %s index creation", Indexer->handle, nm);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating %s index", nm);
dps_snprintf(buf, buf_len, "Req-%s", nm);
req = DpsVarListFindStr(&Indexer->Conf->Vars, buf, NULL);
if (req != NULL) {
dps_snprintf(buf, buf_len, "dbaddr-%s", nm);
dbaddr = DpsVarListFindStr(&Indexer->Conf->Vars, buf, NULL);
if (dbaddr != NULL) {
DpsDBSetAddr(pdb, dbaddr, DPS_OPEN_MODE_READ);
} else {
pdb = db;
}
if (!strcasecmp(ind, "nex8str")) {
if (DPS_OK == DpsSQLLimit8(Indexer, &L8, req, DPS_IFIELD_TYPE_HEX8STR, pdb)) {
MakeNestedIndex(Indexer, &L8, lfname, pdb);
}
} else {
int field_type = DPS_IFIELD_TYPE_INT;
if (!strcasecmp(ind, "strcrc32")) field_type = DPS_IFIELD_TYPE_STRCRC32;
else if (!strcasecmp(ind, "hour")) field_type = DPS_IFIELD_TYPE_HOUR;
else if (!strcasecmp(ind, "hostname")) field_type = DPS_IFIELD_TYPE_HOSTNAME;
else if (!strcasecmp(ind, "char2")) field_type = DPS_IFIELD_TYPE_STR2CRC32;
else if (!strcasecmp(ind, "int")) field_type = DPS_IFIELD_TYPE_INT;
if (DPS_OK == DpsSQLLimit4(Indexer, &L4, req, field_type, db)) {
MakeLinearIndex(Indexer, &L4, lfname, db);
}
}
}
}
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Indexes done.", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Done");
}
}
return DPS_OK;
}
static int MakeNestedIndex(DPS_AGENT *Indexer, const char *field, const char *lim_name, int type, DPS_DB *db) {
DPS_ENV *Conf = Indexer->Conf;
DPS_UINT8URLIDLIST L;
size_t k,prev;
urlid_t *data=NULL;
DPS_UINT8_POS_LEN *ind=NULL;
size_t mind=1000,nind=0;
char fname[PATH_MAX];
int dat_fd=0, ind_fd=0;
int rc=DPS_OK;
const char *vardir = (db->vardir) ? db->vardir : DpsVarListFindStr(&Conf->Vars, "VarDir", DPS_VAR_DIR);
bzero(&L, sizeof(L));
DpsLog(Indexer, DPS_LOG_EXTRA, "MakeNested Index %s, %d records, Item:%x", lim_name, L.nitems, L.Item);
rc = DpsLimit8(Indexer, &L, field, type, db);
DpsLog(Indexer, DPS_LOG_EXTRA, "-MakeNested Index %s [rc:%d], %d records, Item:%x", lim_name, rc, L.nitems, L.Item);
if (rc != DPS_OK) return 1;
if (L.nitems > 1) DpsSort(L.Item, L.nitems, sizeof(DPS_UINT8URLID), (qsort_cmp)cmp_ind8);
data = (urlid_t*)DpsMalloc((L.nitems + 1) * sizeof(urlid_t));
if(!data) {
DpsLog(Indexer, DPS_LOG_ERROR, "Can't alloc %d bytes [%s:%d]", (L.nitems + 1) * sizeof(urlid_t), __FILE__, __LINE__);
goto err1;
}
ind=(DPS_UINT8_POS_LEN*)DpsMalloc(mind*sizeof(DPS_UINT8_POS_LEN));
if(!ind) {
DpsLog(Indexer, DPS_LOG_ERROR, "Can't alloc %d bytes [%s:%d]", mind * sizeof(DPS_UINT8_POS_LEN), __FILE__, __LINE__);
goto err1;
}
prev=0;
for(k=0; k<L.nitems; k++) {
data[k]=L.Item[k].url_id;
if((k==L.nitems-1) || (L.Item[k].hi!=L.Item[prev].hi) || (L.Item[k].lo!=L.Item[prev].lo)) {
if(nind==mind) {
mind+=1000;
ind=(DPS_UINT8_POS_LEN*)DpsRealloc(ind,mind*sizeof(DPS_UINT8_POS_LEN));
if(!ind) {
DpsLog(Indexer, DPS_LOG_ERROR, "Can't alloc %d bytes [%s:%d]", mind * sizeof(DPS_UINT8_POS_LEN), __FILE__, __LINE__);
goto err1;
}
}
/* Fill index */
ind[nind].hi=L.Item[prev].hi;
ind[nind].lo=L.Item[prev].lo;
ind[nind].pos = prev * sizeof(*data);
if (k == L.nitems - 1) ind[nind].len = (k - prev + 1) * sizeof(*data);
else ind[nind].len = (k - prev) * sizeof(*data);
DpsLog(Indexer, DPS_LOG_DEBUG, "%08X%08X - %d %d\n", ind[nind].hi, ind[nind].lo, (int)ind[nind].pos, ind[nind].len);
nind++;
prev=k;
}
}
DPS_FREE(L.Item);
dps_snprintf(fname,sizeof(fname)-1,"%s%c%s%c%s.dat", vardir,DPSSLASH, DPS_TREEDIR,DPSSLASH, lim_name);
if((dat_fd = DpsOpen3(fname, O_CREAT | O_WRONLY | O_TRUNC | DPS_BINARY, DPS_IWRITE)) < 0) {
DpsLog(Indexer, DPS_LOG_ERROR, "Can't open '%s': %s [%s:%d]", fname, strerror(errno), __FILE__, __LINE__);
goto err1;
}
DpsWriteLock(dat_fd);
if((L.nitems * sizeof(*data)) != (size_t)write(dat_fd, data, L.nitems * sizeof(*data))) {
DpsLog(Indexer, DPS_LOG_ERROR, "Can't write '%s': %s [%s:%d]", fname, strerror(errno), __FILE__, __LINE__);
goto err1;
}
DpsUnLock(dat_fd);
DpsClose(dat_fd);
DPS_FREE(data);
dps_snprintf(fname,sizeof(fname)-1,"%s%c%s%c%s.ind", vardir, DPSSLASH,DPS_TREEDIR, DPSSLASH, lim_name);
if((ind_fd = DpsOpen3(fname, O_CREAT | O_WRONLY | O_TRUNC | DPS_BINARY, DPS_IWRITE)) < 0) {
DpsLog(Indexer, DPS_LOG_ERROR, "Can't open '%s': %s [%s:%d]", fname, strerror(errno), __FILE__, __LINE__);
goto err1;
}
DpsWriteLock(ind_fd);
if((nind*sizeof(DPS_UINT8_POS_LEN)) != (size_t)write(ind_fd,ind,nind*sizeof(DPS_UINT8_POS_LEN))) {
DpsLog(Indexer, DPS_LOG_ERROR, "Can't write '%s': %s [%s:%d]", fname, strerror(errno), __FILE__, __LINE__);
goto err1;
}
DpsUnLock(ind_fd);
DpsClose(ind_fd);
DPS_FREE(ind);
return(0);
err1:
DPS_FREE(L.Item);
DPS_FREE(data);
DPS_FREE(ind);
if(dat_fd) DpsClose(dat_fd);
if(ind_fd) DpsClose(ind_fd);
return(1);
}