void
ds_diction_destroy (ds_diction_t diction)
{
ds_term_t term, next;
ds_cursor_t cur;
if (!diction) return;
cur = ds_diction_cursor(diction);
if (!cur) {
perror("ds_diction_destroy: ds_diction_cursor() failed");
return;
}
term = ds_diction_next(cur);
while(term)
{
next = ds_diction_next(cur);
ds_diction_delete(diction, term->key);
term = next;
}
ds_diction_close(cur);
nt_destroy(diction->order);
nt_destroy(diction->chained_order);
free(diction->tbl);
free(diction);
return;
}
int
_ds_getall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction)
{
ds_term_t ds_term;
ds_cursor_t ds_c;
struct _ds_spam_stat stat;
struct _ds_spam_stat *p_stat = &stat;
int ret = 0, x = 0;
if (diction == NULL || CTX == NULL)
return EINVAL;
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
while(ds_term)
{
ds_term->s.spam_hits = 0;
ds_term->s.innocent_hits = 0;
ds_term->s.offset = 0;
x = _ds_get_spamrecord (CTX, ds_term->key, p_stat);
if (!x)
ds_diction_setstat(diction, ds_term->key, p_stat);
else if (x != EFAILURE)
ret = x;
ds_term = ds_diction_next(ds_c);
}
ds_diction_close(ds_c);
if (ret) {
LOGDEBUG("_ds_getall_spamtotals returning %d", ret);
}
return ret;
}
int
_ds_setall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction)
{
ds_term_t ds_term;
ds_cursor_t ds_c;
int ret = EUNKNOWN;
if (diction == NULL || CTX == NULL)
return EINVAL;
if (CTX->operating_mode == DSM_CLASSIFY &&
(CTX->training_mode != DST_TOE ||
(diction->whitelist_token == 0 && (!(CTX->flags & DSF_NOISE)))))
{
return 0;
}
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
while(ds_term)
{
if (!(ds_term->s.status & TST_DIRTY)) {
ds_term = ds_diction_next(ds_c);
continue;
}
if (CTX->training_mode == DST_TOE &&
CTX->classification == DSR_NONE &&
CTX->operating_mode == DSM_CLASSIFY &&
diction->whitelist_token != ds_term->key &&
(!ds_term->name || strncmp(ds_term->name, "bnr.", 4)))
{
ds_term = ds_diction_next(ds_c);
continue;
}
if (ds_term->s.spam_hits > CTX->totals.spam_learned)
ds_term->s.spam_hits = CTX->totals.spam_learned;
if (ds_term->s.innocent_hits > CTX->totals.innocent_learned)
ds_term->s.innocent_hits = CTX->totals.innocent_learned;
if (!_ds_set_spamrecord (CTX, ds_term->key, &ds_term->s))
ret = 0;
ds_term = ds_diction_next(ds_c);
}
ds_diction_close(ds_c);
return ret;
}
int
_ds_setall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction)
{
struct _sqlite_drv_storage *s = (struct _sqlite_drv_storage *) CTX->storage;
struct _ds_spam_stat stat, stat2;
ds_term_t ds_term;
ds_cursor_t ds_c;
buffer *query;
char scratch[1024];
char *err=NULL;
int update_one = 0;
if (s->dbh == NULL)
{
LOGDEBUG ("_ds_setall_spamrecords: invalid database handle (NULL)");
return EINVAL;
}
if (CTX->operating_mode == DSM_CLASSIFY &&
(CTX->training_mode != DST_TOE ||
(diction->whitelist_token == 0 && (!(CTX->flags & DSF_NOISE)))))
return 0;
query = buffer_create (NULL);
if (query == NULL)
{
LOG (LOG_CRIT, ERR_MEM_ALLOC);
return EUNKNOWN;
}
if (s->control_token == 0)
{
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
if (ds_term == NULL)
{
stat.spam_hits = 0;
stat.innocent_hits = 0;
}
else
{
stat.spam_hits = ds_term->s.spam_hits;
stat.innocent_hits = ds_term->s.innocent_hits;
}
ds_diction_close(ds_c);
}
else
{
ds_diction_getstat(diction, s->control_token, &stat);
}
snprintf (scratch, sizeof (scratch),
"update dspam_token_data set last_hit = date('now'), "
"spam_hits = max(0, spam_hits %s %d), "
"innocent_hits = max(0, innocent_hits %s %d) "
"where token in(",
(stat.spam_hits > s->control_sh) ? "+" : "-",
abs (stat.spam_hits - s->control_sh),
(stat.innocent_hits > s->control_ih) ? "+" : "-",
abs (stat.innocent_hits - s->control_ih));
buffer_cat (query, scratch);
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
while(ds_term)
{
int wrote_this = 0;
if (CTX->training_mode == DST_TOE &&
CTX->classification == DSR_NONE &&
CTX->operating_mode == DSM_CLASSIFY &&
diction->whitelist_token != ds_term->key &&
(!ds_term->name || strncmp(ds_term->name, "bnr.", 4)))
{
ds_term = ds_diction_next(ds_c);
continue;
}
if (!(ds_term->s.status & TST_DIRTY)) {
ds_term = ds_diction_next(ds_c);
continue;
}
ds_diction_getstat(diction, ds_term->key, &stat2);
if (!(stat2.status & TST_DISK))
{
char insert[1024];
snprintf(insert, sizeof (insert),
"insert into dspam_token_data(token, spam_hits, "
"innocent_hits, last_hit) values('%" LLU_FMT_SPEC "', %ld, %ld, "
"date('now'))",
ds_term->key,
stat2.spam_hits > 0 ? (long) 1 : (long) 0,
stat2.innocent_hits > 0 ? (long) 1 : (long) 0);
if ((sqlite_exec(s->dbh, insert, NULL, NULL, &err)) != SQLITE_OK)
{
stat2.status |= TST_DISK;
free(err);
}
}
if ((stat2.status & TST_DISK))
{
snprintf (scratch, sizeof (scratch), "'%" LLU_FMT_SPEC "'", ds_term->key);
buffer_cat (query, scratch);
update_one = 1;
wrote_this = 1;
ds_term->s.status |= TST_DISK;
}
ds_term = ds_diction_next(ds_c);
if (ds_term && wrote_this)
buffer_cat (query, ",");
}
ds_diction_close(ds_c);
if (query->used && query->data[strlen (query->data) - 1] == ',')
{
query->used--;
query->data[strlen (query->data) - 1] = 0;
}
buffer_cat (query, ")");
LOGDEBUG("Control: [%ld %ld] [%ld %ld]", s->control_sh, s->control_ih, stat.spam_hits, stat.innocent_hits);
if (update_one)
{
if ((sqlite_exec(s->dbh, query->data, NULL, NULL, &err))!=SQLITE_OK)
{
_sqlite_drv_query_error (err, query->data);
buffer_destroy(query);
return EFAILURE;
}
}
buffer_destroy (query);
return 0;
}
int
_ds_getall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction)
{
struct _sqlite_drv_storage *s = (struct _sqlite_drv_storage *) CTX->storage;
buffer *query;
ds_term_t ds_term;
ds_cursor_t ds_c;
char scratch[1024];
struct _ds_spam_stat stat;
unsigned long long token = 0;
char *err=NULL, **row;
int nrow, ncolumn, get_one = 0, i;
if (s->dbh == NULL)
{
LOGDEBUG ("_ds_getall_spamrecords: invalid database handle (NULL)");
return EINVAL;
}
stat.spam_hits = 0;
stat.innocent_hits = 0;
query = buffer_create (NULL);
if (query == NULL)
{
LOG (LOG_CRIT, ERR_MEM_ALLOC);
return EUNKNOWN;
}
snprintf (scratch, sizeof (scratch),
"select token, spam_hits, innocent_hits "
"from dspam_token_data where token in(");
buffer_cat (query, scratch);
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
while(ds_term)
{
snprintf (scratch, sizeof (scratch), "'%" LLU_FMT_SPEC "'", ds_term->key);
buffer_cat (query, scratch);
ds_term->s.innocent_hits = 0;
ds_term->s.spam_hits = 0;
ds_term->s.probability = 0;
ds_term->s.status &= ~TST_DISK;
ds_term = ds_diction_next(ds_c);
if (ds_term != NULL)
buffer_cat (query, ",");
get_one = 1;
}
ds_diction_close(ds_c);
buffer_cat (query, ")");
#ifdef VERBOSE
LOGDEBUG ("sqlite query length: %ld\n", query->used);
_sqlite_drv_query_error (strdup("VERBOSE DEBUG (INFO ONLY - NOT AN ERROR)"), query->data);
#endif
if (!get_one)
return 0;
if ((sqlite_get_table(s->dbh, query->data, &row, &nrow, &ncolumn, &err))
!=SQLITE_OK)
{
_sqlite_drv_query_error (err, query->data);
buffer_destroy(query);
return EFAILURE;
}
if (nrow < 1) {
sqlite_free_table(row);
buffer_destroy(query);
return 0;
}
if (row == NULL)
return 0;
stat.probability = 0;
stat.status |= TST_DISK;
for(i=1;i<=nrow;i++) {
token = strtoull (row[(i*ncolumn)], NULL, 0);
stat.spam_hits = strtol (row[1+(i*ncolumn)], NULL, 0);
stat.innocent_hits = strtol (row[2+(i*ncolumn)], NULL, 0);
if (stat.spam_hits < 0)
stat.spam_hits = 0;
if (stat.innocent_hits < 0)
stat.innocent_hits = 0;
ds_diction_addstat(diction, token, &stat);
}
sqlite_free_table(row);
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
while(ds_term && !s->control_token) {
if (ds_term->s.spam_hits && ds_term->s.innocent_hits) {
s->control_token = ds_term->key;
s->control_sh = ds_term->s.spam_hits;
s->control_ih = ds_term->s.innocent_hits;
}
ds_term = ds_diction_next(ds_c);
}
ds_diction_close(ds_c);
if (!s->control_token)
{
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
s->control_token = ds_term->key;
s->control_sh = ds_term->s.spam_hits;
s->control_ih = ds_term->s.innocent_hits;
ds_diction_close(ds_c);
}
buffer_destroy (query);
return 0;
}
int _ds_delall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction)
{
struct _sqlite_drv_storage *s = (struct _sqlite_drv_storage *) CTX->storage;
ds_term_t ds_term;
ds_cursor_t ds_c;
buffer *query;
char *err=NULL;
char scratch[1024];
char queryhead[1024];
int writes = 0;
if (diction->items < 1)
return 0;
if (s->dbh == NULL)
{
LOGDEBUG ("_ds_delall_spamrecords: invalid database handle (NULL)");
return EINVAL;
}
query = buffer_create (NULL);
if (query == NULL)
{
LOG (LOG_CRIT, ERR_MEM_ALLOC);
return EUNKNOWN;
}
snprintf (queryhead, sizeof(queryhead),
"delete from dspam_token_data "
"where token in(");
buffer_cat (query, queryhead);
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
while (ds_term)
{
snprintf (scratch, sizeof (scratch), "'%" LLU_FMT_SPEC "'", ds_term->key);
buffer_cat (query, scratch);
ds_term = ds_diction_next(ds_c);
if (writes > 2500 || ds_term == NULL) {
buffer_cat (query, ")");
if ((sqlite_exec(s->dbh, query->data, NULL, NULL, &err))!=SQLITE_OK)
{
_sqlite_drv_query_error (err, query->data);
buffer_destroy(query);
return EFAILURE;
}
buffer_copy(query, queryhead);
writes = 0;
} else {
writes++;
if (ds_term)
buffer_cat (query, ",");
}
}
ds_diction_close(ds_c);
if (writes) {
buffer_cat (query, ")");
if ((sqlite_exec(s->dbh, query->data, NULL, NULL, &err))!=SQLITE_OK)
{
_sqlite_drv_query_error (err, query->data);
buffer_destroy(query);
return EFAILURE;
}
}
buffer_destroy (query);
return 0;
}
