void ds_diction_destroy (ds_diction_t diction) { ds_term_t term, next; ds_cursor_t cur; if (!diction) return; cur = ds_diction_cursor(diction); if (!cur) { perror("ds_diction_destroy: ds_diction_cursor() failed"); return; } term = ds_diction_next(cur); while(term) { next = ds_diction_next(cur); ds_diction_delete(diction, term->key); term = next; } ds_diction_close(cur); nt_destroy(diction->order); nt_destroy(diction->chained_order); free(diction->tbl); free(diction); return; }
int _ds_getall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction) { ds_term_t ds_term; ds_cursor_t ds_c; struct _ds_spam_stat stat; struct _ds_spam_stat *p_stat = &stat; int ret = 0, x = 0; if (diction == NULL || CTX == NULL) return EINVAL; ds_c = ds_diction_cursor(diction); ds_term = ds_diction_next(ds_c); while(ds_term) { ds_term->s.spam_hits = 0; ds_term->s.innocent_hits = 0; ds_term->s.offset = 0; x = _ds_get_spamrecord (CTX, ds_term->key, p_stat); if (!x) ds_diction_setstat(diction, ds_term->key, p_stat); else if (x != EFAILURE) ret = x; ds_term = ds_diction_next(ds_c); } ds_diction_close(ds_c); if (ret) { LOGDEBUG("_ds_getall_spamtotals returning %d", ret); } return ret; }
int _ds_setall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction) { ds_term_t ds_term; ds_cursor_t ds_c; int ret = EUNKNOWN; if (diction == NULL || CTX == NULL) return EINVAL; if (CTX->operating_mode == DSM_CLASSIFY && (CTX->training_mode != DST_TOE || (diction->whitelist_token == 0 && (!(CTX->flags & DSF_NOISE))))) { return 0; } ds_c = ds_diction_cursor(diction); ds_term = ds_diction_next(ds_c); while(ds_term) { if (!(ds_term->s.status & TST_DIRTY)) { ds_term = ds_diction_next(ds_c); continue; } if (CTX->training_mode == DST_TOE && CTX->classification == DSR_NONE && CTX->operating_mode == DSM_CLASSIFY && diction->whitelist_token != ds_term->key && (!ds_term->name || strncmp(ds_term->name, "bnr.", 4))) { ds_term = ds_diction_next(ds_c); continue; } if (ds_term->s.spam_hits > CTX->totals.spam_learned) ds_term->s.spam_hits = CTX->totals.spam_learned; if (ds_term->s.innocent_hits > CTX->totals.innocent_learned) ds_term->s.innocent_hits = CTX->totals.innocent_learned; if (!_ds_set_spamrecord (CTX, ds_term->key, &ds_term->s)) ret = 0; ds_term = ds_diction_next(ds_c); } ds_diction_close(ds_c); return ret; }
int _ds_setall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction) { struct _sqlite_drv_storage *s = (struct _sqlite_drv_storage *) CTX->storage; struct _ds_spam_stat stat, stat2; ds_term_t ds_term; ds_cursor_t ds_c; buffer *query; char scratch[1024]; char *err=NULL; int update_one = 0; if (s->dbh == NULL) { LOGDEBUG ("_ds_setall_spamrecords: invalid database handle (NULL)"); return EINVAL; } if (CTX->operating_mode == DSM_CLASSIFY && (CTX->training_mode != DST_TOE || (diction->whitelist_token == 0 && (!(CTX->flags & DSF_NOISE))))) return 0; query = buffer_create (NULL); if (query == NULL) { LOG (LOG_CRIT, ERR_MEM_ALLOC); return EUNKNOWN; } if (s->control_token == 0) { ds_c = ds_diction_cursor(diction); ds_term = ds_diction_next(ds_c); if (ds_term == NULL) { stat.spam_hits = 0; stat.innocent_hits = 0; } else { stat.spam_hits = ds_term->s.spam_hits; stat.innocent_hits = ds_term->s.innocent_hits; } ds_diction_close(ds_c); } else { ds_diction_getstat(diction, s->control_token, &stat); } snprintf (scratch, sizeof (scratch), "update dspam_token_data set last_hit = date('now'), " "spam_hits = max(0, spam_hits %s %d), " "innocent_hits = max(0, innocent_hits %s %d) " "where token in(", (stat.spam_hits > s->control_sh) ? "+" : "-", abs (stat.spam_hits - s->control_sh), (stat.innocent_hits > s->control_ih) ? "+" : "-", abs (stat.innocent_hits - s->control_ih)); buffer_cat (query, scratch); ds_c = ds_diction_cursor(diction); ds_term = ds_diction_next(ds_c); while(ds_term) { int wrote_this = 0; if (CTX->training_mode == DST_TOE && CTX->classification == DSR_NONE && CTX->operating_mode == DSM_CLASSIFY && diction->whitelist_token != ds_term->key && (!ds_term->name || strncmp(ds_term->name, "bnr.", 4))) { ds_term = ds_diction_next(ds_c); continue; } if (!(ds_term->s.status & TST_DIRTY)) { ds_term = ds_diction_next(ds_c); continue; } ds_diction_getstat(diction, ds_term->key, &stat2); if (!(stat2.status & TST_DISK)) { char insert[1024]; snprintf(insert, sizeof (insert), "insert into dspam_token_data(token, spam_hits, " "innocent_hits, last_hit) values('%" LLU_FMT_SPEC "', %ld, %ld, " "date('now'))", ds_term->key, stat2.spam_hits > 0 ? (long) 1 : (long) 0, stat2.innocent_hits > 0 ? (long) 1 : (long) 0); if ((sqlite_exec(s->dbh, insert, NULL, NULL, &err)) != SQLITE_OK) { stat2.status |= TST_DISK; free(err); } } if ((stat2.status & TST_DISK)) { snprintf (scratch, sizeof (scratch), "'%" LLU_FMT_SPEC "'", ds_term->key); buffer_cat (query, scratch); update_one = 1; wrote_this = 1; ds_term->s.status |= TST_DISK; } ds_term = ds_diction_next(ds_c); if (ds_term && wrote_this) buffer_cat (query, ","); } ds_diction_close(ds_c); if (query->used && query->data[strlen (query->data) - 1] == ',') { query->used--; query->data[strlen (query->data) - 1] = 0; } buffer_cat (query, ")"); LOGDEBUG("Control: [%ld %ld] [%ld %ld]", s->control_sh, s->control_ih, stat.spam_hits, stat.innocent_hits); if (update_one) { if ((sqlite_exec(s->dbh, query->data, NULL, NULL, &err))!=SQLITE_OK) { _sqlite_drv_query_error (err, query->data); buffer_destroy(query); return EFAILURE; } } buffer_destroy (query); return 0; }
int _ds_getall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction) { struct _sqlite_drv_storage *s = (struct _sqlite_drv_storage *) CTX->storage; buffer *query; ds_term_t ds_term; ds_cursor_t ds_c; char scratch[1024]; struct _ds_spam_stat stat; unsigned long long token = 0; char *err=NULL, **row; int nrow, ncolumn, get_one = 0, i; if (s->dbh == NULL) { LOGDEBUG ("_ds_getall_spamrecords: invalid database handle (NULL)"); return EINVAL; } stat.spam_hits = 0; stat.innocent_hits = 0; query = buffer_create (NULL); if (query == NULL) { LOG (LOG_CRIT, ERR_MEM_ALLOC); return EUNKNOWN; } snprintf (scratch, sizeof (scratch), "select token, spam_hits, innocent_hits " "from dspam_token_data where token in("); buffer_cat (query, scratch); ds_c = ds_diction_cursor(diction); ds_term = ds_diction_next(ds_c); while(ds_term) { snprintf (scratch, sizeof (scratch), "'%" LLU_FMT_SPEC "'", ds_term->key); buffer_cat (query, scratch); ds_term->s.innocent_hits = 0; ds_term->s.spam_hits = 0; ds_term->s.probability = 0; ds_term->s.status &= ~TST_DISK; ds_term = ds_diction_next(ds_c); if (ds_term != NULL) buffer_cat (query, ","); get_one = 1; } ds_diction_close(ds_c); buffer_cat (query, ")"); #ifdef VERBOSE LOGDEBUG ("sqlite query length: %ld\n", query->used); _sqlite_drv_query_error (strdup("VERBOSE DEBUG (INFO ONLY - NOT AN ERROR)"), query->data); #endif if (!get_one) return 0; if ((sqlite_get_table(s->dbh, query->data, &row, &nrow, &ncolumn, &err)) !=SQLITE_OK) { _sqlite_drv_query_error (err, query->data); buffer_destroy(query); return EFAILURE; } if (nrow < 1) { sqlite_free_table(row); buffer_destroy(query); return 0; } if (row == NULL) return 0; stat.probability = 0; stat.status |= TST_DISK; for(i=1;i<=nrow;i++) { token = strtoull (row[(i*ncolumn)], NULL, 0); stat.spam_hits = strtol (row[1+(i*ncolumn)], NULL, 0); stat.innocent_hits = strtol (row[2+(i*ncolumn)], NULL, 0); if (stat.spam_hits < 0) stat.spam_hits = 0; if (stat.innocent_hits < 0) stat.innocent_hits = 0; ds_diction_addstat(diction, token, &stat); } sqlite_free_table(row); ds_c = ds_diction_cursor(diction); ds_term = ds_diction_next(ds_c); while(ds_term && !s->control_token) { if (ds_term->s.spam_hits && ds_term->s.innocent_hits) { s->control_token = ds_term->key; s->control_sh = ds_term->s.spam_hits; s->control_ih = ds_term->s.innocent_hits; } ds_term = ds_diction_next(ds_c); } ds_diction_close(ds_c); if (!s->control_token) { ds_c = ds_diction_cursor(diction); ds_term = ds_diction_next(ds_c); s->control_token = ds_term->key; s->control_sh = ds_term->s.spam_hits; s->control_ih = ds_term->s.innocent_hits; ds_diction_close(ds_c); } buffer_destroy (query); return 0; }
int _ds_delall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction) { struct _sqlite_drv_storage *s = (struct _sqlite_drv_storage *) CTX->storage; ds_term_t ds_term; ds_cursor_t ds_c; buffer *query; char *err=NULL; char scratch[1024]; char queryhead[1024]; int writes = 0; if (diction->items < 1) return 0; if (s->dbh == NULL) { LOGDEBUG ("_ds_delall_spamrecords: invalid database handle (NULL)"); return EINVAL; } query = buffer_create (NULL); if (query == NULL) { LOG (LOG_CRIT, ERR_MEM_ALLOC); return EUNKNOWN; } snprintf (queryhead, sizeof(queryhead), "delete from dspam_token_data " "where token in("); buffer_cat (query, queryhead); ds_c = ds_diction_cursor(diction); ds_term = ds_diction_next(ds_c); while (ds_term) { snprintf (scratch, sizeof (scratch), "'%" LLU_FMT_SPEC "'", ds_term->key); buffer_cat (query, scratch); ds_term = ds_diction_next(ds_c); if (writes > 2500 || ds_term == NULL) { buffer_cat (query, ")"); if ((sqlite_exec(s->dbh, query->data, NULL, NULL, &err))!=SQLITE_OK) { _sqlite_drv_query_error (err, query->data); buffer_destroy(query); return EFAILURE; } buffer_copy(query, queryhead); writes = 0; } else { writes++; if (ds_term) buffer_cat (query, ","); } } ds_diction_close(ds_c); if (writes) { buffer_cat (query, ")"); if ((sqlite_exec(s->dbh, query->data, NULL, NULL, &err))!=SQLITE_OK) { _sqlite_drv_query_error (err, query->data); buffer_destroy(query); return EFAILURE; } } buffer_destroy (query); return 0; }