int
_ds_set_signature (DSPAM_CTX * CTX, struct _ds_spam_signature *SIG,
const char *signature)
{
struct _sqlite_drv_storage *s = (struct _sqlite_drv_storage *) CTX->storage;
unsigned long length;
char *mem;
char scratch[1024];
buffer *query;
char *err=NULL;
if (s->dbh == NULL)
{
LOGDEBUG ("_ds_set_signature; invalid database handle (NULL)");
return EINVAL;
}
query = buffer_create (NULL);
if (query == NULL)
{
LOG (LOG_CRIT, ERR_MEM_ALLOC);
return EUNKNOWN;
}
mem = calloc (1, 2 + (257*SIG->length)/254);
if (mem == NULL)
{
LOG (LOG_CRIT, ERR_MEM_ALLOC);
buffer_destroy(query);
return EUNKNOWN;
}
length = sqlite_encode_binary(SIG->data, SIG->length, (unsigned char *) mem);
if (length<0) {
LOG(LOG_ERR, "sqlite_encode_binary() failed on error %d", length);
buffer_destroy(query);
return EFAILURE;
}
snprintf (scratch, sizeof (scratch),
"insert into dspam_signature_data(signature, created_on, data) "
"values(\"%s\", date('now'), '",
signature);
buffer_cat (query, scratch);
buffer_cat (query, mem);
buffer_cat (query, "')");
if ((sqlite_exec(s->dbh, query->data, NULL, NULL, &err))!=SQLITE_OK)
{
_sqlite_drv_query_error (err, query->data);
buffer_destroy(query);
free(mem);
return EFAILURE;
}
free (mem);
buffer_destroy(query);
return 0;
}
char *daemon_getline(THREAD_CTX *TTX, int timeout) {
struct timeval tv;
char *p, *q, *pop;
char buf[1024];
int total_wait = 0;
long recv_len;
fd_set fds;
int i;
pop = pop_buffer(TTX);
while(!pop && total_wait<timeout) {
if (__daemon_run == 0)
return NULL;
total_wait++;
tv.tv_sec = 1;
tv.tv_usec = 0;
FD_ZERO(&fds);
FD_SET(TTX->sockfd, &fds);
i = select(TTX->sockfd+1, &fds, NULL, NULL, &tv);
if (i<=0)
continue;
recv_len = recv(TTX->sockfd, buf, sizeof(buf)-1, 0);
buf[recv_len] = 0;
if (recv_len == 0)
return NULL;
for(p=q=buf,i=0; i<recv_len; i++) {
if (*q) {
*p = *q;
p++;
}
q++;
}
*p = 0;
buffer_cat(TTX->packet_buffer, buf);
pop = pop_buffer(TTX);
}
#ifdef VERBOSE
LOGDEBUG("SRECV: %s", pop);
#endif
return pop;
}
int _ds_degenerate_message(DSPAM_CTX *CTX, buffer * header, buffer * body)
{
char *decode = NULL;
struct nt_node *node_nt, *node_header;
struct nt_c c_nt, c_nt2;
int i = 0;
char heading[1024];
if (! CTX->message)
{
LOG (LOG_WARNING, "_ds_degenerate_message() failed: CTX->message is NULL");
return EUNKNOWN;
}
/* Iterate through each component and create large header/body buffers */
node_nt = c_nt_first (CTX->message->components, &c_nt);
while (node_nt != NULL)
{
struct _ds_message_part *block = (struct _ds_message_part *) node_nt->ptr;
#ifdef VERBOSE
LOGDEBUG ("Processing component %d", i);
#endif
if (! block->headers || ! block->headers->items)
{
#ifdef VERBOSE
LOGDEBUG (" : End of Message Identifier");
#endif
}
else
{
struct _ds_header_field *current_header;
/* Accumulate the headers */
node_header = c_nt_first (block->headers, &c_nt2);
while (node_header != NULL)
{
current_header = (struct _ds_header_field *) node_header->ptr;
snprintf (heading, sizeof (heading),
"%s: %s\n", current_header->heading,
current_header->data);
buffer_cat (header, heading);
node_header = c_nt_next (block->headers, &c_nt2);
}
decode = block->body->data;
if (block->media_type == MT_TEXT ||
block->media_type == MT_MESSAGE ||
block->media_type == MT_UNKNOWN ||
(block->media_type == MT_MULTIPART && !i))
{
/* Accumulate the bodies, skip attachments */
if (
( block->encoding == EN_BASE64
|| block->encoding == EN_QUOTED_PRINTABLE)
&& ! block->original_signed_body)
{
if (block->content_disposition != PCD_ATTACHMENT)
{
LOGDEBUG ("decoding message block from encoding type %d",
block->encoding);
decode = _ds_decode_block (block);
}
}
/* We found a tokenizable body component, add prefilters */
if (decode)
{
char *decode2 = NULL;
char *decode3 = NULL;
/* -- PREFILTERS BEGIN -- */
/* Hexadecimal 8-Bit Encodings */
if (block->encoding == EN_8BIT) {
decode2 = _ds_decode_hex8bit(decode);
} else {
decode2 = strdup(decode);
}
/* HTML-Specific Filters */
if (decode2) {
if (block->media_subtype == MST_HTML) {
decode3 = _ds_strip_html(decode2);
} else {
decode3 = strdup(decode2);
}
free(decode2);
}
/* -- PREFILTERS END -- */
if (decode3) {
buffer_cat (body, decode3);
free(decode3);
}
/* If we've decoded the body, save the original copy */
if (decode != block->body->data)
{
block->original_signed_body = block->body;
block->body = buffer_create (decode);
free (decode);
}
}
}
}
#ifdef VERBOSE
LOGDEBUG ("Getting next message component");
#endif
node_nt = c_nt_next (CTX->message->components, &c_nt);
i++;
} /* while (node_nt != NULL) */
if (header->data == NULL)
buffer_cat (header, " ");
if (body->data == NULL)
buffer_cat (body, " ");
return 0;
}
int
_ds_setall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction)
{
struct _sqlite_drv_storage *s = (struct _sqlite_drv_storage *) CTX->storage;
struct _ds_spam_stat stat, stat2;
ds_term_t ds_term;
ds_cursor_t ds_c;
buffer *query;
char scratch[1024];
char *err=NULL;
int update_one = 0;
if (s->dbh == NULL)
{
LOGDEBUG ("_ds_setall_spamrecords: invalid database handle (NULL)");
return EINVAL;
}
if (CTX->operating_mode == DSM_CLASSIFY &&
(CTX->training_mode != DST_TOE ||
(diction->whitelist_token == 0 && (!(CTX->flags & DSF_NOISE)))))
return 0;
query = buffer_create (NULL);
if (query == NULL)
{
LOG (LOG_CRIT, ERR_MEM_ALLOC);
return EUNKNOWN;
}
if (s->control_token == 0)
{
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
if (ds_term == NULL)
{
stat.spam_hits = 0;
stat.innocent_hits = 0;
}
else
{
stat.spam_hits = ds_term->s.spam_hits;
stat.innocent_hits = ds_term->s.innocent_hits;
}
ds_diction_close(ds_c);
}
else
{
ds_diction_getstat(diction, s->control_token, &stat);
}
snprintf (scratch, sizeof (scratch),
"update dspam_token_data set last_hit = date('now'), "
"spam_hits = max(0, spam_hits %s %d), "
"innocent_hits = max(0, innocent_hits %s %d) "
"where token in(",
(stat.spam_hits > s->control_sh) ? "+" : "-",
abs (stat.spam_hits - s->control_sh),
(stat.innocent_hits > s->control_ih) ? "+" : "-",
abs (stat.innocent_hits - s->control_ih));
buffer_cat (query, scratch);
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
while(ds_term)
{
int wrote_this = 0;
if (CTX->training_mode == DST_TOE &&
CTX->classification == DSR_NONE &&
CTX->operating_mode == DSM_CLASSIFY &&
diction->whitelist_token != ds_term->key &&
(!ds_term->name || strncmp(ds_term->name, "bnr.", 4)))
{
ds_term = ds_diction_next(ds_c);
continue;
}
if (!(ds_term->s.status & TST_DIRTY)) {
ds_term = ds_diction_next(ds_c);
continue;
}
ds_diction_getstat(diction, ds_term->key, &stat2);
if (!(stat2.status & TST_DISK))
{
char insert[1024];
snprintf(insert, sizeof (insert),
"insert into dspam_token_data(token, spam_hits, "
"innocent_hits, last_hit) values('%" LLU_FMT_SPEC "', %ld, %ld, "
"date('now'))",
ds_term->key,
stat2.spam_hits > 0 ? (long) 1 : (long) 0,
stat2.innocent_hits > 0 ? (long) 1 : (long) 0);
if ((sqlite_exec(s->dbh, insert, NULL, NULL, &err)) != SQLITE_OK)
{
stat2.status |= TST_DISK;
free(err);
}
}
if ((stat2.status & TST_DISK))
{
snprintf (scratch, sizeof (scratch), "'%" LLU_FMT_SPEC "'", ds_term->key);
buffer_cat (query, scratch);
update_one = 1;
wrote_this = 1;
ds_term->s.status |= TST_DISK;
}
ds_term = ds_diction_next(ds_c);
if (ds_term && wrote_this)
buffer_cat (query, ",");
}
ds_diction_close(ds_c);
if (query->used && query->data[strlen (query->data) - 1] == ',')
{
query->used--;
query->data[strlen (query->data) - 1] = 0;
}
buffer_cat (query, ")");
LOGDEBUG("Control: [%ld %ld] [%ld %ld]", s->control_sh, s->control_ih, stat.spam_hits, stat.innocent_hits);
if (update_one)
{
if ((sqlite_exec(s->dbh, query->data, NULL, NULL, &err))!=SQLITE_OK)
{
_sqlite_drv_query_error (err, query->data);
buffer_destroy(query);
return EFAILURE;
}
}
buffer_destroy (query);
return 0;
}
int
_ds_getall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction)
{
struct _sqlite_drv_storage *s = (struct _sqlite_drv_storage *) CTX->storage;
buffer *query;
ds_term_t ds_term;
ds_cursor_t ds_c;
char scratch[1024];
struct _ds_spam_stat stat;
unsigned long long token = 0;
char *err=NULL, **row;
int nrow, ncolumn, get_one = 0, i;
if (s->dbh == NULL)
{
LOGDEBUG ("_ds_getall_spamrecords: invalid database handle (NULL)");
return EINVAL;
}
stat.spam_hits = 0;
stat.innocent_hits = 0;
query = buffer_create (NULL);
if (query == NULL)
{
LOG (LOG_CRIT, ERR_MEM_ALLOC);
return EUNKNOWN;
}
snprintf (scratch, sizeof (scratch),
"select token, spam_hits, innocent_hits "
"from dspam_token_data where token in(");
buffer_cat (query, scratch);
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
while(ds_term)
{
snprintf (scratch, sizeof (scratch), "'%" LLU_FMT_SPEC "'", ds_term->key);
buffer_cat (query, scratch);
ds_term->s.innocent_hits = 0;
ds_term->s.spam_hits = 0;
ds_term->s.probability = 0;
ds_term->s.status &= ~TST_DISK;
ds_term = ds_diction_next(ds_c);
if (ds_term != NULL)
buffer_cat (query, ",");
get_one = 1;
}
ds_diction_close(ds_c);
buffer_cat (query, ")");
#ifdef VERBOSE
LOGDEBUG ("sqlite query length: %ld\n", query->used);
_sqlite_drv_query_error (strdup("VERBOSE DEBUG (INFO ONLY - NOT AN ERROR)"), query->data);
#endif
if (!get_one)
return 0;
if ((sqlite_get_table(s->dbh, query->data, &row, &nrow, &ncolumn, &err))
!=SQLITE_OK)
{
_sqlite_drv_query_error (err, query->data);
buffer_destroy(query);
return EFAILURE;
}
if (nrow < 1) {
sqlite_free_table(row);
buffer_destroy(query);
return 0;
}
if (row == NULL)
return 0;
stat.probability = 0;
stat.status |= TST_DISK;
for(i=1;i<=nrow;i++) {
token = strtoull (row[(i*ncolumn)], NULL, 0);
stat.spam_hits = strtol (row[1+(i*ncolumn)], NULL, 0);
stat.innocent_hits = strtol (row[2+(i*ncolumn)], NULL, 0);
if (stat.spam_hits < 0)
stat.spam_hits = 0;
if (stat.innocent_hits < 0)
stat.innocent_hits = 0;
ds_diction_addstat(diction, token, &stat);
}
sqlite_free_table(row);
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
while(ds_term && !s->control_token) {
if (ds_term->s.spam_hits && ds_term->s.innocent_hits) {
s->control_token = ds_term->key;
s->control_sh = ds_term->s.spam_hits;
s->control_ih = ds_term->s.innocent_hits;
}
ds_term = ds_diction_next(ds_c);
}
ds_diction_close(ds_c);
if (!s->control_token)
{
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
s->control_token = ds_term->key;
s->control_sh = ds_term->s.spam_hits;
s->control_ih = ds_term->s.innocent_hits;
ds_diction_close(ds_c);
}
buffer_destroy (query);
return 0;
}
int _ds_delall_spamrecords (DSPAM_CTX * CTX, ds_diction_t diction)
{
struct _sqlite_drv_storage *s = (struct _sqlite_drv_storage *) CTX->storage;
ds_term_t ds_term;
ds_cursor_t ds_c;
buffer *query;
char *err=NULL;
char scratch[1024];
char queryhead[1024];
int writes = 0;
if (diction->items < 1)
return 0;
if (s->dbh == NULL)
{
LOGDEBUG ("_ds_delall_spamrecords: invalid database handle (NULL)");
return EINVAL;
}
query = buffer_create (NULL);
if (query == NULL)
{
LOG (LOG_CRIT, ERR_MEM_ALLOC);
return EUNKNOWN;
}
snprintf (queryhead, sizeof(queryhead),
"delete from dspam_token_data "
"where token in(");
buffer_cat (query, queryhead);
ds_c = ds_diction_cursor(diction);
ds_term = ds_diction_next(ds_c);
while (ds_term)
{
snprintf (scratch, sizeof (scratch), "'%" LLU_FMT_SPEC "'", ds_term->key);
buffer_cat (query, scratch);
ds_term = ds_diction_next(ds_c);
if (writes > 2500 || ds_term == NULL) {
buffer_cat (query, ")");
if ((sqlite_exec(s->dbh, query->data, NULL, NULL, &err))!=SQLITE_OK)
{
_sqlite_drv_query_error (err, query->data);
buffer_destroy(query);
return EFAILURE;
}
buffer_copy(query, queryhead);
writes = 0;
} else {
writes++;
if (ds_term)
buffer_cat (query, ",");
}
}
ds_diction_close(ds_c);
if (writes) {
buffer_cat (query, ")");
if ((sqlite_exec(s->dbh, query->data, NULL, NULL, &err))!=SQLITE_OK)
{
_sqlite_drv_query_error (err, query->data);
buffer_destroy(query);
return EFAILURE;
}
}
buffer_destroy (query);
return 0;
}
buffer * read_sock(THREAD_CTX *TTX, AGENT_CTX *ATX) {
buffer *message;
int body = 0, line = 1;
char *buf;
int strip = _ds_match_attribute(agent_config, "Broken", "lineStripping");
int parseto = _ds_match_attribute(agent_config, "ParseToHeaders", "on");
message = buffer_create(NULL);
if (message == NULL) {
LOG(LOG_CRIT, ERR_MEM_ALLOC);
return NULL;
}
if (_ds_match_attribute(agent_config, "DataSource", "document")) {
buffer_cat(message, ".\n\n");
body = 1;
}
while ((buf = daemon_getline(TTX, 300))!=NULL) {
chomp(buf);
if (!strcmp(buf, ".")) {
free(buf);
return message;
}
if (strip) {
size_t len = strlen(buf);
while (len>1 && buf[len-2]==13) {
buf[len-2] = buf[len-1];
buf[len-1] = 0;
len--;
}
}
if (line > 1 || strncmp (buf, "From QUARANTINE", 15))
{
if (parseto) {
if (buf[0] == 0)
body = 1;
if (!body && !strncasecmp(buf, "To: ", 4))
process_parseto(ATX, buf);
}
if (buffer_cat (message, buf) || buffer_cat(message, "\n"))
{
LOG (LOG_CRIT, ERR_MEM_ALLOC);
goto bail;
}
}
/* Use the original user id if we are reversing a false positive */
if (ATX->source == DSS_ERROR &&
ATX->classification == DSR_ISINNOCENT &&
ATX->operating_mode == DSM_PROCESS &&
!strncasecmp (buf, "X-DSPAM-User: ", 14))
{
char user[MAX_USERNAME_LENGTH];
strlcpy (user, buf + 14, sizeof (user));
chomp (user);
nt_destroy (ATX->users);
ATX->users = nt_create (NT_CHAR);
if (ATX->users == NULL)
{
LOG(LOG_CRIT, ERR_MEM_ALLOC);
goto bail;
}
nt_add (ATX->users, user);
}
free(buf);
line++;
}
return NULL;
bail:
buffer_destroy(message);
return NULL;
}