char *get_str_from_line(Text *text, size_t line)
{
size_t pos_line = 1;
size_t i;
char *buf;
for (i = 0; i < text->size; ++i) {
if (pos_line == line) {
break;
}
if (i == text->gap_start) {
i += GAPSIZE(text) - 1;
continue;
}
if (text->buf[i] == '\n') {
++pos_line;
}
}
buf = calloc(1, NBYTES(text) - i + 1);
if (!buf) return NULL;
if (i <= text->gap_start) {
memcpy(buf, text->buf + i, text->gap_start - i);
memcpy(buf + text->gap_start - i, text->buf + text->text_start, text->size - text->text_start);
} else {
memcpy(buf, text->buf + i, text->size - text->text_start - i);
}
return buf;
}
static struct cache_savepoint *
rspamd_symbols_cache_make_checkpoint (struct rspamd_task *task,
struct symbols_cache *cache)
{
struct cache_savepoint *checkpoint;
if (cache->items_by_id->len != cache->items_by_order->d->len) {
/*
* Cache has been modified, need to resort it
*/
msg_info_cache ("symbols cache has been modified since last check:"
" old items: %ud, new items: %ud",
cache->items_by_order->d->len, cache->items_by_id->len);
rspamd_symbols_cache_resort (cache);
}
checkpoint = rspamd_mempool_alloc0 (task->task_pool, sizeof (*checkpoint));
/* Bit 0: check started, Bit 1: check finished */
checkpoint->processed_bits = rspamd_mempool_alloc0 (task->task_pool,
NBYTES (cache->used_items) * 2);
checkpoint->waitq = g_ptr_array_new ();
g_assert (cache->items_by_order != NULL);
checkpoint->version = cache->items_by_order->d->len;
checkpoint->order = cache->items_by_order;
REF_RETAIN (checkpoint->order);
rspamd_mempool_add_destructor (task->task_pool,
rspamd_symbols_cache_order_unref, checkpoint->order);
rspamd_mempool_add_destructor (task->task_pool,
rspamd_ptr_array_free_hard, checkpoint->waitq);
task->checkpoint = checkpoint;
rspamd_create_metric_result (task, DEFAULT_METRIC);
return checkpoint;
}
static void
target_setup(VALUE t_mol, struct Target * target){
VALUE bit_mat;
VALUE bit_str;
VALUE atom_types;
int i;
atom_types = rb_funcall(t_mol, rb_intern("typ_str"), 0);
Check_Type(atom_types, T_STRING);
target->n_bits = RSTRING(atom_types)->len / sizeof(long);
target->typ = (long *)talloc(target->n_bits * sizeof(long));
memcpy(target->typ, RSTRING(atom_types)->ptr, target->n_bits * sizeof(long));
/*
* Set up adjacency matrix
*/
bit_mat = rb_funcall(t_mol, rb_intern("bit_mat"), 0);
bit_str = rb_funcall(bit_mat, rb_intern("bit_str"), 0);
target->n_bytes = NBYTES(target->n_bits);
target->mat = (long *)talloc(target->n_bytes * target->n_bits * sizeof(long));
memcpy(target->mat, RSTRING(bit_str)->ptr, RSTRING(bit_str)->len);
}
int link_argvalue(Var* arg, APIARGS* aarg)
{
Var* e;
int lup;
void* argmem;
e = arg;
if (V_TYPE(arg) == ID_UNK) {
if ((e = eval(arg)) == NULL) {
printf("No such variable '%s'\n", V_NAME(arg));
return -1;
}
}
if (V_TYPE(e) == ID_STRING) {
if ((aarg->argtype & DTMASK) == DV_UINT8 && (aarg->argtype & PTRBIT) == PTRBIT)
aarg->argval = strdup(V_STRING(e));
else {
printf("Data type mismatch\n");
return -1;
}
} else if (V_TYPE(e) == ID_VAL) {
aarg->argval = calloc(V_DSIZE(e), NBYTES(aarg->argtype & DTMASK));
if ((aarg->argtype & VOLBIT) == 0)
return (typeconvert_args(V_FORMAT(e), V_DATA(e), aarg->argtype & DTMASK, aarg->argval,
V_DSIZE(e)));
}
return 0;
}
void
mergebitset(bitset ss1, bitset ss2, int nbits)
{
int i;
for (i = NBYTES(nbits); --i >= 0; )
*ss1++ |= *ss2++;
}
char *get_str(Text *text)
{
char *buf = calloc(1, NBYTES(text) + 1);
if (!buf) return NULL;
memcpy(buf, text->buf, text->gap_start);
memcpy(buf + text->gap_start, text->buf + text->text_start, text->size - text->text_start);
return buf;
}
/*
* Better Sieve of Atkin.
*
* Uses 1 bit per odd number.
*
* Just some simple optimizations to make it a little better. Still not good.
*
* Time for Pi(10^10) = 97.2s
*/
static WTYPE* sieve_atkin_2(WTYPE end)
{
WTYPE* mem;
size_t x, y, n, sqlimit;
size_t last = (end+1+1)/2;
long loopend, y_limit, dn;
end++;
mem = (WTYPE*) malloc( NWORDS(last) * sizeof(WTYPE) );
assert(mem != 0);
/* mark everything as a composite */
memset(mem, 0xFF, NBYTES(last));
sqlimit = sqrtf(end);
for (x = 1; x <= sqlimit; x++) {
{
size_t xx4 = 4*x*x;
y = 1;
for (n = xx4+1; n <= end; n = xx4+y*y) {
size_t nmod12 = n%12;
if ( (nmod12 == 1) || (nmod12 == 5) )
XOR_ARRAY_BIT(mem,n/2);
y++;
}
}
{
size_t xx3 = 3*x*x;
y = 1;
for (n = xx3+1; n <= end; n = xx3+y*y) {
size_t nmod12 = n%12;
if (nmod12 == 7)
XOR_ARRAY_BIT(mem,n/2);
y++;
}
y = x-1;
while ( y*y >= xx3 )
y--;
for (n = xx3-y*y; y >= 1 && n <= end; n = xx3-y*y) {
size_t nmod12 = n%12;
if (nmod12 == 11)
XOR_ARRAY_BIT(mem,n/2);
y--;
}
}
}
/* Mark all squares of primes as composite */
for (n = 5; n <= sqlimit; n += 2)
if (!IS_SET_ARRAY_BIT(mem,n/2))
for (y = n*n; y <= end; y += 2*n*n)
SET_ARRAY_BIT(mem,y/2);
CLR_ARRAY_BIT(mem, 3/2); /* 3 is prime */
return mem;
}
void
rspamd_re_cache_runtime_destroy (struct rspamd_re_runtime *rt)
{
g_assert (rt != NULL);
g_slice_free1 (NBYTES (rt->cache->nre), rt->checked);
g_slice_free1 (rt->cache->nre, rt->results);
REF_RELEASE (rt->cache);
g_slice_free1 (sizeof (*rt), rt);
}
int
samebitset(bitset ss1, bitset ss2, int nbits)
{
int i;
for (i = NBYTES(nbits); --i >= 0; )
if (*ss1++ != *ss2++)
return 0;
return 1;
}
int typeconvert_args(int srctype, void* src, int dsttype, void* dst, int dsize)
{
int lup, *ip;
short* sp;
float* fp;
double* dp;
char* cp;
if (srctype == dsttype) {
memcpy(dst, src, dsize * NBYTES(dsttype));
return 0;
}
switch (dsttype) {
case DV_UINT8:
cp = (char*)dst;
for (lup = 0; lup < dsize; lup++) {
cp[lup] = saturate_byte(api_extract_int(srctype, src, lup));
}
break;
case DV_INT16:
sp = (short*)dst;
for (lup = 0; lup < dsize; lup++) {
sp[lup] = saturate_short(api_extract_int(srctype, src, lup));
}
break;
case DV_INT32:
ip = (int*)dst;
for (lup = 0; lup < dsize; lup++) {
ip[lup] = api_extract_int(srctype, src, lup);
}
break;
case DV_FLOAT:
fp = (float*)dst;
for (lup = 0; lup < dsize; lup++) {
fp[lup] = api_extract_float(srctype, src, lup);
}
break;
case DV_DOUBLE:
dp = (double*)dst;
for (lup = 0; lup < dsize; lup++) {
dp[lup] = api_extract_double(srctype, src, lup);
}
break;
default:
printf("Unknown data format\n");
return -1;
break;
}
return 0;
}
struct rspamd_re_runtime *
rspamd_re_cache_runtime_new (struct rspamd_re_cache *cache)
{
struct rspamd_re_runtime *rt;
g_assert (cache != NULL);
rt = g_slice_alloc0 (sizeof (*rt));
rt->cache = cache;
REF_RETAIN (cache);
rt->checked = g_slice_alloc0 (NBYTES (cache->nre));
rt->results = g_slice_alloc0 (cache->nre);
return rt;
}
bitset
newbitset(int nbits)
{
int nbytes = NBYTES(nbits);
bitset ss = (char *)PyObject_MALLOC(sizeof(BYTE) * nbytes);
if (ss == NULL)
Py_FatalError("no mem for bitset");
ss += nbytes;
while (--nbytes >= 0)
*--ss = 0;
return ss;
}
static void
printdfas(grammar *g, FILE *fp)
{
dfa *d;
int i, j;
printstates(g, fp);
fprintf(fp, "static dfa dfas[%d] = {\n", g->g_ndfas);
d = g->g_dfa;
for (i = 0; i < g->g_ndfas; i++, d++) {
fprintf(fp, " {%d, \"%s\", %d, %d, states_%d,\n",
d->d_type, d->d_name, d->d_initial, d->d_nstates, i);
fprintf(fp, " \"");
for (j = 0; j < NBYTES(g->g_ll.ll_nlabels); j++)
fprintf(fp, "\\%03o", d->d_first[j] & 0xff);
fprintf(fp, "\"},\n");
}
fprintf(fp, "};\n");
}
/*
* Naive Sieve of Atkin.
*
* Uses 1 bit per odd number.
*
* This is really slow. Just keeping it here as a reference.
*
* Time for Pi(10^10) = 123.5s
*/
static WTYPE* sieve_atkin_naive(WTYPE end)
{
WTYPE* mem;
size_t x, y, n, sqlimit;
size_t last = (end+1+1)/2;
long loopend, y_limit, dn;
end++;
mem = (WTYPE*) malloc( NWORDS(last) * sizeof(WTYPE) );
assert(mem != 0);
/* mark everything as a composite */
memset(mem, 0xFF, NBYTES(last));
sqlimit = sqrt(end);
for (x = 1; x <= sqlimit; x++) {
for (y = 1; y <= sqlimit; y++) {
n = 4*x*x + y*y;
if ( (n <= end) && (n % 12 == 1 || n % 12 == 5) )
XOR_ARRAY_BIT(mem,n/2);
n = 3*x*x + y*y;
if ( (n <= end) && (n % 12 == 7) )
XOR_ARRAY_BIT(mem,n/2);
n = 3*x*x - y*y;
if ( (n <= end) && (x > y) && (n % 12 == 11) )
XOR_ARRAY_BIT(mem,n/2);
}
}
/* Mark all squares of primes as composite */
for (n = 5; n <= sqlimit; n += 2)
if (!IS_SET_ARRAY_BIT(mem,n/2))
for (y = n*n; y <= end; y += 2*n*n)
SET_ARRAY_BIT(mem,y/2);
CLR_ARRAY_BIT(mem, 3/2); /* 3 is prime */
return mem;
}
static void
show(long * l, int h, int w){
int i, j;
int counter = 0;
int n_bytes;
n_bytes = NBYTES(w);
printf(" ");
for(i = 0 ; i < w ; i++){
printf("%d", i % 10);
}
printf("\n");
for(i = 0 ; i < h ; i++){
printf("%3d ", i);
for(j = 0 ; j < n_bytes ; j++){
dump_long(l[counter], (j == n_bytes - 1) ? ((w - 1) % ARCH + 1) : ARCH);
counter++;
}
printf("\n");
}
}
Var* ff_rice(vfuncptr func, Var* arg)
{
Var* obj = NULL;
int x, y, z, nbytes, i, j, k, pos, start, len;
short* in;
unsigned char* out;
int header = 1;
int bits = -1;
Alist alist[4];
alist[0] = make_alist("object", ID_VAL, NULL, &obj);
alist[1] = make_alist("header", DV_INT32, NULL, &header);
alist[2] = make_alist("bits", DV_INT32, NULL, &bits);
alist[3].name = NULL;
if (parse_args(func, arg, alist) == 0) return NULL;
if (obj == NULL) {
parse_error("%s: No object or size specified", func->name);
return NULL;
}
x = GetX(obj);
y = GetY(obj);
z = GetZ(obj);
nbytes = NBYTES(V_FORMAT(obj));
start = 0;
if (bits == -1) bits = nbytes * 8;
if (x > 4096) {
parse_error("%s: Max buffer length is 4096 elements", func->name);
return NULL;
}
if (nbytes > 2) {
parse_error("Only able to compress 2 byte words or less");
return NULL;
}
if (nbytes == 2 && x > 511) {
parse_error("Too many values in a row. Split this up.");
return NULL;
}
in = calloc(x, 2);
out = calloc(x * y * z, 2 * nbytes);
if (header) {
unsigned short sx = x, sy = y, sz = z;
unsigned char sbits = bits;
/* write out header */
if (x > 65535 || y > 65535 || z > 65535) {
parse_error("data block too big for header");
return NULL;
}
memcpy(out, "RICE", 4);
start += 4;
memcpy(out + start, &sx, 2);
start += 2;
memcpy(out + start, &sy, 2);
start += 2;
memcpy(out + start, &sz, 2);
start += 2;
memcpy(out + start, &sbits, 1);
start += 1;
}
for (k = 0; k < z; k++) {
for (j = 0; j < y; j++) {
/*
** Pack a temporary array with a whole line
*/
for (i = 0; i < x; i++) {
pos = cpos(i, j, k, obj);
in[i] = extract_int(obj, pos);
}
len = rice_auto(in, x, bits, out + start, 0);
start += (len + 7) / 8;
}
}
return newVal(BSQ, start, 1, 1, DV_UINT8, out);
}
/*
* Better Sieve of Atkin.
*
* Uses 1 bit per odd number.
*
* From Mike on Programming Praxis. Pretty fast, but not really an improvement
* over a good SoE. Note that the limits aren't handled quite right, so I have
* to add an "if (n <= end)" in front of each XOR.
*
* Time for Pi(10^10) = 53.9s
*/
static WTYPE* sieve_atkin(WTYPE end)
{
WTYPE* mem;
size_t n, s, k;
size_t last = (end+1)/2; /* Extra space allocated */
long loopend, y_limit, dn;
end++;
mem = (WTYPE*) malloc( NWORDS(last) * sizeof(WTYPE) );
assert(mem != 0);
/* mark everything as a composite */
memset(mem, 0xFF, NBYTES(last));
{
long xx3 = 3;
long dxx;
loopend = 12 * (long) sqrtf((end-1)/3.0);
for (dxx = 0; dxx < loopend; dxx += 24) {
xx3 += dxx;
y_limit = (long) (12.0*sqrtf( end - xx3 )) - 36;
n = xx3 + 16;
for (dn = -12; dn < (y_limit+1); dn += 72) {
n += dn;
if (n <= end) XOR_ARRAY_BIT(mem,n/2);
}
n = xx3 + 4;
for (dn = 12; dn < (y_limit+1); dn += 72) {
n += dn;
if (n <= end) XOR_ARRAY_BIT(mem,n/2);
}
}
}
{
long xx4 = 0;
long dxx4;
loopend = 8 * (long) sqrtf((end-1)/4.0) + 4;
for (dxx4 = 4; dxx4 < loopend; dxx4 += 8) {
xx4 += dxx4;
n = xx4 + 1;
if (xx4%3) {
y_limit = 4 * (long)sqrtf( end - xx4 ) - 3;
for (dn = 0; dn < y_limit; dn += 8) {
n += dn;
if (n <= end) XOR_ARRAY_BIT(mem,n/2);
}
} else {
y_limit = 12 * (long)sqrtf( end - xx4 ) - 36;
n = xx4 + 25;
for (dn = -24; dn < (y_limit+1); dn += 72) {
n += dn;
if (n <= end) XOR_ARRAY_BIT(mem,n/2);
}
n = xx4 + 1;
for (dn = 24; dn < (y_limit+1); dn += 72) {
n += dn;
if (n <= end) XOR_ARRAY_BIT(mem,n/2);
}
}
}
}
{
long xx = 1;
long x;
loopend = (long) sqrtf((float)end/2.0) + 1;
for (x = 3; x < loopend; x += 2) {
xx += 4*x - 4;
n = 3*xx;
if (n > end) {
long min_y = (( (long) (sqrtf(n - end)) >>2)<<2);
long yy = min_y * min_y;
n -= yy;
s = 4*min_y + 4;
} else {
s = 4;
}
for (dn = s; dn < 4*x; dn += 8) {
n -= dn;
if ((n <= end) && ((n%12) == 11))
XOR_ARRAY_BIT(mem,n/2);
}
}
gboolean
rspamd_symbols_cache_process_symbols (struct rspamd_task * task,
struct symbols_cache *cache)
{
struct cache_item *item = NULL;
struct cache_savepoint *checkpoint;
gint i;
g_assert (cache != NULL);
if (task->checkpoint == NULL) {
checkpoint = rspamd_mempool_alloc0 (task->task_pool, sizeof (*checkpoint));
/* Bit 0: check started, Bit 1: check finished */
checkpoint->processed_bits = rspamd_mempool_alloc0 (task->task_pool,
NBYTES (cache->used_items) * 2);
checkpoint->waitq = g_ptr_array_new ();
rspamd_mempool_add_destructor (task->task_pool,
rspamd_ptr_array_free_hard, checkpoint->waitq);
task->checkpoint = checkpoint;
rspamd_create_metric_result (task, DEFAULT_METRIC);
if (task->settings) {
const ucl_object_t *wl;
wl = ucl_object_find_key (task->settings, "whitelist");
if (wl != NULL) {
msg_info ("<%s> is whitelisted", task->message_id);
task->flags |= RSPAMD_TASK_FLAG_SKIP;
return TRUE;
}
}
}
else {
checkpoint = task->checkpoint;
}
msg_debug ("symbols processing stage at pass: %d", checkpoint->pass);
if (checkpoint->pass == 0) {
/*
* On the first pass we check symbols that do not have dependencies
* If we figure out symbol that has no dependencies satisfied, then
* we just save it for another pass
*/
for (i = 0; i < (gint)cache->used_items; i ++) {
if (rspamd_symbols_cache_metric_limit (task, checkpoint)) {
msg_info ("<%s> has already scored more than %.2f, so do not "
"plan any more checks", task->message_id,
checkpoint->rs->score);
return TRUE;
}
item = g_ptr_array_index (cache->items_by_order, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
msg_debug ("blocked execution of %d unless deps are resolved",
item->id);
g_ptr_array_add (checkpoint->waitq, item);
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint);
}
}
checkpoint->pass ++;
}
else {
/* We just go through the blocked symbols and check if they are ready */
for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
item = g_ptr_array_index (checkpoint->waitq, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint);
}
}
}
return TRUE;
}