range* rangefunc_has(range_request* rr, range** r)
{
range* ret = range_new(rr);
apr_pool_t* pool = range_request_pool(rr);
const char** tag_names = range_get_hostnames(pool, r[0]);
const char** tag_values = range_get_hostnames(pool, r[1]);
const char* tag_name = tag_names[0];
const char* tag_value = tag_values[0];
const char** all_clusters = _all_clusters(rr);
const char** cluster = all_clusters;
int warn_enabled = range_request_warn_enabled(rr);
if (!cluster) return ret;
range_request_disable_warns(rr);
while (*cluster) {
range* vals = _expand_cluster(rr, *cluster, tag_name);
if (set_get(vals->nodes, tag_value) != NULL) {
range_add(ret, *cluster);
}
cluster++;
}
if (warn_enabled) range_request_enable_warns(rr);
return ret;
}
range* rangefunc_mem(range_request* rr, range** r)
{
range* ret = range_new(rr);
apr_pool_t* pool = range_request_pool(rr);
const char** clusters = range_get_hostnames(pool, r[0]);
const char* cluster = clusters[0];
const char** wanted = range_get_hostnames(pool, r[1]);
range* keys = _expand_cluster(rr, cluster, "KEYS");
const char** all_sections = range_get_hostnames(pool, keys);
const char** p_section = all_sections;
SECTION:
while (*p_section) {
range* r_s = _expand_cluster(rr, cluster, *p_section);
const char** p_wanted = wanted;
while (*p_wanted) {
if (set_get(r_s->nodes, *p_wanted) != NULL) {
range_add(ret, *p_section++);
goto SECTION;
}
++p_wanted;
}
++p_section;
}
return ret;
}
range* rangefunc_get_groups(range_request* rr, range** r)
{
sqlite3* db;
sqlite3_stmt* stmt;
int err;
range* ret = range_new(rr);
apr_pool_t* pool = range_request_pool(rr);
const char** tag_names = range_get_hostnames(pool, r[0]);
const char* tag_name = tag_names[0];
int warn_enabled = range_request_warn_enabled(rr);
if (NULL == tag_name) {
return ret;
}
db = _open_db(rr);
err = sqlite3_prepare(db, GROUPS_SQL, strlen(GROUPS_SQL), &stmt, NULL);
if (err != SQLITE_OK) {
range_request_warn(rr, "?%s: cannot query sqlite db", tag_name );
return ret;
}
assert(err == SQLITE_OK);
sqlite3_bind_text(stmt, 1, tag_name, strlen(tag_name), SQLITE_STATIC);
while(sqlite3_step(stmt) == SQLITE_ROW) {
const char* answer = (const char*)sqlite3_column_text(stmt, 0);
range_add(ret, answer);
}
sqlite3_finalize(stmt);
return ret;
}
range* range_from_match(range_request* rr,
const range* r, const char* regex)
{
range* ret;
int i;
int err_offset;
int ovector[30];
int count;
const char* error;
const char** members;
pcre* re;
apr_pool_t* pool = range_request_pool(rr);
members = range_get_hostnames(pool, r);
ret = range_new(rr);
re = pcre_compile(regex, 0, &error, &err_offset, NULL);
if (!re) {
range_request_warn(rr, "regex [%s] [%s]", regex, error);
return ret;
}
for (i = 0; members[i]; i++) {
count = pcre_exec(re, NULL, members[i],
strlen(members[i]), 0, 0, ovector, 30);
if (count > 0) /* it matched */
range_add(ret, members[i]);
}
pcre_free(re);
return ret;
}
range* rangefunc_get_cluster(range_request* rr, range** r)
{
range* ret = range_new(rr);
if (!validate_range_args(rr, r, 1)) {
return ret;
}
apr_pool_t* pool = range_request_lr_pool(rr);
const char** nodes = range_get_hostnames(pool, r[0]);
const char** p_nodes = nodes;
set* node_cluster = _get_clusters(rr);
while (*p_nodes) {
apr_array_header_t* clusters = set_get_data(node_cluster, *p_nodes);
if (!clusters)
range_request_warn_type(rr, "NO_CLUSTER", *p_nodes);
else {
/* just get one */
const char* cluster = ((const char**)clusters->elts)[0];
assert(cluster);
range_add(ret, cluster);
}
++p_nodes;
}
return ret;
}
range* rangefunc_clusters(range_request* rr, range** r)
{
sqlite3* db;
sqlite3_stmt* stmt;
int err;
range* ret = range_new(rr);
apr_pool_t* pool = range_request_pool(rr);
const char** nodes = range_get_hostnames(pool, r[0]);
const char** p_nodes = nodes;
db = _open_db(rr);
err = sqlite3_prepare(db, CLUSTERS_SQL, strlen(CLUSTERS_SQL), &stmt, NULL);
if (err != SQLITE_OK) {
range_request_warn(rr, "clusters(): cannot query sqlite db");
return ret;
}
while (*p_nodes) {
char * node_name = *p_nodes;
sqlite3_bind_text(stmt, 1, node_name, strlen(node_name), SQLITE_STATIC);
while(sqlite3_step(stmt) == SQLITE_ROW) {
const char* answer = (const char*)sqlite3_column_text(stmt, 0);
range_add(ret, answer);
}
sqlite3_reset(stmt);
sqlite3_clear_bindings(stmt);
++p_nodes;
}
sqlite3_finalize(stmt);
return ret;
}
range* range_from_literal(range_request* rr,
const char* literal)
{
range* r = range_new(rr);
range_add(r, literal);
return r;
}
range* rangefunc_clusters(range_request* rr, range** r)
{
range* ret = range_new(rr);
apr_pool_t* pool = range_request_pool(rr);
const char** nodes = range_get_hostnames(pool, r[0]);
const char** p_nodes = nodes;
set* node_cluster = _get_clusters(rr);
while (*p_nodes) {
apr_array_header_t* clusters = set_get_data(node_cluster, *p_nodes);
if (!clusters)
range_request_warn_type(rr, "NO_CLUSTER", *p_nodes);
else {
/* get all */
int i;
for (i=0; i<clusters->nelts; ++i) {
const char* cluster = ((const char**)clusters->elts)[i];
range_add(ret, cluster);
}
}
++p_nodes;
}
return ret;
}
range* range_from_hostnames(range_request* rr,
const char** names)
{
range* r;
int i;
r = range_new(rr);
for(i = 0; names[i]; i++)
range_add(r, names[i]);
return r;
}
range* _do_has_mem(range_request* rr, range** r, char* sql_query)
{
sqlite3* db;
sqlite3_stmt* stmt;
int err;
range* ret = range_new(rr);
apr_pool_t* pool = range_request_pool(rr);
if (NULL == r[0]) {
// don't attempt anything without arg #1 (key)
return ret;
}
const char** tag_names = range_get_hostnames(pool, r[0]);
const char* tag_name = tag_names[0];
if (NULL == tag_name) {
return ret;
}
const char** tag_values;
const char* tag_value;
if (NULL == r[1]) {
// if we don't have arg #2 (val) then search for keys with empty string value
tag_value = EMPTY_STRING;
} else {
tag_values = range_get_hostnames(pool, r[1]);
tag_value = tag_values[0];
}
const char** all_clusters = _all_clusters(rr);
const char** cluster = all_clusters;
int warn_enabled = range_request_warn_enabled(rr);
db = _open_db(rr);
err = sqlite3_prepare(db, sql_query, strlen(sql_query), &stmt,
NULL);
if (err != SQLITE_OK) {
range_request_warn(rr, "has(%s,%s): cannot query sqlite db", tag_name, tag_value);
return ret;
}
assert(err == SQLITE_OK);
sqlite3_bind_text(stmt, 1, tag_name, strlen(tag_name), SQLITE_STATIC);
sqlite3_bind_text(stmt, 2, tag_value, strlen(tag_value), SQLITE_STATIC);
while(sqlite3_step(stmt) == SQLITE_ROW) {
const char* answer = (const char*)sqlite3_column_text(stmt, 0);
range_add(ret, answer);
}
sqlite3_finalize(stmt);
return ret;
}
range* range_from_braces(range_request* rr,
const range* r1, const range* r2, const range* r3)
{
int i, j, k;
set_element** m1;
set_element** m2;
set_element** m3;
set* temp = NULL;
range* bigrange;
char* bundle;
apr_pool_t* pool = range_request_pool(rr);
if(r1->nodes->members == 0) {
if(!temp) {
temp = set_new(pool, 1);
set_add(temp, "", NULL);
}
m1 = set_members(temp);
} else m1 = set_members(r1->nodes);
if(r2->nodes->members == 0) {
if(!temp) {
temp = set_new(pool, 1);
set_add(temp, "", NULL);
}
m2 = set_members(temp);
} else m2 = set_members(r2->nodes);
if(r3->nodes->members == 0) {
if(!temp) {
temp = set_new(pool, 1);
set_add(temp, "", NULL);
}
m3 = set_members(temp);
} else m3 = set_members(r3->nodes);
bigrange = range_new(rr);
for(i = 0; m1[i]; i++)
for(j = 0; m2[j]; j++)
for(k = 0; m3[k]; k++) {
bundle = apr_pstrcat(pool,
m1[i]->name, m2[j]->name,
m3[k]->name, NULL);
range_add(bigrange, bundle);
}
if (temp) set_destroy(temp);
bigrange->quoted = r1->quoted || r2->quoted || r3->quoted;
return bigrange;
}
static range* _python_function(range_request* rr,
const char* funcname, const range** r)
{
range* result = range_new(rr);
PyObject * pLibcrangeCallFunc;
PyObject * pNodesReturned;
PyObject * pMain = PyImport_AddModule("__main__");
// printf("rr: %p, funcname=%s, range**r = %p\n", rr, funcname, r);
pLibcrangeCallFunc = PyObject_GetAttrString(pMain, "libcrange_call_func");
if (pLibcrangeCallFunc && PyCallable_Check(pLibcrangeCallFunc)) {
PyObject * pRangeFuncName;
PyObject * item;
pRangeFuncName = PyString_FromString(funcname);
PyObject * pFuncArgs;
PyObject * pTempArgList = PyList_New(0);
PyList_Append(pTempArgList, pRangeFuncName);
Py_DECREF(pRangeFuncName);
// build our range** into python function args
const range** p_r = r;
while (*p_r) {
item = range_to_py_array(range_request_pool(rr), *p_r);
PyList_Append(pTempArgList, item);
Py_DECREF(item);
p_r++;
}
pFuncArgs = PyList_AsTuple(pTempArgList);
Py_DECREF(pTempArgList);
// call the function
pNodesReturned = PyObject_CallObject(pLibcrangeCallFunc, pFuncArgs);
Py_DECREF(pFuncArgs);
PyObject *iterator = PyObject_GetIter(pNodesReturned);
if (iterator == NULL) {
printf("ERROR: python function %s ran, but didn't return an iteratable object", funcname);
return result;
}
// an iteratable object was returned, transform it into result
while (item = PyIter_Next(iterator)) {
// PyObject_Print(item, stdout, 0 );
range_add(result, PyString_AsString(item));
Py_DECREF(item);
}
Py_DECREF(pNodesReturned);
}
return result;
}
range* rangefunc_get_groups(range_request* rr, range** r)
{
range* ret = range_new(rr);
if (!validate_range_args(rr, r, 1)) {
return ret;
}
range* n = r[0];
apr_pool_t* pool = range_request_pool(rr);
const char** in_nodes = range_get_hostnames(pool, n);
set* node_group = set_new(pool, 40000);
range* groups_r = _expand_cluster(rr, "GROUPS", "KEYS");
const char** groups = range_get_hostnames(pool, groups_r);
const char** p_group = groups;
while (*p_group) {
range* nodes_r = _expand_cluster(rr, "GROUPS", *p_group);
const char** nodes = range_get_hostnames(pool, nodes_r);
const char** p_nodes = nodes;
while (*p_nodes) {
apr_array_header_t* my_groups = set_get_data(node_group, *p_nodes);
if (!my_groups) {
my_groups = apr_array_make(pool, 4, sizeof(char*));
set_add(node_group, *p_nodes, my_groups);
}
*(const char**)apr_array_push(my_groups) = *p_group;
++p_nodes;
}
++p_group;
}
while (*in_nodes) {
apr_array_header_t* my_groups = set_get_data(node_group, *in_nodes);
if (!my_groups)
range_request_warn_type(rr, "NO_GROUPS", *in_nodes);
else {
int i;
for (i=0; i<my_groups->nelts; ++i)
range_add(ret, ((const char**)my_groups->elts)[i]);
}
in_nodes++;
}
return ret;
}
void range_request_warn_type(range_request* rr, const char* type, const char* node)
{
range* nodes;
if (!rr->warn_enabled) return;
if (!rr->warn_type)
rr->warn_type = set_new(rr->pool, 0);
/* nodes that generated a particular warning type */
nodes = set_get_data(rr->warn_type, type);
if (!nodes) {
nodes = range_new(rr);
set_add(rr->warn_type, type, nodes);
}
range_add(nodes, node);
}
range* rangefunc_allclusters(range_request* rr, range** r)
{
range* ret = range_new(rr);
const char** all_clusters = _all_clusters(rr);
const char** cluster = all_clusters;
int warn_enabled = range_request_warn_enabled(rr);
if (!cluster) return ret;
range_request_disable_warns(rr);
while (*cluster) {
range_add(ret, *cluster);
cluster++;
}
if (warn_enabled) range_request_enable_warns(rr);
return ret;
}
range* range_from_rangeparts(range_request* rr,
const rangeparts* parts)
{
int i;
int f, l, firstlength, lastlength, length;
range* r;
char* pad1 = "";
char* first;
char* last;
char* tmpstr;
apr_pool_t* pool = range_request_pool(rr);
r = range_new(rr);
firstlength = strlen(parts->first);
lastlength = strlen(parts->last);
first = parts->first;
last = parts->last;
if (firstlength > lastlength) {
pad1 = apr_palloc(pool, firstlength - lastlength + 1);
for(i=0; i < (firstlength - lastlength); i++)
pad1[i] = parts->first[i];
pad1[i] = '\0';
first = parts->first + i;
}
f = atoi(first);
l = atoi(last);
length = firstlength > lastlength ? lastlength : firstlength;
for(i=f; i<=l; i++) {
tmpstr = apr_psprintf(pool, "%s%s%0*d%s",
parts->prefix, pad1, length, i, parts->domain);
range_add(r, tmpstr);
}
return r;
}
range* rangefunc_get_admin(range_request* rr, range** r)
{
range* ret = range_new(rr);
if (!validate_range_args(rr, r, 1)) {
return ret;
}
range* n = r[0];
apr_pool_t* pool = range_request_pool(rr);
const char** in_nodes = range_get_hostnames(pool, n);
set* node_admin = set_new(pool, 40000);
range* admins_r = _expand_cluster(rr, "HOSTS", "KEYS");
const char** admins = range_get_hostnames(pool, admins_r);
const char** p_admin = admins;
while (*p_admin) {
range* nodes_r = _expand_cluster(rr, "HOSTS", *p_admin);
const char** nodes = range_get_hostnames(pool, nodes_r);
const char** p_nodes = nodes;
while (*p_nodes) {
set_add(node_admin, *p_nodes, (void*)*p_admin);
++p_nodes;
}
++p_admin;
}
while (*in_nodes) {
const char* admin = set_get_data(node_admin, *in_nodes);
if (!admin) {
range_request_warn_type(rr, "NO_ADMIN", *in_nodes);
}
else {
range_add(ret, admin);
}
in_nodes++;
}
return ret;
}
static void read_ban_ip_list(void)
{
FILE *fp;
int rc,cnt=0;
char buff[100], bots[40], tops[40];
unsigned bot, top;
fp = xyfopen(FILENAME_LIST_BAN, "r");
if (!fp) return;
while(fgets(buff, sizeof buff, fp)) {
rc = sscanf(buff, "%x %x", &bot, &top);
if (rc < 2) {
rc = sscanf(buff, "%s %s", bots, tops);
if (rc < 2) continue;
#if 0
#include <netinet/in.h>
#include <arpa/inet.h>
bot = inet_addr(bots);
if ((int)bot == -1) continue;
top = inet_addr(tops);
if ((int)top == -1) continue;
bot = ntohl(bot); /* could use asc2ipaddr(bots, &bot); */
top = ntohl(top);
#else
if (asc2ipaddr(bots, &bot)) continue;
if (asc2ipaddr(tops, &top)) continue;
#endif
}
cnt += range_add(bot,top);
}
fclose(fp);
Logit("Ipban: Read %d ranges from %s", cnt, filename() );
bot = top = 0xf7000001;
cnt = range_remove(bot,top);
Logit("Ipban: Avoided %d ranges %x-%x", cnt, bot, top );
}
range* rangefunc_group(range_request* rr, range** r)
{
range* ret;
const char** members;
int i, err;
sqlite3* db;
sqlite3_stmt* tag_stmt;
sqlite3_stmt* all_nodes_stmt;
apr_pool_t* pool = range_request_pool(rr);
libcrange* lr = range_request_lr(rr);
ret = range_new(rr);
members = range_get_hostnames(pool, r[0]);
if (!(db = libcrange_get_cache(lr, "sqlite:nodes"))) {
const char* sqlite_db_path = libcrange_getcfg(lr, "sqlitedb");
if (!sqlite_db_path) sqlite_db_path = DEFAULT_SQLITE_DB;
err = sqlite3_open(sqlite_db_path, &db);
if (err != SQLITE_OK) {
fprintf(stderr, "%s: %s\n", sqlite_db_path, sqlite3_errmsg(db));
return ret;
}
libcrange_set_cache(lr, "sqlite:nodes", db);
}
/* prepare our selects */
err = sqlite3_prepare(db, ALL_NODES_SQL, strlen(ALL_NODES_SQL),
&all_nodes_stmt, NULL);
if (err != SQLITE_OK) {
fprintf(stderr, "%s: %s\n", ALL_NODES_SQL, sqlite3_errmsg(db));
abort();
}
err = sqlite3_prepare(db, RANGE_FROM_TAGS, strlen(RANGE_FROM_TAGS),
&tag_stmt, NULL);
assert(err == SQLITE_OK);
/* for each group */
for (i = 0; members[i]; ++i) {
sqlite3_stmt* stmt;
if (strcmp(members[i], "ALL") == 0) {
stmt = all_nodes_stmt;
} else {
stmt = tag_stmt;
/* bind the current group name */
sqlite3_bind_text(tag_stmt, 1, members[i], strlen(members[i]), SQLITE_STATIC);
}
while (sqlite3_step(stmt) == SQLITE_ROW) {
range* this_group;
const char* result = (const char*)sqlite3_column_text(stmt, 0);
if (stmt == all_nodes_stmt) {
range_add(ret, result);
} else {
this_group = do_range_expand(rr, result);
set_union_inplace(ret->nodes, this_group->nodes);
}
}
sqlite3_reset(stmt);
}
sqlite3_finalize(all_nodes_stmt);
sqlite3_finalize(tag_stmt);
return ret;
}
// store file buffer into local storage
// if success == TRUE then "buf" successfuly stored on disc
void cache_mng_store_file_buf (CacheMng *cmng, fuse_ino_t ino, size_t size, off_t off, unsigned char *buf,
cache_mng_on_store_file_buf_cb on_store_file_buf_cb, void *ctx)
{
struct _CacheContext *context;
struct _CacheEntry *entry;
ssize_t res;
int fd;
char path[PATH_MAX];
guint64 old_length, new_length;
guint64 range_size;
time_t now;
range_size = (guint64)(off + size);
// limit the number of cache checks
now = time (NULL);
if (cmng->check_time < now && now - cmng->check_time >= 10) {
// remove data until we have at least size bytes of max_size left
while (cmng->max_size < cmng->size + size && g_queue_peek_tail (cmng->q_lru)) {
entry = (struct _CacheEntry *) g_queue_peek_tail (cmng->q_lru);
cache_mng_remove_file (cmng, entry->ino);
}
cmng->check_time = now;
}
context = cache_context_create (size, ctx);
context->cb.store_cb = on_store_file_buf_cb;
cache_mng_file_name (cmng, path, sizeof (path), ino);
fd = open (path, O_WRONLY|O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
if (fd < 0) {
LOG_err (CMNG_LOG, INO_H"Failed to create / open file for writing! Path: %s", INO_T (ino), path);
if (context->cb.store_cb)
context->cb.store_cb (FALSE, context->user_ctx);
cache_context_destroy (context);
return;
}
res = pwrite(fd, buf, size, off);
close (fd);
entry = g_hash_table_lookup (cmng->h_entries, GUINT_TO_POINTER (ino));
if (!entry) {
entry = cache_entry_create (ino);
g_queue_push_head (cmng->q_lru, entry);
entry->ll_lru = g_queue_peek_head_link (cmng->q_lru);
g_hash_table_insert (cmng->h_entries, GUINT_TO_POINTER (ino), entry);
}
old_length = range_length (entry->avail_range);
range_add (entry->avail_range, off, range_size);
new_length = range_length (entry->avail_range);
if (new_length >= old_length)
cmng->size += new_length - old_length;
else {
LOG_err (CMNG_LOG, INO_H"New length is less than the old length !: %"G_GUINT64_FORMAT" <= %"G_GUINT64_FORMAT,
INO_T (ino), new_length, old_length);
}
// update modification time
entry->modification_time = time (NULL);
context->success = (res == (ssize_t) size);
LOG_debug (CMNG_LOG, INO_H"Written [%"OFF_FMT":%zu] bytes, result: %s",
INO_T (ino), off, size, context->success ? "OK" : "Failed");
context->ev = event_new (application_get_evbase (cmng->app), -1, 0,
cache_write_cb, context);
// fire this event at once
event_active (context->ev, 0, 0);
event_add (context->ev, NULL);
}