cache_t *lru_cache_load(void *arg, inip_file_t *fd, char *grp, data_attr_t *da, int timeout)
{
cache_t *c;
cache_lru_t *cp;
int dt;
if (grp == NULL) grp = "cache-lru";
//** Create the default structure
c = lru_cache_create(arg, da, timeout);
cp = (cache_lru_t *)c->fn.priv;
cache_lock(c);
cp->max_bytes = inip_get_integer(fd, grp, "max_bytes", cp->max_bytes);
cp->dirty_fraction = inip_get_double(fd, grp, "dirty_fraction", cp->dirty_fraction);
cp->dirty_bytes_trigger = cp->dirty_fraction * cp->max_bytes;
c->default_page_size = inip_get_integer(fd, grp, "default_page_size", c->default_page_size);
dt = inip_get_integer(fd, grp, "dirty_max_wait", apr_time_sec(cp->dirty_max_wait));
cp->dirty_max_wait = apr_time_make(dt, 0);
c->max_fetch_fraction = inip_get_double(fd, grp, "max_fetch_fraction", c->max_fetch_fraction);
c->max_fetch_size = c->max_fetch_fraction * cp->max_bytes;
c->write_temp_overflow_fraction = inip_get_double(fd, grp, "write_temp_overflow_fraction", c->write_temp_overflow_fraction);
c->write_temp_overflow_size = c->write_temp_overflow_fraction * cp->max_bytes;
c->n_ppages = inip_get_integer(fd, grp, "ppages", c->n_ppages);
log_printf(0, "COP size=" XOT "\n", c->write_temp_overflow_size);
cache_unlock(c);
return(c);
}
int tbx_ns_chksum_write_flush(tbx_ns_t *ns)
{
char chksum_value[CHKSUM_MAX_SIZE];
int err, n;
tbx_tbuf_t buf;
log_printf(15, "ns_write_chksum_flush: injecting chksum! ns=%d type=%d bytesleft=" I64T " bsize=" I64T "\n",
tbx_ns_getid(ns), tbx_chksum_type(&(ns->write_chksum.chksum)), ns->write_chksum.bytesleft, ns->write_chksum.blocksize);
tbx_log_flush();
if (ns_write_chksum_state(ns) == 0) return(0);
if (ns->write_chksum.bytesleft == ns->write_chksum.blocksize) return(0); //** Nothing to do
n = tbx_chksum_size(&(ns->write_chksum.chksum), CHKSUM_DIGEST_HEX);
tbx_chksum_get(&(ns->write_chksum.chksum), CHKSUM_DIGEST_HEX, chksum_value);
ns->write_chksum.is_running = 0; //** Don't want to get in an endless loop
tbx_tbuf_single(&buf, n, chksum_value);
err = _write_netstream_block(ns, apr_time_now() + apr_time_make(5,0), &buf, 0, n, 0);
ns->write_chksum.is_running = 1;
if (err != 0) {
log_printf(10, "ns_write_chksum_flush: ns=%d Error writing chksum! error=%d\n", tbx_ns_getid(ns), err);
return(err);
}
chksum_value[n] = '\0';
log_printf(15, "ns_write_chksum_flush: ns=%d chksum_value=%s\n", tbx_ns_getid(ns), chksum_value);
log_printf(15, "ns_write_chksum_flush: end of routine! ns=%d\n err=%d", tbx_ns_getid(ns), err);
tbx_log_flush();
return(err);
}
void init_ibp_base_op(ibp_op_t *iop, char *logstr, int timeout_sec, int workload, char *hostport,
int cmp_size, int primary_cmd, int sub_cmd)
{
gop_command_op_t *cmd = &(iop->dop.cmd);
apr_time_t dt;
iop->primary_cmd = primary_cmd;
iop->sub_cmd = sub_cmd;
if (iop->ic->transfer_rate > 0) {
dt = (double)workload / iop->ic->transfer_rate;
if (dt < timeout_sec) dt = timeout_sec;
} else {
dt = timeout_sec;
}
cmd->timeout = apr_time_make(dt, 0);
cmd->retry_count = iop->ic->max_retry;
cmd->workload = workload;
cmd->hostport = hostport;
cmd->cmp_size = cmp_size;
cmd->send_command = NULL;
cmd->send_phase = NULL;
cmd->recv_phase = NULL;
cmd->on_submit = NULL;
cmd->before_exec = NULL;
cmd->destroy_command = NULL;
cmd->coalesced_ops = NULL;
cmd->connect_context = &(iop->ic->cc[primary_cmd]);
tbx_ns_chksum_init(&(iop->ncs));
}
cache_t *lru_cache_create(void *arg, data_attr_t *da, int timeout)
{
cache_t *cache;
cache_lru_t *c;
type_malloc_clear(cache, cache_t, 1);
type_malloc_clear(c, cache_lru_t, 1);
cache->fn.priv = c;
cache_base_create(cache, da, timeout);
cache->shutdown_request = 0;
c->stack = new_stack();
c->waiting_stack = new_stack();
c->pending_free_tasks = new_stack();
c->max_bytes = 100*1024*1024;
c->bytes_used = 0;
c->dirty_fraction = 0.1;
cache->n_ppages = 0;
cache->max_fetch_fraction = 0.1;
cache->max_fetch_size = cache->max_fetch_fraction * c->max_bytes;
cache->write_temp_overflow_used = 0;
cache->write_temp_overflow_fraction = 0.01;
cache->write_temp_overflow_size = cache->write_temp_overflow_fraction * c->max_bytes;
c->dirty_bytes_trigger = c->dirty_fraction * c->max_bytes;
c->dirty_max_wait = apr_time_make(1, 0);
c->flush_in_progress = 0;
c->limbo_pages = 0;
c->free_pending_tables = new_pigeon_coop("free_pending_tables", 50, sizeof(list_t *), cache->mpool, free_pending_table_new, free_pending_table_free);
c->free_page_tables = new_pigeon_coop("free_page_tables", 50, sizeof(page_table_t), cache->mpool, free_page_tables_new, free_page_tables_free);
cache->fn.create_empty_page = lru_create_empty_page;
cache->fn.adjust_dirty = lru_adjust_dirty;
cache->fn.destroy_pages = lru_pages_destroy;
cache->fn.cache_update = lru_update;
cache->fn.cache_miss_tag = lru_miss_tag;
cache->fn.s_page_access = lru_page_access;
cache->fn.s_pages_release = lru_pages_release;
cache->fn.destroy = lru_cache_destroy;
cache->fn.adding_segment = lru_adding_segment;
cache->fn.removing_segment = lru_removing_segment;
cache->fn.get_handle = cache_base_handle;
apr_thread_cond_create(&(c->dirty_trigger), cache->mpool);
thread_create_assert(&(c->dirty_thread), NULL, lru_dirty_thread, (void *)cache, cache->mpool);
return(cache);
}
static void test_anon(abts_case *tc, void *data)
{
apr_proc_t proc;
apr_status_t rv;
apr_shm_t *shm;
apr_size_t retsize;
int cnt, i;
int recvd;
rv = apr_shm_create(&shm, SHARED_SIZE, NULL, p);
APR_ASSERT_SUCCESS(tc, "Error allocating shared memory block", rv);
ABTS_PTR_NOTNULL(tc, shm);
retsize = apr_shm_size_get(shm);
ABTS_INT_EQUAL(tc, SHARED_SIZE, retsize);
boxes = apr_shm_baseaddr_get(shm);
ABTS_PTR_NOTNULL(tc, boxes);
rv = apr_proc_fork(&proc, p);
if (rv == APR_INCHILD) { /* child */
int num = msgwait(5, 0, N_BOXES);
/* exit with the number of messages received so that the parent
* can check that all messages were received.
*/
exit(num);
}
else if (rv == APR_INPARENT) { /* parent */
i = N_BOXES;
cnt = 0;
while (cnt++ < N_MESSAGES) {
if ((i-=3) < 0) {
i += N_BOXES; /* start over at the top */
}
msgput(i, MSG);
apr_sleep(apr_time_make(0, 10000));
}
}
else {
ABTS_FAIL(tc, "apr_proc_fork failed");
}
/* wait for the child */
rv = apr_proc_wait(&proc, &recvd, NULL, APR_WAIT);
ABTS_INT_EQUAL(tc, N_MESSAGES, recvd);
rv = apr_shm_destroy(shm);
APR_ASSERT_SUCCESS(tc, "Error destroying shared memory block", rv);
}
int tbx_ns_chksum_read_flush(tbx_ns_t *ns)
{
char ns_value[CHKSUM_MAX_SIZE], chksum_value[CHKSUM_MAX_SIZE];
int err, n;
tbx_tbuf_t buf;
log_printf(15, "ns_read_chksum_flush: Reading chksum! ns=%d type=%d bleft=" I64T " bsize=" I64T " state=%d\n",
tbx_ns_getid(ns), tbx_chksum_type(&(ns->read_chksum.chksum)), ns->read_chksum.bytesleft, ns->read_chksum.blocksize, ns_read_chksum_state(ns));
tbx_log_flush();
if (ns_read_chksum_state(ns) == 0) return(0);
if (ns->read_chksum.bytesleft == ns->read_chksum.blocksize) return(0); //** Nothing to do
n = tbx_chksum_size(&(ns->read_chksum.chksum), CHKSUM_DIGEST_HEX);
ns->read_chksum.is_running = 0; //** Don't want to get in an endless loop
tbx_tbuf_single(&buf, n, ns_value);
err = _read_netstream_block(ns, apr_time_now() + apr_time_make(5,0), &buf, 0, n, 0);
ns_value[n] = '\0';
ns->read_chksum.is_running = 1;
log_printf(15, "ns_read_chksum_flush: Finished reading chksum! ns=%d\n", tbx_ns_getid(ns));
tbx_log_flush();
if (err != 0) {
log_printf(10, "ns_read_chksum_flush: ns=%d Error reading chksum! error=%d\n", tbx_ns_getid(ns), err);
return(err);
}
tbx_chksum_get(&(ns->read_chksum.chksum), CHKSUM_DIGEST_HEX, chksum_value);
log_printf(15, "ns_read_chksum_flush: after tbx_chksum_get! ns=%d\n", tbx_ns_getid(ns));
tbx_log_flush();
err = (strncmp(chksum_value, ns_value, n) == 0) ? 0 : 1;
log_printf(15, "ns_read_chksum_flush: ns=%d ns_value=%s cmp=%d\n", tbx_ns_getid(ns), ns_value, err);
log_printf(15, "ns_read_chksum_flush: ns=%d chksum_value=%s\n", tbx_ns_getid(ns), chksum_value);
if (err != 0) {
log_printf(1, "ns_read_chksum_flush: ns=%d chksum error!\n", tbx_ns_getid(ns));
log_printf(1, "ns_read_chksum_flush: ns=%d ns_value=%s cmp=%d\n", tbx_ns_getid(ns), ns_value, err);
log_printf(1, "ns_read_chksum_flush: ns=%d chksum_value=%s\n", tbx_ns_getid(ns), chksum_value);
}
log_printf(15, "ns_read_chksum_flush: end of routine! ns=%d\n err=%d", tbx_ns_getid(ns), err);
tbx_log_flush();
return(err);
}
apr_time_t gop_get_end_time(gop_op_generic_t *gop, int *state)
{
apr_time_t end_time;
gop_command_op_t *cmd = &(gop->op->cmd);
if (*state == 0) {
*state = tbx_atomic_get(cmd->on_top);
if (*state == 0) {
end_time = apr_time_now() + apr_time_make(10,0); //** Default to 10 secs while percolating to the top
} else { //** We're on top so use the official end time
end_time = cmd->end_time;
}
} else {
end_time = cmd->end_time; //** This won't change after we're on top so no need to lock
}
return(end_time);
}
static void msgwait(int sleep_sec, int first_box, int last_box)
{
int i;
apr_time_t start = apr_time_now();
apr_interval_time_t sleep_duration = apr_time_from_sec(sleep_sec);
while (apr_time_now() - start < sleep_duration) {
for (i = first_box; i < last_box; i++) {
if (boxes[i].msgavail) {
fprintf(stdout, "received a message in box %d, message was: %s\n",
i, boxes[i].msg);
boxes[i].msgavail = 0; /* reset back to 0 */
}
}
apr_sleep(apr_time_make(0, 10000)); /* 10ms */
}
fprintf(stdout, "done waiting on mailboxes...\n");
}
void default_zsock_config(zsock_context_t *zc)
{
zc->min_idle = apr_time_make(30, 0); //** Connection minimum idle time before disconnecting
zc->min_threads = 1; //** Min and max threads allowed
zc->max_threads = 4; //** Max number of simultaneous connection
zc->max_connections = 128; //** Max number of connections across all connections
zc->max_wait = 30; //** Max time to wait and retry a connection
zc->wait_stable_time = 15; //** Time to wait before opening a new connection for a heavily loaded server
zc->abort_conn_attempts = 4; //** If this many failed connection requests occur in a row we abort
zc->check_connection_interval = 2; //** # of secs to wait between checks if we need more connections
zc->max_retry = 2;
int i;
for (i = 0; i < ZSOCK_MAX_NUM_CMDS; i++) {
zc->cc[i].conn_type = NS_TYPE_ZSOCK;
}
}
void *ds_ibp_warm_thread(apr_thread_t *th, void *data)
{
data_service_fn_t *dsf = (data_service_fn_t *)data;
ds_ibp_priv_t *ds = (ds_ibp_priv_t *)dsf->priv;
apr_time_t max_wait;
char *mcap;
apr_ssize_t hlen;
apr_hash_index_t *hi;
ibp_capset_t *w;
opque_t *q;
int dt, err;
dt = 60;
max_wait = apr_time_make(ds->warm_interval, 0);
apr_thread_mutex_lock(ds->lock);
while (ds->warm_stop == 0) {
//** Generate all the tasks
log_printf(10, "Starting auto-warming run\n");
q = new_opque();
for (hi=apr_hash_first(NULL, ds->warm_table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&mcap, &hlen, (void **)&w);
opque_add(q, new_ibp_modify_alloc_op(ds->ic, mcap, -1, ds->warm_duration, -1, dt));
log_printf(15, " warming: %s\n", mcap);
}
//** Wait until they all complete
err = opque_waitall(q);
log_printf(10, "opque_waitall=%d\n", err);
opque_free(q, OP_DESTROY); //** Clean up. Don;t care if we are successfull or not:)
//** Sleep until the next time or we get an exit request
apr_thread_cond_timedwait(ds->cond, ds->lock, max_wait);
}
apr_thread_mutex_unlock(ds->lock);
log_printf(10, "EXITING auto-warm thread\n");
return(NULL);
}
static int msgwait(int sleep_sec, int first_box, int last_box)
{
int i;
int recvd = 0;
apr_time_t start = apr_time_now();
apr_interval_time_t sleep_duration = apr_time_from_sec(sleep_sec);
while (apr_time_now() - start < sleep_duration) {
for (i = first_box; i < last_box; i++) {
if (boxes[i].msgavail && !strcmp(boxes[i].msg, MSG)) {
recvd++;
boxes[i].msgavail = 0; /* reset back to 0 */
/* reset the msg field. 1024 is a magic number and it should
* be a macro, but I am being lazy.
*/
memset(boxes[i].msg, 0, 1024);
}
}
apr_sleep(apr_time_make(0, 10000)); /* 10ms */
}
return recvd;
}
int sock_connection_request(net_sock_t *nsock, int timeout)
{
apr_int32_t n;
apr_interval_time_t dt;
const apr_pollfd_t *ret_fd;
network_sock_t *sock = (network_sock_t *)nsock;
//dt= apr_time_make(0, 100*1000);
//apr_sleep(dt);
if (sock == NULL) return(-1);
dt = apr_time_make(timeout,0);
n = 0;
apr_pollset_poll(sock->pollset, dt, &n, &ret_fd);
//int i=n;
//log_printf(15, "sock_connection_request: err=%d n=%d APR_SUCCESS=%d\n", err, i, APR_SUCCESS);
if (n == 1) {
return(1);
} else {
return(0);
}
return(-1);
}
NetStream_t *cmd_send(char *host, int port, char *cmd, char **res_buffer, int timeout)
{
int bufsize = 10*1024;
char buffer[bufsize];
char *bstate;
int err, failed;
int n, retry;
double swait;
Net_timeout_t dt;
NetStream_t *ns;
apr_time_t await, sec, usec;
*res_buffer = NULL;
//set_log_level(20);
//int loop =0;
do {
//loop++;
//printf("cmd_send: loop=%d\n", loop);
failed = 0;
//** Make the host connection
retry = 3;
do {
ns = new_netstream();
ns_config_sock(ns, -1);
set_net_timeout(&dt, 60, 0);
err = net_connect(ns, host, port, dt);
if (err != 0) {
destroy_netstream(ns);
printf("get_version: Can't connect to host! host=%s port=%d err=%d\n", host, port, err);
if (retry <= 0) {
printf("cmd_send: Aborting!\n");
return(NULL);
} else {
printf("cmd_send: Sleeping for 1 second and retrying(%d).\n", retry);
sleep(1);
retry--;
}
}
} while ((err != 0) && (retry > -1));
//** Send the command
dt = apr_time_now() + apr_time_make(timeout,0);
n = write_netstream_block(ns, dt, cmd, strlen(cmd));
//** Get the result line
set_net_timeout(&dt, timeout, 0);
n = readline_netstream(ns, buffer, bufsize, dt);
if (n == NS_OK) {
n = atoi(string_token(buffer, " ", &bstate, &err));
if (n != IBP_OK) {
if (n == IBP_E_OUT_OF_SOCKETS) {
swait = atof(string_token(NULL, " ", &bstate, &err));
printf("Depot is busy. Sleeping for %lf sec and retrying.\n", swait);
destroy_netstream(ns);
failed = 1;
sec = swait; usec = (swait-sec) * 1000000;
await = apr_time_make(sec, usec);
// printf("sec=" TT " usec=" TT " await=" TT "\n", sec, usec,await);
apr_sleep(await);
} else {
printf("Error %d returned!\n", n);
destroy_netstream(ns);
return(NULL);
}
}
} else {
//printf("cmd_send: readline_netstream=%d\n", n);
failed = 1;
}
} while (failed == 1);
*res_buffer = strdup(bstate);
return(ns);
}
int fd_connect(int *fd, const char *hostname, int port, int tcpsize, Net_timeout_t timeout)
{
struct sockaddr_in addr;
char in_addr[6];
int sfd, err;
fd_set wfd;
apr_time_t endtime;
Net_timeout_t to;
struct timeval tv;
log_printf(15, "fd_connect: Trying to make connection to Hostname: %s Port: %d\n", hostname, port);
*fd = -1;
if (hostname == NULL) {
log_printf(15, "fd_connect: lookup_host failed. Missing hostname! Port: %d\n",port);
return(1);
}
// Get ip address
if (lookup_host(hostname, in_addr, NULL) != 0) {
log_printf(15, "fd_connect: lookup_host failed. Hostname: %s Port: %d\n", hostname, port);
return(1);
}
// get the socket
sfd = socket(PF_INET, SOCK_STREAM, 0);
*fd = sfd;
if (sfd == -1) {
log_printf(10, "fd_connect: socket open failed. Hostname: %s Port: %d\n", hostname, port);
return(1);
}
// Configure it correctly
int flag=1;
if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (char*)&flag, sizeof(flag)) != 0) {
log_printf(0, "fd_connect: Can't configure SO_REUSEADDR!\n");
}
if (tcpsize > 0) {
log_printf(10, "fd_connect: Setting tcpbufsize=%d\n", tcpsize);
if (setsockopt(sfd, SOL_SOCKET, SO_SNDBUF, (char*)&tcpsize, sizeof(tcpsize)) != 0) {
log_printf(0, "fd_connect: Can't configure SO_SNDBUF to %d!\n", tcpsize);
}
if (setsockopt(sfd, SOL_SOCKET, SO_RCVBUF, (char*)&tcpsize, sizeof(tcpsize)) != 0) {
log_printf(0, "fd_connect: Can't configure SO_RCVBUF to %d!\n", tcpsize);
}
}
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
memcpy(&(addr.sin_addr), &in_addr, 4);
// memset(&(addr.sin_zero), 0, 8);
// Connect it
err = connect(sfd, (struct sockaddr *)&addr, sizeof(struct sockaddr));
if (err != 0) {
if (errno != EINPROGRESS) {
log_printf(0, "fd_connect: connect failed. Hostname: %s Port: %d err=%d errno: %d error: %s\n", hostname, port, err, errno, strerror(errno));
return(1);
}
// err = 0;
}
//Configure the socket for non-blocking I/O
if ((err = fcntl(sfd, F_SETFL, O_NONBLOCK)) == -1) {
log_printf(0, "fd_connect: Can't configure connection for non-blocking I/O!");
}
log_printf(20, "fd_connect: Before select time=" TT "\n", apr_time_now());
endtime = apr_time_now() + apr_time_make(5, 0);
do {
FD_ZERO(&wfd);
FD_SET(sfd, &wfd);
set_net_timeout(&to, 1, 0);
set_timeval(&tv, to);
err = select(sfd+1, NULL, &wfd, NULL, &tv);
log_printf(20, "fd_connect: After select err=%d\n", err);
if (err != 1) {
if (errno != EINPROGRESS) {
log_printf(0, "fd_connect: select failed. Hostname: %s Port: %d select=%d errno: %d error: %s\n", hostname, port, err, errno, strerror(errno));
return(1);
} else {
log_printf(10, "fd_connect: In progress..... time=" TT " Hostname: %s Port: %d select=%d errno: %d error: %s\n", apr_time_now(), hostname, port, err, errno, strerror(errno));
}
}
} while ((err != 1) && (apr_time_now() < endtime));
if (err != 1) { //** There were problems so return with an error
log_printf(0, "fd_connect: Couldn\'t make connection. Hostname: %s Port: %d select=%d errno: %d error: %s\n", hostname, port, err, errno, strerror(errno));
close(sfd);
return(1);
}
*fd = sfd;
return(0);
}
static ngx_inline ngx_int_t
ngx_http_modsecurity_load_headers_out(ngx_http_request_t *r)
{
ngx_http_modsecurity_ctx_t *ctx;
char *data;
request_rec *req;
u_char *content_type;
ngx_uint_t content_type_len;
ngx_http_variable_value_t *vv;
ngx_list_part_t *part;
ngx_table_elt_t *h;
ngx_uint_t i;
char *key, *value;
u_char *buf = NULL;
size_t size = 0;
ctx = ngx_http_get_module_ctx(r, ngx_http_modsecurity);
req = ctx->req;
req->status = r->headers_out.status;
req->status_line = (char *)ngx_pstrdup0(r->pool, &r->headers_out.status_line);
if (r->headers_out.charset.len) {
content_type_len = r->headers_out.content_type.len
+ r->headers_out.charset.len
+ ngx_strlen("; charset=") + 1;
content_type = ngx_palloc(r->pool, content_type_len);
if (content_type == NULL) {
return NGX_ERROR;
}
ngx_snprintf(content_type, content_type_len,
"%V; charset=%V%Z",
&r->headers_out.content_type,
&r->headers_out.charset);
r->headers_out.content_type.data = content_type;
r->headers_out.content_type.len = content_type_len;
}
/* deep copy */
part = &r->headers_out.headers.part;
h = part->elts;
for (i = 0; ; i++) {
if (i >= part->nelts) {
if (part->next == NULL)
break;
part = part->next;
h = part->elts;
i = 0;
}
size += h[i].key.len + h[i].value.len + 2;
buf = ngx_palloc(r->pool, size);
if (buf == NULL) {
return NGX_ERROR;
}
key = (char *)buf;
buf = ngx_cpymem(buf, h[i].key.data, h[i].key.len);
*buf++ = '\0';
value = (char *)buf;
buf = ngx_cpymem(buf, h[i].value.data, h[i].value.len);
*buf++ = '\0';
apr_table_addn(req->headers_out, key, value);
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ModSecurity: load headers out: \"%V: %V\"",
&h[i].key, &h[i].value);
}
for (i = 0; special_headers_out[i].name; i++) {
vv = ngx_http_get_variable(r, &special_headers_out[i].variable_name,
ngx_hash_key(special_headers_out[i].variable_name.data,
special_headers_out[i].variable_name.len));
if (vv && !vv->not_found) {
data = ngx_palloc(r->pool, vv->len + 1);
if (data == NULL) {
return NGX_ERROR;
}
ngx_memcpy(data,vv->data, vv->len);
data[vv->len] = '\0';
apr_table_setn(req->headers_out, special_headers_out[i].name, data);
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ModSecurity: load headers out: \"%s: %s\"",
special_headers_out[i].name, data);
}
}
req->content_type = apr_table_get(ctx->req->headers_out, "Content-Type");
req->content_encoding = apr_table_get(ctx->req->headers_out, "Content-Encoding");
data = (char *)apr_table_get(ctx->req->headers_out, "Content-Languages");
if(data != NULL)
{
ctx->req->content_languages = apr_array_make(ctx->req->pool, 1, sizeof(const char *));
*(const char **)apr_array_push(ctx->req->content_languages) = data;
}
/* req->chunked = r->chunked; may be useless */
req->clength = r->headers_out.content_length_n;
req->mtime = apr_time_make(r->headers_out.last_modified_time, 0);
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ModSecurity: load headers out done");
return NGX_OK;
}
static ngx_inline ngx_int_t
ngx_http_modsecurity_load_request(ngx_http_request_t *r)
{
ngx_http_modsecurity_ctx_t *ctx;
request_rec *req;
size_t root;
ngx_str_t path;
ngx_uint_t port;
struct sockaddr_in *sin;
#if (NGX_HAVE_INET6)
struct sockaddr_in6 *sin6;
#endif
ctx = ngx_http_get_module_ctx(r, ngx_http_modsecurity);
req = ctx->req;
/* request line */
req->method = (char *)ngx_pstrdup0(r->pool, &r->method_name);
/* TODO: how to use ap_method_number_of ?
* req->method_number = ap_method_number_of(req->method);
*/
req->method_number = ngx_http_modsecurity_method_number(r->method);
/* ngx_http_map_uri_to_path() allocates memory for terminating '\0' */
if (ngx_http_map_uri_to_path(r, &path, &root, 0) == NULL) {
return NGX_ERROR;
}
req->filename = (char *) path.data;
req->path_info = req->filename;
req->args = (char *)ngx_pstrdup0(r->pool, &r->args);
req->proto_num = r->http_major *1000 + r->http_minor;
req->protocol = (char *)ngx_pstrdup0(r->pool, &r->http_protocol);
req->request_time = apr_time_make(r->start_sec, r->start_msec);
req->the_request = (char *)ngx_pstrdup0(r->pool, &r->request_line);
req->unparsed_uri = (char *)ngx_pstrdup0(r->pool, &r->unparsed_uri);
req->uri = (char *)ngx_pstrdup0(r->pool, &r->uri);
req->parsed_uri.scheme = "http";
#if (NGX_HTTP_SSL)
if (r->connection->ssl) {
req->parsed_uri.scheme = "https";
}
#endif
req->parsed_uri.path = (char *)ngx_pstrdup0(r->pool, &r->uri);
req->parsed_uri.is_initialized = 1;
switch (r->connection->local_sockaddr->sa_family) {
#if (NGX_HAVE_INET6)
case AF_INET6:
sin6 = (struct sockaddr_in6 *) r->connection->local_sockaddr;
port = ntohs(sin6->sin6_port);
break;
#endif
#if (NGX_HAVE_UNIX_DOMAIN)
case AF_UNIX:
port = 0;
break;
#endif
default: /* AF_INET */
sin = (struct sockaddr_in *) r->connection->local_sockaddr;
port = ntohs(sin->sin_port);
break;
}
req->parsed_uri.port = port;
req->parsed_uri.port_str = ngx_pnalloc(r->pool, sizeof("65535"));
(void) ngx_sprintf((u_char *)req->parsed_uri.port_str, "%ui%c", port, '\0');
req->parsed_uri.query = r->args.len ? req->args : NULL;
req->parsed_uri.dns_looked_up = 0;
req->parsed_uri.dns_resolved = 0;
// req->parsed_uri.password = (char *)ngx_pstrdup0(r->pool, &r->headers_in.passwd);
// req->parsed_uri.user = (char *)ngx_pstrdup0(r->pool, &r->headers_in.user);
req->parsed_uri.fragment = (char *)ngx_pstrdup0(r->pool, &r->exten);
req->hostname = (char *)ngx_pstrdup0(r->pool, (ngx_str_t *)&ngx_cycle->hostname);
req->header_only = r->header_only ? r->header_only : (r->method == NGX_HTTP_HEAD);
return NGX_OK;
}
static ngx_inline ngx_int_t
ngx_http_modsecurity_save_request_body(ngx_http_request_t *r)
{
ngx_http_modsecurity_ctx_t *ctx;
apr_off_t content_length;
ngx_buf_t *buf;
ngx_http_core_srv_conf_t *cscf;
size_t size;
ngx_http_connection_t *hc;
ctx = ngx_http_get_module_ctx(r, ngx_http_modsecurity);
apr_brigade_length(ctx->brigade, 0, &content_length);
if (r->header_in->end - r->header_in->last >= content_length) {
/* use r->header_in */
if (ngx_buf_size(r->header_in)) {
/* move to the end */
ngx_memmove(r->header_in->pos + content_length,
r->header_in->pos,
ngx_buf_size(r->header_in));
}
if (apr_brigade_flatten(ctx->brigade,
(char *)r->header_in->pos,
(apr_size_t *)&content_length) != APR_SUCCESS) {
return NGX_ERROR;
}
apr_brigade_cleanup(ctx->brigade);
r->header_in->last += content_length;
return NGX_OK;
}
if (ngx_buf_size(r->header_in)) {
/*
* ngx_http_set_keepalive will reuse r->header_in if
* (r->header_in != c->buffer && r->header_in.last != r->header_in.end),
* so we need this code block.
* see ngx_http_set_keepalive, ngx_http_alloc_large_header_buffer
*/
cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module);
size = ngx_max(cscf->large_client_header_buffers.size,
(size_t)content_length + ngx_buf_size(r->header_in));
hc = r->http_connection;
#if defined(nginx_version) && nginx_version >= 1011011
if (hc->free && size == cscf->large_client_header_buffers.size) {
buf = hc->free->buf;
#else
if (hc->nfree && size == cscf->large_client_header_buffers.size) {
buf = hc->free[--hc->nfree];
#endif
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ModSecurity: use http free large header buffer: %p %uz",
buf->pos, buf->end - buf->last);
} else if (hc->nbusy < cscf->large_client_header_buffers.num) {
if (hc->busy == NULL) {
hc->busy = ngx_palloc(r->connection->pool,
cscf->large_client_header_buffers.num * sizeof(ngx_buf_t *));
}
if (hc->busy == NULL) {
return NGX_ERROR;
} else {
buf = ngx_create_temp_buf(r->connection->pool, size);
}
} else {
/* TODO: how to deal this case ? */
return NGX_ERROR;
}
} else {
buf = ngx_create_temp_buf(r->pool, (size_t) content_length);
}
if (buf == NULL) {
return NGX_ERROR;
}
if (apr_brigade_flatten(ctx->brigade, (char *)buf->pos,
(apr_size_t *)&content_length) != APR_SUCCESS) {
return NGX_ERROR;
}
apr_brigade_cleanup(ctx->brigade);
buf->last += content_length;
ngx_memcpy(buf->last, r->header_in->pos, ngx_buf_size(r->header_in));
buf->last += ngx_buf_size(r->header_in);
r->header_in = buf;
return NGX_OK;
}
static ngx_inline ngx_int_t
ngx_http_modsecurity_load_headers_out(ngx_http_request_t *r)
{
ngx_http_modsecurity_ctx_t *ctx;
char *data;
request_rec *req;
ngx_http_variable_value_t *vv;
ngx_list_part_t *part;
ngx_table_elt_t *h;
ngx_uint_t i;
char *key, *value;
u_char *buf = NULL;
size_t size = 0;
ctx = ngx_http_get_module_ctx(r, ngx_http_modsecurity);
req = ctx->req;
req->status = r->headers_out.status;
req->status_line = (char *)ngx_pstrdup0(r->pool, &r->headers_out.status_line);
/* deep copy */
part = &r->headers_out.headers.part;
h = part->elts;
for (i = 0; ; i++) {
if (i >= part->nelts) {
if (part->next == NULL)
break;
part = part->next;
h = part->elts;
i = 0;
}
size += h[i].key.len + h[i].value.len + 2;
buf = ngx_palloc(r->pool, size);
if (buf == NULL) {
return NGX_ERROR;
}
key = (char *)buf;
buf = ngx_cpymem(buf, h[i].key.data, h[i].key.len);
*buf++ = '\0';
value = (char *)buf;
buf = ngx_cpymem(buf, h[i].value.data, h[i].value.len);
*buf++ = '\0';
apr_table_addn(req->headers_out, key, value);
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ModSecurity: load headers out: \"%V: %V\"",
&h[i].key, &h[i].value);
}
for (i = 0; special_headers_out[i].name; i++) {
vv = ngx_http_get_variable(r, &special_headers_out[i].variable_name,
ngx_hash_key(special_headers_out[i].variable_name.data,
special_headers_out[i].variable_name.len));
if (vv && !vv->not_found) {
data = ngx_palloc(r->pool, vv->len + 1);
if (data == NULL) {
return NGX_ERROR;
}
ngx_memcpy(data,vv->data, vv->len);
data[vv->len] = '\0';
apr_table_setn(req->headers_out, special_headers_out[i].name, data);
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ModSecurity: load headers out: \"%s: %s\"",
special_headers_out[i].name, data);
}
}
req->content_type = apr_table_get(ctx->req->headers_out, "Content-Type");
req->content_encoding = apr_table_get(ctx->req->headers_out, "Content-Encoding");
data = (char *)apr_table_get(ctx->req->headers_out, "Content-Languages");
if(data != NULL)
{
ctx->req->content_languages = apr_array_make(ctx->req->pool, 1, sizeof(const char *));
*(const char **)apr_array_push(ctx->req->content_languages) = data;
}
/* req->chunked = r->chunked; may be useless */
req->clength = r->headers_out.content_length_n;
req->mtime = apr_time_make(r->headers_out.last_modified_time, 0);
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ModSecurity: load headers out done");
return NGX_OK;
}
static apr_status_t test_anon(apr_pool_t *parpool)
{
apr_status_t rv;
apr_pool_t *pool;
apr_shm_t *shm;
apr_size_t retsize;
pid_t pid;
int cnt, i, exit_int;
rv = apr_pool_create(&pool, parpool);
if (rv != APR_SUCCESS) {
fprintf(stderr, "Error creating child pool\n");
return rv;
}
printf("Creating anonymous shared memory block (%"
APR_SIZE_T_FMT " bytes)........", SHARED_SIZE);
rv = apr_shm_create(&shm, SHARED_SIZE, NULL, pool);
if (rv != APR_SUCCESS) {
fprintf(stderr, "Error allocating shared memory block\n");
return rv;
}
fprintf(stdout, "OK\n");
printf("Checking size...%" APR_SIZE_T_FMT " bytes...",
retsize = apr_shm_size_get(shm));
if (retsize != SHARED_SIZE) {
fprintf(stderr, "Error allocating shared memory block\n");
return rv;
}
fprintf(stdout, "OK\n");
printf("Allocating shared mbox memory for %d boxes ..............",
N_BOXES);
boxes = apr_shm_baseaddr_get(shm);
if (boxes == NULL) {
fprintf(stderr, "Error creating message boxes.\n");
return rv;
}
fprintf(stdout, "OK\n");
printf("Shared Process Test (child/parent)\n");
pid = fork();
if (pid == 0) { /* child */
msgwait(5, 0, N_BOXES);
exit(0);
}
else if (pid > 0) { /* parent */
i = N_BOXES;
cnt = N_MESSAGES;
while (--cnt > 0) {
if ((i-=3) < 0) {
i += N_BOXES; /* start over at the top */
}
msgput(i, "Sending a message\n");
apr_sleep(apr_time_make(0, 10000));
}
}
else {
printf("Error creating a child process\n");
return errno;
}
/* wait for the child */
printf("Waiting for child to exit.\n");
if (waitpid(pid, &exit_int, 0) < 0) {
return errno;
}
printf("Destroying shared memory segment...");
rv = apr_shm_destroy(shm);
if (rv != APR_SUCCESS) {
printf("FAILED\n");
return rv;
}
printf("OK\n");
apr_pool_destroy(pool);
return APR_SUCCESS;
}
static ngx_inline ngx_int_t
ngx_http_modsecurity_load_request(ngx_http_request_t *r)
{
ngx_http_modsecurity_ctx_t *ctx;
request_rec *req;
ngx_str_t str;
size_t root;
ngx_str_t path;
ctx = ngx_http_get_module_ctx(r, ngx_http_modsecurity);
req = ctx->req;
/* request line */
req->method = (char *)ngx_pstrdup0(r->pool, &r->method_name);
/* TODO: how to use ap_method_number_of ?
* req->method_number = ap_method_number_of(req->method);
*/
req->method_number = ngx_http_modsecurity_method_number(r->method);
/* ngx_http_map_uri_to_path() allocates memory for terminating '\0' */
if (ngx_http_map_uri_to_path(r, &path, &root, 0) == NULL) {
return NGX_ERROR;
}
req->filename = (char *) path.data;
req->path_info = req->filename;
req->args = (char *)ngx_pstrdup0(r->pool, &r->args);
req->proto_num = r->http_major *1000 + r->http_minor;
req->protocol = (char *)ngx_pstrdup0(r->pool, &r->http_protocol);
req->request_time = apr_time_make(r->start_sec, r->start_msec);
req->the_request = (char *)ngx_pstrdup0(r->pool, &r->request_line);
req->unparsed_uri = (char *)ngx_pstrdup0(r->pool, &r->unparsed_uri);
req->uri = (char *)ngx_pstrdup0(r->pool, &r->uri);
req->parsed_uri.scheme = "http";
#if (NGX_HTTP_SSL)
if (r->connection->ssl) {
req->parsed_uri.scheme = "https";
}
#endif
req->parsed_uri.path = (char *)ngx_pstrdup0(r->pool, &r->uri);
req->parsed_uri.is_initialized = 1;
str.data = r->port_start;
str.len = r->port_end - r->port_start;
req->parsed_uri.port = ngx_atoi(str.data, str.len);
req->parsed_uri.port_str = (char *)ngx_pstrdup0(r->pool, &str);
req->parsed_uri.query = r->args.len ? req->args : NULL;
req->parsed_uri.dns_looked_up = 0;
req->parsed_uri.dns_resolved = 0;
// req->parsed_uri.password = (char *)ngx_pstrdup0(r->pool, &r->headers_in.passwd);
// req->parsed_uri.user = (char *)ngx_pstrdup0(r->pool, &r->headers_in.user);
req->parsed_uri.fragment = (char *)ngx_pstrdup0(r->pool, &r->exten);
req->hostname = (char *)ngx_pstrdup0(r->pool, (ngx_str_t *)&ngx_cycle->hostname);
req->header_only = r->header_only ? r->header_only : (r->method == NGX_HTTP_HEAD);
return NGX_OK;
}
void *ongoing_heartbeat_thread(apr_thread_t *th, void *data)
{
mq_ongoing_t *on = (mq_ongoing_t *)data;
apr_time_t timeout = apr_time_make(on->check_interval, 0);
op_generic_t *gop;
mq_msg_t *msg;
ongoing_hb_t *oh;
ongoing_table_t *table;
apr_hash_index_t *hi, *hit;
opque_t *q;
char *id;
mq_msg_hash_t *remote_hash;
apr_time_t now;
apr_ssize_t id_len;
int n, k;
char *remote_host_string;
apr_thread_mutex_lock(on->lock);
n = 0;
do {
now = apr_time_now() - apr_time_from_sec(5); //** Give our selves a little buffer
log_printf(5, "Loop Start now=" TT "\n", apr_time_now());
q = new_opque();
// opque_start_execution(q);
for (hit = apr_hash_first(NULL, on->table); hit != NULL; hit = apr_hash_next(hit)) {
apr_hash_this(hit, (const void **)&remote_hash, &id_len, (void **)&table);
k = apr_hash_count(table->table);
if (log_level() > 1) {
remote_host_string = mq_address_to_string(table->remote_host);
log_printf(1, "host=%s count=%d\n", remote_host_string, k);
free(remote_host_string);
}
for (hi = apr_hash_first(NULL, table->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&id, &id_len, (void **)&oh);
log_printf(1, "id=%s now=" TT " next_check=" TT "\n", oh->id, apr_time_sec(apr_time_now()), apr_time_sec(oh->next_check));
if (now > oh->next_check) {
log_printf(1, "id=%s sending HEARTBEAT EXEC SUBMIT nows=" TT " hb=%d\n", oh->id, apr_time_sec(apr_time_now()), oh->heartbeat);
flush_log();
//** Form the message
msg = mq_make_exec_core_msg(table->remote_host, 1);
mq_msg_append_mem(msg, ONGOING_KEY, ONGOING_SIZE, MQF_MSG_KEEP_DATA);
mq_msg_append_mem(msg, oh->id, oh->id_len, MQF_MSG_KEEP_DATA);
mq_msg_append_mem(msg, NULL, 0, MQF_MSG_KEEP_DATA);
//** Make the gop
gop = new_mq_op(on->mqc, msg, ongoing_response_status, NULL, NULL, oh->heartbeat);
gop_set_private(gop, table);
opque_add(q, gop);
oh->in_progress = 1; //** Flag it as in progress so it doesn't get deleted behind the scenes
}
}
}
log_printf(5, "Loop end now=" TT "\n", apr_time_now());
//** Wait for it to complete
apr_thread_mutex_unlock(on->lock);
opque_waitall(q);
apr_thread_mutex_lock(on->lock);
//** Dec the counters
while ((gop = opque_waitany(q)) != NULL) {
log_printf(0, "gid=%d gotone status=%d now=" TT "\n", gop_id(gop), (gop_get_status(gop)).op_status, apr_time_sec(apr_time_now()));
table = gop_get_private(gop);
table->count--;
//** Update the next check
for (hi = apr_hash_first(NULL, table->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&id, &id_len, (void **)&oh);
oh->next_check = apr_time_now() + apr_time_from_sec(oh->heartbeat);
//** Check if we get rid of it
oh->in_progress = 0;
if (oh->count <= 0) { //** Need to delete it
apr_hash_set(table->table, id, id_len, NULL);
free(oh->id);
free(oh);
}
}
gop_free(gop, OP_DESTROY);
}
opque_free(q, OP_DESTROY);
now = apr_time_now();
log_printf(2, "sleeping %d now=" TT "\n", on->check_interval, now);
//** Sleep until time for the next heartbeat or time to exit
if (on->shutdown == 0) apr_thread_cond_timedwait(on->cond, on->lock, timeout);
n = on->shutdown;
now = apr_time_now() - now;
log_printf(2, "main loop bottom n=%d dt=" TT " sec=" TT "\n", n, now, apr_time_sec(now));
} while (n == 0);
log_printf(2, "CLEANUP\n");
for (hit = apr_hash_first(NULL, on->table); hit != NULL; hit = apr_hash_next(hit)) {
apr_hash_this(hit, (const void **)&remote_hash, &id_len, (void **)&table);
for (hi = apr_hash_first(NULL, table->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&id, &id_len, (void **)&oh);
apr_hash_set(table->table, id, id_len, NULL);
free(oh->id);
free(oh);
}
apr_hash_set(on->table, &(table->remote_host_hash), sizeof(mq_msg_hash_t), NULL);
mq_msg_destroy(table->remote_host);
free(table);
}
log_printf(2, "EXITING\n");
apr_thread_mutex_unlock(on->lock);
return(NULL);
}
void check_hportal_connections(host_portal_t *hp)
{
int i, j, total;
int n_newconn = 0;
int64_t curr_workload;
hportal_lock(hp);
curr_workload = hp->workload + hp->executing_workload;
//** Now figure out how many new connections are needed, if any
if (tbx_stack_count(hp->que) == 0) {
n_newconn = 0;
} else if (hp->n_conn < hp->min_conn) {
n_newconn = hp->min_conn - hp->n_conn;
} else {
n_newconn = (curr_workload / hp->context->max_workload) - hp->n_conn;
if (n_newconn < 0) n_newconn = 0;
if ((hp->n_conn+n_newconn) > hp->max_conn) {
n_newconn = hp->max_conn - hp->n_conn;
if (n_newconn < 0) n_newconn = 0;
}
}
i = n_newconn;
if (hp->sleeping_conn > 0) n_newconn = 0; //** IF sleeping don't spawn any more connections
total = n_newconn + hp->n_conn;
if (total > hp->stable_conn) {
if (apr_time_now() > hp->pause_until) {
hp->stable_conn++;
hp->pause_until = apr_time_now();
if (hp->stable_conn > hp->max_conn) {
hp->stable_conn = hp->max_conn;
n_newconn = 0;
} else if (hp->stable_conn == 0) {
hp->stable_conn = 1;
// n_newconn = 1;
n_newconn = (hp->pause_until == 0) ? 1 : 0;
} else {
n_newconn = (hp->n_conn < hp->max_conn) ? 1 : 0;
hp->pause_until = apr_time_now() + apr_time_make(hp->context->wait_stable_time, 0);
}
} else {
if (hp->n_conn > 0) {
n_newconn = 0;
} else if (hp->pause_until == 0) {
n_newconn = 1;
}
}
}
//** Do a check for invalid or down host
if (hp->invalid_host == 1) {
if ((hp->n_conn == 0) && (tbx_stack_count(hp->que) > 0)) n_newconn = 1; //** If no connections create one to sink the command
}
j = (hp->pause_until > apr_time_now()) ? 1 : 0;
log_printf(6, "check_hportal_connections: host=%s n_conn=%d sleeping=%d workload=" I64T " curr_wl=" I64T " exec_wl=" I64T " start_new_conn=%d new_conn=%d stable=%d stack_size=%d pause_until=" TT " now=" TT " pause_until_blocked=%d\n",
hp->skey, hp->n_conn, hp->sleeping_conn, hp->workload, curr_workload, hp->executing_workload, i, n_newconn, hp->stable_conn, tbx_stack_count(hp->que), hp->pause_until, apr_time_now(), j);
//** Update the total # of connections after the operation
//** n_conn is used instead of conn_list to prevent false positives on a dead depot
hp->n_conn = hp->n_conn + n_newconn;
hp->oops_check += n_newconn;
if (n_newconn < 0) hp->oops_neg = -100000;
hportal_unlock(hp);
//** Spawn the new connections if needed **
for (i=0; i<n_newconn; i++) {
spawn_new_connection(hp);
}
}