/**
* Internal method to handle a command that relies
* on a filter name and multiple keys, responses are handled using
* handle_multi_response.
*/
static void handle_filt_multi_key_cmd(
bloom_conn_handler *handle,
char *args, int args_len,
int(*filtmgr_func)(bloom_filtmgr *, char*, char **, int, char*)) {
#define CHECK_ARG_ERR() { \
handle_client_err(handle->conn, (char*)&FILT_KEY_NEEDED, FILT_KEY_NEEDED_LEN); \
return; \
}
// If we have no args, complain.
if (!args) CHECK_ARG_ERR();
// Setup the buffers
char *key_buf[MULTI_OP_SIZE];
char result_buf[MULTI_OP_SIZE];
// Scan all the keys
char *key;
int key_len;
int err = buffer_after_terminator(args, args_len, ' ', &key, &key_len);
if (err || key_len <= 1) CHECK_ARG_ERR();
// Parse any options
char *curr_key = key;
int index = 0;
#define HAS_ANOTHER_KEY() (curr_key && *curr_key != '\0')
while (HAS_ANOTHER_KEY()) {
// Adds a zero terminator to the current key, scans forward
buffer_after_terminator(key, key_len, ' ', &key, &key_len);
// Set the key
key_buf[index] = curr_key;
// Advance to the next key
curr_key = key;
index++;
// If we have filled the buffer, check now
if (index == MULTI_OP_SIZE) {
// Handle the keys now
int res = filtmgr_func(handle->mgr, args, (char**)&key_buf, index, (char*)&result_buf);
res = handle_multi_response(handle, res, index, (char*)&result_buf, !HAS_ANOTHER_KEY());
if (res) return;
// Reset the index
index = 0;
}
}
// Handle any remaining keys
if (index) {
int res = filtmgr_func(handle->mgr, args, key_buf, index, result_buf);
handle_multi_response(handle, res, index, (char*)&result_buf, 1);
}
}
/**
* Invoked by the networking layer when there is new
* data to be handled. The connection handler should
* consume all the input possible, and generate responses
* to all requests.
* @arg handle The connection related information
* @return 0 on success.
*/
int handle_client_connect(statsite_conn_handler *handle) {
// Look for the next command line
char *buf, *key, *val_str, *type_str, *endptr;
metric_type type;
int buf_len, should_free, status, i, after_len;
double val;
while (1) {
status = extract_to_terminator(handle->conn, '\n', &buf, &buf_len, &should_free);
if (status == -1) return 0; // Return if no command is available
// Check for a valid metric
// Scan for the colon
status = buffer_after_terminator(buf, buf_len, ':', &val_str, &after_len);
if (!status) status |= buffer_after_terminator(val_str, after_len, '|', &type_str, &after_len);
if (status == 0) {
// Convert the type
switch (*type_str) {
case 'c':
type = COUNTER;
break;
case 'm':
type = TIMER;
break;
case 'k':
type = KEY_VAL;
break;
default:
type = UNKNOWN;
}
// Convert the value to a double
endptr = NULL;
val = strtod(val_str, &endptr);
// Store the sample if we did the conversion
if (val != 0 || endptr != val_str) {
metrics_add_sample(GLOBAL_METRICS, type, buf, val);
} else {
syslog(LOG_WARNING, "Failed value conversion! Input: %s", val_str);
}
} else {
syslog(LOG_WARNING, "Failed parse metric! Input: %s", buf);
}
// Make sure to free the command buffer if we need to
if (should_free) free(buf);
}
return 0;
}
/**
* Determines the client command.
* @arg cmd_buf A command buffer
* @arg buf_len The length of the buffer
* @arg arg_buf Output. Sets the start address of the command arguments.
* @arg arg_len Output. Sets the length of arg_buf.
* @return The conn_cmd_type enum value.
* UNKNOWN if it doesn't match anything supported, or a proper command.
*/
static conn_cmd_type determine_client_command(char *cmd_buf, int buf_len, char **arg_buf, int *arg_len) {
// Check if we are ending with \r, and remove it.
if (cmd_buf[buf_len-2] == '\r') {
cmd_buf[buf_len-2] = '\0';
buf_len -= 1;
}
// Scan for a space. This will setup the arg_buf and arg_len
// if we do find the terminator. It will also insert a null terminator
// at the space, so we can compare the cmd_buf to the commands.
buffer_after_terminator(cmd_buf, buf_len, ' ', arg_buf, arg_len);
// Search for the command
conn_cmd_type type = UNKNOWN;
#define CMD_MATCH(name) (strcmp(name, cmd_buf) == 0)
if (CMD_MATCH("check") || CMD_MATCH("c")) { type = CHECK; }
else if (CMD_MATCH("multi") || CMD_MATCH("m")) { type = CHECK_MULTI; }
else if (CMD_MATCH("get") || CMD_MATCH("g")) { type = GET; }
else if (CMD_MATCH("set") || CMD_MATCH("s")) { type = SET; }
else if (CMD_MATCH("bulk") || CMD_MATCH("b")) { type = SET_MULTI; }
else if (CMD_MATCH("list") ) { type = LIST; }
else if (CMD_MATCH("info") ) { type = INFO; }
else if (CMD_MATCH("create") ) { type = CREATE; }
else if (CMD_MATCH("drop") ) { type = DROP; }
else if (CMD_MATCH("close") ) { type = CLOSE; }
else if (CMD_MATCH("clear") ) { type = CLEAR; }
else if (CMD_MATCH("flush") ) { type = FLUSH; }
return type;
}
/**
* Internal method to handle a command that relies
* on a filter name and a single key, responses are handled using
* handle_multi_response.
*/
static void handle_filt_cmd(bloom_conn_handler *handle, char *args, int args_len,
int(*filtmgr_func)(bloom_filtmgr *, char*)) {
// If we have no args, complain.
if (!args) {
handle_client_err(handle->conn, (char*)&FILT_NEEDED, FILT_NEEDED_LEN);
return;
}
// Scan past the filter name
char *key;
int key_len;
int after = buffer_after_terminator(args, args_len, ' ', &key, &key_len);
if (after == 0) {
handle_client_err(handle->conn, (char*)&UNEXPECTED_ARGS, UNEXPECTED_ARGS_LEN);
return;
}
// Call into the filter manager
int res = filtmgr_func(handle->mgr, args);
switch (res) {
case 0:
handle_client_resp(handle->conn, (char*)DONE_RESP, DONE_RESP_LEN);
break;
case -1:
handle_client_resp(handle->conn, (char*)FILT_NOT_EXIST, FILT_NOT_EXIST_LEN);
break;
case -2:
handle_client_resp(handle->conn, (char*)FILT_NOT_PROXIED, FILT_NOT_PROXIED_LEN);
break;
default:
INTERNAL_ERROR();
break;
}
}
/**
* Internal method to handle a command that relies
* on a filter name and a single key, responses are handled using
* handle_multi_response.
*/
static void handle_filt_key_cmd(
bloom_conn_handler *handle,
char *args, int args_len,
int(*filtmgr_func)(bloom_filtmgr *, char*, char **, int, char*)) {
#define CHECK_ARG_ERR() { \
handle_client_err(handle->conn, (char*)&FILT_KEY_NEEDED, FILT_KEY_NEEDED_LEN); \
return; \
}
// If we have no args, complain.
if (!args) CHECK_ARG_ERR();
// Scan past the filter name
char *key;
int key_len;
int err = buffer_after_terminator(args, args_len, ' ', &key, &key_len);
if (err || key_len <= 1) CHECK_ARG_ERR();
// Setup the buffers
char *key_buf[] = {key};
char result_buf[1];
// Call into the filter manager
int res = filtmgr_func(handle->mgr, args, (char**)&key_buf, 1, (char*)&result_buf);
handle_multi_response(handle, res, 1, (char*)&result_buf, 1);
}
static void handle_info_cmd(bloom_conn_handler *handle, char *args, int args_len) {
// If we have no args, complain.
if (!args) {
handle_client_err(handle->conn, (char*)&FILT_NEEDED, FILT_NEEDED_LEN);
return;
}
// Scan past the filter name
char *key;
int key_len;
int after = buffer_after_terminator(args, args_len, ' ', &key, &key_len);
if (after == 0) {
handle_client_err(handle->conn, (char*)&UNEXPECTED_ARGS, UNEXPECTED_ARGS_LEN);
return;
}
// Create output buffers
char *output[] = {(char*)&START_RESP, NULL, (char*)&END_RESP};
int lens[] = {START_RESP_LEN, 0, END_RESP_LEN};
// Invoke the callback to get the filter stats
int res = filtmgr_filter_cb(handle->mgr, args, info_filter_cb, &output[1]);
// Check for no filter
if (res != 0) {
switch (res) {
case -1:
handle_client_resp(handle->conn, (char*)FILT_NOT_EXIST, FILT_NOT_EXIST_LEN);
break;
default:
INTERNAL_ERROR();
break;
}
return;
}
// Adjust the buffer size
lens[1] = strlen(output[1]);
// Write out the bufs
send_client_response(handle->conn, (char**)&output, (int*)&lens, 3);
free(output[1]);
}
/**
* Invoked to handle ASCII commands. This is the default
* mode for statsite, to be backwards compatible with statsd
* @arg handle The connection related information
* @return 0 on success.
*/
static int handle_ascii_client_connect(statsite_conn_handler *handle) {
// Look for the next command line
char *buf, *key, *val_str, *type_str, *sample_str, *endptr;
metric_type type;
int buf_len, should_free, status, i, after_len;
double val, sample_rate;
while (1) {
status = extract_to_terminator(handle->conn, '\n', &buf, &buf_len, &should_free);
if (status == -1) return 0; // Return if no command is available
// Check for a valid metric
// Scan for the colon
status = buffer_after_terminator(buf, buf_len, ':', &val_str, &after_len);
if (likely(!status)) status |= buffer_after_terminator(val_str, after_len, '|', &type_str, &after_len);
if (unlikely(status)) {
syslog(LOG_WARNING, "Failed parse metric! Input: %s", buf);
goto ERR_RET;
}
// Convert the type
switch (*type_str) {
case 'c':
type = COUNTER;
break;
case 'm':
type = TIMER;
break;
case 'k':
type = KEY_VAL;
break;
case 'g':
type = GAUGE;
// Check if this is a delta update
switch (*val_str) {
case '+':
// Advance past the + to avoid breaking str2double
val_str++;
case '-':
type = GAUGE_DELTA;
}
break;
case 's':
type = SET;
break;
default:
type = UNKNOWN;
syslog(LOG_WARNING, "Received unknown metric type! Input: %c", *type_str);
goto ERR_RET;
}
// Increment the number of inputs received
if (GLOBAL_CONFIG->input_counter)
metrics_add_sample(GLOBAL_METRICS, COUNTER, GLOBAL_CONFIG->input_counter, 1);
// Fast track the set-updates
if (type == SET) {
metrics_set_update(GLOBAL_METRICS, buf, val_str);
goto END_LOOP;
}
// Convert the value to a double
val = str2double(val_str, &endptr);
if (unlikely(endptr == val_str)) {
syslog(LOG_WARNING, "Failed value conversion! Input: %s", val_str);
goto ERR_RET;
}
// Handle counter sampling if applicable
if (type == COUNTER && !buffer_after_terminator(type_str, after_len, '@', &sample_str, &after_len)) {
sample_rate = str2double(sample_str, &endptr);
if (unlikely(endptr == sample_str)) {
syslog(LOG_WARNING, "Failed sample rate conversion! Input: %s", sample_str);
goto ERR_RET;
}
if (sample_rate > 0 && sample_rate <= 1) {
// Magnify the value
val = val * (1.0 / sample_rate);
}
}
// Store the sample
metrics_add_sample(GLOBAL_METRICS, type, buf, val);
END_LOOP:
// Make sure to free the command buffer if we need to
if (should_free) free(buf);
}
return 0;
ERR_RET:
if (should_free) free(buf);
return -1;
}
static void handle_get_cmd(bloom_conn_handler *handle, char *args, int args_len) {
#define CHECK_KEY_ERR() { \
handle_client_err(handle->conn, (char*)&KEY_NEEDED, FILT_KEY_NEEDED_LEN); \
return; \
}
// If we have no args, complain.
if (!args) CHECK_KEY_ERR();
// List all the filters
bloom_filter_list_head *head;
int res = filtmgr_list_filters(handle->mgr, NULL, &head);
if (res != 0) {
INTERNAL_ERROR();
return;
}
// setup list node
bloom_filter_list *node;
// Scan all the keys
char *key = args;
int key_len = args_len;
// Parse any options
char *curr_key = key;
int index = 0;
// count filters (segmets)
int filters_count = 0;
node = head->head;
while(node) { filters_count++; node = node->next; }
// count keys
int keys_count = 1;
for(int i=0; i<args_len; i++) if(args[i] == ' ') keys_count++;
// Setup the buffers
char *key_buf[keys_count];
char result_buf[keys_count];
// result matrix
char *key_filter_mat[keys_count][filters_count];
memset(key_filter_mat, 0, sizeof(key_filter_mat));
// counter for positive rezults
int key_filter_len[keys_count];
memset(key_filter_len, 0, sizeof(key_filter_len));
node = head->head;
while(node) {
int j = 0;
// Check buffer of keys
#define REQUEST_BUF() { \
int err = filtmgr_check_keys( \
handle->mgr, \
node->filter_name, \
(char**)&key_buf, \
index, \
(char*)&result_buf); \
if (err) { \
INTERNAL_ERROR(); \
return; \
}; \
for(int i=0; i<index; i++, j++) { \
if(result_buf[i] == 1) \
key_filter_mat[j][key_filter_len[j]++] = node->filter_name; \
}; \
index = 0; \
}
#define HAS_ANOTHER_KEY() (curr_key && *curr_key != '\0')
while (HAS_ANOTHER_KEY()) {
// Adds a zero terminator to the current key, scans forward
buffer_after_terminator(key, key_len, ' ', &key, &key_len);
// Set the key
key_buf[index] = curr_key;
// Advance to the next key
curr_key = key;
// If we have filled the buffer, check now
if (index++ == MULTI_OP_SIZE) REQUEST_BUF();
}
// Handle any remaining keys
if (index) REQUEST_BUF();
// revert args termintators back to \32 because buffer_affter_terminatoir() changes it
for(int i=0; i<args_len-1;i++) if(args[i]=='\0') args[i] = ' ';
curr_key = key = args;
key_len = args_len;
index = 0;
node = node->next;
}
// collect key names
char *key_names[keys_count];
char **key_names_it = key_names;
while (HAS_ANOTHER_KEY()) {
*(key_names_it++) = key;
buffer_after_terminator(key, key_len, ' ', &key, &key_len);
curr_key = key;
}
// send key filters
int buf_len = 1024*1024;
char *send_buf = (char*) malloc(buf_len);
for(int i=0; i<keys_count; i++) {
strcat(send_buf, key_names[i]);
int l = key_filter_len[i]-1;
char *d = l>=0 ? "\t" : "\n";
strcat(send_buf, d);
for(int j=0; j<=l; j++) {
if(j) strcat(send_buf, "/");
strcat(send_buf, key_filter_mat[i][j]);
}
strcat(send_buf, "\n");
}
handle_client_resp(handle->conn, send_buf, strlen(send_buf));
// Respond
handle_client_resp(handle->conn, (char*)DONE_RESP, DONE_RESP_LEN);
// Cleanup
filtmgr_cleanup_list(head);
free(send_buf);
}
/**
* Internal command used to handle filter creation.
*/
static void handle_create_cmd(bloom_conn_handler *handle, char *args, int args_len) {
// If we have no args, complain.
if (!args) {
handle_client_err(handle->conn, (char*)&FILT_NEEDED, FILT_NEEDED_LEN);
return;
}
// Scan for options after the filter name
char *options;
int options_len;
int res = buffer_after_terminator(args, args_len, ' ', &options, &options_len);
// Verify the filter name is valid
char *filter_name = args;
if (regexec(&VALID_FILTER_NAMES_RE, filter_name, 0, NULL, 0) != 0) {
handle_client_err(handle->conn, (char*)&BAD_FILT_NAME, BAD_FILT_NAME_LEN);
return;
}
// Parse the options
bloom_config *config = NULL;
int err = 0;
if (res == 0) {
// Make a new config store, copy the current
config = malloc(sizeof(bloom_config));
memcpy(config, handle->config, sizeof(bloom_config));
// Parse any options
char *param = options;
while (param) {
// Adds a zero terminator to the current param, scans forward
buffer_after_terminator(options, options_len, ' ', &options, &options_len);
// Check for the custom params
int match = 0;
match |= sscanf(param, "capacity=%llu", (unsigned long long*)&config->initial_capacity);
match |= sscanf(param, "prob=%lf", &config->default_probability);
match |= sscanf(param, "in_memory=%d", &config->in_memory);
// Check if there was no match
if (!match) {
err = 1;
handle_client_err(handle->conn, (char*)&BAD_ARGS, BAD_ARGS_LEN);
break;
}
// Advance to the next param
param = options;
}
// Validate the params
int invalid_config = 0;
invalid_config |= sane_initial_capacity(config->initial_capacity);
invalid_config |= sane_default_probability(config->default_probability);
invalid_config |= sane_in_memory(config->in_memory);
// Barf if the configs are bad
if (invalid_config) {
err = 1;
handle_client_err(handle->conn, (char*)&BAD_ARGS, BAD_ARGS_LEN);
}
}
// Clean up an leave on errors
if (err) {
if (config) free(config);
return;
}
// Create a new filter
res = filtmgr_create_filter(handle->mgr, filter_name, config);
switch (res) {
case 0:
handle_client_resp(handle->conn, (char*)DONE_RESP, DONE_RESP_LEN);
break;
case -1:
handle_client_resp(handle->conn, (char*)EXISTS_RESP, EXISTS_RESP_LEN);
if (config) free(config);
break;
case -3:
handle_client_resp(handle->conn, (char*)DELETE_IN_PROGRESS, DELETE_IN_PROGRESS_LEN);
if (config) free(config);
break;
default:
INTERNAL_ERROR();
if (config) free(config);
break;
}
}
/**
* Invoked to handle ASCII commands. This is the default
* mode for statsite-proxy, to be backwards compatible with statsd
* @arg handle The connection related information
* @return 0 on success.
*/
static int handle_ascii_client_connect(statsite_proxy_conn_handler *handle) {
// Look for the next command line
char *buf, *val_str, *type_str;
metric_type type;
int buf_len, should_free, status, after_len, res;
while (1) {
status = extract_to_terminator(handle->conn, '\n', &buf, &buf_len, &should_free);
if (status == -1) return 0; // Return if no command is available
// Stack allocate space for message buffer
char* proxy_msg = alloca((buf_len)* sizeof(char));
strncpy(proxy_msg, buf, buf_len);
// Make sure to terminate strings
*(proxy_msg + buf_len-1) = '\n';
// Check for a valid metric
// Scan for the colon
status = buffer_after_terminator(buf, buf_len, ':', &val_str, &after_len);
if (!status) status |= buffer_after_terminator(val_str, after_len, '|', &type_str, &after_len);
if (status == 0) {
// Convert the type
switch (*type_str) {
case 'c':
type = COUNTER;
break;
case 'm':
type = TIMER;
break;
case 'k':
case 'g':
type = KEY_VAL;
break;
default:
type = UNKNOWN;
}
// Route metric via consistent hash based on metric key
void* conn;
res = proxy_get_route_conn(handle->proxy, buf, &conn);
if (res != 0) {
syslog(LOG_WARNING, "Failed to find metric route! Metric key: %s", buf);
}
// Forward metric
res = send_proxy_msg(conn, proxy_msg, buf_len);
if (res != 0) {
syslog(LOG_WARNING, "Failed to route metric route! Metric key: %s", buf);
}
} else {
syslog(LOG_WARNING, "Failed parse metric! Input: %s", buf);
}
// Make sure to free the command buffer if we need to
if (should_free) free(buf);
}
return 0;
}