static void mysql_log_results(noit_module_t *self, noit_check_t *check) {
struct timeval duration, now;
mysql_check_info_t *ci = check->closure;
gettimeofday(&now, NULL);
sub_timeval(now, check->last_fire_time, &duration);
noit_stats_set_whence(check, &now);
noit_stats_set_duration(check, duration.tv_sec * 1000 + duration.tv_usec / 1000);
noit_stats_set_available(check, NP_UNAVAILABLE);
noit_stats_set_state(check, NP_BAD);
if(ci->error) noit_stats_set_status(check, ci->error);
else if(ci->timed_out) noit_stats_set_status(check, "timeout");
else if(ci->rv == 0) {
noit_stats_set_available(check, NP_AVAILABLE);
noit_stats_set_state(check, NP_GOOD);
noit_stats_set_status(check, "no rows, ok");
}
else {
noit_stats_set_available(check, NP_AVAILABLE);
noit_stats_set_state(check, NP_GOOD);
noit_stats_set_status(check, "got rows, ok");
}
if(ci->rv >= 0)
noit_stats_set_metric(check, "row_count", METRIC_INT32, &ci->rv);
if(ci->connect_duration)
noit_stats_set_metric(check, "connect_duration", METRIC_DOUBLE,
ci->connect_duration);
if(ci->query_duration)
noit_stats_set_metric(check, "query_duration", METRIC_DOUBLE,
ci->query_duration);
noit_check_set_stats(check);
}
static void postgres_ingest_stats(postgres_check_info_t *ci) {
if(ci->rv == PGRES_TUPLES_OK) {
/* metrics */
int nrows, ncols, i, j;
nrows = PQntuples(ci->result);
ncols = PQnfields(ci->result);
noit_stats_set_metric(&ci->current, "row_count", METRIC_INT32, &nrows);
for (i=0; i<nrows; i++) {
noitL(nldeb, "postgres: row %d [%d cols]:\n", i, ncols);
if(ncols<2) continue;
if(PQgetisnull(ci->result, i, 0)) continue;
for (j=1; j<ncols; j++) {
Oid coltype;
int iv, *piv;
int64_t lv, *plv;
double dv, *pdv;
char *sv;
char mname[128];
snprintf(mname, sizeof(mname), "%s`%s",
PQgetvalue(ci->result, i, 0), PQfname(ci->result, j));
coltype = PQftype(ci->result, j);
noitL(nldeb, "postgres: col %d (%s) type %d:\n", j, mname, coltype);
switch(coltype) {
case BOOLOID:
if(PQgetisnull(ci->result, i, j)) piv = NULL;
else {
iv = strcmp(PQgetvalue(ci->result, i, j), "f") ? 1 : 0;
piv = &iv;
}
noit_stats_set_metric(&ci->current, mname, METRIC_INT32, piv);
break;
case INT2OID:
case INT4OID:
case INT8OID:
if(PQgetisnull(ci->result, i, j)) plv = NULL;
else {
lv = strtoll(PQgetvalue(ci->result, i, j), NULL, 10);
plv = &lv;
}
noit_stats_set_metric(&ci->current, mname, METRIC_INT64, plv);
case FLOAT4OID:
case FLOAT8OID:
case NUMERICOID:
if(PQgetisnull(ci->result, i, j)) pdv = NULL;
else {
dv = atof(PQgetvalue(ci->result, i, j));
pdv = &dv;
}
noit_stats_set_metric(&ci->current, mname, METRIC_DOUBLE, pdv);
default:
if(PQgetisnull(ci->result, i, j)) sv = NULL;
else sv = PQgetvalue(ci->result, i, j);
noit_stats_set_metric(&ci->current, mname, METRIC_GUESS, sv);
break;
}
}
}
}
}
static int example_initiate(noit_module_t *self, noit_check_t *check,
noit_check_t *cause) {
struct example_check_info *ci = check->closure;
const char *limit = "0";
struct timeval now, diff;
BAIL_ON_RUNNING_CHECK(check);
check->flags |= NP_RUNNING;
mtev_hash_retrieve(check->config, "limit", strlen("limit"), (void **)&limit);
ci->limit = atoi(limit);
mtev_gettimeofday(&now, NULL);
sub_timeval(now, check->last_fire_time, &diff);
noit_stats_set_whence(check, &now);
noit_stats_set_duration(check, diff.tv_sec * 1000 + diff.tv_usec / 1000);
noit_stats_set_available(check, NP_AVAILABLE);
noit_stats_set_status(check, "hello world");
if(ci->limit) {
int value = (int)(lrand48() % ci->limit);
noit_stats_set_metric(check, "random", METRIC_INT32, &value);
noit_stats_set_state(check, NP_GOOD);
}
else {
noit_stats_set_metric(check, "random", METRIC_INT32, NULL);
noit_stats_set_state(check, NP_BAD);
}
noit_check_set_stats(check);
check->flags &= ~NP_RUNNING;
return 0;
}
static void ssh2_log_results(noit_module_t *self, noit_check_t *check) {
struct timeval duration;
ssh2_check_info_t *ci = check->closure;
noit_check_stats_clear(check, &check->stats.inprogress);
gettimeofday(&check->stats.inprogress.whence, NULL);
sub_timeval(check->stats.inprogress.whence, check->last_fire_time, &duration);
check->stats.inprogress.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
check->stats.inprogress.available = ci->available ? NP_AVAILABLE : NP_UNAVAILABLE;
check->stats.inprogress.state = ci->fingerprint[0] ? NP_GOOD : NP_BAD;
if(ci->error) check->stats.inprogress.status = ci->error;
else if(ci->timed_out) check->stats.inprogress.status = "timeout";
else if(ci->fingerprint[0]) check->stats.inprogress.status = ci->fingerprint;
else check->stats.inprogress.status = "internal error";
if(ci->fingerprint[0]) {
u_int32_t mduration = check->stats.inprogress.duration;
noit_stats_set_metric(check, &check->stats.inprogress, "duration", METRIC_UINT32, &mduration);
noit_stats_set_metric(check, &check->stats.inprogress, "fingerprint", METRIC_STRING,
ci->fingerprint);
}
noit_check_set_stats(check, &check->stats.inprogress);
noit_check_stats_clear(check, &check->stats.inprogress);
}
static void postgres_log_results(noit_module_t *self, noit_check_t *check) {
struct timeval duration;
postgres_check_info_t *ci = check->closure;
gettimeofday(&ci->current.whence, NULL);
sub_timeval(ci->current.whence, check->last_fire_time, &duration);
ci->current.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
ci->current.available = NP_UNAVAILABLE;
ci->current.state = NP_BAD;
if(ci->connect_duration)
noit_stats_set_metric(&ci->current, "connect_duration", METRIC_DOUBLE,
ci->connect_duration);
if(ci->query_duration)
noit_stats_set_metric(&ci->current, "query_duration", METRIC_DOUBLE,
ci->query_duration);
if(ci->error) ci->current.status = ci->error;
else if(ci->timed_out) ci->current.status = "timeout";
else if(ci->rv == PGRES_COMMAND_OK) {
ci->current.available = NP_AVAILABLE;
ci->current.state = NP_GOOD;
ci->current.status = "command ok";
}
else if(ci->rv == PGRES_TUPLES_OK) {
ci->current.available = NP_AVAILABLE;
ci->current.state = NP_GOOD;
ci->current.status = "tuples ok";
}
else ci->current.status = "internal error";
noit_check_set_stats(self, check, &ci->current);
}
static void ping_icmp_log_results(noit_module_t *self, noit_check_t *check) {
struct check_info *data;
double avail = 0.0, min = MAXFLOAT, max = 0.0, avg = 0.0, cnt;
int avail_needed = 100;
const char *config_val = NULL;
int i, points = 0;
char human_buffer[256];
struct timeval duration;
noit_check_stats_clear(check, &check->stats.inprogress);
data = (struct check_info *)check->closure;
for(i=0; i<data->expected_count; i++) {
if(data->turnaround[i] >= 0.0) {
points++;
avg += data->turnaround[i];
if(data->turnaround[i] > max) max = data->turnaround[i];
if(data->turnaround[i] < min) min = data->turnaround[i];
}
}
cnt = data->expected_count;
if(points == 0) {
min = 0.0 / 0.0;
max = 0.0 / 0.0;
avg = 0.0 / 0.0;
}
else {
avail = (float)points /cnt;
avg /= (float)points;
}
if(noit_hash_retr_str(check->config, "avail_needed", strlen("avail_needed"),
&config_val))
avail_needed = atoi(config_val);
snprintf(human_buffer, sizeof(human_buffer),
"cnt=%d,avail=%0.0f,min=%0.4f,max=%0.4f,avg=%0.4f",
(int)cnt, 100.0*avail, min, max, avg);
noitL(nldeb, "ping_icmp(%s) [%s]\n", check->target_ip, human_buffer);
gettimeofday(&check->stats.inprogress.whence, NULL);
sub_timeval(check->stats.inprogress.whence, check->last_fire_time, &duration);
check->stats.inprogress.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
check->stats.inprogress.available = (avail > 0.0) ? NP_AVAILABLE : NP_UNAVAILABLE;
check->stats.inprogress.state = (avail < ((float)avail_needed / 100.0)) ? NP_BAD : NP_GOOD;
check->stats.inprogress.status = human_buffer;
noit_stats_set_metric(check, &check->stats.inprogress, "count",
METRIC_INT32, &data->expected_count);
avail *= 100.0;
noit_stats_set_metric(check, &check->stats.inprogress, "available", METRIC_DOUBLE, &avail);
noit_stats_set_metric(check, &check->stats.inprogress, "minimum",
METRIC_DOUBLE, avail > 0.0 ? &min : NULL);
noit_stats_set_metric(check, &check->stats.inprogress, "maximum",
METRIC_DOUBLE, avail > 0.0 ? &max : NULL);
noit_stats_set_metric(check, &check->stats.inprogress, "average",
METRIC_DOUBLE, avail > 0.0 ? &avg : NULL);
noit_check_set_stats(check, &check->stats.inprogress);
noit_check_stats_clear(check, &check->stats.inprogress);
}
static void
update_check(noit_check_t *check, const char *key, char type,
double diff, double sample) {
u_int32_t one = 1, cnt = 1;
char buff[256];
statsd_closure_t *ccl;
stats_t *inprogress;
metric_t *m;
if (sample == 0.0) return; /* would be a div-by-zero */
if (check->closure == NULL) return;
ccl = check->closure;
inprogress = noit_check_get_stats_inprogress(check);
/* First key counts */
snprintf(buff, sizeof(buff), "%s`count", key);
m = noit_stats_get_metric(check, inprogress, buff);
if(!m) ccl->stats_count++;
if(m && m->metric_type == METRIC_UINT32 && m->metric_value.I != NULL) {
(*m->metric_value.I)++;
cnt = *m->metric_value.I;
check_stats_set_metric_hook_invoke(check, inprogress, m);
}
else
noit_stats_set_metric(check, buff, METRIC_UINT32, &one);
/* Next the actual data */
if(type == 'c') {
double v = diff * (1.0 / sample) / (check->period / 1000.0);
snprintf(buff, sizeof(buff), "%s`rate", key);
m = noit_stats_get_metric(check, inprogress, buff);
if(m && m->metric_type == METRIC_DOUBLE && m->metric_value.n != NULL) {
(*m->metric_value.n) += v;
check_stats_set_metric_hook_invoke(check, inprogress, m);
}
else
noit_stats_set_metric(check, buff, METRIC_DOUBLE, &v);
}
snprintf(buff, sizeof(buff), "%s`%s", key,
(type == 'c') ? "counter" : (type == 'g') ? "gauge" : "timing");
m = noit_stats_get_metric(check, inprogress, buff);
if(m && m->metric_type == METRIC_DOUBLE && m->metric_value.n != NULL) {
if(type == 'c') (*m->metric_value.n) += (diff * (1.0/sample));
else {
double new_avg = ((double)(cnt - 1) * (*m->metric_value.n) + diff) / (double)cnt;
(*m->metric_value.n) = new_avg;
}
check_stats_set_metric_hook_invoke(check, inprogress, m);
}
else
noit_stats_set_metric(check, buff, METRIC_DOUBLE, &diff);
}
static int
httptrap_yajl_cb_number(void *ctx, const char * numberVal,
size_t numberLen) {
char val[128];
struct rest_json_payload *json = ctx;
int rv;
if(json->depth<0) return 0;
rv = set_array_key(json);
if(json->last_special_key == 0x2) {
char *str;
str = malloc(numberLen+1);
memcpy(str, numberVal, numberLen);
str[numberLen] = '\0';
NEW_LV(json, str);
return 1;
}
if(rv) return 1;
if(json->last_special_key) return 0;
if(json->keys[json->depth]) {
if(numberLen > sizeof(val)-1) numberLen = sizeof(val)-1;
memcpy(val, numberVal, numberLen);
val[numberLen] = '\0';
noit_stats_set_metric(json->check,
json->keys[json->depth], METRIC_GUESS, val);
if(json->immediate)
noit_stats_log_immediate_metric(json->check,
json->keys[json->depth], METRIC_GUESS, val);
json->cnt++;
}
return 1;
}
static void dns_check_log_results(struct dns_check_info *ci) {
struct timeval duration;
double rtt;
gettimeofday(&ci->current.whence, NULL);
sub_timeval(ci->current.whence, ci->check->last_fire_time, &duration);
rtt = duration.tv_sec * 1000.0 + duration.tv_usec / 1000.0;
ci->current.duration = rtt;
ci->current.state = (ci->error || ci->nrr == 0) ? NP_BAD : NP_GOOD;
ci->current.available = ci->timed_out ? NP_UNAVAILABLE : NP_AVAILABLE;
if(ci->error) {
ci->current.status = strdup(ci->error);
}
else if(!ci->current.status) {
char buff[48];
snprintf(buff, sizeof(buff), "%d %s",
ci->nrr, ci->nrr == 1 ? "record" : "records");
ci->current.status = strdup(buff);
}
noit_stats_set_metric(ci->check, &ci->current, "rtt", METRIC_DOUBLE,
ci->timed_out ? NULL : &rtt);
noit_check_set_stats(ci->check, &ci->current);
if(ci->error) free(ci->error);
if(ci->current.status) free(ci->current.status);
ci->error = NULL;
memset(&ci->current, 0, sizeof(ci->current));
}
static int
httptrap_yajl_cb_boolean(void *ctx, int boolVal) {
int ival, rv;
struct rest_json_payload *json = ctx;
if(json->depth<0) {
_YD("[%3d] cb_boolean [BAD]\n", json->depth);
return 0;
}
rv = set_array_key(json);
if(json->last_special_key == HT_EX_VALUE) {
NEW_LV(json, strdup(boolVal ? "1" : "0"));
_YD("[%3d]*cb_boolean -> %s\n", json->depth, boolVal ? "true" : "false");
return 1;
}
if(json->last_special_key) return 0;
if(rv) return 1;
if(json->keys[json->depth]) {
ival = boolVal ? 1 : 0;
_YD("[%3d] cb_boolean -> %s\n", json->depth, boolVal ? "true" : "false");
noit_stats_set_metric(json->check,
json->keys[json->depth], METRIC_INT32, &ival);
if(json->immediate)
noit_stats_log_immediate_metric(json->check,
json->keys[json->depth], METRIC_INT32, &ival);
json->cnt++;
}
return 1;
}
static int
httptrap_yajl_cb_null(void *ctx) {
struct rest_json_payload *json = ctx;
int rv;
if(json->depth<0) {
_YD("[%3d] cb_null [BAD]\n", json->depth);
return 0;
}
rv = set_array_key(json);
if(json->last_special_key == HT_EX_VALUE) {
_YD("[%3d]*cb_null\n", json->depth);
NEW_LV(json, NULL);
return 1;
}
if(json->last_special_key) return 0;
if(rv) return 1;
if(json->keys[json->depth]) {
_YD("[%3d] cb_null\n", json->depth);
noit_stats_set_metric(json->check,
json->keys[json->depth], METRIC_INT32, NULL);
if(json->immediate)
noit_stats_log_immediate_metric(json->check,
json->keys[json->depth], METRIC_INT32, NULL);
json->cnt++;
}
return 1;
}
static void
histogram_sweep_calculations(struct histogram_config *conf, noit_check_t *check) {
mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
const char *metric_name;
int klen;
void *data;
mtev_hash_table *metrics;
stats_t *c;
double *out_q;
c = noit_check_get_stats_current(check);
/* Only need to do work if it's asked for */
if(!conf->mean && !conf->sum && conf->n_quantiles < 1) return;
metrics = noit_check_get_module_metadata(check, histogram_module_id);
if(!metrics) return;
out_q = alloca(sizeof(double *) * conf->n_quantiles);
u_int64_t s = time(NULL);
while(mtev_hash_next(metrics, &iter, &metric_name, &klen, &data)) {
char mname[1024];
histotier *ht = data;
if(ht->last_aggr == NULL) continue;
if(conf->mean) {
double mean_value;
snprintf(mname, sizeof(mname), "%s:mean", metric_name);
mean_value = hist_approx_mean(ht->last_aggr);
noit_stats_set_metric(check, mname, METRIC_DOUBLE, &mean_value);
}
if(conf->sum) {
double sum;
snprintf(mname, sizeof(mname), "%s:sum", metric_name);
sum = hist_approx_sum(ht->last_aggr);
noit_stats_set_metric(check, mname, METRIC_DOUBLE, &sum);
}
if(conf->n_quantiles) {
if(hist_approx_quantile(ht->last_aggr,
conf->quantiles, conf->n_quantiles, out_q) == 0) {
int i;
for(i=0;i<conf->n_quantiles;i++) {
snprintf(mname, sizeof(mname), "%s:q(%0.5f)", metric_name, conf->quantiles[i]);
noit_stats_set_metric(check, mname, METRIC_DOUBLE, &out_q[i]);
}
}
}
}
}
static void
update_check(noit_check_t *check, const char *key, char type,
double diff, double sample) {
u_int32_t one = 1;
char buff[256];
statsd_closure_t *ccl;
metric_t *m;
if (check->closure == NULL) return;
ccl = check->closure;
/* First key counts */
snprintf(buff, sizeof(buff), "%s`count", key);
m = noit_stats_get_metric(check, &ccl->current, buff);
if(!m) ccl->stats_count++;
if(m && m->metric_type == METRIC_UINT32 && m->metric_value.I != NULL) {
(*m->metric_value.I)++;
check_stats_set_metric_hook_invoke(check, &ccl->current, m);
}
else
noit_stats_set_metric(check, &ccl->current, buff, METRIC_UINT32, &one);
/* Next the actual data */
if(type == 'c') {
double v = diff * (1.0 / sample) / (check->period / 1000.0);
snprintf(buff, sizeof(buff), "%s`rate", key);
m = noit_stats_get_metric(check, &ccl->current, buff);
if(m && m->metric_type == METRIC_DOUBLE && m->metric_value.n != NULL) {
(*m->metric_value.n) += v;
check_stats_set_metric_hook_invoke(check, &ccl->current, m);
}
else
noit_stats_set_metric(check, &ccl->current, buff, METRIC_DOUBLE, &v);
}
snprintf(buff, sizeof(buff), "%s`%s", key,
(type == 'c') ? "counter" : (type == 'g') ? "gauge" : "timing");
m = noit_stats_get_metric(check, &ccl->current, buff);
if(m && m->metric_type == METRIC_DOUBLE && m->metric_value.n != NULL) {
(*m->metric_value.n) = diff;
check_stats_set_metric_hook_invoke(check, &ccl->current, m);
}
else
noit_stats_set_metric(check, &ccl->current, buff, METRIC_DOUBLE, &diff);
}
static int
httptrap_yajl_cb_string(void *ctx, const unsigned char * stringVal,
size_t stringLen) {
struct rest_json_payload *json = ctx;
char val[4096];
int rv;
if(json->depth<0) {
_YD("[%3d] cb_string [BAD]\n", json->depth);
return 0;
}
if(json->last_special_key == HT_EX_TS) /* handle ts */
return 1;
if(json->last_special_key == HT_EX_TAGS) /* handle tag */
return 1;
rv = set_array_key(json);
if(json->last_special_key == HT_EX_TYPE) {
if(stringLen != 1) return 0;
if(*stringVal == 'L' || *stringVal == 'l' ||
*stringVal == 'I' || *stringVal == 'i' ||
*stringVal == 'n' || *stringVal == 's') {
json->last_type = *stringVal;
json->saw_complex_type |= HT_EX_TYPE;
_YD("[%3d] cb_string { _type: %c }\n", json->depth, *stringVal);
return 1;
}
_YD("[%3d] cb_string { bad _type: %.*s }\n", json->depth,
(int)stringLen, stringVal);
return 0;
}
else if(json->last_special_key == HT_EX_VALUE) {
char *str;
str = malloc(stringLen+1);
memcpy(str, stringVal, stringLen);
str[stringLen] = '\0';
NEW_LV(json, str);
_YD("[%3d] cb_string { _value: %s }\n", json->depth, str);
json->saw_complex_type |= HT_EX_VALUE;
return 1;
}
else if(json->last_special_key == HT_EX_TS) return 1;
else if(json->last_special_key == HT_EX_TAGS) return 1;
if(rv) return 1;
if(json->keys[json->depth]) {
if(stringLen > sizeof(val)-1) stringLen = sizeof(val)-1;
memcpy(val, stringVal, stringLen);
val[stringLen] = '\0';
_YD("[%3d] cb_string %s\n", json->depth, val);
noit_stats_set_metric(json->check,
json->keys[json->depth], METRIC_GUESS, val);
if(json->immediate)
noit_stats_log_immediate_metric(json->check,
json->keys[json->depth], METRIC_GUESS, val);
json->cnt++;
}
return 1;
}
static void jobq_thread_helper(eventer_jobq_t *jobq, void *closure) {
int s32;
char buffer[128];
struct threadq_crutch *crutch = (struct threadq_crutch *)closure;
s32 = jobq->concurrency;
if(s32 == 0) return; /* omit if no concurrency */
snprintf(buffer, sizeof(buffer), "%s_threads", jobq->queue_name);
noit_stats_set_metric(crutch->check, &crutch->check->stats.inprogress,
buffer, METRIC_INT32, &s32);
}
static int selfcheck_feed_details(jlog_feed_stats_t *s, void *closure) {
char buff[256];
uint64_t ms;
struct timeval now, diff;
struct threadq_crutch *crutch = (struct threadq_crutch *)closure;
gettimeofday(&now, NULL);
stats_t *tmpstats = &crutch->check->stats.inprogress;
if(s->last_connection.tv_sec > 0) {
sub_timeval(now, s->last_connection, &diff);
ms = diff.tv_sec * 1000 + diff.tv_usec / 1000;
snprintf(buff, sizeof(buff), "feed`%s`last_connection_ms", s->feed_name);
noit_stats_set_metric(crutch->check, tmpstats, buff, METRIC_UINT64, &ms);
}
if(s->last_checkpoint.tv_sec > 0) {
sub_timeval(now, s->last_checkpoint, &diff);
ms = diff.tv_sec * 1000 + diff.tv_usec / 1000;
snprintf(buff, sizeof(buff), "feed`%s`last_checkpoint_ms", s->feed_name);
noit_stats_set_metric(crutch->check, tmpstats, buff, METRIC_UINT64, &ms);
}
return 1;
}
static int
httptrap_yajl_cb_string(void *ctx, const unsigned char * stringVal,
size_t stringLen) {
struct rest_json_payload *json = ctx;
char val[4096];
int rv;
if(json->depth<0) return 0;
rv = set_array_key(json);
if(json->last_special_key == 0x1) {
if(stringLen != 1) return 0;
if(*stringVal == 'L' || *stringVal == 'l' ||
*stringVal == 'I' || *stringVal == 'i' ||
*stringVal == 'n' || *stringVal == 's') {
json->last_type = *stringVal;
json->saw_complex_type |= 0x1;
return 1;
}
return 0;
}
else if(json->last_special_key == 0x2) {
char *str;
str = malloc(stringLen+1);
memcpy(str, stringVal, stringLen);
str[stringLen] = '\0';
NEW_LV(json, str);
json->saw_complex_type |= 0x2;
return 1;
}
if(rv) return 1;
if(json->keys[json->depth]) {
if(stringLen > sizeof(val)-1) stringLen = sizeof(val)-1;
memcpy(val, stringVal, stringLen);
val[stringLen] = '\0';
noit_stats_set_metric(json->check,
json->keys[json->depth], METRIC_GUESS, val);
if(json->immediate)
noit_stats_log_immediate_metric(json->check,
json->keys[json->depth], METRIC_GUESS, val);
json->cnt++;
}
return 1;
}
static void selfcheck_log_results(noit_module_t *self, noit_check_t *check) {
char buff[128];
u_int64_t u64;
int64_t s64;
int32_t s32;
struct threadq_crutch crutch;
struct timeval duration, epoch, diff;
selfcheck_info_t *ci = check->closure;
crutch.check = check;
noit_check_stats_clear(check, &check->stats.inprogress);
gettimeofday(&check->stats.inprogress.whence, NULL);
sub_timeval(check->stats.inprogress.whence, check->last_fire_time, &duration);
check->stats.inprogress.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
check->stats.inprogress.available = NP_UNAVAILABLE;
check->stats.inprogress.state = NP_BAD;
if(ci->timed_out) check->stats.inprogress.status = "timeout";
else {
check->stats.inprogress.available = NP_AVAILABLE;
check->stats.inprogress.state = NP_GOOD;
check->stats.inprogress.status = "ok";
}
/* Set all the metrics here */
s64 = (int64_t)ci->logsize;
noit_stats_set_metric(check, &check->stats.inprogress, "feed_bytes", METRIC_INT64, &s64);
s32 = noit_poller_check_count();
noit_stats_set_metric(check, &check->stats.inprogress, "check_cnt", METRIC_INT32, &s32);
s32 = noit_poller_transient_check_count();
noit_stats_set_metric(check, &check->stats.inprogress, "transient_cnt", METRIC_INT32, &s32);
if(eventer_get_epoch(&epoch)) s64 = 0;
else {
sub_timeval(check->stats.inprogress.whence, epoch, &diff);
s64 = diff.tv_sec;
}
noit_stats_set_metric(check, &check->stats.inprogress, "uptime", METRIC_INT64, &s64);
eventer_jobq_process_each(jobq_thread_helper, &crutch);
noit_build_version(buff, sizeof(buff));
noit_stats_set_metric(check, &check->stats.inprogress, "version", METRIC_STRING, buff);
u64 = noit_check_completion_count();
noit_stats_set_metric(check, &check->stats.inprogress, "checks_run", METRIC_UINT64, &u64);
/* feed pull info */
noit_jlog_foreach_feed_stats(selfcheck_feed_details, &crutch);
noit_check_set_stats(check, &check->stats.inprogress);
noit_check_stats_clear(check, &check->stats.inprogress);
}
static int
httptrap_yajl_cb_null(void *ctx) {
struct rest_json_payload *json = ctx;
int rv;
if(json->depth<0) return 0;
rv = set_array_key(json);
if(json->last_special_key == 0x2) {
NEW_LV(json, NULL);
return 1;
}
if(json->last_special_key) return 0;
if(rv) return 1;
if(json->keys[json->depth]) {
noit_stats_set_metric(json->check,
json->keys[json->depth], METRIC_INT32, NULL);
if(json->immediate)
noit_stats_log_immediate_metric(json->check,
json->keys[json->depth], METRIC_INT32, NULL);
json->cnt++;
}
return 1;
}
static int
httptrap_yajl_cb_boolean(void *ctx, int boolVal) {
int ival, rv;
struct rest_json_payload *json = ctx;
mtevL(mtev_error, "BOOL\n");
if(json->depth<0) return 0;
rv = set_array_key(json);
if(json->last_special_key == 0x2) {
NEW_LV(json, strdup(boolVal ? "1" : "0"));
return 1;
}
if(json->last_special_key) return 0;
if(rv) return 1;
if(json->keys[json->depth]) {
ival = boolVal ? 1 : 0;
noit_stats_set_metric(json->check,
json->keys[json->depth], METRIC_INT32, &ival);
if(json->immediate)
noit_stats_log_immediate_metric(json->check,
json->keys[json->depth], METRIC_INT32, &ival);
json->cnt++;
}
return 1;
}
static void external_log_results(noit_module_t *self, noit_check_t *check) {
external_data_t *data;
struct check_info *ci;
stats_t current;
struct timeval duration;
noit_check_stats_clear(¤t);
data = noit_module_get_userdata(self);
ci = (struct check_info *)check->closure;
noitL(data->nldeb, "external(%s) (timeout: %d, exit: %x)\n",
check->target, ci->timedout, ci->exit_code);
gettimeofday(¤t.whence, NULL);
sub_timeval(current.whence, check->last_fire_time, &duration);
current.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
if(ci->timedout) {
current.available = NP_UNAVAILABLE;
current.state = NP_BAD;
}
else if(WEXITSTATUS(ci->exit_code) == 3) {
current.available = NP_UNKNOWN;
current.state = NP_UNKNOWN;
}
else {
current.available = NP_AVAILABLE;
current.state = (WEXITSTATUS(ci->exit_code) == 0) ? NP_GOOD : NP_BAD;
}
/* Hack the output into metrics */
if(ci->output && ci->matcher) {
int rc, len, startoffset = 0;
int ovector[30];
len = strlen(ci->output);
noitL(data->nldeb, "going to match output at %d/%d\n", startoffset, len);
while((rc = pcre_exec(ci->matcher, NULL, ci->output, len, startoffset, 0,
ovector, sizeof(ovector)/sizeof(*ovector))) > 0) {
char metric[128];
char value[128];
startoffset = ovector[1];
noitL(data->nldeb, "matched at offset %d\n", rc);
if(pcre_copy_named_substring(ci->matcher, ci->output, ovector, rc,
"key", metric, sizeof(metric)) > 0 &&
pcre_copy_named_substring(ci->matcher, ci->output, ovector, rc,
"value", value, sizeof(value)) > 0) {
/* We're able to extract something... */
noit_stats_set_metric(¤t, metric, METRIC_GUESS, value);
}
noitL(data->nldeb, "going to match output at %d/%d\n", startoffset, len);
}
noitL(data->nldeb, "match failed.... %d\n", rc);
}
current.status = ci->output;
noit_check_set_stats(self, check, ¤t);
/* If we didn't exit normally, or we core, or we have stderr to report...
* provide a full report.
*/
if((WTERMSIG(ci->exit_code) != SIGQUIT && WTERMSIG(ci->exit_code) != 0) ||
WCOREDUMP(ci->exit_code) ||
(ci->error && *ci->error)) {
char uuid_str[37];
uuid_unparse_lower(check->checkid, uuid_str);
noitL(data->nlerr, "external/%s: (sig:%d%s) [%s]\n", uuid_str,
WTERMSIG(ci->exit_code), WCOREDUMP(ci->exit_code)?", cored":"",
ci->error ? ci->error : "");
}
}
static void
noit_stats_set_metric_ignore_whence(noit_check_t *check,
const char *name, metric_type_t type,
const void *value, const struct timeval *ignored) {
noit_stats_set_metric(check, name, type, value);
}
static void dns_cb(struct dns_ctx *ctx, void *result, void *data) {
int r = dns_status(ctx);
int len, i;
struct dns_check_info *ci = data;
struct dns_parse p;
struct dns_rr rr;
unsigned nrr;
unsigned char dn[DNS_MAXDN];
const unsigned char *pkt, *cur, *end;
char *result_str[MAX_RR] = { NULL };
char *result_combined = NULL;
/* If out ci isn't active, we must have timed out already */
if(!__isactive_ci(ci)) {
if(result) free(result);
return;
}
ci->timed_out = 0;
/* If we don't have a result, explode */
if (!result) {
ci->error = strdup(dns_strerror(r));
goto cleanup;
}
/* Process the packet */
pkt = result; end = pkt + r; cur = dns_payload(pkt);
dns_getdn(pkt, &cur, end, dn, sizeof(dn));
dns_initparse(&p, NULL, pkt, cur, end);
p.dnsp_qcls = 0;
p.dnsp_qtyp = 0;
nrr = 0;
while((r = dns_nextrr(&p, &rr)) > 0) {
if (!dns_dnequal(dn, rr.dnsrr_dn)) continue;
if ((ci->query_ctype == DNS_C_ANY || ci->query_ctype == rr.dnsrr_cls) &&
(ci->query_rtype == DNS_T_ANY || ci->query_rtype == rr.dnsrr_typ))
++nrr;
else if (rr.dnsrr_typ == DNS_T_CNAME && !nrr) {
if (dns_getdn(pkt, &rr.dnsrr_dptr, end,
p.dnsp_dnbuf, sizeof(p.dnsp_dnbuf)) <= 0 ||
rr.dnsrr_dptr != rr.dnsrr_dend) {
ci->error = strdup("protocol error");
break;
}
else {
int32_t on = 1;
/* This actually updates what we're looking for */
dns_dntodn(p.dnsp_dnbuf, ci->dn, sizeof(dn));
noit_stats_set_metric(ci->check, &ci->current, "cname", METRIC_INT32, &on);
/* Now follow the leader */
noitL(nldeb, "%s. CNAME %s.\n", dns_dntosp(dn),
dns_dntosp(p.dnsp_dnbuf));
dns_dntodn(p.dnsp_dnbuf, dn, sizeof(dn));
noitL(nldeb, " ---> '%s'\n", dns_dntosp(dn));
}
}
}
if (!r && !nrr) {
ci->error = strdup("no data");
}
dns_rewind(&p, NULL);
p.dnsp_qtyp = ci->query_rtype == DNS_T_ANY ? 0 : ci->query_rtype;
p.dnsp_qcls = ci->query_ctype == DNS_C_ANY ? 0 : ci->query_ctype;
while(dns_nextrr(&p, &rr) && ci->nrr < MAX_RR)
decode_rr(ci, &p, &rr, &result_str[ci->nrr]);
if(ci->sort)
qsort(result_str, ci->nrr, sizeof(*result_str), cstring_cmp);
/* calculate the length and allocate on the stack */
len = 0;
for(i=0; i<ci->nrr; i++) len += strlen(result_str[i]) + 2;
result_combined = alloca(len);
result_combined[0] = '\0';
/* string it together */
len = 0;
for(i=0; i<ci->nrr; i++) {
int slen;
if(i) { memcpy(result_combined + len, ", ", 2); len += 2; }
slen = strlen(result_str[i]);
memcpy(result_combined + len, result_str[i], slen);
len += slen;
result_combined[len] = '\0';
free(result_str[i]); /* free as we go */
}
noit_stats_set_metric(ci->check, &ci->current, "answer", METRIC_STRING, result_combined);
cleanup:
if(result) free(result);
if(ci->timeout_event) {
eventer_t e = eventer_remove(ci->timeout_event);
ci->timeout_event = NULL;
if(e) eventer_free(e);
}
ci->check->flags &= ~NP_RUNNING;
dns_check_log_results(ci);
__deactivate_ci(ci);
}
static void decode_rr(struct dns_check_info *ci, struct dns_parse *p,
struct dns_rr *rr, char **output) {
char buff[DNS_MAXDN], *txt_str = buff, *c;
u_int32_t ttl, vu;
int32_t vs;
int totalsize;
const unsigned char *pkt = p->dnsp_pkt;
const unsigned char *end = p->dnsp_end;
const unsigned char *dptr = rr->dnsrr_dptr;
const unsigned char *dend = rr->dnsrr_dend;
unsigned char *dn = rr->dnsrr_dn;
const unsigned char *tmp;
/* Not interested unless it is the answer to my exact question */
if (!dns_dnequal(ci->dn, dn)) return;
if (!p->dnsp_rrl && !rr->dnsrr_dn[0] && rr->dnsrr_typ == DNS_T_OPT) {
/* We don't handle EDNS0 OPT records */
goto decode_err;
}
noitL(nldeb, "%s. %u %s %s\n", dns_dntosp(dn), rr->dnsrr_ttl,
dns_classname(rr->dnsrr_cls),
dns_typename(rr->dnsrr_typ));
ttl = rr->dnsrr_ttl;
noit_stats_set_metric(ci->check, &ci->current, "ttl", METRIC_UINT32, &ttl);
switch(rr->dnsrr_typ) {
case DNS_T_A:
if (rr->dnsrr_dsz != 4) goto decode_err;
snprintf(buff, sizeof(buff), "%d.%d.%d.%d",
dptr[0], dptr[1], dptr[2], dptr[3]);
break;
case DNS_T_AAAA:
if (rr->dnsrr_dsz != 16) goto decode_err;
inet_ntop(AF_INET6, dptr, buff, sizeof(buff));
break;
case DNS_T_TXT:
totalsize = 0;
for(tmp = dptr; tmp < dend; totalsize += *tmp, tmp += *tmp + 1)
if(tmp + *tmp + 1 > dend) goto decode_err;
/* worst case: every character escaped + '\0' */
txt_str = alloca(totalsize * 3 + 1);
if(!txt_str) goto decode_err;
c = txt_str;
for(tmp = dptr; tmp < dend; tmp += *tmp + 1)
c = encode_txt(c, tmp+1, *tmp);
break;
case DNS_T_MX:
snprintf(buff, sizeof(buff), "%d ", dns_get16(dptr));
tmp = dptr + 2;
if(dns_getdn(pkt, &tmp, end, dn, DNS_MAXDN) <= 0 || tmp != dend)
goto decode_err;
dns_dntop(dn, buff + strlen(buff), sizeof(buff) - strlen(buff));
break;
case DNS_T_SOA:
if(dns_getdn(pkt, &dptr, end, dn, DNS_MAXDN) <= 0) goto decode_err;
dns_dntop(dn, buff, sizeof(buff));
noit_stats_set_metric(ci->check, &ci->current, "name-server", METRIC_STRING, buff);
if(dns_getdn(pkt, &dptr, end, dn, DNS_MAXDN) <= 0) goto decode_err;
dns_dntop(dn, buff, sizeof(buff));
noit_stats_set_metric(ci->check, &ci->current, "email-addr", METRIC_STRING, buff);
if(dptr + 5 * sizeof(u_int32_t) != dend) goto decode_err;
vu = dns_get32(dptr); dptr += sizeof(u_int32_t);
noit_stats_set_metric(ci->check, &ci->current, "serial", METRIC_UINT32, &vu);
/* the serial is what we elect to store as the "answer" as text...
* because it rarely changes and that seems the most interesting thing
* to track change-log-style.
*/
snprintf(buff, sizeof(buff), "%u", vu);
vs = dns_get32(dptr); dptr += sizeof(int32_t);
noit_stats_set_metric(ci->check, &ci->current, "refresh", METRIC_UINT32, &vs);
vs = dns_get32(dptr); dptr += sizeof(int32_t);
noit_stats_set_metric(ci->check, &ci->current, "retry", METRIC_UINT32, &vs);
vs = dns_get32(dptr); dptr += sizeof(int32_t);
noit_stats_set_metric(ci->check, &ci->current, "expiry", METRIC_UINT32, &vs);
vs = dns_get32(dptr); dptr += sizeof(int32_t);
noit_stats_set_metric(ci->check, &ci->current, "minimum", METRIC_UINT32, &vs);
break;
case DNS_T_CNAME:
case DNS_T_PTR:
case DNS_T_NS:
case DNS_T_MB:
case DNS_T_MD:
case DNS_T_MF:
case DNS_T_MG:
case DNS_T_MR:
if(dns_getdn(pkt, &dptr, end, dn, DNS_MAXDN) <= 0) goto decode_err;
dns_dntop(dn, buff, sizeof(buff));
break;
default:
break;
}
if(*output) {
int newlen = strlen(*output) + strlen(", ") + strlen(buff) + 1;
char *newstr;
newstr = malloc(newlen);
snprintf(newstr, newlen, "%s, %s", *output, buff);
free(*output);
*output = newstr;
}
else
*output = strdup(txt_str);
ci->nrr++;
return;
decode_err:
ci->error = strdup("RR decode error");
return;
}
static int
populate_stats_from_resmon_formatted_json(noit_check_t *check,
stats_t *s, struct json_object *o,
const char *prefix) {
int count = 0;
char keybuff[256];
#define MKKEY(fmt, arg) do { \
if(prefix) snprintf(keybuff, sizeof(keybuff), "%s`" fmt, prefix, arg); \
else snprintf(keybuff, sizeof(keybuff), fmt, arg); \
} while(0)
if(o == NULL) {
if(prefix) {
noit_stats_set_metric(check, s, prefix, METRIC_STRING, NULL);
count++;
}
return count;
}
switch(json_object_get_type(o)) {
/* sub callers */
case json_type_array:
{
int i, alen = json_object_array_length(o);
for(i=0;i<alen;i++) {
struct json_object *item = json_object_array_get_idx(o, i);
MKKEY("%d", i);
count += populate_stats_from_resmon_formatted_json(check, s, item, keybuff);
}
}
break;
case json_type_object:
{
struct lh_table *lh;
struct lh_entry *el;
struct json_object *has_type = NULL, *has_value = NULL;
lh = json_object_get_object(o);
lh_foreach(lh, el) {
if(!strcmp(el->k, "_type")) has_type = (struct json_object *)el->v;
else if(!strcmp(el->k, "_value")) has_value = (struct json_object *)el->v;
else {
struct json_object *item = (struct json_object *)el->v;
MKKEY("%s", (const char *)el->k);
count += populate_stats_from_resmon_formatted_json(check, s, item, keybuff);
}
}
if(prefix && has_type && has_value &&
json_object_is_type(has_type, json_type_string) &&
json_object_is_type(has_value, json_type_string)) {
const char *type_str = json_object_get_string(has_type);
const char *value_str = json_object_get_string(has_value);
switch(*type_str) {
case METRIC_INT32:
case METRIC_UINT32:
case METRIC_INT64:
case METRIC_UINT64:
case METRIC_DOUBLE:
case METRIC_STRING:
noit_stats_set_metric_coerce(check, s, prefix,
(metric_type_t)*type_str, value_str);
count++;
default:
break;
}
}
break;
}
/* directs */
case json_type_string:
if(prefix) {
noit_stats_set_metric(check, s, prefix, METRIC_GUESS,
(char *)json_object_get_string(o));
count++;
}
break;
case json_type_boolean:
if(prefix) {
int val = json_object_get_boolean(o) ? 1 : 0;
noit_stats_set_metric(check, s, prefix, METRIC_INT32, &val);
count++;
}
break;
case json_type_null:
if(prefix) {
noit_stats_set_metric(check, s, prefix, METRIC_STRING, NULL);
count++;
}
break;
case json_type_double:
if(prefix) {
double val = json_object_get_double(o);
noit_stats_set_metric(check, s, prefix, METRIC_DOUBLE, &val);
count++;
}
break;
case json_type_int:
if(prefix) {
int64_t i64;
uint64_t u64;
switch(json_object_get_int_overflow(o)) {
case json_overflow_int:
i64 = json_object_get_int(o);
noit_stats_set_metric(check, s, prefix, METRIC_INT64, &i64);
count++;
break;
case json_overflow_int64:
i64 = json_object_get_int64(o);
noit_stats_set_metric(check, s, prefix, METRIC_INT64, &i64);
count++;
break;
case json_overflow_uint64:
u64 = json_object_get_uint64(o);
noit_stats_set_metric(check, s, prefix, METRIC_UINT64, &u64);
count++;
break;
}
}
}
return count;
}
static void mysql_ingest_stats(mysql_check_info_t *ci) {
if(ci->rv > 0) {
/* metrics */
int nrows, ncols, i, j;
nrows = ci->rv;
ncols = mysql_num_fields(ci->result);
MYSQL_FIELD *cdesc = mysql_fetch_fields(ci->result);
for (i=0; i<nrows; i++) {
mtevL(nldeb, "mysql: row %d [%d cols]:\n", i, ncols);
if(ncols<2) continue;
MYSQL_ROW row = mysql_fetch_row(ci->result);
if(NULL == row[0]) continue;
for (j=1; j<ncols; j++) {
enum enum_field_types coltype;
int iv, *piv;
int64_t lv, *plv;
double dv, *pdv;
char *sv;
char mname[128];
snprintf(mname, sizeof(mname), "%s`%s", row[0], cdesc[j].name);
coltype = cdesc[j].type;
mtevL(nldeb, "mysql: col %d (%s) type %d:\n", j, mname, coltype);
switch(coltype) {
case FIELD_TYPE_TINY:
case FIELD_TYPE_SHORT:
case FIELD_TYPE_LONG:
if(!row[j]) piv = NULL;
else {
iv = strtol(row[j], NULL, 10);
piv = &iv;
}
noit_stats_set_metric(ci->check, mname, METRIC_INT32, piv);
break;
case FIELD_TYPE_INT24:
case FIELD_TYPE_LONGLONG:
if(!row[j]) plv = NULL;
else {
lv = strtoll(row[j], NULL, 10);
plv = &lv;
}
noit_stats_set_metric(ci->check, mname, METRIC_INT64, plv);
break;
case FIELD_TYPE_DECIMAL:
case FIELD_TYPE_FLOAT:
case FIELD_TYPE_DOUBLE:
if(!row[j]) pdv = NULL;
else {
dv = atof(row[j]);
pdv = &dv;
}
noit_stats_set_metric(ci->check, mname, METRIC_DOUBLE, pdv);
break;
default:
if(!row[j]) sv = NULL;
else sv = row[j];
noit_stats_set_metric(ci->check, mname, METRIC_GUESS, sv);
break;
}
}
}
}
}
static int
noit_lua_set_metric(lua_State *L) {
noit_check_t *check;
noit_lua_check_info_t *ci;
const char *metric_name;
metric_type_t metric_type;
double __n = 0.0;
int32_t __i = 0;
u_int32_t __I = 0;
int64_t __l = 0;
u_int64_t __L = 0;
if(lua_gettop(L) != 2) luaL_error(L, "wrong number of arguments");
check = lua_touserdata(L, lua_upvalueindex(1));
ci = check->closure;
if(!lua_isstring(L, 1)) luaL_error(L, "argument #1 must be a string");
metric_name = lua_tostring(L, 1);
metric_type = lua_tointeger(L, lua_upvalueindex(2));
if(lua_isnil(L, 2)) {
noit_stats_set_metric(check, &ci->current, metric_name, metric_type, NULL);
lua_pushboolean(L, 1);
return 1;
}
switch(metric_type) {
case METRIC_INT32:
case METRIC_UINT32:
case METRIC_INT64:
case METRIC_UINT64:
case METRIC_DOUBLE:
if(!lua_isnumber(L, 2)) {
noit_stats_set_metric(check, &ci->current, metric_name, metric_type, NULL);
lua_pushboolean(L, 0);
return 1;
}
default:
break;
}
switch(metric_type) {
case METRIC_GUESS:
case METRIC_STRING:
noit_stats_set_metric(check, &ci->current, metric_name, metric_type,
(void *)lua_tostring(L, 2));
break;
case METRIC_INT32:
__i = strtol(lua_tostring(L, 2), NULL, 10);
noit_stats_set_metric(check, &ci->current, metric_name, metric_type, &__i);
break;
case METRIC_UINT32:
__I = strtoul(lua_tostring(L, 2), NULL, 10);
noit_stats_set_metric(check, &ci->current, metric_name, metric_type, &__I);
break;
case METRIC_INT64:
__l = strtoll(lua_tostring(L, 2), NULL, 10);
noit_stats_set_metric(check, &ci->current, metric_name, metric_type, &__l);
break;
case METRIC_UINT64:
__L = strtoull(lua_tostring(L, 2), NULL, 10);
noit_stats_set_metric(check, &ci->current, metric_name, metric_type, &__L);
break;
case METRIC_DOUBLE:
__n = luaL_optnumber(L, 2, 0);
noit_stats_set_metric(check, &ci->current, metric_name, metric_type, &__n);
break;
}
lua_pushboolean(L, 1);
return 1;
}
