static void ssh2_log_results(noit_module_t *self, noit_check_t *check) {
struct timeval duration;
ssh2_check_info_t *ci = check->closure;
noit_check_stats_clear(check, &check->stats.inprogress);
gettimeofday(&check->stats.inprogress.whence, NULL);
sub_timeval(check->stats.inprogress.whence, check->last_fire_time, &duration);
check->stats.inprogress.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
check->stats.inprogress.available = ci->available ? NP_AVAILABLE : NP_UNAVAILABLE;
check->stats.inprogress.state = ci->fingerprint[0] ? NP_GOOD : NP_BAD;
if(ci->error) check->stats.inprogress.status = ci->error;
else if(ci->timed_out) check->stats.inprogress.status = "timeout";
else if(ci->fingerprint[0]) check->stats.inprogress.status = ci->fingerprint;
else check->stats.inprogress.status = "internal error";
if(ci->fingerprint[0]) {
u_int32_t mduration = check->stats.inprogress.duration;
noit_stats_set_metric(check, &check->stats.inprogress, "duration", METRIC_UINT32, &mduration);
noit_stats_set_metric(check, &check->stats.inprogress, "fingerprint", METRIC_STRING,
ci->fingerprint);
}
noit_check_set_stats(check, &check->stats.inprogress);
noit_check_stats_clear(check, &check->stats.inprogress);
}
static void ping_icmp_log_results(noit_module_t *self, noit_check_t *check) {
struct check_info *data;
double avail = 0.0, min = MAXFLOAT, max = 0.0, avg = 0.0, cnt;
int avail_needed = 100;
const char *config_val = NULL;
int i, points = 0;
char human_buffer[256];
struct timeval duration;
noit_check_stats_clear(check, &check->stats.inprogress);
data = (struct check_info *)check->closure;
for(i=0; i<data->expected_count; i++) {
if(data->turnaround[i] >= 0.0) {
points++;
avg += data->turnaround[i];
if(data->turnaround[i] > max) max = data->turnaround[i];
if(data->turnaround[i] < min) min = data->turnaround[i];
}
}
cnt = data->expected_count;
if(points == 0) {
min = 0.0 / 0.0;
max = 0.0 / 0.0;
avg = 0.0 / 0.0;
}
else {
avail = (float)points /cnt;
avg /= (float)points;
}
if(noit_hash_retr_str(check->config, "avail_needed", strlen("avail_needed"),
&config_val))
avail_needed = atoi(config_val);
snprintf(human_buffer, sizeof(human_buffer),
"cnt=%d,avail=%0.0f,min=%0.4f,max=%0.4f,avg=%0.4f",
(int)cnt, 100.0*avail, min, max, avg);
noitL(nldeb, "ping_icmp(%s) [%s]\n", check->target_ip, human_buffer);
gettimeofday(&check->stats.inprogress.whence, NULL);
sub_timeval(check->stats.inprogress.whence, check->last_fire_time, &duration);
check->stats.inprogress.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
check->stats.inprogress.available = (avail > 0.0) ? NP_AVAILABLE : NP_UNAVAILABLE;
check->stats.inprogress.state = (avail < ((float)avail_needed / 100.0)) ? NP_BAD : NP_GOOD;
check->stats.inprogress.status = human_buffer;
noit_stats_set_metric(check, &check->stats.inprogress, "count",
METRIC_INT32, &data->expected_count);
avail *= 100.0;
noit_stats_set_metric(check, &check->stats.inprogress, "available", METRIC_DOUBLE, &avail);
noit_stats_set_metric(check, &check->stats.inprogress, "minimum",
METRIC_DOUBLE, avail > 0.0 ? &min : NULL);
noit_stats_set_metric(check, &check->stats.inprogress, "maximum",
METRIC_DOUBLE, avail > 0.0 ? &max : NULL);
noit_stats_set_metric(check, &check->stats.inprogress, "average",
METRIC_DOUBLE, avail > 0.0 ? &avg : NULL);
noit_check_set_stats(check, &check->stats.inprogress);
noit_check_stats_clear(check, &check->stats.inprogress);
}
static void dns_check_log_results(struct dns_check_info *ci) {
struct timeval duration;
double rtt;
char buff[48];
gettimeofday(&ci->check->stats.inprogress.whence, NULL);
sub_timeval(ci->check->stats.inprogress.whence, ci->check->last_fire_time, &duration);
rtt = duration.tv_sec * 1000.0 + duration.tv_usec / 1000.0;
ci->check->stats.inprogress.duration = rtt;
ci->check->stats.inprogress.state = (ci->error || ci->nrr == 0) ? NP_BAD : NP_GOOD;
ci->check->stats.inprogress.available = ci->timed_out ? NP_UNAVAILABLE : NP_AVAILABLE;
if(ci->check->stats.inprogress.status) {
free(ci->check->stats.inprogress.status);
ci->check->stats.inprogress.status = NULL;
}
if(ci->error) {
ci->check->stats.inprogress.status = ci->error;
}
else if(!ci->check->stats.inprogress.status) {
snprintf(buff, sizeof(buff), "%d %s",
ci->nrr, ci->nrr == 1 ? "record" : "records");
ci->check->stats.inprogress.status = buff;
}
noit_stats_set_metric(ci->check, &ci->check->stats.inprogress, "rtt", METRIC_DOUBLE,
ci->timed_out ? NULL : &rtt);
noit_check_set_stats(ci->check, &ci->check->stats.inprogress);
if(ci->error) free(ci->error);
ci->error = NULL;
noit_check_stats_clear(ci->check, &ci->check->stats.inprogress);
}
static void selfcheck_log_results(noit_module_t *self, noit_check_t *check) {
char buff[128];
u_int64_t u64;
int64_t s64;
int32_t s32;
struct threadq_crutch crutch;
struct timeval duration, epoch, diff;
selfcheck_info_t *ci = check->closure;
crutch.check = check;
noit_check_stats_clear(check, &check->stats.inprogress);
gettimeofday(&check->stats.inprogress.whence, NULL);
sub_timeval(check->stats.inprogress.whence, check->last_fire_time, &duration);
check->stats.inprogress.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
check->stats.inprogress.available = NP_UNAVAILABLE;
check->stats.inprogress.state = NP_BAD;
if(ci->timed_out) check->stats.inprogress.status = "timeout";
else {
check->stats.inprogress.available = NP_AVAILABLE;
check->stats.inprogress.state = NP_GOOD;
check->stats.inprogress.status = "ok";
}
/* Set all the metrics here */
s64 = (int64_t)ci->logsize;
noit_stats_set_metric(check, &check->stats.inprogress, "feed_bytes", METRIC_INT64, &s64);
s32 = noit_poller_check_count();
noit_stats_set_metric(check, &check->stats.inprogress, "check_cnt", METRIC_INT32, &s32);
s32 = noit_poller_transient_check_count();
noit_stats_set_metric(check, &check->stats.inprogress, "transient_cnt", METRIC_INT32, &s32);
if(eventer_get_epoch(&epoch)) s64 = 0;
else {
sub_timeval(check->stats.inprogress.whence, epoch, &diff);
s64 = diff.tv_sec;
}
noit_stats_set_metric(check, &check->stats.inprogress, "uptime", METRIC_INT64, &s64);
eventer_jobq_process_each(jobq_thread_helper, &crutch);
noit_build_version(buff, sizeof(buff));
noit_stats_set_metric(check, &check->stats.inprogress, "version", METRIC_STRING, buff);
u64 = noit_check_completion_count();
noit_stats_set_metric(check, &check->stats.inprogress, "checks_run", METRIC_UINT64, &u64);
/* feed pull info */
noit_jlog_foreach_feed_stats(selfcheck_feed_details, &crutch);
noit_check_set_stats(check, &check->stats.inprogress);
noit_check_stats_clear(check, &check->stats.inprogress);
}
static int
statsd_submit(noit_module_t *self, noit_check_t *check,
noit_check_t *cause) {
statsd_closure_t *ccl;
struct timeval duration;
statsd_mod_config_t *conf;
conf = noit_module_get_userdata(self);
if(!conf->primary_active) conf->check = NULL;
if(0 == memcmp(conf->primary, check->checkid, sizeof(uuid_t))) {
conf->check = check;
if(NOIT_CHECK_DISABLED(check) || NOIT_CHECK_KILLED(check)) {
conf->check = NULL;
return 0;
}
}
/* We are passive, so we don't do anything for transient checks */
if(check->flags & NP_TRANSIENT) return 0;
if(!check->closure) {
ccl = check->closure = calloc(1, sizeof(*ccl));
ccl->self = self;
memset(&check->stats.inprogress, 0, sizeof(check->stats.inprogress));
} else {
// Don't count the first run
char human_buffer[256];
ccl = (statsd_closure_t*)check->closure;
gettimeofday(&check->stats.inprogress.whence, NULL);
sub_timeval(check->stats.inprogress.whence, check->last_fire_time, &duration);
check->stats.inprogress.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
snprintf(human_buffer, sizeof(human_buffer),
"dur=%d,run=%d,stats=%d", check->stats.inprogress.duration,
check->generation, ccl->stats_count);
noitL(nldeb, "statsd(%s) [%s]\n", check->target, human_buffer);
// Not sure what to do here
check->stats.inprogress.available = (ccl->stats_count > 0) ?
NP_AVAILABLE : NP_UNAVAILABLE;
check->stats.inprogress.state = (ccl->stats_count > 0) ?
NP_GOOD : NP_BAD;
check->stats.inprogress.status = human_buffer;
if(check->last_fire_time.tv_sec)
noit_check_passive_set_stats(check, &check->stats.inprogress);
memcpy(&check->last_fire_time, &check->stats.inprogress.whence, sizeof(duration));
}
ccl->stats_count = 0;
noit_check_stats_clear(check, &check->stats.inprogress);
return 0;
}
static void external_log_results(noit_module_t *self, noit_check_t *check) {
external_data_t *data;
struct check_info *ci;
stats_t current;
struct timeval duration;
noit_check_stats_clear(¤t);
data = noit_module_get_userdata(self);
ci = (struct check_info *)check->closure;
noitL(data->nldeb, "external(%s) (timeout: %d, exit: %x)\n",
check->target, ci->timedout, ci->exit_code);
gettimeofday(¤t.whence, NULL);
sub_timeval(current.whence, check->last_fire_time, &duration);
current.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
if(ci->timedout) {
current.available = NP_UNAVAILABLE;
current.state = NP_BAD;
}
else if(WEXITSTATUS(ci->exit_code) == 3) {
current.available = NP_UNKNOWN;
current.state = NP_UNKNOWN;
}
else {
current.available = NP_AVAILABLE;
current.state = (WEXITSTATUS(ci->exit_code) == 0) ? NP_GOOD : NP_BAD;
}
/* Hack the output into metrics */
if(ci->output && ci->matcher) {
int rc, len, startoffset = 0;
int ovector[30];
len = strlen(ci->output);
noitL(data->nldeb, "going to match output at %d/%d\n", startoffset, len);
while((rc = pcre_exec(ci->matcher, NULL, ci->output, len, startoffset, 0,
ovector, sizeof(ovector)/sizeof(*ovector))) > 0) {
char metric[128];
char value[128];
startoffset = ovector[1];
noitL(data->nldeb, "matched at offset %d\n", rc);
if(pcre_copy_named_substring(ci->matcher, ci->output, ovector, rc,
"key", metric, sizeof(metric)) > 0 &&
pcre_copy_named_substring(ci->matcher, ci->output, ovector, rc,
"value", value, sizeof(value)) > 0) {
/* We're able to extract something... */
noit_stats_set_metric(¤t, metric, METRIC_GUESS, value);
}
noitL(data->nldeb, "going to match output at %d/%d\n", startoffset, len);
}
noitL(data->nldeb, "match failed.... %d\n", rc);
}
current.status = ci->output;
noit_check_set_stats(self, check, ¤t);
/* If we didn't exit normally, or we core, or we have stderr to report...
* provide a full report.
*/
if((WTERMSIG(ci->exit_code) != SIGQUIT && WTERMSIG(ci->exit_code) != 0) ||
WCOREDUMP(ci->exit_code) ||
(ci->error && *ci->error)) {
char uuid_str[37];
uuid_unparse_lower(check->checkid, uuid_str);
noitL(data->nlerr, "external/%s: (sig:%d%s) [%s]\n", uuid_str,
WTERMSIG(ci->exit_code), WCOREDUMP(ci->exit_code)?", cored":"",
ci->error ? ci->error : "");
}
}
static void
clear_closure(noit_check_t *check, statsd_closure_t *ccl) {
ccl->stats_count = 0;
noit_check_stats_clear(check, &ccl->current);
}
static int test_abort_drive_session(eventer_t e, int mask, void *closure,
struct timeval *passed_now) {
struct timespec rqtp;
struct timeval target_time, now, diff;
double i, r;
test_abort_check_info_t *ci = closure;
noit_check_t *check = ci->check;
if(mask & (EVENTER_READ | EVENTER_WRITE)) {
/* this case is impossible from the eventer. It is called as
* such on the synchronous completion of the event.
*/
noit_check_stats_clear(check, &check->stats.inprogress);
check->stats.inprogress.available = NP_AVAILABLE;
check->stats.inprogress.state = ci->timed_out ? NP_BAD : NP_GOOD;
noitL(nlerr, "test_abort: EVENTER_READ | EVENTER_WRITE\n");
noit_check_set_stats(check, &check->stats.inprogress);
noit_check_stats_clear(check, &check->stats.inprogress);
check->flags &= ~NP_RUNNING;
return 0;
}
switch(mask) {
case EVENTER_ASYNCH_WORK:
noitL(nlerr, "test_abort: EVENTER_ASYNCH_WORK\n");
r = modf(ci->timeout, &i);
ci->timed_out = 1;
if(ci->ignore_signals) { /* compuational loop */
double trash = 1.0;
gettimeofday(&now, NULL);
diff.tv_sec = (int)i;
diff.tv_usec = (int)(r * 1000000.0);
add_timeval(now, diff, &target_time);
do {
for(i=0; i<100000; i++) {
trash += drand48();
trash = log(trash);
trash += 1.1;
trash = exp(trash);
}
gettimeofday(&now, NULL);
sub_timeval(target_time, now, &diff);
} while(diff.tv_sec >= 0 && diff.tv_usec >= 0);
}
else {
rqtp.tv_sec = (int)i;
rqtp.tv_nsec = (int)(r * 1000000000.0);
nanosleep(&rqtp,NULL);
}
noitL(nlerr, "test_abort: EVENTER_ASYNCH_WORK (done)\n");
ci->timed_out = 0;
return 0;
break;
case EVENTER_ASYNCH_CLEANUP:
/* This sets us up for a completion call. */
noitL(nlerr, "test_abort: EVENTER_ASYNCH_CLEANUP\n");
e->mask = EVENTER_READ | EVENTER_WRITE;
break;
default:
abort();
}
return 0;
}
/* Handling of results */
static void noit_snmp_log_results(noit_module_t *self, noit_check_t *check,
struct snmp_pdu *pdu) {
struct check_info *info = check->closure;
struct variable_list *vars;
struct timeval duration;
char buff[128];
stats_t current;
int nresults = 0;
noit_check_stats_clear(check, ¤t);
if(pdu)
for(vars = pdu->variables; vars; vars = vars->next_variable)
nresults++;
gettimeofday(¤t.whence, NULL);
sub_timeval(current.whence, check->last_fire_time, &duration);
current.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
current.available = pdu ? NP_AVAILABLE : NP_UNAVAILABLE;
current.state = (nresults == info->noids) ? NP_GOOD : NP_BAD;
snprintf(buff, sizeof(buff), "%d/%d gets", nresults, info->noids);
current.status = buff;
/* We have no results over which to iterate. */
if(!pdu) {
noit_check_set_stats(check, ¤t);
return;
}
/* manipulate the information ourselves */
nresults = 0;
for(vars = pdu->variables; vars; vars = vars->next_variable) {
char *sp;
int oid_idx;
double float_conv;
u_int64_t u64;
int64_t i64;
char *endptr;
char varbuff[256];
/* find the oid to which this is the response */
oid_idx = nresults; /* our current idx is the most likely */
if(info->oids[oid_idx].oidlen != vars->name_length ||
memcmp(info->oids[oid_idx].oid, vars->name,
vars->name_length * sizeof(oid))) {
/* Not the most obvious guess */
for(oid_idx = info->noids - 1; oid_idx >= 0; oid_idx--) {
if(info->oids[oid_idx].oidlen == vars->name_length &&
memcmp(info->oids[oid_idx].oid, vars->name,
vars->name_length * sizeof(oid))) break;
}
}
if(oid_idx < 0) {
snprint_variable(varbuff, sizeof(varbuff),
vars->name, vars->name_length, vars);
noitL(nlerr, "Unexpected oid results to %s`%s`%s: %s\n",
check->target, check->module, check->name, varbuff);
nresults++;
continue;
}
#define SETM(a,b) noit_stats_set_metric(check, ¤t, \
info->oids[oid_idx].confname, a, b)
if(info->oids[oid_idx].type_should_override) {
snprint_value(varbuff, sizeof(varbuff), vars->name, vars->name_length, vars);
sp = strchr(varbuff, ' ');
if(sp) sp++;
noit_stats_set_metric_coerce(check, ¤t, info->oids[oid_idx].confname,
info->oids[oid_idx].type_override,
sp);
}
else {
switch(vars->type) {
case ASN_OCTET_STR:
sp = malloc(1 + vars->val_len);
memcpy(sp, vars->val.string, vars->val_len);
sp[vars->val_len] = '\0';
SETM(METRIC_STRING, sp);
free(sp);
break;
case ASN_INTEGER:
case ASN_GAUGE:
SETM(METRIC_INT32, vars->val.integer);
break;
case ASN_TIMETICKS:
case ASN_COUNTER:
SETM(METRIC_UINT32, vars->val.integer);
break;
case ASN_INTEGER64:
printI64(varbuff, vars->val.counter64);
i64 = strtoll(varbuff, &endptr, 10);
SETM(METRIC_INT64, (varbuff == endptr) ? NULL : &i64);
break;
case ASN_COUNTER64:
printU64(varbuff, vars->val.counter64);
u64 = strtoull(varbuff, &endptr, 10);
SETM(METRIC_UINT64, (varbuff == endptr) ? NULL : &u64);
break;
case ASN_FLOAT:
if(vars->val.floatVal) float_conv = *(vars->val.floatVal);
SETM(METRIC_DOUBLE, vars->val.floatVal ? &float_conv : NULL);
break;
case ASN_DOUBLE:
SETM(METRIC_DOUBLE, vars->val.doubleVal);
break;
case SNMP_NOSUCHOBJECT:
case SNMP_NOSUCHINSTANCE:
SETM(METRIC_STRING, NULL);
break;
default:
snprint_variable(varbuff, sizeof(varbuff), vars->name, vars->name_length, vars);
/* Advance passed the first space and use that unless there
* is no space or we have no more string left.
*/
sp = strchr(varbuff, ' ');
if(sp) sp++;
SETM(METRIC_STRING, (sp && *sp) ? sp : NULL);
}
}
nresults++;
}
noit_check_set_stats(check, ¤t);
}
static int postgres_drive_session(eventer_t e, int mask, void *closure,
struct timeval *now) {
const char *dsn, *sql;
char sql_buff[8192];
char dsn_buff[512];
struct timeval t1, t2, diff;
postgres_check_info_t *ci = closure;
noit_check_t *check = ci->check;
if(mask & (EVENTER_READ | EVENTER_WRITE)) {
/* this case is impossible from the eventer. It is called as
* such on the synchronous completion of the event.
*/
postgres_log_results(ci->self, ci->check);
postgres_cleanup(ci->self, ci->check);
check->flags &= ~NP_RUNNING;
return 0;
}
switch(mask) {
case EVENTER_ASYNCH_WORK:
noit_check_stats_clear(&ci->current);
ci->connect_duration = NULL;
ci->query_duration = NULL;
FETCH_CONFIG_OR(dsn, "");
noit_check_interpolate(dsn_buff, sizeof(dsn_buff), dsn,
&ci->attrs, check->config);
ci->conn = PQconnectdb(dsn_buff);
if(!ci->conn) AVAIL_BAIL("PQconnectdb failed");
if(PQstatus(ci->conn) != CONNECTION_OK)
AVAIL_BAIL(PQerrorMessage(ci->conn));
FETCH_CONFIG_OR(sql, "");
noit_check_interpolate(sql_buff, sizeof(sql_buff), sql,
&ci->attrs, check->config);
gettimeofday(&t1, NULL);
sub_timeval(t1, check->last_fire_time, &diff);
ci->connect_duration_d = diff.tv_sec * 1000.0 + diff.tv_usec / 1000.0;
ci->connect_duration = &ci->connect_duration_d;
ci->result = PQexec(ci->conn, sql_buff);
gettimeofday(&t2, NULL);
sub_timeval(t2, t1, &diff);
ci->query_duration_d = diff.tv_sec * 1000.0 + diff.tv_usec / 1000.0;
ci->query_duration = &ci->query_duration_d;
if(!ci->result) AVAIL_BAIL("PQexec failed");
ci->rv = PQresultStatus(ci->result);
switch(ci->rv) {
case PGRES_TUPLES_OK:
postgres_ingest_stats(ci);
case PGRES_COMMAND_OK:
break;
default:
AVAIL_BAIL(PQresultErrorMessage(ci->result));
}
if(ci->result) {
PGresult *result_swap = ci->result;
ci->result = NULL;
PQclear(result_swap);
}
if(ci->conn) {
PGconn *conn_swap = ci->conn;
ci->conn = NULL;
PQfinish(conn_swap);
}
ci->timed_out = 0;
return 0;
break;
case EVENTER_ASYNCH_CLEANUP:
/* This sets us up for a completion call. */
e->mask = EVENTER_READ | EVENTER_WRITE;
break;
default:
abort();
}
return 0;
}