int main(int argc, char **argv)
{
bool argerr = false;
bool follow = false;
bool use_stdin = false;
long num = -1;
char *matcher = "TRUE";
char *filename = NULL;
output_function ofn = output_text;
int c;
while ((c = getopt(argc, argv, "tchfn:m:")) != -1) {
switch (c) {
case 't':
ofn = output_text;
break;
case 'c':
ofn = NULL;
break;
case 'h':
ofn = output_heka;
break;
case 'f':
follow = true;
break;
case 'n':
num = strtol(optarg, NULL, 10);
if (num < 0) argerr = true;
break;
case 'm':
matcher = optarg;
break;
default:
argerr = true;
break;
}
}
if (argc - optind == 1) {
filename = argv[optind];
use_stdin = strcmp("-", filename) == 0;
} else {
argerr = true;
}
if (argerr) {
log_cb(NULL, NULL, 0,
"usage: %s [-t|-c|-h] [-m message_matcher] [-f] [-n #] <FILE>\n"
"description:\n"
" -t output the messages in text format (default)\n"
" -c only output the message count\n"
" -h output the messages as a Heka protobuf stream\n"
" -f output appended data as the file grows\n"
" -n output the last # of messages (simple header check so not "
"100%% accurate)\n"
" -m message_matcher expression (default \"TRUE\")\n"
" FILE name of the file to cat or '-' for stdin\n"
"notes:\n"
" All output is written to stdout and all log/error messages are "
"written to stderr.\n",
argv[0]);
return EXIT_FAILURE;
}
lsb_message_matcher *mm = lsb_create_message_matcher(matcher);
if (!mm) {
log_cb(NULL, NULL, 0, "invalid message matcher: %s", matcher);
return EXIT_FAILURE;
}
FILE *fh = stdin;
if (!use_stdin) {
fh = fopen(filename, "r");
if (!fh) {
log_cb(NULL, NULL, 0, "error opening: %s", filename);
return EXIT_FAILURE;
}
if (num >= 0) {
move_to_offset(fh, num);
}
}
size_t discarded_bytes;
size_t bytes_read = 0;
size_t pcnt = 0;
size_t mcnt = 0;
lsb_input_buffer ib;
lsb_init_input_buffer(&ib, 1024 * 1024 * 1024);
lsb_heka_message msg;
lsb_init_heka_message(&msg, 8);
do {
if (lsb_find_heka_message(&msg, &ib, true, &discarded_bytes, &logger)) {
if (lsb_eval_message_matcher(mm, &msg)) {
if (ofn) {
ofn(&msg);
}
++mcnt;
}
++pcnt;
} else {
bytes_read = read_file(fh, &ib);
}
if (bytes_read == 0 && follow && !use_stdin) {
sleep(1);
}
} while (bytes_read > 0 || follow);
lsb_free_heka_message(&msg);
lsb_free_input_buffer(&ib);
lsb_destroy_message_matcher(mm);
if (!use_stdin) {
fclose(fh);
}
if (ofn) {
log_cb(NULL, NULL, 0, "Processed: %zu, matched: %zu messages\n", pcnt,
mcnt);
} else {
printf("Processed: %zu, matched: %zu messages\n", pcnt, mcnt);
}
}
static hs_output_plugin*
create_output_plugin(lsb_message_match_builder *mmb, const hs_config *cfg,
hs_sandbox_config *sbc)
{
char *state_file = NULL;
char lua_file[HS_MAX_PATH];
if (!hs_get_fqfn(sbc->dir, sbc->filename, lua_file, sizeof(lua_file))) {
hs_log(NULL, g_module, 3, "%s failed to construct the lua_file path",
sbc->cfg_name);
return NULL;
}
hs_output_plugin *p = calloc(1, sizeof(hs_output_plugin));
if (!p) {
hs_log(NULL, g_module, 2, "%s hs_output_plugin memory allocation failed",
sbc->cfg_name);
return NULL;
}
if (pthread_mutex_init(&p->cp_lock, NULL)) {
free(p);
hs_log(NULL, g_module, 3, "%s pthread_mutex_init failed", sbc->cfg_name);
return NULL;
}
p->list_index = -1;
p->sequence_id = 1;
p->ticker_interval = sbc->ticker_interval;
int stagger = p->ticker_interval > 60 ? 60 : p->ticker_interval;
// distribute when the timer_events will fire
if (stagger) {
p->ticker_expires = time(NULL) + rand() % stagger;
}
if (sbc->async_buffer_size > 0) {
p->async_len = sbc->async_buffer_size;
p->async_cp = calloc(p->async_len, sizeof(hs_checkpoint_pair));
if (!p->async_cp) {
destroy_output_plugin(p);
hs_log(NULL, g_module, 2, "%s async buffer memory allocation failed",
sbc->cfg_name);
return NULL;
}
}
p->mm = lsb_create_message_matcher(mmb, sbc->message_matcher);
if (!p->mm) {
hs_log(NULL, g_module, 3, "%s invalid message_matcher: %s", sbc->cfg_name,
sbc->message_matcher);
destroy_output_plugin(p);
return NULL;
}
size_t len = strlen(sbc->cfg_name) + 1;
p->name = malloc(len);
if (!p->name) {
hs_log(NULL, g_module, 2, "%s name memory allocation failed",
sbc->cfg_name);
destroy_output_plugin(p);
}
memcpy(p->name, sbc->cfg_name, len);
if (sbc->preserve_data) {
size_t len = strlen(cfg->output_path) + strlen(sbc->cfg_name) + 7;
state_file = malloc(len);
if (!state_file) {
hs_log(NULL, g_module, 2, "%s state_file memory allocation failed",
sbc->cfg_name);
destroy_output_plugin(p);
return NULL;
}
int ret = snprintf(state_file, len, "%s/%s.data", cfg->output_path,
sbc->cfg_name);
if (ret < 0 || ret > (int)len - 1) {
hs_log(NULL, g_module, 3, "%s failed to construct the state_file path",
sbc->cfg_name);
free(state_file);
destroy_output_plugin(p);
return NULL;
}
}
lsb_output_buffer ob;
if (lsb_init_output_buffer(&ob, 8 * 1024)) {
hs_log(NULL, g_module, 3, "%s configuration memory allocation failed",
sbc->cfg_name);
free(state_file);
destroy_output_plugin(p);
return NULL;
}
if (!hs_get_full_config(&ob, 'o', cfg, sbc)) {
hs_log(NULL, g_module, 3, "%s hs_get_full_config failed", sbc->cfg_name);
lsb_free_output_buffer(&ob);
free(state_file);
destroy_output_plugin(p);
return NULL;
}
lsb_logger logger = {.context = NULL, .cb = hs_log};
p->hsb = lsb_heka_create_output(p, lua_file, state_file, ob.buf, &logger,
update_checkpoint_callback);
lsb_free_output_buffer(&ob);
free(sbc->cfg_lua);
sbc->cfg_lua = NULL;
free(state_file);
if (!p->hsb) {
destroy_output_plugin(p);
hs_log(NULL, g_module, 3, "%s lsb_heka_create_output failed",
sbc->cfg_name);
return NULL;
}
return p;
}
