void
write_read_check(avro_schema_t writers_schema, avro_datum_t datum,
avro_schema_t readers_schema, avro_datum_t expected, char *type)
{
avro_datum_t datum_out;
int validate;
for (validate = 0; validate <= 1; validate++) {
reader = avro_reader_memory(buf, sizeof(buf));
writer = avro_writer_memory(buf, sizeof(buf));
if (!expected) {
expected = datum;
}
/* Validating read/write */
if (avro_write_data
(writer, validate ? writers_schema : NULL, datum)) {
fprintf(stderr, "Unable to write %s validate=%d\n %s\n",
type, validate, avro_strerror());
exit(EXIT_FAILURE);
}
int64_t size =
avro_size_data(writer, validate ? writers_schema : NULL,
datum);
if (size != avro_writer_tell(writer)) {
fprintf(stderr,
"Unable to calculate size %s validate=%d "
"(%"PRId64" != %"PRId64")\n %s\n",
type, validate, size, avro_writer_tell(writer),
avro_strerror());
exit(EXIT_FAILURE);
}
if (avro_read_data
(reader, writers_schema, readers_schema, &datum_out)) {
fprintf(stderr, "Unable to read %s validate=%d\n %s\n",
type, validate, avro_strerror());
fprintf(stderr, " %s\n", avro_strerror());
exit(EXIT_FAILURE);
}
if (!avro_datum_equal(expected, datum_out)) {
fprintf(stderr,
"Unable to encode/decode %s validate=%d\n %s\n",
type, validate, avro_strerror());
exit(EXIT_FAILURE);
}
avro_reader_dump(reader, stderr);
avro_datum_decref(datum_out);
avro_reader_free(reader);
avro_writer_free(writer);
}
}
static int write_header(avro_file_writer_t w)
{
int rval;
uint8_t version = 1;
/* TODO: remove this static buffer */
avro_writer_t schema_writer;
char schema_buf[64 * 1024];
const avro_encoding_t *enc = &avro_binary_encoding;
/* Generate random sync */
generate_sync(w);
check(rval, avro_write(w->writer, "Obj", 3));
check(rval, avro_write(w->writer, &version, 1));
check(rval, enc->write_long(w->writer, 3));
check(rval, enc->write_string(w->writer, "avro.sync"));
check(rval, enc->write_bytes(w->writer, w->sync, sizeof(w->sync)));
check(rval, enc->write_string(w->writer, "avro.codec"));
check(rval, enc->write_bytes(w->writer, "null", 4));
check(rval, enc->write_string(w->writer, "avro.schema"));
schema_writer = avro_writer_memory(schema_buf, sizeof(schema_buf));
rval = avro_schema_to_json(w->writers_schema, schema_writer);
if (rval) {
avro_writer_free(schema_writer);
return rval;
}
check(rval,
enc->write_bytes(w->writer, schema_buf,
avro_writer_tell(schema_writer)));
check(rval, enc->write_long(w->writer, 0));
return write_sync(w);
}
int main(void)
{
int pass;
for (pass = 0 ; json_schemas[pass] ; pass++) {
int rval = 0;
size_t len;
static char buf[4096];
avro_writer_t writer;
avro_file_writer_t file_writer;
avro_file_reader_t file_reader;
avro_schema_t schema = NULL;
avro_schema_error_t error = NULL;
char outpath[64];
const char *json_schema = json_schemas[pass];
printf("pass %d with schema %s\n", pass, json_schema);
check(rval, avro_schema_from_json(json_schema, strlen(json_schema),
&schema, &error));
avro_value_iface_t *iface = avro_generic_class_from_schema(schema);
avro_value_t val;
avro_generic_value_new(iface, &val);
avro_value_t out;
avro_generic_value_new(iface, &out);
/* create the val */
avro_value_reset(&val);
avro_value_set_string(&val, "test-1691");
/* Write value to file */
snprintf(outpath, sizeof(outpath), "test-1691-%d.avro", pass);
/* create the writers */
writer = avro_writer_memory(buf, sizeof(buf));
check(rval, avro_file_writer_create(outpath, schema, &file_writer));
check(rval, avro_value_write(writer, &val));
len = avro_writer_tell(writer);
check(rval, avro_file_writer_append_encoded(file_writer, buf, len));
check(rval, avro_file_writer_close(file_writer));
/* Read the value back */
check(rval, avro_file_reader(outpath, &file_reader));
check(rval, avro_file_reader_read_value(file_reader, &out));
if (!avro_value_equal(&val, &out)) {
fprintf(stderr, "fail!\n");
exit(EXIT_FAILURE);
}
fprintf(stderr, "pass %d: ok: schema %s\n", pass, json_schema);
check(rval, avro_file_reader_close(file_reader));
remove(outpath);
}
exit(EXIT_SUCCESS);
}
/*
typedef struct {
int jobid;
int vpid;
} process_name_t;
typedef struct {
char *en_vars;
char *args;
char *host_name;
process_name_t proc_name;
} launch_context_t;
typedef struct {
bool is_successful;
process_name_t proc_name;
} launch_response_t;
*/
static void build_launch_response(launch_response_t *launch_response_array, int array_size, avro_slice_t **slice)
{
char filename[FILE_NAME_LEN];
char buf[BUFFER_SIZE];
long len = 0;
avro_schema_t schema;
avro_value_iface_t *iface;
avro_value_t record;
avro_value_t results_value, LaunchResult_value, is_successful_value, name_value, jobid_value, vpid_value;
size_t index;
int i;
avro_writer_t writer;
sprintf(filename, "%s/%s", SCHEMA_PATH, "LaunchResponseRecordAvro.avsc");
init_schema(filename, &schema);
iface = avro_generic_class_from_schema(schema);
avro_generic_value_new(iface, &record);
avro_value_get_by_name(&record, "results", &results_value, &index);
for (i = 0; i < array_size; i++) {
avro_value_append(&results_value, &LaunchResult_value, &index);
avro_value_get_by_name(&LaunchResult_value, "is_successful", &is_successful_value, &index);
avro_value_set_boolean(&is_successful_value, launch_response_array[i].is_successful);
avro_value_get_by_name(&LaunchResult_value, "name", &name_value, &index);
avro_value_get_by_name(&name_value, "jobid", &jobid_value, &index);
avro_value_set_int(&jobid_value, launch_response_array[i].proc_name.jobid);
avro_value_get_by_name(&name_value, "vpid", &vpid_value, &index);
avro_value_set_int(&vpid_value, launch_response_array[i].proc_name.vpid);
}
/* create a writer with memory buffer */
writer = avro_writer_memory(buf, sizeof(buf));
/* write record to writer (buffer) */
if (avro_value_write(writer, &record)) {
fprintf(stderr, "Unable to write record to memory buffer\n");
fprintf(stderr, "Error: %s\n", avro_strerror());
exit(1);
}
avro_writer_flush(writer);
len = avro_writer_tell(writer);
//avro_generic_value_free(&record);
avro_value_iface_decref(iface);
avro_schema_decref(schema);
*slice = xmalloc(sizeof(avro_slice_t));
(*slice)->buffer = xmalloc(len);
(*slice)->len = len;
memcpy((*slice)->buffer, buf, len);
}
static int file_write_block(avro_file_writer_t w)
{
const avro_encoding_t *enc = &avro_binary_encoding;
int rval;
if (w->block_count) {
int64_t blocklen = avro_writer_tell(w->datum_writer);
/* Write the block count */
check(rval, enc->write_long(w->writer, w->block_count));
/* Write the block length */
check(rval, enc->write_long(w->writer, blocklen));
/* Write the block */
check(rval, avro_write(w->writer, w->datum_buffer, blocklen));
/* Write the sync marker */
check(rval, write_sync(w));
/* Reset the datum writer */
avro_writer_reset(w->datum_writer);
w->block_count = 0;
}
return 0;
}
void kafka_cache_purge(struct chained_cache *queue[], int index, int safe_action)
{
struct pkt_primitives *data = NULL;
struct pkt_bgp_primitives *pbgp = NULL;
struct pkt_nat_primitives *pnat = NULL;
struct pkt_mpls_primitives *pmpls = NULL;
struct pkt_tunnel_primitives *ptun = NULL;
char *pcust = NULL;
struct pkt_vlen_hdr_primitives *pvlen = NULL;
struct pkt_bgp_primitives empty_pbgp;
struct pkt_nat_primitives empty_pnat;
struct pkt_mpls_primitives empty_pmpls;
struct pkt_tunnel_primitives empty_ptun;
char *empty_pcust = NULL;
char src_mac[18], dst_mac[18], src_host[INET6_ADDRSTRLEN], dst_host[INET6_ADDRSTRLEN], ip_address[INET6_ADDRSTRLEN];
char rd_str[SRVBUFLEN], misc_str[SRVBUFLEN], dyn_kafka_topic[SRVBUFLEN], *orig_kafka_topic = NULL;
int i, j, stop, batch_idx, is_topic_dyn = FALSE, qn = 0, ret, saved_index = index;
int mv_num = 0, mv_num_save = 0;
time_t start, duration;
pid_t writer_pid = getpid();
char *json_buf = NULL;
int json_buf_off = 0;
#ifdef WITH_AVRO
avro_writer_t avro_writer;
char *avro_buf = NULL;
int avro_buffer_full = FALSE;
#endif
p_kafka_init_host(&kafkap_kafka_host, config.kafka_config_file);
/* setting some defaults */
if (!config.sql_host) config.sql_host = default_kafka_broker_host;
if (!config.kafka_broker_port) config.kafka_broker_port = default_kafka_broker_port;
if (!config.sql_table) config.sql_table = default_kafka_topic;
else {
if (strchr(config.sql_table, '$')) {
is_topic_dyn = TRUE;
orig_kafka_topic = config.sql_table;
}
}
if (config.amqp_routing_key_rr) orig_kafka_topic = config.sql_table;
p_kafka_init_topic_rr(&kafkap_kafka_host);
p_kafka_set_topic_rr(&kafkap_kafka_host, config.amqp_routing_key_rr);
empty_pcust = malloc(config.cpptrs.len);
if (!empty_pcust) {
Log(LOG_ERR, "ERROR ( %s/%s ): Unable to malloc() empty_pcust. Exiting.\n", config.name, config.type);
exit_plugin(1);
}
memset(&empty_pbgp, 0, sizeof(struct pkt_bgp_primitives));
memset(&empty_pnat, 0, sizeof(struct pkt_nat_primitives));
memset(&empty_pmpls, 0, sizeof(struct pkt_mpls_primitives));
memset(&empty_ptun, 0, sizeof(struct pkt_tunnel_primitives));
memset(empty_pcust, 0, config.cpptrs.len);
p_kafka_connect_to_produce(&kafkap_kafka_host);
p_kafka_set_broker(&kafkap_kafka_host, config.sql_host, config.kafka_broker_port);
if (!is_topic_dyn && !config.amqp_routing_key_rr) p_kafka_set_topic(&kafkap_kafka_host, config.sql_table);
p_kafka_set_partition(&kafkap_kafka_host, config.kafka_partition);
p_kafka_set_key(&kafkap_kafka_host, config.kafka_partition_key, config.kafka_partition_keylen);
if (config.message_broker_output & PRINT_OUTPUT_JSON) p_kafka_set_content_type(&kafkap_kafka_host, PM_KAFKA_CNT_TYPE_STR);
else if (config.message_broker_output & PRINT_OUTPUT_AVRO) p_kafka_set_content_type(&kafkap_kafka_host, PM_KAFKA_CNT_TYPE_BIN);
else {
Log(LOG_ERR, "ERROR ( %s/%s ): Unsupported kafka_output value specified. Exiting.\n", config.name, config.type);
exit_plugin(1);
}
for (j = 0, stop = 0; (!stop) && P_preprocess_funcs[j]; j++)
stop = P_preprocess_funcs[j](queue, &index, j);
Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - START (PID: %u) ***\n", config.name, config.type, writer_pid);
start = time(NULL);
if (config.print_markers) {
if (config.message_broker_output & PRINT_OUTPUT_JSON || config.message_broker_output & PRINT_OUTPUT_AVRO) {
void *json_obj;
char *json_str;
json_obj = compose_purge_init_json(config.name, writer_pid);
if (json_obj) json_str = compose_json_str(json_obj);
if (json_str) {
Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str);
ret = p_kafka_produce_data(&kafkap_kafka_host, json_str, strlen(json_str));
free(json_str);
json_str = NULL;
}
}
}
if (config.message_broker_output & PRINT_OUTPUT_JSON) {
if (config.sql_multi_values) {
json_buf = malloc(config.sql_multi_values);
if (!json_buf) {
Log(LOG_ERR, "ERROR ( %s/%s ): malloc() failed (json_buf). Exiting ..\n", config.name, config.type);
exit_plugin(1);
}
else memset(json_buf, 0, config.sql_multi_values);
}
}
else if (config.message_broker_output & PRINT_OUTPUT_AVRO) {
#ifdef WITH_AVRO
if (!config.avro_buffer_size) config.avro_buffer_size = LARGEBUFLEN;
avro_buf = malloc(config.avro_buffer_size);
if (!avro_buf) {
Log(LOG_ERR, "ERROR ( %s/%s ): malloc() failed (avro_buf). Exiting ..\n", config.name, config.type);
exit_plugin(1);
}
else memset(avro_buf, 0, config.avro_buffer_size);
avro_writer = avro_writer_memory(avro_buf, config.avro_buffer_size);
#endif
}
for (j = 0; j < index; j++) {
void *json_obj;
char *json_str;
if (queue[j]->valid != PRINT_CACHE_COMMITTED) continue;
data = &queue[j]->primitives;
if (queue[j]->pbgp) pbgp = queue[j]->pbgp;
else pbgp = &empty_pbgp;
if (queue[j]->pnat) pnat = queue[j]->pnat;
else pnat = &empty_pnat;
if (queue[j]->pmpls) pmpls = queue[j]->pmpls;
else pmpls = &empty_pmpls;
if (queue[j]->ptun) ptun = queue[j]->ptun;
else ptun = &empty_ptun;
if (queue[j]->pcust) pcust = queue[j]->pcust;
else pcust = empty_pcust;
if (queue[j]->pvlen) pvlen = queue[j]->pvlen;
else pvlen = NULL;
if (queue[j]->valid == PRINT_CACHE_FREE) continue;
if (config.message_broker_output & PRINT_OUTPUT_JSON) {
#ifdef WITH_JANSSON
json_t *json_obj = json_object();
int idx;
for (idx = 0; idx < N_PRIMITIVES && cjhandler[idx]; idx++) cjhandler[idx](json_obj, queue[j]);
add_writer_name_and_pid_json(json_obj, config.name, writer_pid);
json_str = compose_json_str(json_obj);
#endif
}
else if (config.message_broker_output & PRINT_OUTPUT_AVRO) {
#ifdef WITH_AVRO
avro_value_iface_t *avro_iface = avro_generic_class_from_schema(avro_acct_schema);
avro_value_t avro_value = compose_avro(config.what_to_count, config.what_to_count_2, queue[j]->flow_type,
&queue[j]->primitives, pbgp, pnat, pmpls, ptun, pcust, pvlen, queue[j]->bytes_counter,
queue[j]->packet_counter, queue[j]->flow_counter, queue[j]->tcp_flags,
&queue[j]->basetime, queue[j]->stitch, avro_iface);
size_t avro_value_size;
add_writer_name_and_pid_avro(avro_value, config.name, writer_pid);
avro_value_sizeof(&avro_value, &avro_value_size);
if (avro_value_size > config.avro_buffer_size) {
Log(LOG_ERR, "ERROR ( %s/%s ): AVRO: insufficient buffer size (avro_buffer_size=%u)\n",
config.name, config.type, config.avro_buffer_size);
Log(LOG_ERR, "ERROR ( %s/%s ): AVRO: increase value or look for avro_buffer_size in CONFIG-KEYS document.\n\n",
config.name, config.type);
exit_plugin(1);
}
else if (avro_value_size >= (config.avro_buffer_size - avro_writer_tell(avro_writer))) {
avro_buffer_full = TRUE;
j--;
}
else if (avro_value_write(avro_writer, &avro_value)) {
Log(LOG_ERR, "ERROR ( %s/%s ): AVRO: unable to write value: %s\n",
config.name, config.type, avro_strerror());
exit_plugin(1);
}
else {
mv_num++;
}
avro_value_decref(&avro_value);
avro_value_iface_decref(avro_iface);
#else
if (config.debug) Log(LOG_DEBUG, "DEBUG ( %s/%s ): compose_avro(): AVRO object not created due to missing --enable-avro\n", config.name, config.type);
#endif
}
if (config.message_broker_output & PRINT_OUTPUT_JSON) {
char *tmp_str = NULL;
if (json_str && config.sql_multi_values) {
int json_strlen = (strlen(json_str) ? (strlen(json_str) + 1) : 0);
if (json_strlen >= (config.sql_multi_values - json_buf_off)) {
if (json_strlen >= config.sql_multi_values) {
Log(LOG_ERR, "ERROR ( %s/%s ): kafka_multi_values not large enough to store JSON elements. Exiting ..\n", config.name, config.type);
exit(1);
}
tmp_str = json_str;
json_str = json_buf;
}
else {
strcat(json_buf, json_str);
mv_num++;
string_add_newline(json_buf);
json_buf_off = strlen(json_buf);
free(json_str);
json_str = NULL;
}
}
if (json_str) {
if (is_topic_dyn) {
P_handle_table_dyn_strings(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, queue[j]);
p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic);
}
if (config.amqp_routing_key_rr) {
P_handle_table_dyn_rr(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, &kafkap_kafka_host.topic_rr);
p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic);
}
Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str);
ret = p_kafka_produce_data(&kafkap_kafka_host, json_str, strlen(json_str));
if (config.sql_multi_values) {
json_str = tmp_str;
strcpy(json_buf, json_str);
mv_num_save = mv_num;
mv_num = 1;
string_add_newline(json_buf);
json_buf_off = strlen(json_buf);
}
free(json_str);
json_str = NULL;
if (!ret) {
if (!config.sql_multi_values) qn++;
else qn += mv_num_save;
}
else break;
}
}
else if (config.message_broker_output & PRINT_OUTPUT_AVRO) {
#ifdef WITH_AVRO
if (!config.sql_multi_values || (mv_num >= config.sql_multi_values) || avro_buffer_full) {
if (is_topic_dyn) {
P_handle_table_dyn_strings(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, queue[j]);
p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic);
}
if (config.amqp_routing_key_rr) {
P_handle_table_dyn_rr(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, &kafkap_kafka_host.topic_rr);
p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic);
}
ret = p_kafka_produce_data(&kafkap_kafka_host, avro_buf, avro_writer_tell(avro_writer));
avro_writer_reset(avro_writer);
avro_buffer_full = FALSE;
mv_num_save = mv_num;
mv_num = 0;
if (!ret) qn += mv_num_save;
else break;
}
#endif
}
}
if (config.sql_multi_values) {
if (config.message_broker_output & PRINT_OUTPUT_JSON) {
if (json_buf && json_buf_off) {
/* no handling of dyn routing keys here: not compatible */
Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_buf);
ret = p_kafka_produce_data(&kafkap_kafka_host, json_buf, strlen(json_buf));
if (!ret) qn += mv_num;
}
}
else if (config.message_broker_output & PRINT_OUTPUT_AVRO) {
#ifdef WITH_AVRO
if (avro_writer_tell(avro_writer)) {
ret = p_kafka_produce_data(&kafkap_kafka_host, avro_buf, avro_writer_tell(avro_writer));
avro_writer_free(avro_writer);
if (!ret) qn += mv_num;
}
#endif
}
}
duration = time(NULL)-start;
if (config.print_markers) {
if (config.message_broker_output & PRINT_OUTPUT_JSON || config.message_broker_output & PRINT_OUTPUT_AVRO) {
void *json_obj;
char *json_str;
json_obj = compose_purge_close_json(config.name, writer_pid, qn, saved_index, duration);
if (json_obj) json_str = compose_json_str(json_obj);
if (json_str) {
sleep(1); /* Let's give a small delay to facilitate purge_close being
the last message in batch in case of partitioned topics */
Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str);
ret = p_kafka_produce_data(&kafkap_kafka_host, json_str, strlen(json_str));
free(json_str);
json_str = NULL;
}
}
}
p_kafka_close(&kafkap_kafka_host, FALSE);
Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - END (PID: %u, QN: %u/%u, ET: %u) ***\n",
config.name, config.type, writer_pid, qn, saved_index, duration);
if (config.sql_trigger_exec && !safe_action) P_trigger_exec(config.sql_trigger_exec);
if (empty_pcust) free(empty_pcust);
if (json_buf) free(json_buf);
#ifdef WITH_AVRO
if (avro_buf) free(avro_buf);
#endif
}
static void build_heartbeat_response(completed_proc_t *completed_proc_array, int array_size, avro_slice_t **slice)
{
char filename[FILE_NAME_LEN];
char buf[BUFFER_SIZE];
long len = 0;
avro_schema_t schema;
avro_value_iface_t *iface;
avro_value_t record;
avro_value_t completed_processes_value, ProcessStatus_value;
avro_value_t name_value, ProcessName_value, jobid_value, vpid_value;
avro_value_t state_value;
avro_value_t exit_value_value;
size_t index;
int i;
avro_writer_t writer;
sprintf(filename, "%s/%s", SCHEMA_PATH, "HeartBeatResponseRecordAvro.avsc");
init_schema(filename, &schema);
iface = avro_generic_class_from_schema(schema);
avro_generic_value_new(iface, &record);
avro_value_get_by_name(&record, "completed_processes", &completed_processes_value, &index);
for (i = 0; i < array_size; i++) {
avro_value_append(&completed_processes_value, &ProcessStatus_value, &index);
avro_value_get_by_name(&ProcessStatus_value, "name", &name_value, &index);
avro_value_get_by_name(&name_value, "jobid", &jobid_value, &index);
avro_value_set_int(&jobid_value, completed_proc_array[i].proc_name.jobid);
avro_value_get_by_name(&name_value, "vpid", &vpid_value, &index);
avro_value_set_int(&vpid_value, completed_proc_array[i].proc_name.vpid);
avro_value_get_by_name(&ProcessStatus_value, "state", &state_value, &index);
avro_value_set_enum(&state_value, completed_proc_array[i].proc_state);
avro_value_get_by_name(&ProcessStatus_value, "exit_value", &exit_value_value, &index);
avro_value_set_int(&exit_value_value, completed_proc_array[i].exit_value);
}
/* create a writer with memory buffer */
writer = avro_writer_memory(buf, sizeof(buf));
/* write record to writer (buffer) */
if (avro_value_write(writer, &record)) {
fprintf(stderr, "Unable to write record to memory buffer\n");
fprintf(stderr, "Error: %s\n", avro_strerror());
exit(1);
}
avro_writer_flush(writer);
len = avro_writer_tell(writer);
//avro_generic_value_free(&record);
avro_value_iface_decref(iface);
avro_schema_decref(schema);
*slice = xmalloc(sizeof(avro_slice_t));
(*slice)->buffer = xmalloc(len);
(*slice)->len = len;
memcpy((*slice)->buffer, buf, len);
}
