void push_data_to_kafka(char *push_data)
{
int partition = 0;
const char* topic = "test";
//strcpy(push_data, "helloworld");
wrapper_Info test_info;
if (PRODUCER_INIT_SUCCESS == producer_init(partition, topic, "192.168.1.68:9092", &test_msg_delivered, &test_info))
printf("producer init success!\n");
else
printf("producer init failed\n");
size_t len = strlen(push_data);
if (push_data[len - 1] == '\n')
push_data[--len] = '\0';
if (PUSH_DATA_SUCCESS == producer_push_data(push_data, strlen(push_data), &test_info))
printf("push data success: %s\n", push_data);
else
printf("push data failed %s\n", push_data);
printf("-----------------------------------------\n");
producer_close(&test_info);
}
bool keysig_search(int method, uint8_t *enc_buf, size_t buf_sz,
keysig_notify_fn_t notify, void *user, int nr_threads)
{
/* init producer */
producer_init();
/* init search */
keysig_search_init();
pthread_mutex_init(&g_keysig_search.mutex, NULL);
g_keysig_search.enc_buf = enc_buf;
g_keysig_search.enc_buf_sz = buf_sz;
g_keysig_search.found_keysig = false;
/* get methods */
routine_t worker_fn = NULL;
routine_t producer_fn = NULL;
if(method == KEYSIG_SEARCH_ASCII_HEX)
{
worker_fn = hex_worker;
producer_fn = hex_producer;
}
/* create workers */
pthread_t *worker = malloc(sizeof(pthread_t) * nr_threads);
pthread_t producer;
for(int i = 0; i < nr_threads; i++)
pthread_create(&worker[i], NULL, worker_fn, NULL);
pthread_create(&producer, NULL, producer_fn, NULL);
/* wait for all threads */
pthread_join(producer, NULL);
for(int i = 0; i < nr_threads; i++)
pthread_join(worker[i], NULL);
free(worker);
if(g_keysig_search.found_keysig)
notify(user, g_keysig_search.key, g_keysig_search.sig);
return g_keysig_search.found_keysig;
}
//Create Producer client and set producer group name
void m_producer_init( char* producerName){
producer_init(producerName);
}
/*
* Main Producer Loop
* ------------------
*/
int
producer(const char *fullTopicName, const char *backupTopicName,
const char *fname, const char *pipe_fname)
{
int ret_val;
char key_content[keySize];
FILE *fp, *pipe_fp;
int pipe_fd;
char *line = NULL;
size_t len = 0;
ssize_t read;
int msg_idx = 0;
char *linebuf = NULL;
char *keybuf = NULL;
struct timeval now, last_now;
int newrate, rate = STARTING_RATE;
int n_sent_this_sec = 0;
long tdiff;
const char *cur_topic = fullTopicName;
char namebuf[100];
streams_topic_partition_t topic;
streams_config_t config;
streams_producer_t producer;
ret_val = producer_init(cur_topic, &topic, &config, &producer);
if (EXIT_SUCCESS != ret_val) {
DPRINTF("producer_init() failed\n");
return (ret_val);
}
fp = fopen(fname, "ro");
if (fp == NULL) {
DPRINTF("open of file %s failed\n", fname);
return (-1);
}
DPRINTF("opening pipe %s\n", pipe_fname);
pipe_fd = open(pipe_fname, O_RDWR|O_NONBLOCK);
if (pipe_fd < 0) {
DPRINTF("open of file %s failed\n", pipe_fname);
return (-1);
}
pipe_fp = fdopen(pipe_fd, "r");
DPRINTF("\nSTEP 4: The producer will create, send, "
"and flush now\n");
/* send initial metrics */
ret_val = send_metrics(cur_topic, fullTopicName, backupTopicName, rate);
if (EXIT_SUCCESS != ret_val) {
IPRINTF("send_metrics() failed\n");
return (ret_val);
}
gettimeofday(&last_now, NULL);
while ((read = getline(&line, &len, fp)) != -1) {
/*
* check if we've sent enough,
* if so wait it out so we keep to the specified rate
*/
if (n_sent_this_sec >= rate) {
/* send new metrics */
ret_val = send_metrics(cur_topic, fullTopicName, backupTopicName, rate);
if (EXIT_SUCCESS != ret_val) {
IPRINTF("send_metrics() failed\n");
return (ret_val);
}
gettimeofday(&now, NULL);
tdiff = tvdiff(&last_now, &now);
printf("time diff %lu rate %d\n", tdiff, rate);
if (tdiff < 1000) {
DPRINTF("produced %d messages in %lums, sleeping\n",
rate, tdiff);
usleep((1000 - tdiff) * 1000);
} else if (tdiff >= 1000) {
DPRINTF("warning: falling behind, took %lums to write"
" %d messages\n", tdiff, rate);
}
gettimeofday(&last_now, NULL);
n_sent_this_sec = 0;
/* check the pipe for a rate change */
DPRINTF("checking pipe\n");
ret_val = is_pipe_ready(fileno(pipe_fp));
if (ret_val < 0) {
IPRINTF("is_pipe_ready() failed\n");
return (ret_val);
}
if (ret_val > 0) {
(void) fscanf(pipe_fp, "%d", &newrate);
IPRINTF("new rate: %d\n", newrate);
/* if we need to failover, toggle connection to the other cluster */
if (newrate == FAIL_OVER_CODE || newrate == FAIL_BACK_CODE) {
IPRINTF("failing over to %s cluster\n",
newrate == FAIL_OVER_CODE ? "backup" : "primary");
ret_val = producer_shutdown(&topic, &config, &producer);
if (EXIT_SUCCESS != ret_val) {
DPRINTF("trying to failover: shutdown() failed\n");
return (ret_val);
}
cur_topic =
newrate == FAIL_OVER_CODE ? backupTopicName : fullTopicName;
ret_val = producer_init(cur_topic, &topic, &config, &producer);
if (EXIT_SUCCESS != ret_val) {
DPRINTF("trying to failover: init() failed\n");
return (ret_val);
}
/* this was an out-of-band code so leave rate unchanged */
} else {
/* this was a rate request, set the new rate to it */
rate = newrate;
/* send new metrics */
ret_val =
send_metrics(cur_topic,
fullTopicName, backupTopicName, rate);
if (EXIT_SUCCESS != ret_val) {
IPRINTF("send_metrics() failed\n");
return (ret_val);
}
}
}
continue;
}
printf("sending inc %d\n", n_sent_this_sec);
n_sent_this_sec++;
/* DPRINTF("Retrieved line of length %zu :\n", read); */
/* DPRINTF("%s", line); */
/* Create a unique key and value for each message. */
char key_content[keySize];
sprintf(key_content, "Key_%d", msg_idx);
linebuf = malloc(strlen(line) + 1);
keybuf = malloc(strlen(key_content) + 1);
strcpy(keybuf, key_content);
strcpy(linebuf, line);
/* Create a record that contains the message. */
streams_producer_record_t record;
ret_val = streams_producer_record_create(
topic, keybuf, strlen(keybuf) + 1,
linebuf, strlen(linebuf) + 1, &record);
if (EXIT_SUCCESS != ret_val) {
DPRINTF("streams_producer_record_create() failed\n");
return (ret_val);
}
/* Buffer the message. */
/* DPRINTF("calling streams_producer_send()\n"); */
ret_val =
streams_producer_send(producer,
record, producerCallback, NULL);
if (EXIT_SUCCESS != ret_val) {
DPRINTF("streams_producer_send() failed\n");
return (ret_val);
}
msg_idx++;
/* Flush the message.
* this is the synchronous approach but reduces throughput
*
ret_val = streams_producer_flush(producer);
if (EXIT_SUCCESS != ret_val) {
DPRINTF("streams_producer_flush() failed\n");
return (ret_val);
}
DPRINTF("Produced: MESSAGE %d: ", msg_idx);
*/
}
fclose(fp);
if (line)
free(line);
/* send 0 metric now that we're shuttong down */
ret_val = send_metrics(cur_topic, fullTopicName, backupTopicName, 0);
if (EXIT_SUCCESS != ret_val) {
IPRINTF("send_metrics() failed\n");
return (ret_val);
}
return (EXIT_SUCCESS);
}
