예제 #1
0
rd_kafka_topic_t *test_create_producer_topic (rd_kafka_t *rk,
	const char *topic, ...) {
	rd_kafka_topic_t *rkt;
	rd_kafka_topic_conf_t *topic_conf;
	char errstr[512];
	va_list ap;
	const char *name, *val;

	test_conf_init(NULL, &topic_conf, 20);

	va_start(ap, topic);
	while ((name = va_arg(ap, const char *)) &&
	       (val = va_arg(ap, const char *))) {
		if (rd_kafka_topic_conf_set(topic_conf, name, val,
			errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK)
			TEST_FAIL("Conf failed: %s\n", errstr);
	}
	va_end(ap);

	/* Make sure all replicas are in-sync after producing
	 * so that consume test wont fail. */
        rd_kafka_topic_conf_set(topic_conf, "request.required.acks", "-1",
                                errstr, sizeof(errstr));


	rkt = rd_kafka_topic_new(rk, topic, topic_conf);
	if (!rkt)
		TEST_FAIL("Failed to create topic: %s\n",
                          rd_kafka_err2str(rd_kafka_errno2err(errno)));

	return rkt;

}
예제 #2
0
파일: main.c 프로젝트: eugpermar/rb_monitor
static void parse_rdkafka_keyval_config(rd_kafka_conf_t *rk_conf,
					rd_kafka_topic_conf_t *rkt_conf,
					const char *key,
					const char *value) {
	// Extracted from Magnus Edenhill's kafkacat
	rd_kafka_conf_res_t res;
	char errstr[512];

	const char *name = key + strlen(CONFIG_RDKAFKA_KEY);

	res = RD_KAFKA_CONF_UNKNOWN;
	/* Try "topic." prefixed properties on topic
	 * conf first, and then fall through to global if
	 * it didnt match a topic configuration property. */
	if (!strncmp(name, "topic.", strlen("topic."))) {
		res = rd_kafka_topic_conf_set(rkt_conf,
					      name + strlen("topic."),
					      value,
					      errstr,
					      sizeof(errstr));
	}

	if (res == RD_KAFKA_CONF_UNKNOWN) {
		res = rd_kafka_conf_set(
				rk_conf, name, value, errstr, sizeof(errstr));
	}

	if (res != RD_KAFKA_CONF_OK) {
		rdlog(LOG_ERR, "rdkafka: %s", errstr);
	}
}
예제 #3
0
void test_topic_conf_set (rd_kafka_topic_conf_t *tconf,
                          const char *name, const char *val) {
        char errstr[512];
        if (rd_kafka_topic_conf_set(tconf, name, val, errstr, sizeof(errstr)) !=
            RD_KAFKA_CONF_OK)
                TEST_FAIL("Failed to set topic config \"%s\"=\"%s\": %s\n",
                          name, val, errstr);
}
예제 #4
0
int main (int argc, char **argv) {

        if (argc < 0 /* always false */) {
                rd_kafka_version();
                rd_kafka_version_str();
                rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
                rd_kafka_errno2err(EINVAL);
                rd_kafka_conf_new();
                rd_kafka_conf_destroy(NULL);
                rd_kafka_conf_dup(NULL);
                rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_conf_set_dr_cb(NULL, NULL);
                rd_kafka_conf_set_error_cb(NULL, NULL);
                rd_kafka_conf_set_stats_cb(NULL, NULL);
                rd_kafka_conf_set_opaque(NULL, NULL);
                rd_kafka_conf_dump(NULL, NULL);
                rd_kafka_topic_conf_dump(NULL, NULL);
                rd_kafka_conf_dump_free(NULL, 0);
                rd_kafka_conf_properties_show(NULL);
                rd_kafka_topic_conf_new();
                rd_kafka_topic_conf_dup(NULL);
                rd_kafka_topic_conf_destroy(NULL);
                rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_topic_conf_set_opaque(NULL, NULL);
                rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
                rd_kafka_topic_partition_available(NULL, 0);
                rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_new(0, NULL, NULL, 0);
                rd_kafka_destroy(NULL);
                rd_kafka_name(NULL);
                rd_kafka_topic_new(NULL, NULL, NULL);
                rd_kafka_topic_destroy(NULL);
                rd_kafka_topic_name(NULL);
                rd_kafka_message_destroy(NULL);
                rd_kafka_message_errstr(NULL);
                rd_kafka_consume_start(NULL, 0, 0);
                rd_kafka_consume_stop(NULL, 0);
                rd_kafka_consume(NULL, 0, 0);
                rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
                rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
                rd_kafka_offset_store(NULL, 0, 0);
                rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
                rd_kafka_poll(NULL, 0);
                rd_kafka_brokers_add(NULL, NULL);
                rd_kafka_set_logger(NULL, NULL);
                rd_kafka_set_log_level(NULL, 0);
                rd_kafka_log_print(NULL, 0, NULL, NULL);
                rd_kafka_log_syslog(NULL, 0, NULL, NULL);
                rd_kafka_outq_len(NULL);
                rd_kafka_dump(NULL, NULL);
                rd_kafka_thread_cnt();
                rd_kafka_wait_destroyed(0);
        }


	return 0;
}
예제 #5
0
void p_kafka_apply_topic_config(struct p_kafka_host *kafka_host)
{
  FILE *file;
  char buf[SRVBUFLEN], errstr[SRVBUFLEN], *key, *value;
  int lineno = 1, ret;

  if (kafka_host && kafka_host->config_file && kafka_host->topic_cfg) {
    if ((file = fopen(kafka_host->config_file, "r")) == NULL) {
      Log(LOG_WARNING, "WARN ( %s/%s ): [%s] file not found. librdkafka topic configuration not loaded.\n", config.name, config.type, kafka_host->config_file);
      return;
    }
    else Log(LOG_INFO, "INFO ( %s/%s ): [%s] Reading librdkafka topic configuration.\n", config.name, config.type, kafka_host->config_file);

    while (!feof(file)) {
      if (fgets(buf, SRVBUFLEN, file)) {
        if ((ret = p_kafka_parse_config_entry(buf, "topic", &key, &value)) > 0) {
          ret = rd_kafka_topic_conf_set(kafka_host->topic_cfg, key, value, errstr, sizeof(errstr));
          if (ret != RD_KAFKA_CONF_OK) {
            Log(LOG_WARNING, "WARN ( %s/%s ): [%s:%u] key=%s value=%s failed: %s\n",
                config.name, config.type, kafka_host->config_file, lineno, key, value, errstr);
          }
        }
        else {
          if (ret == ERR) {
            Log(LOG_WARNING, "WARN ( %s/%s ): [%s:%u] Line malformed. Ignored.", config.name, config.type, kafka_host->config_file, lineno);
            continue;
          }
        }
      }

      lineno++;
    }

    fclose(file);
  }
}
예제 #6
0
/**
 * Creates and sets up kafka configuration objects.
 * Will read "test.conf" file if it exists.
 */
void test_conf_init (rd_kafka_conf_t **conf, rd_kafka_topic_conf_t **topic_conf,
		     int timeout) {
	FILE *fp;
	char buf[512];
	int line = 0;
	const char *test_conf = getenv("RDKAFKA_TEST_CONF") ? : "test.conf";
	char errstr[512];

	/* Limit the test run time. */
	alarm(timeout);
	signal(SIGALRM, sig_alarm);

	*conf = rd_kafka_conf_new();
	*topic_conf = rd_kafka_topic_conf_new();

	rd_kafka_conf_set_error_cb(*conf, test_error_cb);

	/* Open and read optional local test configuration file, if any. */
	if (!(fp = fopen(test_conf, "r"))) {
		if (errno == ENOENT)
			TEST_FAIL("%s not found\n", test_conf);
		else
			TEST_FAIL("Failed to read %s: %s",
				  test_conf, strerror(errno));
	}

	while (fgets(buf, sizeof(buf)-1, fp)) {
		char *t;
		char *b = buf;
		rd_kafka_conf_res_t res;
		char *name, *val;

		line++;
		if ((t = strchr(b, '\n')))
			*t = '\0';

		if (*b == '#' || !*b)
			continue;

		if (!(t = strchr(b, '=')))
			TEST_FAIL("%s:%i: expected name=value format\n",
				  test_conf, line);
		
		name = b;
		*t = '\0';
		val = t+1;
		
		if (!strncmp(name, "topic.", strlen("topic."))) {
			name += strlen("topic.");
			res = rd_kafka_topic_conf_set(*topic_conf,
						      name, val,
						      errstr, sizeof(errstr));
		} else
			res = rd_kafka_conf_set(*conf,
						name, val,
						errstr, sizeof(errstr));

		if (res != RD_KAFKA_CONF_OK)
			TEST_FAIL("%s:%i: %s\n",
				  test_conf, line, errstr);
	}

	fclose(fp);
}
예제 #7
0
static void test_read_conf_file (const char *conf_path,
                                 rd_kafka_conf_t *conf,
                                 rd_kafka_topic_conf_t *topic_conf,
                                 int *timeoutp) {
        FILE *fp;
	char buf[512];
	int line = 0;

#ifndef _MSC_VER
	fp = fopen(conf_path, "r");
#else
	fp = NULL;
	errno = fopen_s(&fp, conf_path, "r");
#endif
	if (!fp) {
		if (errno == ENOENT) {
			TEST_SAY("Test config file %s not found\n", conf_path);
                        return;
		} else
			TEST_FAIL("Failed to read %s: %s",
				  conf_path, strerror(errno));
	}

	while (fgets(buf, sizeof(buf)-1, fp)) {
		char *t;
		char *b = buf;
		rd_kafka_conf_res_t res = RD_KAFKA_CONF_UNKNOWN;
		char *name, *val;
                char errstr[512];

		line++;
		if ((t = strchr(b, '\n')))
			*t = '\0';

		if (*b == '#' || !*b)
			continue;

		if (!(t = strchr(b, '=')))
			TEST_FAIL("%s:%i: expected name=value format\n",
				  conf_path, line);

		name = b;
		*t = '\0';
		val = t+1;

                if (!strcmp(name, "test.timeout.multiplier")) {
                        TEST_LOCK();
                        test_timeout_multiplier = strtod(val, NULL);
                        TEST_UNLOCK();
                        *timeoutp = tmout_multip((*timeoutp)*1000) / 1000;
                        res = RD_KAFKA_CONF_OK;
                } else if (!strcmp(name, "test.topic.prefix")) {
					rd_snprintf(test_topic_prefix, sizeof(test_topic_prefix),
						"%s", val);
				    res = RD_KAFKA_CONF_OK;
                } else if (!strcmp(name, "test.topic.random")) {
                        if (!strcmp(val, "true") ||
                            !strcmp(val, "1"))
                                test_topic_random = 1;
                        else
                                test_topic_random = 0;
                        res = RD_KAFKA_CONF_OK;
                } else if (!strcmp(name, "test.concurrent.max")) {
                        TEST_LOCK();
                        test_concurrent_max = strtod(val, NULL);
                        TEST_UNLOCK();
                        res = RD_KAFKA_CONF_OK;
                } else if (!strncmp(name, "topic.", strlen("topic."))) {
			name += strlen("topic.");
                        if (topic_conf)
                                res = rd_kafka_topic_conf_set(topic_conf,
                                                              name, val,
                                                              errstr,
                                                              sizeof(errstr));
                        else
                                res = RD_KAFKA_CONF_OK;
                        name -= strlen("topic.");
                }

                if (res == RD_KAFKA_CONF_UNKNOWN) {
                        if (conf)
                                res = rd_kafka_conf_set(conf,
                                                        name, val,
                                                        errstr, sizeof(errstr));
                        else
                                res = RD_KAFKA_CONF_OK;
                }

		if (res != RD_KAFKA_CONF_OK)
			TEST_FAIL("%s:%i: %s\n",
				  conf_path, line, errstr);
	}

	fclose(fp);
}
예제 #8
0
int main_0008_reqacks (int argc, char **argv) {
	int partition = 0;
	int r;
	rd_kafka_t *rk;
	rd_kafka_topic_t *rkt;
	rd_kafka_conf_t *conf;
	rd_kafka_topic_conf_t *topic_conf;
	char errstr[512];
	char msg[128];
	int msgcnt = 100;
	int i;
        int reqacks;
        int idbase = 0;
        const char *topic = NULL;

	TEST_SAY("\033[33mNOTE! This test requires at "
                 "least 3 brokers!\033[0m\n");

	TEST_SAY("\033[33mNOTE! This test requires "
		 "default.replication.factor=3 to be configured on "
		 "all brokers!\033[0m\n");

        /* Try different request.required.acks settings (issue #75) */
        for (reqacks = -1 ; reqacks <= 1 ; reqacks++) {
                char tmp[10];

                test_conf_init(&conf, &topic_conf, 10);

                if (!topic)
                        topic = test_mk_topic_name("0008", 0);

                rd_snprintf(tmp, sizeof(tmp), "%i", reqacks);

                if (rd_kafka_topic_conf_set(topic_conf, "request.required.acks",
                                            tmp, errstr, sizeof(errstr)) !=
                    RD_KAFKA_CONF_OK)
                        TEST_FAIL("%s", errstr);

                /* Set delivery report callback */
                rd_kafka_conf_set_dr_cb(conf, dr_cb);

                /* Create kafka instance */
                rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
                                  errstr, sizeof(errstr));
                if (!rk)
                        TEST_FAIL("Failed to create rdkafka instance: %s\n",
                                  errstr);

                TEST_SAY("Created    kafka instance %s with required acks %i\n",
                         rd_kafka_name(rk), reqacks);

                rkt = rd_kafka_topic_new(rk, topic, topic_conf);
                if (!rkt)
                        TEST_FAIL("Failed to create topic: %s\n",
                                  rd_strerror(errno));

                /* Produce messages */
                for (i = 0 ; i < msgcnt ; i++) {
                        int *msgidp = malloc(sizeof(*msgidp));
                        *msgidp = idbase + i;
                        rd_snprintf(msg, sizeof(msg),
                                 "%s test message #%i (acks=%i)",
                                 argv[0], *msgidp, reqacks);
                        r = rd_kafka_produce(rkt, partition,
                                             RD_KAFKA_MSG_F_COPY,
                                             msg, strlen(msg), NULL, 0, msgidp);
                        if (r == -1)
                                TEST_FAIL("Failed to produce message #%i: %s\n",
                                          *msgidp, rd_strerror(errno));
                }

                TEST_SAY("Produced %i messages, waiting for deliveries\n",
                         msgcnt);

                /* Wait for messages to time out */
                while (rd_kafka_outq_len(rk) > 0)
                        rd_kafka_poll(rk, 50);

                if (fails)
                        TEST_FAIL("%i failures, see previous errors", fails);

                if (msgid_next != idbase + msgcnt)
                        TEST_FAIL("Still waiting for messages: "
                                  "next %i != end %i\n",
                                  msgid_next, msgcnt);
                idbase += i;

                /* Destroy topic */
                rd_kafka_topic_destroy(rkt);

                /* Destroy rdkafka instance */
                TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
                rd_kafka_destroy(rk);
        }

	return 0;
}
예제 #9
0
int main (int argc, char **argv) {
	char *brokers = "localhost";
	char mode = 'C';
	char *topic = NULL;
	const char *key = NULL;
	int partition = RD_KAFKA_PARTITION_UA; /* random */
	int opt;
	int msgcnt = -1;
	int sendflags = 0;
	char *msgpattern = "librdkafka_performance testing!";
	int msgsize = strlen(msgpattern);
	const char *debug = NULL;
	rd_ts_t now;
	char errstr[512];
	uint64_t seq = 0;
	int seed = time(NULL);
	rd_kafka_topic_t *rkt;
	rd_kafka_conf_t *conf;
	rd_kafka_topic_conf_t *topic_conf;
	const char *compression = "no";
	int64_t start_offset = 0;
	int batch_size = 0;

	/* Kafka configuration */
	conf = rd_kafka_conf_new();
	rd_kafka_conf_set_error_cb(conf, err_cb);
	rd_kafka_conf_set_dr_cb(conf, msg_delivered);

	/* Producer config */
	rd_kafka_conf_set(conf, "queue.buffering.max.messages", "500000",
			  NULL, 0);
	rd_kafka_conf_set(conf, "message.send.max.retries", "3", NULL, 0);
	rd_kafka_conf_set(conf, "retry.backoff.ms", "500", NULL, 0);
	
	/* Consumer config */
	/* Tell rdkafka to (try to) maintain 1M messages
	 * in its internal receive buffers. This is to avoid
	 * application -> rdkafka -> broker  per-message ping-pong
	 * latency.
	 * The larger the local queue, the higher the performance.
	 * Try other values with: ... -X queued.min.messages=1000
	 */
	rd_kafka_conf_set(conf, "queued.min.messages", "1000000", NULL, 0);



	/* Kafka topic configuration */
	topic_conf = rd_kafka_topic_conf_new();
	rd_kafka_topic_conf_set(topic_conf, "message.timeout.ms", "5000",
				NULL, 0);

	while ((opt =
		getopt(argc, argv,
		       "PCt:p:b:s:k:c:fi:Dd:m:S:x:R:a:z:o:X:B:eT:q")) != -1) {
		switch (opt) {
		case 'P':
		case 'C':
			mode = opt;
			break;
		case 't':
			topic = optarg;
			break;
		case 'p':
			partition = atoi(optarg);
			break;
		case 'b':
			brokers = optarg;
			break;
		case 's':
			msgsize = atoi(optarg);
			break;
		case 'k':
			key = optarg;
			break;
		case 'c':
			msgcnt = atoi(optarg);
			break;
		case 'D':
			sendflags |= RD_KAFKA_MSG_F_FREE;
			break;
		case 'i':
			dispintvl = atoi(optarg);
			break;
		case 'm':
			msgpattern = optarg;
			break;
		case 'S':
			seq = strtoull(optarg, NULL, 10);
			do_seq = 1;
			break;
		case 'x':
			exit_after = atoi(optarg);
			break;
		case 'R':
			seed = atoi(optarg);
			break;
		case 'a':
			if (rd_kafka_topic_conf_set(topic_conf,
						    "request.required.acks",
						    optarg,
						    errstr, sizeof(errstr)) !=
			    RD_KAFKA_CONF_OK) {
				fprintf(stderr, "%% %s\n", errstr);
				exit(1);
			}
			break;
		case 'B':
			batch_size = atoi(optarg);
			break;
		case 'z':
			if (rd_kafka_conf_set(conf, "compression.codec",
					      optarg,
					      errstr, sizeof(errstr)) !=
			    RD_KAFKA_CONF_OK) {
				fprintf(stderr, "%% %s\n", errstr);
				exit(1);
			}
			compression = optarg;
			break;
		case 'o':
			start_offset = strtoll(optarg, NULL, 10);
			break;
		case 'e':
			exit_eof = 1;
			break;
		case 'd':
			debug = optarg;
			break;
		case 'X':
		{
			char *name, *val;
			rd_kafka_conf_res_t res;

			if (!strcmp(optarg, "list") ||
			    !strcmp(optarg, "help")) {
				rd_kafka_conf_properties_show(stdout);
				exit(0);
			}

			name = optarg;
			if (!(val = strchr(name, '='))) {
				fprintf(stderr, "%% Expected "
					"-X property=value, not %s\n", name);
				exit(1);
			}

			*val = '\0';
			val++;

			res = RD_KAFKA_CONF_UNKNOWN;
			/* Try "topic." prefixed properties on topic
			 * conf first, and then fall through to global if
			 * it didnt match a topic configuration property. */
			if (!strncmp(name, "topic.", strlen("topic.")))
				res = rd_kafka_topic_conf_set(topic_conf,
							      name+
							      strlen("topic"),
							      val,
							      errstr,
							      sizeof(errstr));

			if (res == RD_KAFKA_CONF_UNKNOWN)
				res = rd_kafka_conf_set(conf, name, val,
							errstr, sizeof(errstr));

			if (res != RD_KAFKA_CONF_OK) {
				fprintf(stderr, "%% %s\n", errstr);
				exit(1);
			}
		}
		break;

		case 'T':
			if (rd_kafka_conf_set(conf, "statistics.interval.ms",
					      optarg, errstr, sizeof(errstr)) !=
			    RD_KAFKA_CONF_OK) {
				fprintf(stderr, "%% %s\n", errstr);
				exit(1);
			}
			rd_kafka_conf_set_stats_cb(conf, stats_cb);
			break;

		case 'q':
			quiet = 1;
			break;

		default:
			goto usage;
		}
	}

	if (!topic || optind != argc) {
	usage:
		fprintf(stderr,
			"Usage: %s [-C|-P] -t <topic> "
			"[-p <partition>] [-b <broker,broker..>] [options..]\n"
			"\n"
			" Options:\n"
			"  -C | -P      Consumer or Producer mode\n"
			"  -t <topic>   Topic to fetch / produce\n"
			"  -p <num>     Partition (defaults to random)\n"
			"  -b <brokers> Broker address list (host[:port],..)\n"
			"  -s <size>    Message size (producer)\n"
			"  -k <key>     Message key (producer)\n"
			"  -c <cnt>     Messages to transmit/receive\n"
			"  -D           Copy/Duplicate data buffer (producer)\n"
			"  -i <ms>      Display interval\n"
			"  -m <msg>     Message payload pattern\n"
			"  -S <start>   Send a sequence number starting at "
			"<start> as payload\n"
			"  -R <seed>    Random seed value (defaults to time)\n"
			"  -a <acks>    Required acks (producer): "
			"-1, 0, 1, >1\n"
			"  -B <size>    Consume batch size (# of msgs)\n"
			"  -z <codec>   Enable compression:\n"
			"               none|gzip|snappy\n"
			"  -o <offset>  Start offset (consumer)\n"
			"  -d [facs..]  Enable debugging contexts:\n"
			"               %s\n"
			"  -X <prop=name> Set arbitrary librdkafka "
			"configuration property\n"
			"               Properties prefixed with \"topic.\" "
			"will be set on topic object.\n"
			"               Use '-X list' to see the full list\n"
			"               of supported properties.\n"
			"  -T <intvl>   Enable statistics from librdkafka at "
			"specified interval (ms)\n"
			"  -q           Be more quiet\n"
			"\n"
			" In Consumer mode:\n"
			"  consumes messages and prints thruput\n"
			"  If -B <..> is supplied the batch consumer\n"
			"  mode is used, else the callback mode is used.\n"
			"\n"
			" In Producer mode:\n"
			"  writes messages of size -s <..> and prints thruput\n"
			"\n",
			argv[0],
			RD_KAFKA_DEBUG_CONTEXTS);
		exit(1);
	}


	dispintvl *= 1000; /* us */

	printf("%% Using random seed %i\n", seed);
	srand(seed);
	signal(SIGINT, stop);
	signal(SIGUSR1, sig_usr1);


	if (debug &&
	    rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
	    RD_KAFKA_CONF_OK) {
		printf("%% Debug configuration failed: %s: %s\n",
		       errstr, debug);
		exit(1);
	}

	/* Socket hangups are gracefully handled in librdkafka on socket error
	 * without the use of signals, so SIGPIPE should be ignored by the
	 * calling program. */
	signal(SIGPIPE, SIG_IGN);

	if (msgcnt != -1)
		forever = 0;

	if (mode == 'P') {
		/*
		 * Producer
		 */
		char *sbuf;
		char *pbuf;
		int outq;
		int i;
		int keylen = key ? strlen(key) : 0;
		off_t rof = 0;
		size_t plen = strlen(msgpattern);

		if (do_seq) {
			if (msgsize < strlen("18446744073709551615: ")+1)
				msgsize = strlen("18446744073709551615: ")+1;
			/* Force duplication of payload */
			sendflags |= RD_KAFKA_MSG_F_FREE;
		}

		sbuf = malloc(msgsize);

		/* Copy payload content to new buffer */
		while (rof < msgsize) {
			size_t xlen = RD_MIN(msgsize-rof, plen);
			memcpy(sbuf+rof, msgpattern, xlen);
			rof += xlen;
		}

		if (msgcnt == -1)
			printf("%% Sending messages of size %i bytes\n",
			       msgsize);
		else
			printf("%% Sending %i messages of size %i bytes\n",
			       msgcnt, msgsize);

		/* Create Kafka handle */
		if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
 					errstr, sizeof(errstr)))) {
			fprintf(stderr,
				"%% Failed to create Kafka producer: %s\n",
				errstr);
			exit(1);
		}

		if (debug)
			rd_kafka_set_log_level(rk, 7);

		/* Add broker(s) */
		if (rd_kafka_brokers_add(rk, brokers) < 1) {
			fprintf(stderr, "%% No valid brokers specified\n");
			exit(1);
		}

		/* Explicitly create topic to avoid per-msg lookups. */
		rkt = rd_kafka_topic_new(rk, topic, topic_conf);

		cnt.t_start = rd_clock();

		while (run && (msgcnt == -1 || cnt.msgs < msgcnt)) {
			/* Send/Produce message. */

			if (do_seq) {
				snprintf(sbuf, msgsize-1, "%"PRIu64": ", seq);
				seq++;
			}

			if (sendflags & RD_KAFKA_MSG_F_FREE) {
				/* Duplicate memory */
				pbuf = malloc(msgsize);
				memcpy(pbuf, sbuf, msgsize);
			} else
				pbuf = sbuf;

			cnt.tx++;
			while (run &&
			       rd_kafka_produce(rkt, partition,
						sendflags, pbuf, msgsize,
						key, keylen, NULL) == -1) {
				if (!quiet || errno != ENOBUFS)
					printf("produce error: %s%s\n",
					       strerror(errno),
					       errno == ENOBUFS ?
					       " (backpressure)":"");
				cnt.tx_err++;
				if (errno != ENOBUFS) {
					run = 0;
					break;
				}
				now = rd_clock();
				if (cnt.t_last + dispintvl <= now) {
					printf("%% Backpressure %i "
					       "(tx %"PRIu64", "
					       "txerr %"PRIu64")\n",
					       rd_kafka_outq_len(rk),
					       cnt.tx, cnt.tx_err);
					cnt.t_last = now;
				}
				/* Poll to handle delivery reports */
				rd_kafka_poll(rk, 10);
			}

			msgs_wait_cnt++;
			cnt.msgs++;
			cnt.bytes += msgsize;

			print_stats(mode, 0, compression);

			/* Must poll to handle delivery reports */
			rd_kafka_poll(rk, 0);
			
		}

		forever = 0;
		printf("All messages produced, "
		       "now waiting for %li deliveries\n",
		       msgs_wait_cnt);
		rd_kafka_dump(stdout, rk);

		/* Wait for messages to be delivered */
		i = 0;
		while (run && rd_kafka_poll(rk, 1000) != -1) {
			if (!(i++ % (dispintvl/1000)))
				printf("%% Waiting for %li, "
				       "%i messages in outq "
				       "to be sent. Abort with Ctrl-c\n",
				       msgs_wait_cnt,
				       rd_kafka_outq_len(rk));
		}


		outq = rd_kafka_outq_len(rk);
		printf("%% %i messages in outq\n", outq);
		cnt.msgs -= outq;
		cnt.bytes -= msgsize * outq;

		cnt.t_end = t_end;

		if (cnt.tx_err > 0)
			printf("%% %"PRIu64" backpressures for %"PRIu64
			       " produce calls: %.3f%% backpressure rate\n",
			       cnt.tx_err, cnt.tx,
			       ((double)cnt.tx_err / (double)cnt.tx) * 100.0);

		rd_kafka_dump(stdout, rk);

		/* Destroy the handle */
		rd_kafka_destroy(rk);

	} else if (mode == 'C') {
		/*
		 * Consumer
		 */

		rd_kafka_message_t **rkmessages = NULL;

#if 0 /* Future API */
		/* The offset storage file is optional but its presence
		 * avoids starting all over from offset 0 again when
		 * the program restarts.
		 * ZooKeeper functionality will be implemented in future
		 * versions and then the offset will be stored there instead. */
		conf.consumer.offset_file = "."; /* current directory */

		/* Indicate to rdkafka that the application is responsible
		 * for storing the offset. This allows the application to
		 * successfully handle a message before storing the offset.
		 * If this flag is not set rdkafka will store the offset
		 * just prior to returning the message from rd_kafka_consume().
		 */
		conf.flags |= RD_KAFKA_CONF_F_APP_OFFSET_STORE;
#endif

		/* Create Kafka handle */
		if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
					errstr, sizeof(errstr)))) {
			fprintf(stderr,
				"%% Failed to create Kafka producer: %s\n",
				errstr);
			exit(1);
		}

		if (debug)
			rd_kafka_set_log_level(rk, 7);

		/* Add broker(s) */
		if (rd_kafka_brokers_add(rk, brokers) < 1) {
			fprintf(stderr, "%% No valid brokers specified\n");
			exit(1);
		}

		/* Create topic to consume from */
		rkt = rd_kafka_topic_new(rk, topic, topic_conf);

		/* Batch consumer */
		if (batch_size)
			rkmessages = malloc(sizeof(*rkmessages) * batch_size);

		/* Start consuming */
		if (rd_kafka_consume_start(rkt, partition, start_offset) == -1){
			fprintf(stderr, "%% Failed to start consuming: %s\n",
				strerror(errno));
			exit(1);
		}
		
		cnt.t_start = rd_clock();
		while (run && (msgcnt == -1 || msgcnt > cnt.msgs)) {
			/* Consume messages.
			 * A message may either be a real message, or
			 * an error signaling (if rkmessage->err is set).
			 */
			uint64_t latency;
			int r;

			latency = rd_clock();
			
			if (batch_size) {
				int i;

				/* Batch fetch mode */
				r = rd_kafka_consume_batch(rkt, partition,
							   1000,
							   rkmessages,
							   batch_size);
				if (r != -1) {
					for (i = 0 ; i < r ; i++) {
						msg_consume(rkmessages[i],NULL);
						rd_kafka_message_destroy(
							rkmessages[i]);
					}
				}
			} else {
				/* Callback mode */
				r = rd_kafka_consume_callback(rkt, partition,
							      1000/*timeout*/,
							      msg_consume,
							      NULL);
			}

			cnt.t_latency += rd_clock() - latency;
			
			if (r == -1)
				fprintf(stderr, "%% Error: %s\n",
					strerror(errno));

			print_stats(mode, 0, compression);

			/* Poll to handle stats callbacks */
			rd_kafka_poll(rk, 0);
		}
		cnt.t_end = rd_clock();

		/* Stop consuming */
		rd_kafka_consume_stop(rkt, partition);

		/* Destroy topic */
		rd_kafka_topic_destroy(rkt);

		if (batch_size)
			free(rkmessages);

		/* Destroy the handle */
		rd_kafka_destroy(rk);

	}

	print_stats(mode, 1, compression);

	if (cnt.t_latency && cnt.msgs)
		printf("%% Average application fetch latency: %"PRIu64"us\n",
		       cnt.t_latency / cnt.msgs);

	/* Let background threads clean up and terminate cleanly. */
	rd_kafka_wait_destroyed(2000);

	return 0;
}
예제 #10
0
int main (int argc, char **argv) {
	char *topic = "rdkafkatest1";
	int partition = 99; /* non-existent */
	int r;
	rd_kafka_t *rk;
	rd_kafka_topic_t *rkt;
	rd_kafka_conf_t *conf;
	rd_kafka_topic_conf_t *topic_conf;
	char errstr[512];
	char msg[128];
	int msgcnt = 10;
	time_t t_start, t_spent;
	int i;

	/* Socket hangups are gracefully handled in librdkafka on socket error
	 * without the use of signals, so SIGPIPE should be ignored by the
	 * calling program. */
	signal(SIGPIPE, SIG_IGN);


	test_conf_init(&conf, &topic_conf, 10);

	/* Set message.timeout.ms configuration for topic */
	if (rd_kafka_topic_conf_set(topic_conf, "message.timeout.ms", "2000",
				    errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK)
		TEST_FAIL("%s\n", errstr);

	/* Set delivery report callback */
	rd_kafka_conf_set_dr_cb(conf, dr_cb);

	/* Create kafka instance */
	rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
			  errstr, sizeof(errstr));
	if (!rk)
		TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);

	TEST_SAY("Created    kafka instance %s\n", rd_kafka_name(rk));

	rkt = rd_kafka_topic_new(rk, topic, topic_conf);
	if (!rkt)
		TEST_FAIL("Failed to create topic: %s\n",
			  strerror(errno));

	/* Produce a message */
	for (i = 0 ; i < msgcnt ; i++) {
		int *msgidp = malloc(sizeof(*msgidp));
		*msgidp = i;
		msgs_wait |= (1 << i);
		snprintf(msg, sizeof(msg), "%s test message #%i", argv[0], i);
		r = rd_kafka_produce(rkt, partition, RD_KAFKA_MSG_F_COPY,
				     msg, strlen(msg), NULL, 0, msgidp);
		if (r == -1)
			TEST_FAIL("Failed to produce message #%i: %s\n",
				  i, strerror(errno));
	}

	t_start = time(NULL);

	/* Wait for messages to time out */
	while (rd_kafka_outq_len(rk) > 0)
		rd_kafka_poll(rk, 50);

	t_spent = time(NULL) - t_start;

	if (msgs_wait != 0)
		TEST_FAIL("Still waiting for messages: 0x%x\n", msgs_wait);

	if (t_spent > 5 /* 2000ms+cruft*/)
		TEST_FAIL("Messages timed out too slowly (%i seconds > 5)\n",
			  (int)t_spent);

	/* Destroy topic */
	rd_kafka_topic_destroy(rkt);
		
	/* Destroy rdkafka instance */
	TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
	rd_kafka_destroy(rk);

	/* Wait for everything to be cleaned up since broker destroys are
	 * handled in its own thread. */
	test_wait_exit(10);

	/* If we havent failed at this point then
	 * there were no threads leaked */
	return 0;
}
예제 #11
0
파일: kafka.c 프로젝트: dwieland/phpkafka
void kafka_consume_all(rd_kafka_t *rk, zval *return_value, const char *topic, const char *offset, int item_count)
{
    char errstr[512];
    rd_kafka_topic_t *rkt;
    rd_kafka_topic_conf_t *conf;
    const struct rd_kafka_metadata *meta = NULL;
    rd_kafka_queue_t *rkqu = NULL;
    int current, p, i = 0;
    int32_t partition = 0;
    int64_t start;
    struct consume_cb_params cb_params = {item_count, return_value, NULL, 0, 0, 0};
    //check for NULL pointers, all arguments are required!
    if (rk == NULL || return_value == NULL || topic == NULL || offset == NULL || strlen(offset) == 0)
        return;

    if (!strcmp(offset, "end"))
        start = RD_KAFKA_OFFSET_END;
    else if (!strcmp(offset, "beginning"))
        start = RD_KAFKA_OFFSET_BEGINNING;
    else if (!strcmp(offset, "stored"))
        start = RD_KAFKA_OFFSET_STORED;
    else
        start = strtoll(offset, NULL, 10);

    /* Topic configuration */
    conf = rd_kafka_topic_conf_new();

    /* Disable autocommit, queue_consume sets offsets automatically */
    if (rd_kafka_topic_conf_set(conf, "auto.commit.enable", "false", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK)
    {
        if (log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(
                LOG_WARNING,
                "failed to turn autocommit off consuming %d messages (start offset %"PRId64") from topic %s: %s",
                item_count,
                start,
                topic,
                errstr
            );
        }
        cb_params.auto_commit = 1;
    }
    /* Create topic */
    rkt = rd_kafka_topic_new(rk, topic, conf);
    if (!rkt)
    {
        if (log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(LOG_INFO, "phpkafka - Failed to read %s from %"PRId64" (%s)", topic, start, offset);
        }
        return;
    }
    rkqu = rd_kafka_queue_new(rk);
    if (RD_KAFKA_RESP_ERR_NO_ERROR == rd_kafka_metadata(rk, 0, rkt, &meta, 5))
    {
        p = meta->topics->partition_cnt;
        cb_params.partition_ends = calloc(sizeof *cb_params.partition_ends, p);
        if (cb_params.partition_ends == NULL)
        {
            if (log_level)
            {
                openlog("phpkafka", 0, LOG_USER);
                syslog(LOG_INFO, "phpkafka - Failed to read %s from %"PRId64" (%s)", topic, start, offset);
            }
            rd_kafka_metadata_destroy(meta);
            meta = NULL;
            rd_kafka_queue_destroy(rkqu);
            rd_kafka_topic_destroy(rkt);
            return;
        }
        cb_params.eop = p;
        for (i=0;i<p;++i)
        {
            partition = meta->topics[0].partitions[i].id;
            if (rd_kafka_consume_start_queue(rkt, partition, start, rkqu))
            {
                if (log_level)
                {
                    openlog("phpkafka", 0, LOG_USER);
                    syslog(LOG_ERR,
                        "Failed to start consuming topic %s [%"PRId32"]: %s",
                        topic, partition, offset
                    );
                }
                continue;
            }
        }
        while(cb_params.read_count && cb_params.eop)
            rd_kafka_consume_callback_queue(rkqu, 200, queue_consume, &cb_params);
        free(cb_params.partition_ends);
        cb_params.partition_ends = NULL;
        for (i=0;i<p;++i)
        {
            partition = meta->topics[0].partitions[i].id;
            rd_kafka_consume_stop(rkt, partition);
        }
        rd_kafka_metadata_destroy(meta);
        meta = NULL;
        rd_kafka_queue_destroy(rkqu);
        while(rd_kafka_outq_len(rk) > 0)
            rd_kafka_poll(rk, 50);
        rd_kafka_topic_destroy(rkt);
    }
    if (meta)
        rd_kafka_metadata_destroy(meta);
}
예제 #12
0
static PyObject *
RdkHandle_configure(RdkHandle *self, PyObject *args, PyObject *kwds)
{
    char *keywords[] = {"conf", "topic_conf", NULL};
    PyObject *conf = NULL;
    PyObject *topic_conf = NULL;
    if (! PyArg_ParseTupleAndKeywords(args,
                                      kwds,
                                      "|OO",
                                      keywords,
                                      &conf,
                                      &topic_conf)) {
        return NULL;
    }

    if (RdkHandle_safe_lock(self, /* check_running= */ 0)) return NULL;
    if ((conf && topic_conf) || (!conf && !topic_conf)) {
        return set_pykafka_error(
            "RdKafkaException",
            "You need to specify *either* `conf` *or* `topic_conf`.");
    }
    if (self->rdk_handle) {
        return set_pykafka_error(
            "RdKafkaException",
            "Cannot configure: seems instance was started already?");
    }

    Py_BEGIN_ALLOW_THREADS  /* avoid callbacks deadlocking */
        if (! self->rdk_conf) {
            self->rdk_conf = rd_kafka_conf_new();
            rd_kafka_conf_set_log_cb(self->rdk_conf, logging_callback);
        }
        if (! self->rdk_topic_conf) {
            self->rdk_topic_conf = rd_kafka_topic_conf_new();
        }
    Py_END_ALLOW_THREADS

    PyObject *retval = Py_None;
    PyObject *conf_or_topic_conf = topic_conf ? topic_conf : conf;
    Py_ssize_t i, len = PyList_Size(conf_or_topic_conf);
    for (i = 0; i != len; ++i) {
        PyObject *conf_pair = PyList_GetItem(conf_or_topic_conf, i);
        const char *name = NULL;
        const char *value =  NULL;
        if (! PyArg_ParseTuple(conf_pair, "ss", &name, &value)) {
            retval = NULL;
            break;
        }
        char errstr[512];
        rd_kafka_conf_res_t res;
        Py_BEGIN_ALLOW_THREADS  /* avoid callbacks deadlocking */
            if (topic_conf) {
                res = rd_kafka_topic_conf_set(
                    self->rdk_topic_conf, name, value, errstr, sizeof(errstr));
            } else {
                res = rd_kafka_conf_set(
                    self->rdk_conf, name, value, errstr, sizeof(errstr));
            }
        Py_END_ALLOW_THREADS
        if (res != RD_KAFKA_CONF_OK) {
            retval = set_pykafka_error("RdKafkaException", errstr);
            break;
        }
    }

    if (RdkHandle_unlock(self)) return NULL;
    Py_XINCREF(retval);
    return retval;
}
int main (int argc, char **argv) {
        char mode = 'C';
	char *brokers = "localhost:9092";
	int opt;
	rd_kafka_conf_t *conf;
	rd_kafka_topic_conf_t *topic_conf;
	char errstr[512];
	const char *debug = NULL;
	int do_conf_dump = 0;
	char tmp[16];
        rd_kafka_resp_err_t err;
        char *group = NULL;
        rd_kafka_topic_partition_list_t *topics;
        int i;

	quiet = !isatty(STDIN_FILENO);

	/* Kafka configuration */
	conf = rd_kafka_conf_new();

        /* Set logger */
        rd_kafka_conf_set_log_cb(conf, logger);

	/* Quick termination */
	snprintf(tmp, sizeof(tmp), "%i", SIGIO);
	rd_kafka_conf_set(conf, "internal.termination.signal", tmp, NULL, 0);

	/* Topic configuration */
	topic_conf = rd_kafka_topic_conf_new();

	while ((opt = getopt(argc, argv, "g:b:qd:eX:As:DO")) != -1) {
		switch (opt) {
		case 'b':
			brokers = optarg;
			break;
                case 'g':
                        group = optarg;
                        break;
		case 'e':
			exit_eof = 1;
			break;
		case 'd':
			debug = optarg;
			break;
		case 'q':
			quiet = 1;
			break;
		case 'A':
			output = OUTPUT_RAW;
			break;
		case 'X':
		{
			char *name, *val;
			rd_kafka_conf_res_t res;

			if (!strcmp(optarg, "list") ||
			    !strcmp(optarg, "help")) {
				rd_kafka_conf_properties_show(stdout);
				exit(0);
			}

			if (!strcmp(optarg, "dump")) {
				do_conf_dump = 1;
				continue;
			}

			name = optarg;
			if (!(val = strchr(name, '='))) {
				fprintf(stderr, "%% Expected "
					"-X property=value, not %s\n", name);
				exit(1);
			}

			*val = '\0';
			val++;

			res = RD_KAFKA_CONF_UNKNOWN;
			/* Try "topic." prefixed properties on topic
			 * conf first, and then fall through to global if
			 * it didnt match a topic configuration property. */
			if (!strncmp(name, "topic.", strlen("topic.")))
				res = rd_kafka_topic_conf_set(topic_conf,
							      name+
							      strlen("topic."),
							      val,
							      errstr,
							      sizeof(errstr));

			if (res == RD_KAFKA_CONF_UNKNOWN)
				res = rd_kafka_conf_set(conf, name, val,
							errstr, sizeof(errstr));

			if (res != RD_KAFKA_CONF_OK) {
				fprintf(stderr, "%% %s\n", errstr);
				exit(1);
			}
		}
		break;

                case 'D':
                case 'O':
                        mode = opt;
                        break;

		default:
			goto usage;
		}
	}


	if (do_conf_dump) {
		const char **arr;
		size_t cnt;
		int pass;

		for (pass = 0 ; pass < 2 ; pass++) {
			if (pass == 0) {
				arr = rd_kafka_conf_dump(conf, &cnt);
				printf("# Global config\n");
			} else {
				printf("# Topic config\n");
				arr = rd_kafka_topic_conf_dump(topic_conf,
							       &cnt);
			}

			for (i = 0 ; i < (int)cnt ; i += 2)
				printf("%s = %s\n",
				       arr[i], arr[i+1]);

			printf("\n");

			rd_kafka_conf_dump_free(arr, cnt);
		}

		exit(0);
	}


	if (strchr("OC", mode) && optind == argc) {
	usage:
		fprintf(stderr,
			"Usage: %s [options] <topic[:part]> <topic[:part]>..\n"
			"\n"
			"librdkafka version %s (0x%08x)\n"
			"\n"
			" Options:\n"
                        "  -g <group>      Consumer group (%s)\n"
			"  -b <brokers>    Broker address (%s)\n"
			"  -e              Exit consumer when last message\n"
			"                  in partition has been received.\n"
                        "  -D              Describe group.\n"
                        "  -O              Get commmitted offset(s)\n"
			"  -d [facs..]     Enable debugging contexts:\n"
			"                  %s\n"
			"  -q              Be quiet\n"
			"  -A              Raw payload output (consumer)\n"
			"  -X <prop=name> Set arbitrary librdkafka "
			"configuration property\n"
			"               Properties prefixed with \"topic.\" "
			"will be set on topic object.\n"
			"               Use '-X list' to see the full list\n"
			"               of supported properties.\n"
			"\n"
			"\n",
			argv[0],
			rd_kafka_version_str(), rd_kafka_version(),
                        group, brokers,
			RD_KAFKA_DEBUG_CONTEXTS);
		exit(1);
	}


	signal(SIGINT, stop);
	signal(SIGUSR1, sig_usr1);

	if (debug &&
	    rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
	    RD_KAFKA_CONF_OK) {
		fprintf(stderr, "%% Debug configuration failed: %s: %s\n",
			errstr, debug);
		exit(1);
	}

        /*
         * Client/Consumer group
         */

        if (strchr("CO", mode)) {
                /* Consumer groups require a group id */
                if (!group)
                        group = "rdkafka_consumer_example";
                if (rd_kafka_conf_set(conf, "group.id", group,
                                      errstr, sizeof(errstr)) !=
                    RD_KAFKA_CONF_OK) {
                        fprintf(stderr, "%% %s\n", errstr);
                        exit(1);
                }

                /* Consumer groups always use broker based offset storage */
                if (rd_kafka_topic_conf_set(topic_conf, "offset.store.method",
                                            "broker",
                                            errstr, sizeof(errstr)) !=
                    RD_KAFKA_CONF_OK) {
                        fprintf(stderr, "%% %s\n", errstr);
                        exit(1);
                }

                /* Set default topic config for pattern-matched topics. */
                rd_kafka_conf_set_default_topic_conf(conf, topic_conf);

                /* Callback called on partition assignment changes */
                rd_kafka_conf_set_rebalance_cb(conf, rebalance_cb);
        }

        /* Create Kafka handle */
        if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
                                errstr, sizeof(errstr)))) {
                fprintf(stderr,
                        "%% Failed to create new consumer: %s\n",
                        errstr);
                exit(1);
        }

        rd_kafka_set_log_level(rk, LOG_DEBUG);

        /* Add brokers */
        if (rd_kafka_brokers_add(rk, brokers) == 0) {
                fprintf(stderr, "%% No valid brokers specified\n");
                exit(1);
        }


        if (mode == 'D') {
                int r;
                /* Describe groups */
                r = describe_groups(rk, group);

                rd_kafka_destroy(rk);
                exit(r == -1 ? 1 : 0);
        }

        /* Redirect rd_kafka_poll() to consumer_poll() */
        rd_kafka_poll_set_consumer(rk);

        topics = rd_kafka_topic_partition_list_new(argc - optind);
        for (i = optind ; i < argc ; i++) {
                /* Parse "topic[:part] */
                char *topic = argv[i];
                char *t;
                int32_t partition = -1;

                if ((t = strstr(topic, ":"))) {
                        *t = '\0';
                        partition = atoi(t+1);
                }

                rd_kafka_topic_partition_list_add(topics, topic, partition);
        }

        if (mode == 'O') {
                /* Offset query */

                err = rd_kafka_position(rk, topics, 5000);
                if (err) {
                        fprintf(stderr, "%% Failed to fetch offsets: %s\n",
                                rd_kafka_err2str(err));
                        exit(1);
                }

                for (i = 0 ; i < topics->cnt ; i++) {
                        rd_kafka_topic_partition_t *p = &topics->elems[i];
                        printf("Topic \"%s\" partition %"PRId32,
                               p->topic, p->partition);
                        if (p->err)
                                printf(" error %s",
                                       rd_kafka_err2str(p->err));
                        else {
                                printf(" offset %"PRId64"",
                                       p->offset);

                                if (p->metadata_size)
                                        printf(" (%d bytes of metadata)",
                                               (int)p->metadata_size);
                        }
                        printf("\n");
                }

                goto done;
        }


        if ((err = rd_kafka_subscribe(rk, topics))) {
                fprintf(stderr, "%% Failed to start consuming topics: %s\n",
                        rd_kafka_err2str(err));
                exit(1);
        }

        while (run) {
                rd_kafka_message_t *rkmessage;

                rkmessage = rd_kafka_consumer_poll(rk, 1000);
                if (rkmessage) {
                        msg_consume(rkmessage, NULL);
                        rd_kafka_message_destroy(rkmessage);
                }
        }

done:
        err = rd_kafka_consumer_close(rk);
        if (err)
                fprintf(stderr, "%% Failed to close consumer: %s\n",
                        rd_kafka_err2str(err));
        else
                fprintf(stderr, "%% Consumer closed\n");

        rd_kafka_topic_partition_list_destroy(topics);

        /* Destroy handle */
        rd_kafka_destroy(rk);

	/* Let background threads clean up and terminate cleanly. */
	run = 5;
	while (run-- > 0 && rd_kafka_wait_destroyed(1000) == -1)
		printf("Waiting for librdkafka to decommission\n");
	if (run <= 0)
		rd_kafka_dump(stdout, rk);

	return 0;
}
예제 #14
0
파일: main.c 프로젝트: DemandCube/Sparkngin
/**
 * Configuration file reader callback.
 *
 * Set a single configuration property 'name' using value 'val'.
 * Returns 0 on success, and -1 on error in which case 'errstr' will
 * contain an error string.
 */
static int conf_set (const char *name, const char *val,
		     char *errstr, size_t errstr_size,
		     const char *path, int line, void *opaque) {
	rd_kafka_conf_res_t res;

	if (!val) {
		snprintf(errstr, errstr_size, "Expected \"key=value\" format");
		return -1;
	}

	/* Kafka configuration: let librdkafka handle it. */
	if (!strncmp(name, "kafka.", 6)) {
		const char *kname = name + 6;

		if (!strncmp(kname, "topic.", 6)) {
			/* Kafka topic configuration. */
			res = rd_kafka_topic_conf_set(conf.rkt_conf,
						      kname+6, val,
						      errstr, errstr_size);
		} else {
			/* Kafka global configuration */
			res = rd_kafka_conf_set(conf.rk_conf, kname, val,
						errstr, errstr_size);
		}

		if (res == RD_KAFKA_CONF_OK) {
			return 0;
		} else if (res != RD_KAFKA_CONF_UNKNOWN) {
			return -1;
		}

		/* FALLTHRU for unknown configuration properties */
	}

	/* Configuration options */
	if (!strcmp(name, "zeromq.socket.type")) {
		if (!strcmp(val, "rep") || !strcmp(val, "reply"))
			conf.zmq_socket_type = ZMQ_REP;
		else if (!strcmp(val, "sub") || !strcmp(val, "subscribe"))
			conf.zmq_socket_type = ZMQ_SUB;
		else if (!strcmp(val, "pull"))
			conf.zmq_socket_type = ZMQ_PULL;
		else {
			snprintf(errstr, errstr_size,
				 "Unknown zeromq.socket.type \"%s\", "
				 "try one of: reply, subscribe, pull", val);
			return -1;
		}

	} else if (!strcmp(name, "zeromq.identity")) {
		if (conf.zmq_id)
			free(conf.zmq_id);
		conf.zmq_id = strdup(val);

	} else if (!strcmp(name, "zeromq.bind")) {
		if (conf.zmq_connect) {
			snprintf(errstr, errstr_size,
				 "zeromq.bind and zeromq.connect are "
				 "mutually exclusive");
			return -1;
		}
		if (conf.zmq_bind)
			free(conf.zmq_bind);
		conf.zmq_bind = strdup(val);

	} else if (!strcmp(name, "zeromq.connect")) {
		if (conf.zmq_bind) {
			snprintf(errstr, errstr_size,
				 "zeromq.bind and zeromq.connect are "
				 "mutually exclusive");
			return -1;
		}
		if (conf.zmq_connect)
			free(conf.zmq_connect);
		conf.zmq_connect = strdup(val);

	} else if (!strcmp(name, "zeromq.subscribe")) {
		if (conf.zmq_subscriptions)
			free(conf.zmq_subscriptions);
		conf.zmq_subscriptions_cnt =
			ezd_csv2array(&conf.zmq_subscriptions, val);

	} else if (!strcmp(name, "kafka.topic"))
		conf.kafka_topic = strdup(val);

	else if (!strcmp(name, "kafka.partition"))
		conf.kafka_partition = atoi(val);

	else if (!strcmp(name, "log.kafka.msg.error")) {
                if (ezd_str_tof(val))
                        conf.flags |= CONF_F_LOG_KAFKA_MSG_ERROR;
                else
                        conf.flags &= ~CONF_F_LOG_KAFKA_MSG_ERROR;

	} else if (!strcmp(name, "log.level"))
		conf.log_level = atoi(val);

	else if (!strcmp(name, "daemonize"))
		conf.daemonize = ezd_str_tof(val);

	else {
		snprintf(errstr, errstr_size,
			 "Invalid configuration property \"%s\"", name);
		return -1;
	}


	return 0;
}
예제 #15
0
/**
 * Creates and sets up kafka configuration objects.
 * Will read "test.conf" file if it exists.
 */
void test_conf_init (rd_kafka_conf_t **conf, rd_kafka_topic_conf_t **topic_conf,
		     int timeout) {
	FILE *fp;
	char buf[512];
	int line = 0;
	const char *test_conf =
#ifndef _MSC_VER
		getenv("RDKAFKA_TEST_CONF") ? getenv("RDKAFKA_TEST_CONF") : 
#endif
		"test.conf";
	char errstr[512];

	test_init();

        if (conf) {
                *conf = rd_kafka_conf_new();
                rd_kafka_conf_set_error_cb(*conf, test_error_cb);

#ifdef SIGIO
                /* Quick termination */
                rd_snprintf(buf, sizeof(buf), "%i", SIGIO);
                rd_kafka_conf_set(*conf, "internal.termination.signal",
                                  buf, NULL, 0);
                signal(SIGIO, SIG_IGN);
#endif
        }

	if (topic_conf)
		*topic_conf = rd_kafka_topic_conf_new();

	/* Open and read optional local test configuration file, if any. */
#ifndef _MSC_VER
	fp = fopen(test_conf, "r");
#else
	fp = NULL;
	errno = fopen_s(&fp, test_conf, "r");
#endif
	if (!fp) {
		if (errno == ENOENT)
			TEST_FAIL("%s not found\n", test_conf);
		else
			TEST_FAIL("Failed to read %s: errno %i",
				  test_conf, errno);
	}

	while (fgets(buf, sizeof(buf)-1, fp)) {
		char *t;
		char *b = buf;
		rd_kafka_conf_res_t res = RD_KAFKA_CONF_UNKNOWN;
		char *name, *val;

		line++;
		if ((t = strchr(b, '\n')))
			*t = '\0';

		if (*b == '#' || !*b)
			continue;

		if (!(t = strchr(b, '=')))
			TEST_FAIL("%s:%i: expected name=value format\n",
				  test_conf, line);

		name = b;
		*t = '\0';
		val = t+1;

                if (!strcmp(name, "test.timeout.multiplier")) {
                        timeout = (int)((float)timeout * strtod(val, NULL));
                        res = RD_KAFKA_CONF_OK;
                } else if (!strcmp(name, "test.topic.prefix")) {
					rd_snprintf(test_topic_prefix, sizeof(test_topic_prefix),
						"%s", val);
				    res = RD_KAFKA_CONF_OK;
                } else if (!strcmp(name, "test.topic.random")) {
                        if (!strcmp(val, "true") ||
                            !strcmp(val, "1"))
                                test_topic_random = 1;
                        else
                                test_topic_random = 0;
                        res = RD_KAFKA_CONF_OK;
                } else if (!strncmp(name, "topic.", strlen("topic."))) {
			name += strlen("topic.");
                        if (topic_conf)
                                res = rd_kafka_topic_conf_set(*topic_conf,
                                                              name, val,
                                                              errstr,
                                                              sizeof(errstr));
                        else
                                res = RD_KAFKA_CONF_OK;
                        name -= strlen("topic.");
                }

                if (res == RD_KAFKA_CONF_UNKNOWN) {
                        if (conf)
                                res = rd_kafka_conf_set(*conf,
                                                        name, val,
                                                        errstr, sizeof(errstr));
                        else
                                res = RD_KAFKA_CONF_OK;
                }

		if (res != RD_KAFKA_CONF_OK)
			TEST_FAIL("%s:%i: %s\n",
				  test_conf, line, errstr);
	}

	fclose(fp);

        if (timeout) {
                /* Limit the test run time. */
#ifndef _MSC_VER
                alarm(timeout);
                signal(SIGALRM, sig_alarm);
#endif
        }
}
예제 #16
0
/**
 * Produces 'msgcnt' messages split over 'partition_cnt' partitions.
 */
static void produce_null_messages (uint64_t testid, const char *topic,
                                   int partition_cnt, int msgcnt) {
	int r;
	rd_kafka_t *rk;
	rd_kafka_topic_t *rkt;
	rd_kafka_conf_t *conf;
	rd_kafka_topic_conf_t *topic_conf;
	char errstr[512];
	int i;
	int32_t partition;
	int msgid = 0;

	test_conf_init(&conf, &topic_conf, 20);

	rd_kafka_conf_set_dr_cb(conf, dr_cb);

        /* Make sure all replicas are in-sync after producing
         * so that consume test wont fail. */
        rd_kafka_topic_conf_set(topic_conf, "request.required.acks", "-1",
                                errstr, sizeof(errstr));

	/* Create kafka instance */
	rk = test_create_handle(RD_KAFKA_PRODUCER, conf);

	rkt = rd_kafka_topic_new(rk, topic, topic_conf);
	if (!rkt)
		TEST_FAIL("Failed to create topic: %s\n",
			  rd_kafka_err2str(rd_kafka_last_error()));

        /* Produce messages */
	prod_msg_remains = msgcnt;
	for (partition = 0 ; partition < partition_cnt ; partition++) {
		int batch_cnt = msgcnt / partition_cnt;

		for (i = 0 ; i < batch_cnt ; i++) {
                        char key[128];
			rd_snprintf(key, sizeof(key),
				 "testid=%"PRIu64", partition=%i, msg=%i",
				 testid, (int)partition, msgid);
                        r = rd_kafka_produce(rkt, partition, 0,
                                             NULL, 0,
                                             key, strlen(key),
                                             NULL);
                        if (r == -1)
                                TEST_FAIL("Failed to produce message %i "
                                          "to partition %i: %s",
                                          msgid, (int)partition,
                                          rd_kafka_err2str(rd_kafka_last_error()));
			msgid++;
		}
        }


        TEST_SAY("Produced %d messages to %d partition(s), "
                 "waiting for deliveries\n", msgcnt, partition_cnt);
	/* Wait for messages to be delivered */
	while (rd_kafka_outq_len(rk) > 0)
		rd_kafka_poll(rk, 100);

	if (fails)
		TEST_FAIL("%i failures, see previous errors", fails);

	if (prod_msg_remains != 0)
		TEST_FAIL("Still waiting for %i messages to be produced",
			  prod_msg_remains);
        else
                TEST_SAY("All messages delivered\n");

	/* Destroy topic */
	rd_kafka_topic_destroy(rkt);

	/* Destroy rdkafka instance */
	TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
	rd_kafka_destroy(rk);
}
예제 #17
0
static void producer_argparse (int argc, char **argv) {
  char errstr[512];
  char **left_argv;
  int opt;
  int option_index = 0;
  int left_argc = 0;

  while ((opt = getopt_long(argc, argv,
                            "b:p:d:k:c:z:uTX:E:vq",
                            producer_long_options,
                            &option_index)) != -1) {
    switch (opt) {
    case 'p':
      conf.partition = atoi(optarg);
      break;
    case 'b':
      conf.brokers = optarg;
      break;
    case 'z':
      if (rd_kafka_conf_set(conf.rk_conf,
                "compression.codec", optarg,
                errstr, sizeof(errstr)) !=
          RD_KAFKA_CONF_OK)
        FATAL("%s", errstr);
      break;
    case 'd':
      conf.delim = parse_delim(optarg);
      break;
    case 'k':
      conf.key_delim = parse_delim(optarg);
      conf.flags |= CONF_F_KEY_DELIM;
      break;
    case 'c':
      conf.msg_cnt = strtoll(optarg, NULL, 10);
      break;
    case 'q':
      conf.verbosity = 0;
      break;
    case 'v':
      conf.verbosity++;
      break;
    case 'T':
      conf.flags |= CONF_F_TEE;
      break;
    case 'u':
      setbuf(stdout, NULL);
      break;
    case 'E':
      conf.fd_err = fopen(optarg, "a");
      if (conf.fd_err == NULL)
        FATAL("Couldn't open error file %s: %s\n", optarg, strerror(errno));
      break;
    case 'X':
    {
      char *name, *val;
      rd_kafka_conf_res_t res;

      if (!strcmp(optarg, "list") ||
          !strcmp(optarg, "help")) {
        rd_kafka_conf_properties_show(stdout);
        exit(0);
      }

      if (!strcmp(optarg, "dump")) {
        conf.conf_dump = 1;
        continue;
      }

      name = optarg;
      if (!(val = strchr(name, '='))) {
        fprintf(stderr, "%% Expected "
          "-X property=value, not %s, "
          "use -X list to display available "
          "properties\n", name);
        exit(1);
      }

      *val = '\0';
      val++;

      res = RD_KAFKA_CONF_UNKNOWN;
      /* Try "topic." prefixed properties on topic
       * conf first, and then fall through to global if
       * it didnt match a topic configuration property. */
      if (!strncmp(name, "topic.", strlen("topic.")))
        res = rd_kafka_topic_conf_set(conf.rkt_conf,
                    name+
                    strlen("topic."),
                    val,
                    errstr,
                    sizeof(errstr));

      if (res == RD_KAFKA_CONF_UNKNOWN)
        res = rd_kafka_conf_set(conf.rk_conf, name, val,
              errstr, sizeof(errstr));

      if (res != RD_KAFKA_CONF_OK)
        FATAL("%s", errstr);
    }
    break;

    default:
      usage(argv[0], 1, NULL);
      break;
    }
  }

  /* Validate topic */
  if (argc - optind == 0)
    usage(argv[0], 1, "topic missing");
  else
    conf.topic = argv[optind++];

  /* Validate broker list */
  if (rd_kafka_conf_set(conf.rk_conf, "metadata.broker.list",
            conf.brokers, errstr, sizeof(errstr)) !=
      RD_KAFKA_CONF_OK)
    usage(argv[0], 1, errstr);

  /* Set message error log file descriptor */
  if(conf.fd_err == NULL)
    conf.fd_err = stderr;

  /* Retrieve input files */
  left_argc = argc - optind;
  conf.n_inputs = (left_argc == 0) ? 1 : left_argc;
  left_argv = &argv[optind];

  conf.inputs = calloc(conf.n_inputs, sizeof(char *));

  if (left_argc == 0 ||
      (left_argc == 1 && !strcmp(left_argv[0], "-"))) {
    /* Defaults to stdin if no file is passed or if single '-' */
    conf.inputs[0] = stdin;
  } else {
    for (int i = 0; i < conf.n_inputs; i++) {
      FILE *fd = fopen(left_argv[i], "r");

      if (fd == NULL)
        FATAL("Failed to open %s: %s\n", left_argv[i], strerror(errno));

      conf.inputs[i] = fd;
    }
  }
}
예제 #18
0
/* transmitter worker */
void *kaf_worker(void *thread_id) {
  char buf[MAX_BUF], *b;
  int rc=-1, nr, len, l, count=0,kr;

  /* kafka connection setup */
  char errstr[512];
  rd_kafka_t *k;
  rd_kafka_topic_t *t;
  rd_kafka_conf_t *conf;
  rd_kafka_topic_conf_t *topic_conf;
  int partition = RD_KAFKA_PARTITION_UA;
  char *key = NULL;
  int keylen = key ? strlen(key) : 0;

  /* set up global options */
  conf = rd_kafka_conf_new();
  rd_kafka_conf_set_error_cb(conf, err_cb);
  //rd_kafka_conf_set_throttle_cb(conf, throttle_cb);
  rd_kafka_conf_set_stats_cb(conf, stats_cb);
  kr = rd_kafka_conf_set(conf, "statistics.interval.ms", "60000", errstr, sizeof(errstr));
  if (kr != RD_KAFKA_CONF_OK) {
    fprintf(stderr,"error: rd_kafka_conf_set: statistics.interval.ms 60000 => %s\n", errstr);
    goto done;
  }
  char **opt=NULL;
  while( (opt=(char **)utarray_next(CF.rdkafka_options,opt))) {
    char *eq = strchr(*opt,'=');
    if (eq == NULL) {
      fprintf(stderr,"error: specify rdkafka params as key=value\n");
      goto done;
    }
    char *k = strdup(*opt), *v;
    k[eq-*opt] = '\0';
    v = &k[eq-*opt + 1];
    if (CF.verbose) fprintf(stderr,"setting %s %s\n", k, v); 
    kr = rd_kafka_conf_set(conf, k, v, errstr, sizeof(errstr));
    if (kr != RD_KAFKA_CONF_OK) {
      fprintf(stderr,"error: rd_kafka_conf_set: %s %s => %s\n", k, v, errstr);
      goto done;
    }
    free(k);
  }

  
  /* set up topic options */
  topic_conf = rd_kafka_topic_conf_new();
  opt=NULL;
  while( (opt=(char **)utarray_next(CF.rdkafka_topic_options,opt))) {
    char *eq = strchr(*opt,'=');
    if (eq == NULL) {
      fprintf(stderr,"error: specify rdkafka topic params as key=value\n");
      goto done;
    }
    char *k = strdup(*opt), *v;
    k[eq-*opt] = '\0';
    v = &k[eq-*opt + 1];
    if (CF.verbose) fprintf(stderr,"setting %s %s\n", k, v); 
    kr = rd_kafka_topic_conf_set(topic_conf, k, v, errstr, sizeof(errstr));
    if (kr != RD_KAFKA_CONF_OK) {
      fprintf(stderr,"error: rd_kafka_conf_set: %s %s => %s\n", k, v, errstr);
      goto done;
    }
    free(k);
  }


  k = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr));
  if (k == NULL) {
    fprintf(stderr, "rd_kafka_new: %s\n", errstr);
    goto done;
  }

  if (rd_kafka_brokers_add(k, CF.broker) < 1) {
    fprintf(stderr, "invalid broker\n");
    goto done;
  }

  t = rd_kafka_topic_new(k, CF.topic, topic_conf);

  while (CF.shutdown == 0) {
    len = nn_recv(CF.egress_socket_pull, buf, MAX_BUF, 0);
    if (len < 0) {
      fprintf(stderr,"nn_recv: %s\n", nn_strerror(errno));
      goto done;
    }

    rc = rd_kafka_produce(t, partition, RD_KAFKA_MSG_F_COPY, buf, len, 
                     key, keylen, NULL);
    if (rc == -1) {
      fprintf(stderr,"rd_kafka_produce: %s %s\n", 
        rd_kafka_err2str( rd_kafka_errno2err(errno)), 
        ((errno == ENOBUFS) ? "(backpressure)" : ""));
      goto done;
    }

    // cause rdkafka to invoke optional callbacks (msg delivery reports or error)
    if ((++count % 1000) == 0) rd_kafka_poll(k, 0);
    
    if (thread_id == 0) {
      /* only emit these stats from the first worker thread (not thread safe) */
      ts_add(CF.kaf_bytes_ts, CF.now, &len);
      ts_add(CF.kaf_msgs_ts, CF.now, NULL);
    }
  }

  rc = 0;

 done:
  CF.shutdown = 1;
  return NULL;
}
예제 #19
0
static void kafka_config_topic(rd_kafka_conf_t *conf, oconfig_item_t *ci) /* {{{ */
{
    int                          status;
    int                          i;
    struct kafka_topic_context  *tctx;
    char                        *key;
    char                        *val;
    char                         callback_name[DATA_MAX_NAME_LEN];
    char                         errbuf[1024];
    user_data_t                  ud;
	oconfig_item_t              *child;
    rd_kafka_conf_res_t          ret;

	if ((tctx = calloc(1, sizeof (*tctx))) == NULL) {
		ERROR ("write_kafka plugin: calloc failed.");
        return;
	}

    tctx->escape_char = '.';
    tctx->store_rates = 1;

    rd_kafka_conf_set_log_cb(conf, kafka_log);
    if ((tctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
                                    errbuf, sizeof(errbuf))) == NULL) {
        sfree(tctx);
        ERROR("write_kafka plugin: cannot create kafka handle.");
        return;
    }
    conf = NULL;

    if ((tctx->conf = rd_kafka_topic_conf_new()) == NULL) {
        rd_kafka_destroy(tctx->kafka);
        sfree(tctx);
        ERROR ("write_kafka plugin: cannot create topic configuration.");
        return;
    }

    if (ci->values_num != 1) {
        WARNING("kafka topic name needed.");
        goto errout;
    }

    if (ci->values[0].type != OCONFIG_TYPE_STRING) {
        WARNING("kafka topic needs a string argument.");
        goto errout;
    }

    if ((tctx->topic_name = strdup(ci->values[0].value.string)) == NULL) {
        ERROR("write_kafka plugin: cannot copy topic name.");
        goto errout;
    }

	for (i = 0; i < ci->children_num; i++) {
		/*
		 * The code here could be simplified but makes room
		 * for easy adding of new options later on.
		 */
		child = &ci->children[i];
		status = 0;

		if (strcasecmp ("Property", child->key) == 0) {
			if (child->values_num != 2) {
				WARNING("kafka properties need both a key and a value.");
                goto errout;
			}
			if (child->values[0].type != OCONFIG_TYPE_STRING ||
			    child->values[1].type != OCONFIG_TYPE_STRING) {
				WARNING("kafka properties needs string arguments.");
                goto errout;
			}
            key = child->values[0].value.string;
            val = child->values[0].value.string;
            ret = rd_kafka_topic_conf_set(tctx->conf,key, val,
                                          errbuf, sizeof(errbuf));
            if (ret != RD_KAFKA_CONF_OK) {
				WARNING("cannot set kafka topic property %s to %s: %s.",
                        key, val, errbuf);
                goto errout;
			}

        } else if (strcasecmp ("Key", child->key) == 0)  {
            char *tmp_buf = NULL;
            status = cf_util_get_string(child, &tmp_buf);
            if (status != 0) {
                WARNING("write_kafka plugin: invalid key supplied");
                break;
            }

            if (strcasecmp(tmp_buf, "Random") != 0) {
                tctx->has_key = 1;
                tctx->key = crc32_buffer((u_char *)tmp_buf, strlen(tmp_buf));
            }
            sfree(tmp_buf);

        } else if (strcasecmp ("Format", child->key) == 0) {
            status = cf_util_get_string(child, &key);
            if (status != 0)
                goto errout;

            assert(key != NULL);

            if (strcasecmp(key, "Command") == 0) {

                tctx->format = KAFKA_FORMAT_COMMAND;

            } else if (strcasecmp(key, "Graphite") == 0) {
                tctx->format = KAFKA_FORMAT_GRAPHITE;

            } else if (strcasecmp(key, "Json") == 0) {
                tctx->format = KAFKA_FORMAT_JSON;

            } else {
                WARNING ("write_kafka plugin: Invalid format string: %s",
                         key);
            }
            sfree(key);

        } else if (strcasecmp ("StoreRates", child->key) == 0) {
            status = cf_util_get_boolean (child, &tctx->store_rates);
            (void) cf_util_get_flag (child, &tctx->graphite_flags,
                                     GRAPHITE_STORE_RATES);

        } else if (strcasecmp ("GraphiteSeparateInstances", child->key) == 0) {
            status = cf_util_get_flag (child, &tctx->graphite_flags,
                                       GRAPHITE_SEPARATE_INSTANCES);

        } else if (strcasecmp ("GraphiteAlwaysAppendDS", child->key) == 0) {
            status = cf_util_get_flag (child, &tctx->graphite_flags,
                                       GRAPHITE_ALWAYS_APPEND_DS);

        } else if (strcasecmp ("GraphitePrefix", child->key) == 0) {
            status = cf_util_get_string (child, &tctx->prefix);
        } else if (strcasecmp ("GraphitePostfix", child->key) == 0) {
            status = cf_util_get_string (child, &tctx->postfix);
        } else if (strcasecmp ("GraphiteEscapeChar", child->key) == 0) {
            char *tmp_buff = NULL;
            status = cf_util_get_string (child, &tmp_buff);
            if (strlen (tmp_buff) > 1)
                WARNING ("write_kafka plugin: The option \"GraphiteEscapeChar\" handles "
                        "only one character. Others will be ignored.");
            tctx->escape_char = tmp_buff[0];
            sfree (tmp_buff);
        } else {
            WARNING ("write_kafka plugin: Invalid directive: %s.", child->key);
        }

        if (status != 0)
            break;
    }

    rd_kafka_topic_conf_set_partitioner_cb(tctx->conf, kafka_partition);
    rd_kafka_topic_conf_set_opaque(tctx->conf, tctx);

    if ((tctx->topic = rd_kafka_topic_new(tctx->kafka, tctx->topic_name,
                                       tctx->conf)) == NULL) {
        ERROR("write_kafka plugin: cannot create topic.");
        goto errout;
    }
    tctx->conf = NULL;

    ssnprintf(callback_name, sizeof(callback_name),
              "write_kafka/%s", tctx->topic_name);

    ud.data = tctx;
    ud.free_func = kafka_topic_context_free;

	status = plugin_register_write (callback_name, kafka_write, &ud);
	if (status != 0) {
		WARNING ("write_kafka plugin: plugin_register_write (\"%s\") "
				"failed with status %i.",
				callback_name, status);
        goto errout;
    }
    return;
 errout:
    if (conf != NULL)
        rd_kafka_conf_destroy(conf);
    if (tctx->kafka != NULL)
        rd_kafka_destroy(tctx->kafka);
    if (tctx->topic != NULL)
        rd_kafka_topic_destroy(tctx->topic);
    if (tctx->topic_name != NULL)
        free(tctx->topic_name);
    if (tctx->conf != NULL)
        rd_kafka_topic_conf_destroy(tctx->conf);
    sfree(tctx);
} /* }}} int kafka_config_topic */
int main (int argc, char **argv) {
	rd_kafka_topic_t *rkt;
	char *brokers = "localhost:9092";
	char mode = 'C';
	char *topic = NULL;
	int partition = RD_KAFKA_PARTITION_UA;
	int opt;
	rd_kafka_conf_t *conf;
	rd_kafka_topic_conf_t *topic_conf;
	char errstr[512];
	const char *debug = NULL;
	int64_t start_offset = 0;
        int report_offsets = 0;
	int do_conf_dump = 0;

	quiet = !isatty(STDIN_FILENO);

	/* Kafka configuration */
	conf = rd_kafka_conf_new();

	/* Topic configuration */
	topic_conf = rd_kafka_topic_conf_new();

	while ((opt = getopt(argc, argv, "PCLt:p:b:z:qd:o:eX:A")) != -1) {
		switch (opt) {
		case 'P':
		case 'C':
                case 'L':
			mode = opt;
			break;
		case 't':
			topic = optarg;
			break;
		case 'p':
			partition = atoi(optarg);
			break;
		case 'b':
			brokers = optarg;
			break;
		case 'z':
			if (rd_kafka_conf_set(conf, "compression.codec",
					      optarg,
					      errstr, sizeof(errstr)) !=
			    RD_KAFKA_CONF_OK) {
				fprintf(stderr, "%% %s\n", errstr);
				exit(1);
			}
			break;
		case 'o':
			if (!strcmp(optarg, "end"))
				start_offset = RD_KAFKA_OFFSET_END;
			else if (!strcmp(optarg, "beginning"))
				start_offset = RD_KAFKA_OFFSET_BEGINNING;
			else if (!strcmp(optarg, "stored"))
				start_offset = RD_KAFKA_OFFSET_STORED;
                        else if (!strcmp(optarg, "report"))
                                report_offsets = 1;
			else
				start_offset = strtoll(optarg, NULL, 10);
			break;
		case 'e':
			exit_eof = 1;
			break;
		case 'd':
			debug = optarg;
			break;
		case 'q':
			quiet = 1;
			break;
		case 'A':
			output = OUTPUT_RAW;
			break;
		case 'X':
		{
			char *name, *val;
			rd_kafka_conf_res_t res;

			if (!strcmp(optarg, "list") ||
			    !strcmp(optarg, "help")) {
				rd_kafka_conf_properties_show(stdout);
				exit(0);
			}

			if (!strcmp(optarg, "dump")) {
				do_conf_dump = 1;
				continue;
			}

			name = optarg;
			if (!(val = strchr(name, '='))) {
				fprintf(stderr, "%% Expected "
					"-X property=value, not %s\n", name);
				exit(1);
			}

			*val = '\0';
			val++;

			res = RD_KAFKA_CONF_UNKNOWN;
			/* Try "topic." prefixed properties on topic
			 * conf first, and then fall through to global if
			 * it didnt match a topic configuration property. */
			if (!strncmp(name, "topic.", strlen("topic.")))
				res = rd_kafka_topic_conf_set(topic_conf,
							      name+
							      strlen("topic."),
							      val,
							      errstr,
							      sizeof(errstr));

			if (res == RD_KAFKA_CONF_UNKNOWN)
				res = rd_kafka_conf_set(conf, name, val,
							errstr, sizeof(errstr));

			if (res != RD_KAFKA_CONF_OK) {
				fprintf(stderr, "%% %s\n", errstr);
				exit(1);
			}
		}
		break;

		default:
			goto usage;
		}
	}


	if (do_conf_dump) {
		const char **arr;
		size_t cnt;
		int pass;

		for (pass = 0 ; pass < 2 ; pass++) {
			int i;

			if (pass == 0) {
				arr = rd_kafka_conf_dump(conf, &cnt);
				printf("# Global config\n");
			} else {
				printf("# Topic config\n");
				arr = rd_kafka_topic_conf_dump(topic_conf,
							       &cnt);
			}

			for (i = 0 ; i < cnt ; i += 2)
				printf("%s = %s\n",
				       arr[i], arr[i+1]);

			printf("\n");

			rd_kafka_conf_dump_free(arr, cnt);
		}

		exit(0);
	}


	if (optind != argc || (mode != 'L' && !topic)) {
	usage:
		fprintf(stderr,
			"Usage: %s -C|-P|-L -t <topic> "
			"[-p <partition>] [-b <host1:port1,host2:port2,..>]\n"
			"\n"
			"librdkafka version %s (0x%08x)\n"
			"\n"
			" Options:\n"
			"  -C | -P         Consumer or Producer mode\n"
                        "  -L              Metadata list mode\n"
			"  -t <topic>      Topic to fetch / produce\n"
			"  -p <num>        Partition (random partitioner)\n"
			"  -b <brokers>    Broker address (localhost:9092)\n"
			"  -z <codec>      Enable compression:\n"
			"                  none|gzip|snappy\n"
			"  -o <offset>     Start offset (consumer)\n"
                        "  -o report       Report message offsets (producer)\n"
			"  -e              Exit consumer when last message\n"
			"                  in partition has been received.\n"
			"  -d [facs..]     Enable debugging contexts:\n"
			"  -q              Be quiet\n"
			"  -A              Raw payload output (consumer)\n"
			"                  %s\n"
			"  -X <prop=name> Set arbitrary librdkafka "
			"configuration property\n"
			"               Properties prefixed with \"topic.\" "
			"will be set on topic object.\n"
			"               Use '-X list' to see the full list\n"
			"               of supported properties.\n"
			"\n"
			" In Consumer mode:\n"
			"  writes fetched messages to stdout\n"
			" In Producer mode:\n"
			"  reads messages from stdin and sends to broker\n"
                        " In List mode:\n"
                        "  queries broker for metadata information, "
                        "topic is optional.\n"
			"\n"
			"\n"
			"\n",
			argv[0],
			rd_kafka_version_str(), rd_kafka_version(),
			RD_KAFKA_DEBUG_CONTEXTS);
		exit(1);
	}


	signal(SIGINT, stop);
	signal(SIGUSR1, sig_usr1);

	if (debug &&
	    rd_kafka_conf_set(conf, "debug", debug, errstr, sizeof(errstr)) !=
	    RD_KAFKA_CONF_OK) {
		fprintf(stderr, "%% Debug configuration failed: %s: %s\n",
			errstr, debug);
		exit(1);
	}

	if (mode == 'P') {
		/*
		 * Producer
		 */
		char buf[2048];
		int sendcnt = 0;

		/* Set up a message delivery report callback.
		 * It will be called once for each message, either on successful
		 * delivery to broker, or upon failure to deliver to broker. */

                /* If offset reporting (-o report) is enabled, use the
                 * richer dr_msg_cb instead. */
                if (report_offsets) {
                        rd_kafka_topic_conf_set(topic_conf,
                                                "produce.offset.report",
                                                "true", errstr, sizeof(errstr));
                        rd_kafka_conf_set_dr_msg_cb(conf, msg_delivered2);
                } else
                        rd_kafka_conf_set_dr_cb(conf, msg_delivered);

		/* Create Kafka handle */
		if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
					errstr, sizeof(errstr)))) {
			fprintf(stderr,
				"%% Failed to create new producer: %s\n",
				errstr);
			exit(1);
		}

		/* Set logger */
		rd_kafka_set_logger(rk, logger);
		rd_kafka_set_log_level(rk, LOG_DEBUG);

		/* Add brokers */
		if (rd_kafka_brokers_add(rk, brokers) == 0) {
			fprintf(stderr, "%% No valid brokers specified\n");
			exit(1);
		}

		/* Create topic */
		rkt = rd_kafka_topic_new(rk, topic, topic_conf);

		if (!quiet)
			fprintf(stderr,
				"%% Type stuff and hit enter to send\n");

		while (run && fgets(buf, sizeof(buf), stdin)) {
			size_t len = strlen(buf);
			if (buf[len-1] == '\n')
				buf[--len] = '\0';

			/* Send/Produce message. */
			if (rd_kafka_produce(rkt, partition,
					     RD_KAFKA_MSG_F_COPY,
					     /* Payload and length */
					     buf, len,
					     /* Optional key and its length */
					     NULL, 0,
					     /* Message opaque, provided in
					      * delivery report callback as
					      * msg_opaque. */
					     NULL) == -1) {
				fprintf(stderr,
					"%% Failed to produce to topic %s "
					"partition %i: %s\n",
					rd_kafka_topic_name(rkt), partition,
					rd_kafka_err2str(
						rd_kafka_errno2err(errno)));
				/* Poll to handle delivery reports */
				rd_kafka_poll(rk, 0);
				continue;
			}

			if (!quiet)
				fprintf(stderr, "%% Sent %zd bytes to topic "
					"%s partition %i\n",
				len, rd_kafka_topic_name(rkt), partition);
			sendcnt++;
			/* Poll to handle delivery reports */
			rd_kafka_poll(rk, 0);
		}

		/* Poll to handle delivery reports */
		rd_kafka_poll(rk, 0);

		/* Wait for messages to be delivered */
		while (run && rd_kafka_outq_len(rk) > 0)
			rd_kafka_poll(rk, 100);

		/* Destroy the handle */
		rd_kafka_destroy(rk);

	} else if (mode == 'C') {
		/*
		 * Consumer
		 */

		/* Create Kafka handle */
		if (!(rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf,
					errstr, sizeof(errstr)))) {
			fprintf(stderr,
				"%% Failed to create new consumer: %s\n",
				errstr);
			exit(1);
		}

		/* Set logger */
		rd_kafka_set_logger(rk, logger);
		rd_kafka_set_log_level(rk, LOG_DEBUG);

		/* Add brokers */
		if (rd_kafka_brokers_add(rk, brokers) == 0) {
			fprintf(stderr, "%% No valid brokers specified\n");
			exit(1);
		}

		/* Create topic */
		rkt = rd_kafka_topic_new(rk, topic, topic_conf);

		/* Start consuming */
		if (rd_kafka_consume_start(rkt, partition, start_offset) == -1){
			fprintf(stderr, "%% Failed to start consuming: %s\n",
				rd_kafka_err2str(rd_kafka_errno2err(errno)));
			exit(1);
		}

		while (run) {
			rd_kafka_message_t *rkmessage;

			/* Consume single message.
			 * See rdkafka_performance.c for high speed
			 * consuming of messages. */
			rkmessage = rd_kafka_consume(rkt, partition, 1000);
			if (!rkmessage) /* timeout */
				continue;

			msg_consume(rkmessage, NULL);

			/* Return message to rdkafka */
			rd_kafka_message_destroy(rkmessage);
		}

		/* Stop consuming */
		rd_kafka_consume_stop(rkt, partition);

		rd_kafka_topic_destroy(rkt);

		rd_kafka_destroy(rk);

        } else if (mode == 'L') {
                rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;

		/* Create Kafka handle */
		if (!(rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
					errstr, sizeof(errstr)))) {
			fprintf(stderr,
				"%% Failed to create new producer: %s\n",
				errstr);
			exit(1);
		}

		/* Set logger */
		rd_kafka_set_logger(rk, logger);
		rd_kafka_set_log_level(rk, LOG_DEBUG);

		/* Add brokers */
		if (rd_kafka_brokers_add(rk, brokers) == 0) {
			fprintf(stderr, "%% No valid brokers specified\n");
			exit(1);
		}

                /* Create topic */
                if (topic)
                        rkt = rd_kafka_topic_new(rk, topic, topic_conf);
                else
                        rkt = NULL;

                while (run) {
                        const struct rd_kafka_metadata *metadata;

                        /* Fetch metadata */
                        err = rd_kafka_metadata(rk, rkt ? 0 : 1, rkt,
                                                &metadata, 5000);
                        if (err != RD_KAFKA_RESP_ERR_NO_ERROR) {
                                fprintf(stderr,
                                        "%% Failed to acquire metadata: %s\n",
                                        rd_kafka_err2str(err));
                                run = 0;
                                break;
                        }

                        metadata_print(topic, metadata);

                        rd_kafka_metadata_destroy(metadata);
                        run = 0;
                }

		/* Destroy the handle */
		rd_kafka_destroy(rk);

                /* Exit right away, dont wait for background cleanup, we haven't
                 * done anything important anyway. */
                exit(err ? 2 : 0);
        }

	/* Let background threads clean up and terminate cleanly. */
	rd_kafka_wait_destroyed(2000);

	return 0;
}
예제 #21
0
/**
 * Produces 'msgcnt' messages split over 'partition_cnt' partitions.
 */
static void produce_messages (uint64_t testid, const char *topic,
			      int partition_cnt, int msgcnt) {
	int r;
	rd_kafka_t *rk;
	rd_kafka_topic_t *rkt;
	rd_kafka_conf_t *conf;
	rd_kafka_topic_conf_t *topic_conf;
	char errstr[512];
	char msg[128];
	int failcnt = 0;
	int i;
        rd_kafka_message_t *rkmessages;
	int32_t partition;
	int msgid = 0;

	test_conf_init(&conf, &topic_conf, 20);

	rd_kafka_conf_set_dr_cb(conf, dr_cb);

        /* Make sure all replicas are in-sync after producing
         * so that consume test wont fail. */
        rd_kafka_topic_conf_set(topic_conf, "request.required.acks", "-1",
                                errstr, sizeof(errstr));

	/* Create kafka instance */
	rk = test_create_handle(RD_KAFKA_PRODUCER, conf);

	rkt = rd_kafka_topic_new(rk, topic, topic_conf);
	if (!rkt)
		TEST_FAIL("Failed to create topic: %s\n",
			  rd_strerror(errno));

        /* Create messages. */
	prod_msg_remains = msgcnt;
        rkmessages = calloc(sizeof(*rkmessages), msgcnt / partition_cnt);
	for (partition = 0 ; partition < partition_cnt ; partition++) {
		int batch_cnt = msgcnt / partition_cnt;

		for (i = 0 ; i < batch_cnt ; i++) {
			rd_snprintf(msg, sizeof(msg),
				 "testid=%"PRIu64", partition=%i, msg=%i",
				 testid, (int)partition, msgid);
			rkmessages[i].payload   = rd_strdup(msg);
			rkmessages[i].len       = strlen(msg);
			msgid++;
		}

		TEST_SAY("Start produce to partition %i: msgs #%d..%d\n",
			 (int)partition, msgid-batch_cnt, msgid);
		/* Produce batch for this partition */
		r = rd_kafka_produce_batch(rkt, partition, RD_KAFKA_MSG_F_FREE,
					    rkmessages, batch_cnt);
		if (r == -1)
			TEST_FAIL("Failed to produce "
				  "batch for partition %i: %s",
				  (int)partition,
				  rd_kafka_err2str(rd_kafka_errno2err(errno)));

		/* Scan through messages to check for errors. */
		for (i = 0 ; i < batch_cnt ; i++) {
			if (rkmessages[i].err) {
				failcnt++;
				if (failcnt < 100)
					TEST_SAY("Message #%i failed: %s\n",
						 i,
						 rd_kafka_err2str(rkmessages[i].
								  err));
			}
		}

		/* All messages should've been produced. */
		if (r < batch_cnt) {
			TEST_SAY("Not all messages were accepted "
				 "by produce_batch(): %i < %i\n", r, batch_cnt);

			if (batch_cnt - r != failcnt)
				TEST_SAY("Discrepency between failed "
					 "messages (%i) "
					 "and return value %i (%i - %i)\n",
					 failcnt, batch_cnt - r, batch_cnt, r);
			TEST_FAIL("%i/%i messages failed\n",
				  batch_cnt - r, batch_cnt);
		}

		TEST_SAY("Produced %i messages to partition %i, "
			 "waiting for deliveries\n", r, partition);
	}


        free(rkmessages);

	/* Wait for messages to be delivered */
	while (rd_kafka_outq_len(rk) > 0)
		rd_kafka_poll(rk, 100);

	if (fails)
		TEST_FAIL("%i failures, see previous errors", fails);

	if (prod_msg_remains != 0)
		TEST_FAIL("Still waiting for %i messages to be produced",
			  prod_msg_remains);

	/* Destroy topic */
	rd_kafka_topic_destroy(rkt);

	/* Destroy rdkafka instance */
	TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
	rd_kafka_destroy(rk);
}
예제 #22
0
int main_0006_symbols (int argc, char **argv) {

        if (argc < 0 /* always false */) {
                rd_kafka_version();
                rd_kafka_version_str();
		rd_kafka_get_debug_contexts();
		rd_kafka_get_err_descs(NULL, NULL);
                rd_kafka_err2str(RD_KAFKA_RESP_ERR_NO_ERROR);
		rd_kafka_err2name(RD_KAFKA_RESP_ERR_NO_ERROR);
                rd_kafka_errno2err(EINVAL);
                rd_kafka_errno();
		rd_kafka_last_error();
                rd_kafka_conf_new();
                rd_kafka_conf_destroy(NULL);
                rd_kafka_conf_dup(NULL);
                rd_kafka_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_conf_set_dr_cb(NULL, NULL);
                rd_kafka_conf_set_dr_msg_cb(NULL, NULL);
                rd_kafka_conf_set_error_cb(NULL, NULL);
                rd_kafka_conf_set_stats_cb(NULL, NULL);
                rd_kafka_conf_set_log_cb(NULL, NULL);
                rd_kafka_conf_set_socket_cb(NULL, NULL);
		rd_kafka_conf_set_rebalance_cb(NULL, NULL);
		rd_kafka_conf_set_offset_commit_cb(NULL, NULL);
		rd_kafka_conf_set_throttle_cb(NULL, NULL);
		rd_kafka_conf_set_default_topic_conf(NULL, NULL);
		rd_kafka_conf_get(NULL, NULL, NULL, NULL);
#ifndef _MSC_VER
		rd_kafka_conf_set_open_cb(NULL, NULL);
#endif
		rd_kafka_conf_set_opaque(NULL, NULL);
                rd_kafka_opaque(NULL);
                rd_kafka_conf_dump(NULL, NULL);
                rd_kafka_topic_conf_dump(NULL, NULL);
                rd_kafka_conf_dump_free(NULL, 0);
                rd_kafka_conf_properties_show(NULL);
                rd_kafka_topic_conf_new();
                rd_kafka_topic_conf_dup(NULL);
                rd_kafka_topic_conf_destroy(NULL);
                rd_kafka_topic_conf_set(NULL, NULL, NULL, NULL, 0);
                rd_kafka_topic_conf_set_opaque(NULL, NULL);
		rd_kafka_topic_conf_get(NULL, NULL, NULL, NULL);
                rd_kafka_topic_conf_set_partitioner_cb(NULL, NULL);
                rd_kafka_topic_partition_available(NULL, 0);
		rd_kafka_topic_opaque(NULL);
                rd_kafka_msg_partitioner_random(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_msg_partitioner_consistent(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_msg_partitioner_consistent_random(NULL, NULL, 0, 0, NULL, NULL);
                rd_kafka_new(0, NULL, NULL, 0);
                rd_kafka_destroy(NULL);
                rd_kafka_name(NULL);
		rd_kafka_memberid(NULL);
                rd_kafka_topic_new(NULL, NULL, NULL);
                rd_kafka_topic_destroy(NULL);
                rd_kafka_topic_name(NULL);
                rd_kafka_message_destroy(NULL);
                rd_kafka_message_errstr(NULL);
		rd_kafka_message_timestamp(NULL, NULL);
                rd_kafka_consume_start(NULL, 0, 0);
                rd_kafka_consume_stop(NULL, 0);
                rd_kafka_consume(NULL, 0, 0);
                rd_kafka_consume_batch(NULL, 0, 0, NULL, 0);
                rd_kafka_consume_callback(NULL, 0, 0, NULL, NULL);
                rd_kafka_offset_store(NULL, 0, 0);
                rd_kafka_produce(NULL, 0, 0, NULL, 0, NULL, 0, NULL);
                rd_kafka_produce_batch(NULL, 0, 0, NULL, 0);
                rd_kafka_poll(NULL, 0);
                rd_kafka_brokers_add(NULL, NULL);
                /* DEPRECATED: rd_kafka_set_logger(NULL, NULL); */
                rd_kafka_set_log_level(NULL, 0);
                rd_kafka_log_print(NULL, 0, NULL, NULL);
#ifndef _MSC_VER
                rd_kafka_log_syslog(NULL, 0, NULL, NULL);
#endif
                rd_kafka_outq_len(NULL);
                rd_kafka_dump(NULL, NULL);
                rd_kafka_thread_cnt();
                rd_kafka_wait_destroyed(0);
                rd_kafka_metadata(NULL, 0, NULL, NULL, 0);
                rd_kafka_metadata_destroy(NULL);
                rd_kafka_queue_destroy(NULL);
                rd_kafka_consume_start_queue(NULL, 0, 0, NULL);
                rd_kafka_consume_queue(NULL, 0);
                rd_kafka_consume_batch_queue(NULL, 0, NULL, 0);
                rd_kafka_consume_callback_queue(NULL, 0, NULL, NULL);
                rd_kafka_seek(NULL, 0, 0, 0);
                rd_kafka_yield(NULL);
                rd_kafka_mem_free(NULL, NULL);
                rd_kafka_list_groups(NULL, NULL, NULL, 0);
                rd_kafka_group_list_destroy(NULL);

		/* KafkaConsumer API */
		rd_kafka_subscribe(NULL, NULL);
		rd_kafka_unsubscribe(NULL);
		rd_kafka_subscription(NULL, NULL);
		rd_kafka_consumer_poll(NULL, 0);
		rd_kafka_consumer_close(NULL);
		rd_kafka_assign(NULL, NULL);
		rd_kafka_assignment(NULL, NULL);
		rd_kafka_commit(NULL, NULL, 0);
		rd_kafka_commit_message(NULL, NULL, 0);
                rd_kafka_committed(NULL, NULL, 0);
		rd_kafka_position(NULL, NULL);

		/* TopicPartition */
		rd_kafka_topic_partition_list_new(0);
		rd_kafka_topic_partition_list_destroy(NULL);
		rd_kafka_topic_partition_list_add(NULL, NULL, 0);
		rd_kafka_topic_partition_list_add_range(NULL, NULL, 0, 0);
		rd_kafka_topic_partition_list_del(NULL, NULL, 0);
		rd_kafka_topic_partition_list_del_by_idx(NULL, 0);
		rd_kafka_topic_partition_list_copy(NULL);
		rd_kafka_topic_partition_list_set_offset(NULL, NULL, 0, 0);
		rd_kafka_topic_partition_list_find(NULL, NULL, 0);
		rd_kafka_query_watermark_offsets(NULL, NULL, 0, NULL, NULL, 0);
		rd_kafka_get_watermark_offsets(NULL, NULL, 0, NULL, NULL);
        }


	return 0;
}
예제 #23
0
파일: kafka.c 프로젝트: dwieland/phpkafka
int kafka_produce_report(rd_kafka_t *r, const char *topic, char *msg, int msg_len, long timeout)
{
    char errstr[512];
    rd_kafka_topic_t *rkt = NULL;
    int partition = RD_KAFKA_PARTITION_UA;
    rd_kafka_topic_conf_t *conf = NULL;
    struct produce_cb_params pcb = {1, 0, 0, 0, 0, NULL};

    if (r == NULL)
    {
        if (log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(LOG_ERR, "No connection provided to produce to topic %s", topic);
        }
        return -2;
    }

    /* Topic configuration */
    conf = rd_kafka_topic_conf_new();

    rd_kafka_topic_conf_set(conf,"produce.offset.report", "true", errstr, sizeof errstr );

    char timeoutStr[64];
    snprintf(timeoutStr, 64, "%lu", timeout);
    if (rd_kafka_topic_conf_set(conf, "message.timeout.ms", timeoutStr, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK)
    {
        if (log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(
                LOG_ERR,
                "Failed to configure topic param 'message.timeout.ms' to %lu before producing; config err was: %s",
                timeout,
                errstr
            );
        }
        rd_kafka_topic_conf_destroy(conf);
        return -3;
    }

    //callback already set in kafka_set_connection
    rkt = rd_kafka_topic_new(r, topic, conf);
    if (!rkt)
    {
        if (log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(LOG_ERR, "Failed to open topic %s", topic);
        }
        rd_kafka_topic_conf_destroy(conf);
        return -1;
    }

    //begin producing:
    if (rd_kafka_produce(rkt, partition, RD_KAFKA_MSG_F_COPY, msg, msg_len,NULL, 0,&pcb) == -1)
    {
        if (log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(LOG_ERR, "Failed to produce message: %s", rd_kafka_err2str(rd_kafka_errno2err(errno)));
        }
        //handle delivery response (callback)
        rd_kafka_poll(rk, 0);
        rd_kafka_topic_destroy(rkt);
        return -1;
    }
    rd_kafka_poll(rk, 0);
    while(pcb.msg_count && rd_kafka_outq_len(r) > 0)
        rd_kafka_poll(r, 10);
    rd_kafka_topic_destroy(rkt);
    return 0;
}
예제 #24
0
파일: kafka.c 프로젝트: dwieland/phpkafka
int kafka_produce_batch(rd_kafka_t *r, char *topic, char **msg, int *msg_len, int msg_cnt, int report, long timeout)
{
    char errstr[512];
    rd_kafka_topic_t *rkt;
    struct produce_cb_params pcb = {msg_cnt, 0, 0, 0, 0, NULL};
    void *opaque;
    int partition = RD_KAFKA_PARTITION_UA;
    int i,
        err_cnt = 0;

    if (report)
        opaque = &pcb;
    else
        opaque = NULL;
    rd_kafka_topic_conf_t *topic_conf;

    if (r == NULL)
    {
        if (log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(LOG_ERR, "phpkafka - no connection to produce to topic: %s", topic);
        }
        return -2;
    }

    /* Topic configuration */
    topic_conf = rd_kafka_topic_conf_new();

    char timeoutStr[64];
    snprintf(timeoutStr, 64, "%lu", timeout);
    if (rd_kafka_topic_conf_set(topic_conf, "message.timeout.ms", timeoutStr, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK)
    {
        if (log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(
                LOG_ERR,
                "Failed to configure topic param 'message.timeout.ms' to %lu before producing; config err was: %s",
                timeout,
                errstr
            );
        }
        rd_kafka_topic_conf_destroy(topic_conf);
        return -3;
    }

    /* Create topic */
    rkt = rd_kafka_topic_new(r, topic, topic_conf);

    //do we have VLA?
    rd_kafka_message_t *messages = calloc(sizeof *messages, msg_cnt);
    if (messages == NULL)
    {//fallback to individual produce calls
        for (i=0;i<msg_cnt;++i)
        {
            if (rd_kafka_produce(rkt, partition, RD_KAFKA_MSG_F_COPY, msg[i], msg_len[i], NULL, 0, opaque) == -1)
            {
                if (log_level)
                {
                    openlog("phpkafka", 0, LOG_USER);
                    syslog(LOG_INFO, "phpkafka - %% Failed to produce to topic %s "
                        "partition %i: %s",
                        rd_kafka_topic_name(rkt), partition,
                        rd_kafka_err2str(
                        rd_kafka_errno2err(errno)));
                }
            }
        }
    }
    else
    {
        for (i=0;i<msg_cnt;++i)
        {
            messages[i].payload = msg[i];
            messages[i].len = msg_len[i];
        }
        i = rd_kafka_produce_batch(rkt, partition, RD_KAFKA_MSG_F_COPY, messages, msg_cnt);
        if (i < msg_cnt)
        {
            if (log_level)
            {
                openlog("phpkafka", 0, LOG_USER);
                syslog(LOG_WARNING, "Failed to queue full message batch, %d of %d were put in queue", i, msg_cnt);
            }
        }
        err_cnt = msg_cnt - i;
        free(messages);
        messages = NULL;
    }
    /* Poll to handle delivery reports */
    rd_kafka_poll(r, 0);

    /* Wait for messages to be delivered */
    while (report && pcb.msg_count && rd_kafka_outq_len(r) > 0)
        rd_kafka_poll(r, 10);

    //set global to NULL again
    rd_kafka_topic_destroy(rkt);
    if (report)
        err_cnt = pcb.err_count;
    return err_cnt;
}
예제 #25
0
int main (int argc, char **argv) {
    rd_kafka_t *rk;
    rd_kafka_topic_t *rkt;
    rd_kafka_conf_t *ignore_conf, *conf, *conf2;
    rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2;
    char errstr[512];
    const char **arr_orig, **arr_dup;
    size_t cnt_orig, cnt_dup;
    int i;
    const char *topic;
    static const char *gconfs[] = {
        "message.max.bytes", "12345", /* int property */
        "client.id", "my id", /* string property */
        "debug", "topic,metadata", /* S2F property */
        "compression.codec", "gzip", /* S2I property */
        NULL
    };
    static const char *tconfs[] = {
        "request.required.acks", "-1", /* int */
        "auto.commit.enable", "false", /* bool */
        "auto.offset.reset", "error",  /* S2I */
        "offset.store.path", "my/path", /* string */
        NULL
    };

    test_conf_init(&ignore_conf, &ignore_topic_conf, 10);
    rd_kafka_conf_destroy(ignore_conf);
    rd_kafka_topic_conf_destroy(ignore_topic_conf);

    topic = test_mk_topic_name("generic", 0);

    /* Set up a global config object */
    conf = rd_kafka_conf_new();

    rd_kafka_conf_set_dr_cb(conf, dr_cb);
    rd_kafka_conf_set_error_cb(conf, error_cb);

    for (i = 0 ; gconfs[i] ; i += 2) {
        if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1],
                              errstr, sizeof(errstr)) !=
                RD_KAFKA_CONF_OK)
            TEST_FAIL("%s\n", errstr);
    }

    /* Set up a topic config object */
    tconf = rd_kafka_topic_conf_new();

    rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner);
    rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef);

    for (i = 0 ; tconfs[i] ; i += 2) {
        if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1],
                                    errstr, sizeof(errstr)) !=
                RD_KAFKA_CONF_OK)
            TEST_FAIL("%s\n", errstr);
    }


    /* Verify global config */
    arr_orig = rd_kafka_conf_dump(conf, &cnt_orig);
    conf_verify(__LINE__, arr_orig, cnt_orig, gconfs);

    /* Verify copied global config */
    conf2 = rd_kafka_conf_dup(conf);
    arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup);
    conf_verify(__LINE__, arr_dup, cnt_dup, gconfs);
    conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup);
    rd_kafka_conf_dump_free(arr_orig, cnt_orig);
    rd_kafka_conf_dump_free(arr_dup, cnt_dup);

    /* Verify topic config */
    arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig);
    conf_verify(__LINE__, arr_orig, cnt_orig, tconfs);

    /* Verify copied topic config */
    tconf2 = rd_kafka_topic_conf_dup(tconf);
    arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup);
    conf_verify(__LINE__, arr_dup, cnt_dup, tconfs);
    conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup);
    rd_kafka_conf_dump_free(arr_orig, cnt_orig);
    rd_kafka_conf_dump_free(arr_dup, cnt_dup);


    /*
     * Create kafka instances using original and copied confs
     */

    /* original */
    rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf,
                      errstr, sizeof(errstr));
    if (!rk)
        TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);

    rkt = rd_kafka_topic_new(rk, topic, tconf);
    if (!rkt)
        TEST_FAIL("Failed to create topic: %s\n",
                  strerror(errno));

    rd_kafka_topic_destroy(rkt);
    rd_kafka_destroy(rk);

    /* copied */
    rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf2,
                      errstr, sizeof(errstr));
    if (!rk)
        TEST_FAIL("Failed to create rdkafka instance: %s\n", errstr);

    rkt = rd_kafka_topic_new(rk, topic, tconf2);
    if (!rkt)
        TEST_FAIL("Failed to create topic: %s\n",
                  strerror(errno));

    rd_kafka_topic_destroy(rkt);
    rd_kafka_destroy(rk);


    /* Wait for everything to be cleaned up since broker destroys are
     * handled in its own thread. */
    test_wait_exit(2);

    /* If we havent failed at this point then
     * there were no threads leaked */
    return 0;
}
예제 #26
0
파일: kafka.c 프로젝트: dwieland/phpkafka
int kafka_produce(rd_kafka_t *r, char* topic, char* msg, int msg_len, int report, long timeout)
{

    char errstr[512];
    rd_kafka_topic_t *rkt;
    struct produce_cb_params pcb = {1, 0, 0, 0, 0, NULL};
    void *opaque;
    int partition = RD_KAFKA_PARTITION_UA;

    //decide whether to pass callback params or not...
    if (report)
        opaque = &pcb;
    else
        opaque = NULL;

    rd_kafka_topic_conf_t *topic_conf;

    if (r == NULL)
    {
        if (log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(LOG_ERR, "phpkafka - no connection to produce to topic: %s", topic);
        }
        return -2;
    }

    /* Topic configuration */
    topic_conf = rd_kafka_topic_conf_new();

    char timeoutStr[64];
    snprintf(timeoutStr, 64, "%lu", timeout);
    if (rd_kafka_topic_conf_set(topic_conf, "message.timeout.ms", timeoutStr, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK)
    {
        if (log_level)
        {
            openlog("phpkafka", 0, LOG_USER);
            syslog(
                LOG_ERR,
                "Failed to configure topic param 'message.timeout.ms' to %lu before producing; config err was: %s",
                timeout,
                errstr
            );
        }
        rd_kafka_topic_conf_destroy(topic_conf);
        return -3;
    }

    /* Create topic */
    rkt = rd_kafka_topic_new(r, topic, topic_conf);

    if (rd_kafka_produce(rkt, partition,
                     RD_KAFKA_MSG_F_COPY,
                     /* Payload and length */
                     msg, msg_len,
                     /* Optional key and its length */
                     NULL, 0,
                     /* Message opaque, provided in
                      * delivery report callback as
                      * msg_opaque. */
                     opaque) == -1) {
       if (log_level) {
           openlog("phpkafka", 0, LOG_USER);
           syslog(LOG_INFO, "phpkafka - %% Failed to produce to topic %s "
               "partition %i: %s",
               rd_kafka_topic_name(rkt), partition,
               rd_kafka_err2str(
               rd_kafka_errno2err(errno)));
        }
       rd_kafka_topic_destroy(rkt);
       return -1;
    }

    /* Poll to handle delivery reports */
    rd_kafka_poll(r, 0);

    /* Wait for messages to be delivered */
    while (report && pcb.msg_count && rd_kafka_outq_len(r) > 0)
      rd_kafka_poll(r, 10);

    //set global to NULL again
    rd_kafka_topic_destroy(rkt);
    return 0;
}
예제 #27
0
int main_0004_conf (int argc, char **argv) {
	rd_kafka_t *rk;
	rd_kafka_topic_t *rkt;
	rd_kafka_conf_t *ignore_conf, *conf, *conf2;
	rd_kafka_topic_conf_t *ignore_topic_conf, *tconf, *tconf2;
	char errstr[512];
	const char **arr_orig, **arr_dup;
	size_t cnt_orig, cnt_dup;
	int i;
        const char *topic;
	static const char *gconfs[] = {
		"message.max.bytes", "12345", /* int property */
		"client.id", "my id", /* string property */
		"debug", "topic,metadata", /* S2F property */
		"topic.blacklist", "__.*", /* #778 */
                "auto.offset.reset", "earliest", /* Global->Topic fallthru */
#if WITH_ZLIB
		"compression.codec", "gzip", /* S2I property */
#endif
		NULL
	};
	static const char *tconfs[] = {
		"request.required.acks", "-1", /* int */
		"auto.commit.enable", "false", /* bool */
		"auto.offset.reset", "error",  /* S2I */
		"offset.store.path", "my/path", /* string */
		NULL
	};

	test_conf_init(&ignore_conf, &ignore_topic_conf, 10);
	rd_kafka_conf_destroy(ignore_conf);
	rd_kafka_topic_conf_destroy(ignore_topic_conf);

        topic = test_mk_topic_name("0004", 0);

	/* Set up a global config object */
	conf = rd_kafka_conf_new();

	rd_kafka_conf_set_dr_cb(conf, dr_cb);
	rd_kafka_conf_set_error_cb(conf, error_cb);

	for (i = 0 ; gconfs[i] ; i += 2) {
		if (rd_kafka_conf_set(conf, gconfs[i], gconfs[i+1],
				      errstr, sizeof(errstr)) !=
		    RD_KAFKA_CONF_OK)
			TEST_FAIL("%s\n", errstr);
	}

	/* Set up a topic config object */
	tconf = rd_kafka_topic_conf_new();

	rd_kafka_topic_conf_set_partitioner_cb(tconf, partitioner);
	rd_kafka_topic_conf_set_opaque(tconf, (void *)0xbeef);

	for (i = 0 ; tconfs[i] ; i += 2) {
		if (rd_kafka_topic_conf_set(tconf, tconfs[i], tconfs[i+1],
				      errstr, sizeof(errstr)) !=
		    RD_KAFKA_CONF_OK)
			TEST_FAIL("%s\n", errstr);
	}


	/* Verify global config */
	arr_orig = rd_kafka_conf_dump(conf, &cnt_orig);
	conf_verify(__LINE__, arr_orig, cnt_orig, gconfs);

	/* Verify copied global config */
	conf2 = rd_kafka_conf_dup(conf);
	arr_dup = rd_kafka_conf_dump(conf2, &cnt_dup);
	conf_verify(__LINE__, arr_dup, cnt_dup, gconfs);
	conf_cmp("global", arr_orig, cnt_orig, arr_dup, cnt_dup);
	rd_kafka_conf_dump_free(arr_orig, cnt_orig);
	rd_kafka_conf_dump_free(arr_dup, cnt_dup);

	/* Verify topic config */
	arr_orig = rd_kafka_topic_conf_dump(tconf, &cnt_orig);
	conf_verify(__LINE__, arr_orig, cnt_orig, tconfs);

	/* Verify copied topic config */
	tconf2 = rd_kafka_topic_conf_dup(tconf);
	arr_dup = rd_kafka_topic_conf_dump(tconf2, &cnt_dup);
	conf_verify(__LINE__, arr_dup, cnt_dup, tconfs);
	conf_cmp("topic", arr_orig, cnt_orig, arr_dup, cnt_dup);
	rd_kafka_conf_dump_free(arr_orig, cnt_orig);
	rd_kafka_conf_dump_free(arr_dup, cnt_dup);


	/*
	 * Create kafka instances using original and copied confs
	 */

	/* original */
	rk = test_create_handle(RD_KAFKA_PRODUCER, conf);

	rkt = rd_kafka_topic_new(rk, topic, tconf);
	if (!rkt)
		TEST_FAIL("Failed to create topic: %s\n",
			  rd_strerror(errno));

	rd_kafka_topic_destroy(rkt);
	rd_kafka_destroy(rk);

	/* copied */
	rk = test_create_handle(RD_KAFKA_PRODUCER, conf2);

	rkt = rd_kafka_topic_new(rk, topic, tconf2);
	if (!rkt)
		TEST_FAIL("Failed to create topic: %s\n",
			  rd_strerror(errno));

	rd_kafka_topic_destroy(rkt);
	rd_kafka_destroy(rk);


	/* Incremental S2F property.
	 * NOTE: The order of fields returned in get() is hardcoded here. */
	{
		static const char *s2fs[] = {
			"generic,broker,queue,cgrp",
			"generic,broker,queue,cgrp",

			"-broker,+queue,topic",
			"generic,topic,queue,cgrp",

			"-all,security,-fetch,+metadata",
			"metadata,security",

			NULL
		};

		TEST_SAY("Incremental S2F tests\n");
		conf = rd_kafka_conf_new();

		for (i = 0 ; s2fs[i] ; i += 2) {
			const char *val;

			TEST_SAY("  Set: %s\n", s2fs[i]);
			test_conf_set(conf, "debug", s2fs[i]);
			val = test_conf_get(conf, "debug");
			TEST_SAY("  Now: %s\n", val);

			if (strcmp(val, s2fs[i+1]))
				TEST_FAIL_LATER("\n"
						"Expected: %s\n"
						"     Got: %s",
						s2fs[i+1], val);
		}
		rd_kafka_conf_destroy(conf);
	}

	/* Canonical int values, aliases, s2i-verified strings */
	{
		static const struct {
			const char *prop;
			const char *val;
			const char *exp;
			int is_global;
		} props[] = {
			{ "request.required.acks", "0", "0" },
			{ "request.required.acks", "-1", "-1" },
			{ "request.required.acks", "1", "1" },
			{ "acks", "3", "3" }, /* alias test */
			{ "request.required.acks", "393", "393" },
			{ "request.required.acks", "bad", NULL },
			{ "request.required.acks", "all", "-1" },
                        { "request.required.acks", "all", "-1", 1/*fallthru*/ },
			{ "acks", "0", "0" }, /* alias test */
#if WITH_SASL
			{ "sasl.mechanisms", "GSSAPI", "GSSAPI", 1 },
			{ "sasl.mechanisms", "PLAIN", "PLAIN", 1  },
			{ "sasl.mechanisms", "GSSAPI,PLAIN", NULL, 1  },
			{ "sasl.mechanisms", "", NULL, 1  },
#endif
			{ NULL }
		};

		TEST_SAY("Canonical tests\n");
		tconf = rd_kafka_topic_conf_new();
		conf = rd_kafka_conf_new();

		for (i = 0 ; props[i].prop ; i++) {
			char dest[64];
			size_t destsz;
			rd_kafka_conf_res_t res;

			TEST_SAY("  Set: %s=%s expect %s (%s)\n",
				 props[i].prop, props[i].val, props[i].exp,
                                 props[i].is_global ? "global":"topic");


			/* Set value */
			if (props[i].is_global)
				res = rd_kafka_conf_set(conf,
						      props[i].prop,
						      props[i].val,
						      errstr, sizeof(errstr));
			else
				res = rd_kafka_topic_conf_set(tconf,
							      props[i].prop,
							      props[i].val,
							      errstr,
							      sizeof(errstr));
			if ((res == RD_KAFKA_CONF_OK ? 1:0) !=
			    (props[i].exp ? 1:0))
				TEST_FAIL("Expected %s, got %s",
					  props[i].exp ? "success" : "failure",
					  (res == RD_KAFKA_CONF_OK ? "OK" :
					   (res == RD_KAFKA_CONF_INVALID ? "INVALID" :
					    "UNKNOWN")));

			if (!props[i].exp)
				continue;

			/* Get value and compare to expected result */
			destsz = sizeof(dest);
			if (props[i].is_global)
				res = rd_kafka_conf_get(conf,
							props[i].prop,
							dest, &destsz);
			else
				res = rd_kafka_topic_conf_get(tconf,
							      props[i].prop,
							      dest, &destsz);
			TEST_ASSERT(res == RD_KAFKA_CONF_OK,
				    ".._conf_get(%s) returned %d",
                                    props[i].prop, res);

			TEST_ASSERT(!strcmp(props[i].exp, dest),
				    "Expected \"%s\", got \"%s\"",
				    props[i].exp, dest);
		}
		rd_kafka_topic_conf_destroy(tconf);
		rd_kafka_conf_destroy(conf);
	}

	return 0;
}
예제 #28
0
int main_0018_cgrp_term (int argc, char **argv) {
	const char *topic = test_mk_topic_name(__FUNCTION__, 1);
#define _CONS_CNT 2
	rd_kafka_t *rk_p, *rk_c[_CONS_CNT];
        rd_kafka_topic_t *rkt_p;
	int msg_cnt = 1000;
	int msg_base = 0;
        int partition_cnt = 2;
        int partition;
	uint64_t testid;
        rd_kafka_topic_conf_t *default_topic_conf;
	rd_kafka_topic_partition_list_t *topics;
	rd_kafka_resp_err_t err;
	test_timing_t t_assign, t_consume;
	char errstr[512];
	int i;

	testid = test_id_generate();

	/* Produce messages */
	rk_p = test_create_producer();
	rkt_p = test_create_producer_topic(rk_p, topic, NULL);

        for (partition = 0 ; partition < partition_cnt ; partition++) {
                test_produce_msgs(rk_p, rkt_p, testid, partition,
                                  msg_base+(partition*msg_cnt), msg_cnt,
				  NULL, 0);
        }

	rd_kafka_topic_destroy(rkt_p);
	rd_kafka_destroy(rk_p);


        test_conf_init(NULL, &default_topic_conf,
		       (test_session_timeout_ms * 3) / 1000);
        if (rd_kafka_topic_conf_set(default_topic_conf, "auto.offset.reset",
				    "smallest", errstr, sizeof(errstr)) !=
	    RD_KAFKA_CONF_OK)
		TEST_FAIL("%s\n", errstr);

	/* Fill in topic subscription set */
	topics = rd_kafka_topic_partition_list_new(1);
	rd_kafka_topic_partition_list_add(topics, topic, -1);

	/* Create consumers and start subscription */
	for (i = 0 ; i < _CONS_CNT ; i++) {
		rk_c[i] = test_create_consumer(topic/*group_id*/,
					       rebalance_cb, NULL,
					       rd_kafka_topic_conf_dup(
						       default_topic_conf),
					       NULL);

		err = rd_kafka_poll_set_consumer(rk_c[i]);
		if (err)
			TEST_FAIL("poll_set_consumer: %s\n",
				  rd_kafka_err2str(err));

		err = rd_kafka_subscribe(rk_c[i], topics);
		if (err)
			TEST_FAIL("subscribe: %s\n", rd_kafka_err2str(err));
	}

        rd_kafka_topic_conf_destroy(default_topic_conf);

        rd_kafka_topic_partition_list_destroy(topics);


	/* Wait for both consumers to get an assignment */
	TIMING_START(&t_assign, "WAIT.ASSIGN");
	while (assign_cnt < _CONS_CNT)
		consume_all(rk_c, _CONS_CNT, msg_cnt,
			    test_session_timeout_ms + 3000);
	TIMING_STOP(&t_assign);

	/* Now close one of the consumers, this will cause a rebalance. */
	TEST_SAY("Closing down 1/%d consumer(s): %s\n", _CONS_CNT,
		 rd_kafka_name(rk_c[0]));
	err = rd_kafka_consumer_close(rk_c[0]);
	if (err)
		TEST_FAIL("consumer_close failed: %s\n", rd_kafka_err2str(err));
	rd_kafka_destroy(rk_c[0]);
	rk_c[0] = NULL;

	/* Let remaining consumers run for a while to take over the now
	 * lost partitions. */

	if (assign_cnt != _CONS_CNT-1)
		TEST_FAIL("assign_cnt %d, should be %d\n",
			  assign_cnt, _CONS_CNT-1);

	TIMING_START(&t_consume, "CONSUME.WAIT");
	consume_all(rk_c, _CONS_CNT, msg_cnt, test_session_timeout_ms + 3000);
	TIMING_STOP(&t_consume);

	TEST_SAY("Closing remaining consumers\n");
	for (i = 0 ; i < _CONS_CNT ; i++) {
		test_timing_t t_close;
                rd_kafka_topic_partition_list_t *sub;
                int j;

		if (!rk_c[i])
			continue;

                /* Query subscription */
                err = rd_kafka_subscription(rk_c[i], &sub);
                if (err)
                        TEST_FAIL("%s: subscription() failed: %s\n",
                                  rd_kafka_name(rk_c[i]),
                                  rd_kafka_err2str(err));
                TEST_SAY("%s: subscription (%d):\n",
                         rd_kafka_name(rk_c[i]), sub->cnt);
                for (j = 0 ; j < sub->cnt ; j++)
                        TEST_SAY(" %s\n", sub->elems[j].topic);
                rd_kafka_topic_partition_list_destroy(sub);

                /* Run an explicit unsubscribe() (async) prior to close()
                 * to trigger race condition issues on termination. */
                TEST_SAY("Unsubscribing instance %s\n", rd_kafka_name(rk_c[i]));
                err = rd_kafka_unsubscribe(rk_c[i]);
                if (err)
                        TEST_FAIL("%s: unsubscribe failed: %s\n",
                                  rd_kafka_name(rk_c[i]),
                                  rd_kafka_err2str(err));

		TEST_SAY("Closing %s\n", rd_kafka_name(rk_c[i]));
		TIMING_START(&t_close, "CONSUMER.CLOSE");
		err = rd_kafka_consumer_close(rk_c[i]);
		TIMING_STOP(&t_close);
		if (err)
			TEST_FAIL("consumer_close failed: %s\n",
				  rd_kafka_err2str(err));

		rd_kafka_destroy(rk_c[i]);
		rk_c[i] = NULL;
	}

	TEST_SAY("%d/%d messages consumed\n", consumed_msg_cnt, msg_cnt);
	if (consumed_msg_cnt < msg_cnt)
		TEST_FAIL("Only %d/%d messages were consumed\n",
			  consumed_msg_cnt, msg_cnt);
	else if (consumed_msg_cnt > msg_cnt)
		TEST_SAY("At least %d/%d messages were consumed "
			 "multiple times\n",
			 consumed_msg_cnt - msg_cnt, msg_cnt);
	
	return 0;
}