/**
* Send auth message with framing.
* This is a blocking call.
*/
static int rd_kafka_sasl_send (rd_kafka_transport_t *rktrans,
const void *payload, int len,
char *errstr, int errstr_size) {
struct msghdr msg = RD_ZERO_INIT;
struct iovec iov[1];
int32_t hdr;
char *frame;
int total_len = sizeof(hdr) + len;
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASL",
"Send SASL frame to broker (%d bytes)", len);
frame = rd_malloc(total_len);
hdr = htobe32(len);
memcpy(frame, &hdr, sizeof(hdr));
if (payload)
memcpy(frame+sizeof(hdr), payload, len);
msg.msg_iov = iov;
msg.msg_iovlen = 1;
iov[0].iov_base = frame;
iov[0].iov_len = total_len;
/* Simulate blocking behaviour on non-blocking socket..
* FIXME: This isn't optimal but is highly unlikely to stall since
* the socket buffer will most likely not be exceeded. */
do {
int r;
r = rd_kafka_transport_sendmsg(rktrans, &msg,
errstr, errstr_size);
if (r == -1) {
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASL",
"SASL send of %d bytes failed: %s",
total_len, errstr);
free(frame);
return -1;
}
if (r == total_len)
break;
/* Avoid busy-looping */
rd_usleep(10*1000, NULL);
/* Shave off written bytes */
total_len -= r;
iov[0].iov_base = ((char *)(iov[0].iov_base)) + r;
iov[0].iov_len = total_len;
} while (total_len > 0);
free(frame);
return 0;
}
/**
* @brief Perform \p itcnt iterations of HMAC() on the given buffer \p in
* using \p salt, writing the output into \p out which must be
* at least EVP_MAX_MD_SIZE. Actual size is updated in \p *outsize.
* @returns 0 on success, else -1
*/
static int
rd_kafka_sasl_scram_Hi (rd_kafka_transport_t *rktrans,
const rd_chariov_t *in,
const rd_chariov_t *salt,
int itcnt, rd_chariov_t *out) {
const EVP_MD *evp =
rktrans->rktrans_rkb->rkb_rk->rk_conf.sasl.scram_evp;
unsigned int ressize = 0;
unsigned char tempres[EVP_MAX_MD_SIZE];
unsigned char *saltplus;
int i;
/* U1 := HMAC(str, salt + INT(1)) */
saltplus = rd_alloca(salt->size + 4);
memcpy(saltplus, salt->ptr, salt->size);
saltplus[salt->size] = 0;
saltplus[salt->size+1] = 0;
saltplus[salt->size+2] = 0;
saltplus[salt->size+3] = 1;
/* U1 := HMAC(str, salt + INT(1)) */
if (!HMAC(evp,
(const unsigned char *)in->ptr, (int)in->size,
saltplus, salt->size+4,
tempres, &ressize)) {
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SCRAM",
"HMAC priming failed");
return -1;
}
memcpy(out->ptr, tempres, ressize);
/* Ui-1 := HMAC(str, Ui-2) .. */
for (i = 1 ; i < itcnt ; i++) {
unsigned char tempdest[EVP_MAX_MD_SIZE];
int j;
if (unlikely(!HMAC(evp,
(const unsigned char *)in->ptr, (int)in->size,
tempres, ressize,
tempdest, NULL))) {
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SCRAM",
"Hi() HMAC #%d/%d failed", i, itcnt);
return -1;
}
/* U1 XOR U2 .. */
for (j = 0 ; j < (int)ressize ; j++) {
out->ptr[j] ^= tempdest[j];
tempres[j] = tempdest[j];
}
}
out->size = ressize;
return 0;
}
/**
* Execute kinit to refresh ticket.
*
* Returns 0 on success, -1 on error.
*
* Locality: any
*/
static int rd_kafka_sasl_kinit_refresh (rd_kafka_broker_t *rkb) {
rd_kafka_t *rk = rkb->rkb_rk;
int r;
char *cmd;
char errstr[128];
if (!rk->rk_conf.sasl.kinit_cmd ||
!strstr(rk->rk_conf.sasl.mechanisms, "GSSAPI"))
return 0; /* kinit not configured */
/* Build kinit refresh command line using string rendering and config */
cmd = rd_string_render(rk->rk_conf.sasl.kinit_cmd,
errstr, sizeof(errstr),
render_callback, rkb);
if (!cmd) {
rd_rkb_log(rkb, LOG_ERR, "SASLREFRESH",
"Failed to construct kinit command "
"from sasl.kerberos.kinit.cmd template: %s",
errstr);
return -1;
}
/* Execute kinit */
rd_rkb_dbg(rkb, SECURITY, "SASLREFRESH",
"Refreshing SASL keys with command: %s", cmd);
mtx_lock(&rd_kafka_sasl_kinit_lock);
r = system(cmd);
mtx_unlock(&rd_kafka_sasl_kinit_lock);
if (r == -1) {
rd_rkb_log(rkb, LOG_ERR, "SASLREFRESH",
"SASL key refresh failed: Failed to execute %s",
cmd);
rd_free(cmd);
return -1;
} else if (WIFSIGNALED(r)) {
rd_rkb_log(rkb, LOG_ERR, "SASLREFRESH",
"SASL key refresh failed: %s: received signal %d",
cmd, WTERMSIG(r));
rd_free(cmd);
return -1;
} else if (WIFEXITED(r) && WEXITSTATUS(r) != 0) {
rd_rkb_log(rkb, LOG_ERR, "SASLREFRESH",
"SASL key refresh failed: %s: exited with code %d",
cmd, WEXITSTATUS(r));
rd_free(cmd);
return -1;
}
rd_free(cmd);
rd_rkb_dbg(rkb, SECURITY, "SASLREFRESH", "SASL key refreshed");
return 0;
}
/**
* @brief Perform HMAC(key,str) and stores the result in \p out
* which must be at least EVP_MAX_MD_SIZE.
* @returns 0 on success, else -1
*/
static int
rd_kafka_sasl_scram_HMAC (rd_kafka_transport_t *rktrans,
const rd_chariov_t *key,
const rd_chariov_t *str,
rd_chariov_t *out) {
const EVP_MD *evp =
rktrans->rktrans_rkb->rkb_rk->rk_conf.sasl.scram_evp;
unsigned int outsize;
//printf("HMAC KEY: %s\n", rd_base64_encode(key));
//printf("HMAC STR: %s\n", rd_base64_encode(str));
if (!HMAC(evp,
(const unsigned char *)key->ptr, (int)key->size,
(const unsigned char *)str->ptr, (int)str->size,
(unsigned char *)out->ptr, &outsize)) {
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SCRAM",
"HMAC failed");
return -1;
}
out->size = outsize;
//printf("HMAC OUT: %s\n", rd_base64_encode(out));
return 0;
}
/**
* Verify SSL handshake was valid.
*/
static int rd_kafka_transport_ssl_verify (rd_kafka_transport_t *rktrans) {
long int rl;
X509 *cert;
cert = SSL_get_peer_certificate(rktrans->rktrans_ssl);
X509_free(cert);
if (!cert) {
rd_kafka_broker_fail(rktrans->rktrans_rkb, LOG_ERR,
RD_KAFKA_RESP_ERR__SSL,
"Broker did not provide a certificate");
return -1;
}
if ((rl = SSL_get_verify_result(rktrans->rktrans_ssl)) != X509_V_OK) {
rd_kafka_broker_fail(rktrans->rktrans_rkb, LOG_ERR,
RD_KAFKA_RESP_ERR__SSL,
"Failed to verify broker certificate: %s",
X509_verify_cert_error_string(rl));
return -1;
}
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SSLVERIFY",
"Broker SSL certificate verified");
return 0;
}
static RD_UNUSED int rd_kafka_sasl_cb_canon (sasl_conn_t *conn,
void *context,
const char *in, unsigned inlen,
unsigned flags,
const char *user_realm,
char *out, unsigned out_max,
unsigned *out_len) {
rd_kafka_transport_t *rktrans = context;
if (strstr(rktrans->rktrans_rkb->rkb_rk->rk_conf.
sasl.mechanisms, "GSSAPI")) {
*out_len = rd_snprintf(out, out_max, "%s",
rktrans->rktrans_rkb->rkb_rk->
rk_conf.sasl.principal);
} else if (!strcmp(rktrans->rktrans_rkb->rkb_rk->rk_conf.
sasl.mechanisms, "PLAIN")) {
*out_len = rd_snprintf(out, out_max, "%.*s", inlen, in);
} else
out = NULL;
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL",
"CB_CANON: flags 0x%x, \"%.*s\" @ \"%s\": returning \"%.*s\"",
flags, (int)inlen, in, user_realm, (int)(*out_len), out);
return out ? SASL_OK : SASL_FAIL;
}
/**
* TCP connection established.
* Set up socket options, SSL, etc.
*
* Locality: broker thread
*/
static void rd_kafka_transport_connected (rd_kafka_transport_t *rktrans) {
rd_kafka_broker_t *rkb = rktrans->rktrans_rkb;
rd_rkb_dbg(rkb, BROKER, "CONNECT",
"Connected to %s",
rd_sockaddr2str(rkb->rkb_addr_last,
RD_SOCKADDR2STR_F_PORT |
RD_SOCKADDR2STR_F_FAMILY));
/* Set socket send & receive buffer sizes if configuerd */
if (rkb->rkb_rk->rk_conf.socket_sndbuf_size != 0) {
if (setsockopt(rktrans->rktrans_s, SOL_SOCKET, SO_SNDBUF,
(void *)&rkb->rkb_rk->rk_conf.socket_sndbuf_size,
sizeof(rkb->rkb_rk->rk_conf.
socket_sndbuf_size)) == SOCKET_ERROR)
rd_rkb_log(rkb, LOG_WARNING, "SNDBUF",
"Failed to set socket send "
"buffer size to %i: %s",
rkb->rkb_rk->rk_conf.socket_sndbuf_size,
socket_strerror(socket_errno));
}
if (rkb->rkb_rk->rk_conf.socket_rcvbuf_size != 0) {
if (setsockopt(rktrans->rktrans_s, SOL_SOCKET, SO_RCVBUF,
(void *)&rkb->rkb_rk->rk_conf.socket_rcvbuf_size,
sizeof(rkb->rkb_rk->rk_conf.
socket_rcvbuf_size)) == SOCKET_ERROR)
rd_rkb_log(rkb, LOG_WARNING, "RCVBUF",
"Failed to set socket receive "
"buffer size to %i: %s",
rkb->rkb_rk->rk_conf.socket_rcvbuf_size,
socket_strerror(socket_errno));
}
#if WITH_SSL
if (rkb->rkb_proto == RD_KAFKA_PROTO_SSL ||
rkb->rkb_proto == RD_KAFKA_PROTO_SASL_SSL) {
char errstr[512];
/* Set up SSL connection.
* This is also an asynchronous operation so dont
* propagate to broker_connect_done() just yet. */
if (rd_kafka_transport_ssl_connect(rkb, rktrans,
errstr,
sizeof(errstr)) == -1) {
rd_kafka_transport_connect_done(rktrans, errstr);
return;
}
return;
}
#endif
/* Propagate connect success */
rd_kafka_transport_connect_done(rktrans, NULL);
}
static int rd_kafka_sasl_cb_log (void *context, int level, const char *message){
rd_kafka_transport_t *rktrans = context;
if (level >= LOG_DEBUG)
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL",
"%s", message);
else
rd_rkb_log(rktrans->rktrans_rkb, level, "LIBSASL",
"%s", message);
return SASL_OK;
}
static int rd_kafka_sasl_cb_getrealm (void *context, int id,
const char **availrealms,
const char **result) {
rd_kafka_transport_t *rktrans = context;
*result = *availrealms;
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL",
"CB_GETREALM: id 0x%x: returning %s", id, *result);
return SASL_OK;
}
/**
* @brief the kernel SO_ERROR in \p errp for the given transport.
* @returns 0 if getsockopt() was succesful (and \p and errp can be trusted),
* else -1 in which case \p errp 's value is undefined.
*/
static int rd_kafka_transport_get_socket_error (rd_kafka_transport_t *rktrans,
int *errp) {
socklen_t intlen = sizeof(*errp);
if (getsockopt(rktrans->rktrans_s, SOL_SOCKET,
SO_ERROR, (void *)errp, &intlen) == -1) {
rd_rkb_dbg(rktrans->rktrans_rkb, BROKER, "SO_ERROR",
"Failed to get socket error: %s",
socket_strerror(socket_errno));
return -1;
}
return 0;
}
static int rd_kafka_sasl_cb_chalprompt (void *context, int id,
const char *challenge,
const char *prompt,
const char *defres,
const char **result, unsigned *len) {
rd_kafka_transport_t *rktrans = context;
*result = "min_chalprompt";
*len = strlen(*result);
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL",
"CB_CHALPROMPT: id 0x%x, challenge %s, prompt %s, "
"default %s: returning %s",
id, challenge, prompt, defres, *result);
return SASL_OK;
}
/**
*
* libsasl callbacks
*
*/
static RD_UNUSED int
rd_kafka_sasl_cb_getopt (void *context, const char *plugin_name,
const char *option,
const char **result, unsigned *len) {
rd_kafka_transport_t *rktrans = context;
if (!strcmp(option, "client_mech_list"))
*result = "GSSAPI";
if (!strcmp(option, "canon_user_plugin"))
*result = "INTERNAL";
if (*result && len)
*len = strlen(*result);
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL",
"CB_GETOPT: plugin %s, option %s: returning %s",
plugin_name, option, *result);
return SASL_OK;
}
static int rd_kafka_sasl_cb_getsecret (sasl_conn_t *conn, void *context,
int id, sasl_secret_t **psecret) {
rd_kafka_transport_t *rktrans = context;
const char *password;
password = rktrans->rktrans_rkb->rkb_rk->rk_conf.sasl.password;
if (!password) {
*psecret = NULL;
} else {
size_t passlen = strlen(password);
*psecret = rd_realloc(*psecret, sizeof(**psecret) + passlen);
(*psecret)->len = passlen;
memcpy((*psecret)->data, password, passlen);
}
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL",
"CB_GETSECRET: id 0x%x: returning %s",
id, *psecret ? "(hidden)":"NULL");
return SASL_OK;
}
static int rd_kafka_sasl_cb_getsimple (void *context, int id,
const char **result, unsigned *len) {
rd_kafka_transport_t *rktrans = context;
switch (id)
{
case SASL_CB_USER:
case SASL_CB_AUTHNAME:
*result = rktrans->rktrans_rkb->rkb_rk->rk_conf.sasl.username;
break;
default:
*result = NULL;
break;
}
if (len)
*len = *result ? strlen(*result) : 0;
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL",
"CB_GETSIMPLE: id 0x%x: returning %s", id, *result);
return *result ? SASL_OK : SASL_FAIL;
}
/**
* @brief Build client-final-message
* @returns -1 on error.
*/
static int
rd_kafka_sasl_scram_build_client_final_message (
rd_kafka_transport_t *rktrans,
const rd_chariov_t *salt,
const char *server_nonce,
const rd_chariov_t *server_first_msg,
int itcnt, rd_chariov_t *out) {
struct rd_kafka_sasl_scram_state *state = rktrans->rktrans_sasl.state;
const rd_kafka_conf_t *conf = &rktrans->rktrans_rkb->rkb_rk->rk_conf;
rd_chariov_t SaslPassword =
{ .ptr = conf->sasl.password,
.size = strlen(conf->sasl.password) };
rd_chariov_t SaltedPassword =
{ .ptr = rd_alloca(EVP_MAX_MD_SIZE) };
rd_chariov_t ClientKey =
{ .ptr = rd_alloca(EVP_MAX_MD_SIZE) };
rd_chariov_t ServerKey =
{ .ptr = rd_alloca(EVP_MAX_MD_SIZE) };
rd_chariov_t StoredKey =
{ .ptr = rd_alloca(EVP_MAX_MD_SIZE) };
rd_chariov_t AuthMessage = RD_ZERO_INIT;
rd_chariov_t ClientSignature =
{ .ptr = rd_alloca(EVP_MAX_MD_SIZE) };
rd_chariov_t ServerSignature =
{ .ptr = rd_alloca(EVP_MAX_MD_SIZE) };
const rd_chariov_t ClientKeyVerbatim =
{ .ptr = "Client Key", .size = 10 };
const rd_chariov_t ServerKeyVerbatim =
{ .ptr = "Server Key", .size = 10 };
rd_chariov_t ClientProof =
{ .ptr = rd_alloca(EVP_MAX_MD_SIZE) };
rd_chariov_t client_final_msg_wo_proof;
char *ClientProofB64;
int i;
/* Constructing the ClientProof attribute (p):
*
* p = Base64-encoded ClientProof
* SaltedPassword := Hi(Normalize(password), salt, i)
* ClientKey := HMAC(SaltedPassword, "Client Key")
* StoredKey := H(ClientKey)
* AuthMessage := client-first-message-bare + "," +
* server-first-message + "," +
* client-final-message-without-proof
* ClientSignature := HMAC(StoredKey, AuthMessage)
* ClientProof := ClientKey XOR ClientSignature
* ServerKey := HMAC(SaltedPassword, "Server Key")
* ServerSignature := HMAC(ServerKey, AuthMessage)
*/
/* SaltedPassword := Hi(Normalize(password), salt, i) */
if (rd_kafka_sasl_scram_Hi(
rktrans, &SaslPassword, salt,
itcnt, &SaltedPassword) == -1)
return -1;
/* ClientKey := HMAC(SaltedPassword, "Client Key") */
if (rd_kafka_sasl_scram_HMAC(
rktrans, &SaltedPassword, &ClientKeyVerbatim,
&ClientKey) == -1)
return -1;
/* StoredKey := H(ClientKey) */
if (rd_kafka_sasl_scram_H(rktrans, &ClientKey, &StoredKey) == -1)
return -1;
/* client-final-message-without-proof */
rd_kafka_sasl_scram_build_client_final_message_wo_proof(
state, server_nonce, &client_final_msg_wo_proof);
/* AuthMessage := client-first-message-bare + "," +
* server-first-message + "," +
* client-final-message-without-proof */
AuthMessage.size =
state->first_msg_bare.size + 1 +
server_first_msg->size + 1 +
client_final_msg_wo_proof.size;
AuthMessage.ptr = rd_alloca(AuthMessage.size+1);
rd_snprintf(AuthMessage.ptr, AuthMessage.size+1,
"%.*s,%.*s,%.*s",
(int)state->first_msg_bare.size, state->first_msg_bare.ptr,
(int)server_first_msg->size, server_first_msg->ptr,
(int)client_final_msg_wo_proof.size,
client_final_msg_wo_proof.ptr);
/*
* Calculate ServerSignature for later verification when
* server-final-message is received.
*/
/* ServerKey := HMAC(SaltedPassword, "Server Key") */
if (rd_kafka_sasl_scram_HMAC(
rktrans, &SaltedPassword, &ServerKeyVerbatim,
&ServerKey) == -1) {
rd_free(client_final_msg_wo_proof.ptr);
return -1;
}
/* ServerSignature := HMAC(ServerKey, AuthMessage) */
if (rd_kafka_sasl_scram_HMAC(rktrans, &ServerKey,
&AuthMessage, &ServerSignature) == -1) {
rd_free(client_final_msg_wo_proof.ptr);
return -1;
}
/* Store the Base64 encoded ServerSignature for quick comparison */
state->ServerSignatureB64 = rd_base64_encode(&ServerSignature);
/*
* Continue with client-final-message
*/
/* ClientSignature := HMAC(StoredKey, AuthMessage) */
if (rd_kafka_sasl_scram_HMAC(rktrans, &StoredKey,
&AuthMessage, &ClientSignature) == -1) {
rd_free(client_final_msg_wo_proof.ptr);
return -1;
}
/* ClientProof := ClientKey XOR ClientSignature */
assert(ClientKey.size == ClientSignature.size);
for (i = 0 ; i < (int)ClientKey.size ; i++)
ClientProof.ptr[i] = ClientKey.ptr[i] ^ ClientSignature.ptr[i];
ClientProof.size = ClientKey.size;
/* Base64 encoded ClientProof */
ClientProofB64 = rd_base64_encode(&ClientProof);
/* Construct client-final-message */
out->size = client_final_msg_wo_proof.size +
strlen(",p=") + strlen(ClientProofB64);
out->ptr = rd_malloc(out->size + 1);
rd_snprintf(out->ptr, out->size+1,
"%.*s,p=%s",
(int)client_final_msg_wo_proof.size,
client_final_msg_wo_proof.ptr,
ClientProofB64);
rd_free(ClientProofB64);
rd_free(client_final_msg_wo_proof.ptr);
return 0;
}
/**
* @brief Handle first message from server
*
* Parse server response which looks something like:
* "r=fyko+d2lbbFgONR....,s=QSXCR+Q6sek8bf92,i=4096"
*
* @returns -1 on error.
*/
static int
rd_kafka_sasl_scram_handle_server_first_message (rd_kafka_transport_t *rktrans,
const rd_chariov_t *in,
rd_chariov_t *out,
char *errstr,
size_t errstr_size) {
struct rd_kafka_sasl_scram_state *state = rktrans->rktrans_sasl.state;
char *server_nonce;
rd_chariov_t salt_b64, salt;
char *itcntstr;
const char *endptr;
int itcnt;
char *attr_m;
/* Mandatory future extension check */
if ((attr_m = rd_kafka_sasl_scram_get_attr(
in, 'm', NULL, NULL, 0))) {
rd_snprintf(errstr, errstr_size,
"Unsupported mandatory SCRAM extension");
rd_free(attr_m);
return -1;
}
/* Server nonce */
if (!(server_nonce = rd_kafka_sasl_scram_get_attr(
in, 'r',
"Server nonce in server-first-message",
errstr, errstr_size)))
return -1;
if (strlen(server_nonce) <= state->cnonce.size ||
strncmp(state->cnonce.ptr, server_nonce, state->cnonce.size)) {
rd_snprintf(errstr, errstr_size,
"Server/client nonce mismatch in "
"server-first-message");
rd_free(server_nonce);
return -1;
}
/* Salt (Base64) */
if (!(salt_b64.ptr = rd_kafka_sasl_scram_get_attr(
in, 's',
"Salt in server-first-message",
errstr, errstr_size))) {
rd_free(server_nonce);
return -1;
}
salt_b64.size = strlen(salt_b64.ptr);
/* Convert Salt to binary */
if (rd_base64_decode(&salt_b64, &salt) == -1) {
rd_snprintf(errstr, errstr_size,
"Invalid Base64 Salt in server-first-message");
rd_free(server_nonce);
rd_free(salt_b64.ptr);
}
rd_free(salt_b64.ptr);
/* Iteration count (as string) */
if (!(itcntstr = rd_kafka_sasl_scram_get_attr(
in, 'i',
"Iteration count in server-first-message",
errstr, errstr_size))) {
rd_free(server_nonce);
rd_free(salt.ptr);
return -1;
}
/* Iteration count (as int) */
errno = 0;
itcnt = (int)strtoul(itcntstr, (char **)&endptr, 10);
if (itcntstr == endptr || *endptr != '\0' || errno != 0 ||
itcnt > 1000000) {
rd_snprintf(errstr, errstr_size,
"Invalid value (not integer or too large) "
"for Iteration count in server-first-message");
rd_free(server_nonce);
rd_free(salt.ptr);
rd_free(itcntstr);
return -1;
}
rd_free(itcntstr);
/* Build client-final-message */
if (rd_kafka_sasl_scram_build_client_final_message(
rktrans, &salt, server_nonce, in, itcnt, out) == -1) {
rd_snprintf(errstr, errstr_size,
"Failed to build SCRAM client-final-message");
rd_free(salt.ptr);
rd_free(server_nonce);
return -1;
}
rd_free(server_nonce);
rd_free(salt.ptr);
return 0;
}
/**
* @brief Handle server-final-message
*
* This is the end of authentication and the SCRAM state
* will be freed at the end of this function regardless of
* authentication outcome.
*
* @returns -1 on failure
*/
static int
rd_kafka_sasl_scram_handle_server_final_message (
rd_kafka_transport_t *rktrans,
const rd_chariov_t *in,
char *errstr, size_t errstr_size) {
struct rd_kafka_sasl_scram_state *state = rktrans->rktrans_sasl.state;
char *attr_v, *attr_e;
if ((attr_e = rd_kafka_sasl_scram_get_attr(
in, 'e', "server-error in server-final-message",
errstr, errstr_size))) {
/* Authentication failed */
rd_snprintf(errstr, errstr_size,
"SASL SCRAM authentication failed: "
"broker responded with %s",
attr_e);
rd_free(attr_e);
return -1;
} else if ((attr_v = rd_kafka_sasl_scram_get_attr(
in, 'v', "verifier in server-final-message",
errstr, errstr_size))) {
const rd_kafka_conf_t *conf;
/* Authentication succesful on server,
* but we need to verify the ServerSignature too. */
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY | RD_KAFKA_DBG_BROKER,
"SCRAMAUTH",
"SASL SCRAM authentication succesful on server: "
"verifying ServerSignature");
if (strcmp(attr_v, state->ServerSignatureB64)) {
rd_snprintf(errstr, errstr_size,
"SASL SCRAM authentication failed: "
"ServerSignature mismatch "
"(server's %s != ours %s)",
attr_v, state->ServerSignatureB64);
rd_free(attr_v);
return -1;
}
rd_free(attr_v);
conf = &rktrans->rktrans_rkb->rkb_rk->rk_conf;
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY | RD_KAFKA_DBG_BROKER,
"SCRAMAUTH",
"Authenticated as %s using %s",
conf->sasl.username,
conf->sasl.mechanisms);
rd_kafka_sasl_auth_done(rktrans);
return 0;
} else {
rd_snprintf(errstr, errstr_size,
"SASL SCRAM authentication failed: "
"no verifier or server-error returned from broker");
return -1;
}
}
/**
* @brief Build client-first-message
*/
static void
rd_kafka_sasl_scram_build_client_first_message (
rd_kafka_transport_t *rktrans,
rd_chariov_t *out) {
char *sasl_username;
struct rd_kafka_sasl_scram_state *state = rktrans->rktrans_sasl.state;
const rd_kafka_conf_t *conf = &rktrans->rktrans_rkb->rkb_rk->rk_conf;
rd_kafka_sasl_scram_generate_nonce(&state->cnonce);
sasl_username = rd_kafka_sasl_safe_string(conf->sasl.username);
out->size = strlen("n,,n=,r=") + strlen(sasl_username) +
state->cnonce.size;
out->ptr = rd_malloc(out->size+1);
rd_snprintf(out->ptr, out->size+1,
"n,,n=%s,r=%.*s",
sasl_username,
(int)state->cnonce.size, state->cnonce.ptr);
rd_free(sasl_username);
/* Save client-first-message-bare (skip gs2-header) */
state->first_msg_bare.size = out->size-3;
state->first_msg_bare.ptr = rd_memdup(out->ptr+3,
state->first_msg_bare.size);
}
/**
* @brief SASL SCRAM client state machine
* @returns -1 on failure (errstr set), else 0.
*/
static int rd_kafka_sasl_scram_fsm (rd_kafka_transport_t *rktrans,
const rd_chariov_t *in,
char *errstr, size_t errstr_size) {
static const char *state_names[] = {
"client-first-message",
"server-first-message",
"client-final-message",
};
struct rd_kafka_sasl_scram_state *state = rktrans->rktrans_sasl.state;
rd_chariov_t out = RD_ZERO_INIT;
int r = -1;
rd_ts_t ts_start = rd_clock();
int prev_state = state->state;
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASLSCRAM",
"SASL SCRAM client in state %s",
state_names[state->state]);
switch (state->state)
{
case RD_KAFKA_SASL_SCRAM_STATE_CLIENT_FIRST_MESSAGE:
rd_dassert(!in); /* Not expecting any server-input */
rd_kafka_sasl_scram_build_client_first_message(rktrans, &out);
state->state = RD_KAFKA_SASL_SCRAM_STATE_SERVER_FIRST_MESSAGE;
break;
case RD_KAFKA_SASL_SCRAM_STATE_SERVER_FIRST_MESSAGE:
rd_dassert(in); /* Requires server-input */
if (rd_kafka_sasl_scram_handle_server_first_message(
rktrans, in, &out, errstr, errstr_size) == -1)
return -1;
state->state = RD_KAFKA_SASL_SCRAM_STATE_CLIENT_FINAL_MESSAGE;
break;
case RD_KAFKA_SASL_SCRAM_STATE_CLIENT_FINAL_MESSAGE:
rd_dassert(in); /* Requires server-input */
r = rd_kafka_sasl_scram_handle_server_final_message(
rktrans, in, errstr, errstr_size);
break;
}
if (out.ptr) {
r = rd_kafka_sasl_send(rktrans, out.ptr, (int)out.size,
errstr, errstr_size);
rd_free(out.ptr);
}
ts_start = (rd_clock() - ts_start) / 1000;
if (ts_start >= 100)
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SCRAM",
"SASL SCRAM state %s handled in %"PRId64"ms",
state_names[prev_state], ts_start);
return r;
}
/**
* @brief Handle received frame from broker.
*/
static int rd_kafka_sasl_scram_recv (rd_kafka_transport_t *rktrans,
const void *buf, size_t size,
char *errstr, size_t errstr_size) {
const rd_chariov_t in = { .ptr = (char *)buf, .size = size };
return rd_kafka_sasl_scram_fsm(rktrans, &in, errstr, errstr_size);
}
/**
* @brief Initialize and start SASL SCRAM (builtin) authentication.
*
* Returns 0 on successful init and -1 on error.
*
* @locality broker thread
*/
static int rd_kafka_sasl_scram_client_new (rd_kafka_transport_t *rktrans,
const char *hostname,
char *errstr, size_t errstr_size) {
struct rd_kafka_sasl_scram_state *state;
state = rd_calloc(1, sizeof(*state));
state->state = RD_KAFKA_SASL_SCRAM_STATE_CLIENT_FIRST_MESSAGE;
rktrans->rktrans_sasl.state = state;
/* Kick off the FSM */
return rd_kafka_sasl_scram_fsm(rktrans, NULL, errstr, errstr_size);
}
/**
* @brief Validate SCRAM config and look up the hash function
*/
static int rd_kafka_sasl_scram_conf_validate (rd_kafka_t *rk,
char *errstr,
size_t errstr_size) {
const char *mech = rk->rk_conf.sasl.mechanisms;
if (!rk->rk_conf.sasl.username || !rk->rk_conf.sasl.password) {
rd_snprintf(errstr, errstr_size,
"sasl.username and sasl.password must be set");
return -1;
}
if (!strcmp(mech, "SCRAM-SHA-1")) {
rk->rk_conf.sasl.scram_evp = EVP_sha1();
rk->rk_conf.sasl.scram_H = SHA1;
rk->rk_conf.sasl.scram_H_size = SHA_DIGEST_LENGTH;
} else if (!strcmp(mech, "SCRAM-SHA-256")) {
rk->rk_conf.sasl.scram_evp = EVP_sha256();
rk->rk_conf.sasl.scram_H = SHA256;
rk->rk_conf.sasl.scram_H_size = SHA256_DIGEST_LENGTH;
} else if (!strcmp(mech, "SCRAM-SHA-512")) {
rk->rk_conf.sasl.scram_evp = EVP_sha512();
rk->rk_conf.sasl.scram_H = SHA512;
rk->rk_conf.sasl.scram_H_size = SHA512_DIGEST_LENGTH;
} else {
rd_snprintf(errstr, errstr_size,
"Unsupported hash function: %s "
"(try SCRAM-SHA-512)",
mech);
return -1;
}
return 0;
}
const struct rd_kafka_sasl_provider rd_kafka_sasl_scram_provider = {
.name = "SCRAM (builtin)",
.client_new = rd_kafka_sasl_scram_client_new,
.recv = rd_kafka_sasl_scram_recv,
.close = rd_kafka_sasl_scram_close,
.conf_validate = rd_kafka_sasl_scram_conf_validate,
};
/**
* @brief Decompress MessageSet, pass the uncompressed MessageSet to
* the MessageSet reader.
*/
static rd_kafka_resp_err_t
rd_kafka_msgset_reader_decompress (rd_kafka_msgset_reader_t *msetr,
int MsgVersion, int Attributes,
int64_t Timestamp, int64_t Offset,
const void *compressed,
size_t compressed_size) {
struct iovec iov = { .iov_base = NULL, .iov_len = 0 };
rd_kafka_toppar_t *rktp = msetr->msetr_rktp;
int codec = Attributes & RD_KAFKA_MSG_ATTR_COMPRESSION_MASK;
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
rd_kafka_buf_t *rkbufz;
switch (codec)
{
#if WITH_ZLIB
case RD_KAFKA_COMPRESSION_GZIP:
{
uint64_t outlenx = 0;
/* Decompress Message payload */
iov.iov_base = rd_gz_decompress(compressed, (int)compressed_size,
&outlenx);
if (unlikely(!iov.iov_base)) {
rd_rkb_dbg(msetr->msetr_rkb, MSG, "GZIP",
"Failed to decompress Gzip "
"message at offset %"PRId64
" of %"PRIusz" bytes: "
"ignoring message",
Offset, compressed_size);
err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
goto err;
}
iov.iov_len = (size_t)outlenx;
}
break;
#endif
#if WITH_SNAPPY
case RD_KAFKA_COMPRESSION_SNAPPY:
{
const char *inbuf = compressed;
size_t inlen = compressed_size;
int r;
static const unsigned char snappy_java_magic[] =
{ 0x82, 'S','N','A','P','P','Y', 0 };
static const size_t snappy_java_hdrlen = 8+4+4;
/* snappy-java adds its own header (SnappyCodec)
* which is not compatible with the official Snappy
* implementation.
* 8: magic, 4: version, 4: compatible
* followed by any number of chunks:
* 4: length
* ...: snappy-compressed data. */
if (likely(inlen > snappy_java_hdrlen + 4 &&
!memcmp(inbuf, snappy_java_magic, 8))) {
/* snappy-java framing */
char errstr[128];
inbuf = inbuf + snappy_java_hdrlen;
inlen -= snappy_java_hdrlen;
iov.iov_base = rd_kafka_snappy_java_uncompress(
inbuf, inlen,
&iov.iov_len,
errstr, sizeof(errstr));
if (unlikely(!iov.iov_base)) {
rd_rkb_dbg(msetr->msetr_rkb, MSG, "SNAPPY",
"%s [%"PRId32"]: "
"Snappy decompression for message "
"at offset %"PRId64" failed: %s: "
"ignoring message",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
Offset, errstr);
err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
goto err;
}
} else {
/* No framing */
/* Acquire uncompressed length */
if (unlikely(!rd_kafka_snappy_uncompressed_length(
inbuf, inlen, &iov.iov_len))) {
rd_rkb_dbg(msetr->msetr_rkb, MSG, "SNAPPY",
"Failed to get length of Snappy "
"compressed payload "
"for message at offset %"PRId64
" (%"PRIusz" bytes): "
"ignoring message",
Offset, inlen);
err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
goto err;
}
/* Allocate output buffer for uncompressed data */
iov.iov_base = rd_malloc(iov.iov_len);
if (unlikely(!iov.iov_base)) {
rd_rkb_dbg(msetr->msetr_rkb, MSG, "SNAPPY",
"Failed to allocate Snappy "
"decompress buffer of size %"PRIusz
"for message at offset %"PRId64
" (%"PRIusz" bytes): %s: "
"ignoring message",
iov.iov_len, Offset, inlen,
rd_strerror(errno));
err = RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE;
goto err;
}
/* Uncompress to outbuf */
if (unlikely((r = rd_kafka_snappy_uncompress(
inbuf, inlen, iov.iov_base)))) {
rd_rkb_dbg(msetr->msetr_rkb, MSG, "SNAPPY",
"Failed to decompress Snappy "
"payload for message at offset "
"%"PRId64" (%"PRIusz" bytes): %s: "
"ignoring message",
Offset, inlen,
rd_strerror(-r/*negative errno*/));
rd_free(iov.iov_base);
err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
goto err;
}
}
}
break;
#endif
case RD_KAFKA_COMPRESSION_LZ4:
{
err = rd_kafka_lz4_decompress(msetr->msetr_rkb,
/* Proper HC? */
MsgVersion >= 1 ? 1 : 0,
Offset,
/* @warning Will modify compressed
* if no proper HC */
(char *)compressed,
compressed_size,
&iov.iov_base, &iov.iov_len);
if (err)
goto err;
}
break;
default:
rd_rkb_dbg(msetr->msetr_rkb, MSG, "CODEC",
"%s [%"PRId32"]: Message at offset %"PRId64
" with unsupported "
"compression codec 0x%x: message ignored",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
Offset, (int)codec);
err = RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED;
goto err;
}
rd_assert(iov.iov_base);
/*
* Decompression successful
*/
/* Create a new buffer pointing to the uncompressed
* allocated buffer (outbuf) and let messages keep a reference to
* this new buffer. */
rkbufz = rd_kafka_buf_new_shadow(iov.iov_base, iov.iov_len, rd_free);
rkbufz->rkbuf_rkb = msetr->msetr_rkbuf->rkbuf_rkb;
rd_kafka_broker_keep(rkbufz->rkbuf_rkb);
/* In MsgVersion v0..1 the decompressed data contains
* an inner MessageSet, pass it to a new MessageSet reader.
*
* For MsgVersion v2 the decompressed data are the list of messages.
*/
if (MsgVersion <= 1) {
/* Pass decompressed data (inner Messageset)
* to new instance of the MessageSet parser. */
rd_kafka_msgset_reader_t inner_msetr;
rd_kafka_msgset_reader_init(&inner_msetr,
rkbufz,
msetr->msetr_rktp,
msetr->msetr_tver,
&msetr->msetr_rkq);
if (MsgVersion == 1) {
/* postproc() will convert relative to
* absolute offsets */
inner_msetr.msetr_relative_offsets = 1;
inner_msetr.msetr_outer.offset = Offset;
/* Apply single LogAppendTime timestamp for
* all messages. */
if (Attributes & RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME) {
inner_msetr.msetr_outer.tstype =
RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME;
inner_msetr.msetr_outer.timestamp = Timestamp;
}
}
/* Parse the inner MessageSet */
err = rd_kafka_msgset_reader_run(&inner_msetr);
} else {
/* MsgVersion 2 */
rd_kafka_buf_t *orig_rkbuf = msetr->msetr_rkbuf;
/* Temporarily replace read buffer with uncompressed buffer */
msetr->msetr_rkbuf = rkbufz;
/* Read messages */
err = rd_kafka_msgset_reader_msgs_v2(msetr);
/* Restore original buffer */
msetr->msetr_rkbuf = orig_rkbuf;
}
/* Loose our refcnt of the uncompressed rkbuf.
* Individual messages/rko's will have their own reference. */
rd_kafka_buf_destroy(rkbufz);
return err;
err:
/* Enqueue error messsage:
* Create op and push on temporary queue. */
rd_kafka_q_op_err(&msetr->msetr_rkq, RD_KAFKA_OP_CONSUMER_ERR,
err, msetr->msetr_tver->version, rktp, Offset,
"Decompression (codec 0x%x) of message at %"PRIu64
" of %"PRIu64" bytes failed: %s",
codec, Offset, compressed_size, rd_kafka_err2str(err));
return err;
}
/**
* @brief Message parser for MsgVersion v0..1
*
* @returns RD_KAFKA_RESP_ERR_NO_ERROR on success or on single-message errors,
* or any other error code when the MessageSet parser should stop
* parsing (such as for partial Messages).
*/
static rd_kafka_resp_err_t
rd_kafka_msgset_reader_msg_v0_1 (rd_kafka_msgset_reader_t *msetr) {
rd_kafka_buf_t *rkbuf = msetr->msetr_rkbuf;
rd_kafka_toppar_t *rktp = msetr->msetr_rktp;
rd_kafka_broker_t *rkb = msetr->msetr_rkb;
struct {
int64_t Offset; /* MessageSet header */
int32_t MessageSize; /* MessageSet header */
uint32_t Crc;
int8_t MagicByte; /* MsgVersion */
int8_t Attributes;
int64_t Timestamp; /* v1 */
} hdr; /* Message header */
rd_kafkap_bytes_t Key;
rd_kafkap_bytes_t Value;
int32_t Value_len;
rd_kafka_op_t *rko;
size_t hdrsize = 6; /* Header size following MessageSize */
rd_slice_t crc_slice;
rd_kafka_msg_t *rkm;
int relative_offsets = 0;
const char *reloff_str = "";
/* Only log decoding errors if protocol debugging enabled. */
int log_decode_errors = (rkbuf->rkbuf_rkb->rkb_rk->rk_conf.debug &
RD_KAFKA_DBG_PROTOCOL) ? LOG_DEBUG : 0;
size_t message_end;
rd_kafka_buf_read_i64(rkbuf, &hdr.Offset);
rd_kafka_buf_read_i32(rkbuf, &hdr.MessageSize);
message_end = rd_slice_offset(&rkbuf->rkbuf_reader) + hdr.MessageSize;
rd_kafka_buf_read_i32(rkbuf, &hdr.Crc);
if (!rd_slice_narrow_copy_relative(&rkbuf->rkbuf_reader, &crc_slice,
hdr.MessageSize - 4))
rd_kafka_buf_check_len(rkbuf, hdr.MessageSize - 4);
rd_kafka_buf_read_i8(rkbuf, &hdr.MagicByte);
rd_kafka_buf_read_i8(rkbuf, &hdr.Attributes);
if (hdr.MagicByte == 1) { /* MsgVersion */
rd_kafka_buf_read_i64(rkbuf, &hdr.Timestamp);
hdrsize += 8;
/* MsgVersion 1 has relative offsets for compressed MessageSets*/
if (!(hdr.Attributes & RD_KAFKA_MSG_ATTR_COMPRESSION_MASK) &&
msetr->msetr_relative_offsets) {
relative_offsets = 1;
reloff_str = "relative ";
}
} else
hdr.Timestamp = 0;
/* Verify MessageSize */
if (unlikely(hdr.MessageSize < (ssize_t)hdrsize))
rd_kafka_buf_parse_fail(rkbuf,
"Message at %soffset %"PRId64
" MessageSize %"PRId32
" < hdrsize %"PRIusz,
reloff_str,
hdr.Offset, hdr.MessageSize, hdrsize);
/* Early check for partial messages */
rd_kafka_buf_check_len(rkbuf, hdr.MessageSize - hdrsize);
if (rkb->rkb_rk->rk_conf.check_crcs) {
/* Verify CRC32 if desired. */
uint32_t calc_crc;
calc_crc = rd_slice_crc32(&crc_slice);
rd_dassert(rd_slice_remains(&crc_slice) == 0);
if (unlikely(hdr.Crc != calc_crc)) {
/* Propagate CRC error to application and
* continue with next message. */
rd_kafka_q_op_err(&msetr->msetr_rkq,
RD_KAFKA_OP_CONSUMER_ERR,
RD_KAFKA_RESP_ERR__BAD_MSG,
msetr->msetr_tver->version,
rktp,
hdr.Offset,
"Message at %soffset %"PRId64
" (%"PRId32" bytes) "
"failed CRC32 check "
"(original 0x%"PRIx32" != "
"calculated 0x%"PRIx32")",
reloff_str, hdr.Offset,
hdr.MessageSize, hdr.Crc, calc_crc);
rd_kafka_buf_skip_to(rkbuf, message_end);
rd_atomic64_add(&rkb->rkb_c.rx_err, 1);
/* Continue with next message */
return RD_KAFKA_RESP_ERR_NO_ERROR;
}
}
/* Extract key */
rd_kafka_buf_read_bytes(rkbuf, &Key);
/* Extract Value */
rd_kafka_buf_read_bytes(rkbuf, &Value);
Value_len = RD_KAFKAP_BYTES_LEN(&Value);
/* MessageSets may contain offsets earlier than we
* requested (compressed MessageSets in particular),
* drop the earlier messages.
* Note: the inner offset may only be trusted for
* absolute offsets. KIP-31 introduced
* ApiVersion 2 that maintains relative offsets
* of compressed messages and the base offset
* in the outer message is the offset of
* the *LAST* message in the MessageSet.
* This requires us to assign offsets
* after all messages have been read from
* the messageset, and it also means
* we cant perform this offset check here
* in that case. */
if (!relative_offsets &&
hdr.Offset < rktp->rktp_offsets.fetch_offset)
return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue with next msg */
/* Handle compressed MessageSet */
if (unlikely(hdr.Attributes & RD_KAFKA_MSG_ATTR_COMPRESSION_MASK))
return rd_kafka_msgset_reader_decompress(
msetr, hdr.MagicByte, hdr.Attributes, hdr.Timestamp,
hdr.Offset, Value.data, Value_len);
/* Pure uncompressed message, this is the innermost
* handler after all compression and cascaded
* MessageSets have been peeled off. */
/* Create op/message container for message. */
rko = rd_kafka_op_new_fetch_msg(&rkm, rktp, msetr->msetr_tver->version,
rkbuf,
hdr.Offset,
(size_t)RD_KAFKAP_BYTES_LEN(&Key),
RD_KAFKAP_BYTES_IS_NULL(&Key) ?
NULL : Key.data,
(size_t)RD_KAFKAP_BYTES_LEN(&Value),
RD_KAFKAP_BYTES_IS_NULL(&Value) ?
NULL : Value.data);
/* Assign message timestamp.
* If message was in a compressed MessageSet and the outer/wrapper
* Message.Attribute had a LOG_APPEND_TIME set, use the
* outer timestamp */
if (msetr->msetr_outer.tstype == RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME) {
rkm->rkm_timestamp = msetr->msetr_outer.timestamp;
rkm->rkm_tstype = msetr->msetr_outer.tstype;
} else if (hdr.MagicByte >= 1 && hdr.Timestamp) {
rkm->rkm_timestamp = hdr.Timestamp;
if (hdr.Attributes & RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME)
rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME;
else
rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_CREATE_TIME;
}
/* Enqueue message on temporary queue */
rd_kafka_q_enq(&msetr->msetr_rkq, rko);
msetr->msetr_msgcnt++;
return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue */
err_parse:
/* Count all parse errors as partial message errors. */
rd_atomic64_add(&msetr->msetr_rkb->rkb_c.rx_partial, 1);
return rkbuf->rkbuf_err;
}
/**
* @brief Message parser for MsgVersion v2
*/
static rd_kafka_resp_err_t
rd_kafka_msgset_reader_msg_v2 (rd_kafka_msgset_reader_t *msetr) {
rd_kafka_buf_t *rkbuf = msetr->msetr_rkbuf;
rd_kafka_toppar_t *rktp = msetr->msetr_rktp;
struct {
int64_t Length;
int64_t MsgAttributes; /* int8_t, but int64 req. for varint */
int64_t TimestampDelta;
int64_t OffsetDelta;
int64_t Offset; /* Absolute offset */
rd_kafkap_bytes_t Key;
rd_kafkap_bytes_t Value;
int64_t HeaderCnt;
} hdr;
rd_kafka_op_t *rko;
rd_kafka_msg_t *rkm;
/* Only log decoding errors if protocol debugging enabled. */
int log_decode_errors = (rkbuf->rkbuf_rkb->rkb_rk->rk_conf.debug &
RD_KAFKA_DBG_PROTOCOL) ? LOG_DEBUG : 0;
size_t message_end;
rd_kafka_buf_read_varint(rkbuf, &hdr.Length);
message_end = rd_slice_offset(&rkbuf->rkbuf_reader)+(size_t)hdr.Length;
rd_kafka_buf_read_varint(rkbuf, &hdr.MsgAttributes);
rd_kafka_buf_read_varint(rkbuf, &hdr.TimestampDelta);
rd_kafka_buf_read_varint(rkbuf, &hdr.OffsetDelta);
hdr.Offset = msetr->msetr_v2_hdr->BaseOffset + hdr.OffsetDelta;
/* Skip message if outdated */
if (hdr.Offset < rktp->rktp_offsets.fetch_offset) {
rd_rkb_dbg(msetr->msetr_rkb, MSG, "MSG",
"Skip offset %"PRId64" < fetch_offset %"PRId64,
hdr.Offset, rktp->rktp_offsets.fetch_offset);
rd_kafka_buf_skip_to(rkbuf, message_end);
return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue with next msg */
}
rd_kafka_buf_read_bytes_varint(rkbuf, &hdr.Key);
rd_kafka_buf_read_bytes_varint(rkbuf, &hdr.Value);
/* Ignore headers for now */
rd_kafka_buf_skip_to(rkbuf, message_end);
/* Create op/message container for message. */
rko = rd_kafka_op_new_fetch_msg(&rkm,
rktp, msetr->msetr_tver->version, rkbuf,
hdr.Offset,
(size_t)RD_KAFKAP_BYTES_LEN(&hdr.Key),
RD_KAFKAP_BYTES_IS_NULL(&hdr.Key) ?
NULL : hdr.Key.data,
(size_t)RD_KAFKAP_BYTES_LEN(&hdr.Value),
RD_KAFKAP_BYTES_IS_NULL(&hdr.Value) ?
NULL : hdr.Value.data);
/* Set timestamp.
*
* When broker assigns the timestamps (LOG_APPEND_TIME) it will
* assign the same timestamp for all messages in a MessageSet
* using MaxTimestamp.
*/
if ((msetr->msetr_v2_hdr->Attributes &
RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME) ||
(hdr.MsgAttributes & RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME)) {
rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME;
rkm->rkm_timestamp = msetr->msetr_v2_hdr->MaxTimestamp;
} else {
rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_CREATE_TIME;
rkm->rkm_timestamp =
msetr->msetr_v2_hdr->BaseTimestamp + hdr.TimestampDelta;
}
/* Enqueue message on temporary queue */
rd_kafka_q_enq(&msetr->msetr_rkq, rko);
msetr->msetr_msgcnt++;
return RD_KAFKA_RESP_ERR_NO_ERROR;
err_parse:
/* Count all parse errors as partial message errors. */
rd_atomic64_add(&msetr->msetr_rkb->rkb_c.rx_partial, 1);
return rkbuf->rkbuf_err;
}
/**
* Initiate asynchronous connection attempt.
*
* Locality: broker thread
*/
rd_kafka_transport_t *rd_kafka_transport_connect (rd_kafka_broker_t *rkb,
const rd_sockaddr_inx_t *sinx,
char *errstr,
int errstr_size) {
rd_kafka_transport_t *rktrans;
int s = -1;
int on = 1;
rkb->rkb_addr_last = sinx;
s = rkb->rkb_rk->rk_conf.socket_cb(sinx->in.sin_family,
SOCK_STREAM, IPPROTO_TCP,
rkb->rkb_rk->rk_conf.opaque);
if (s == -1) {
rd_snprintf(errstr, errstr_size, "Failed to create socket: %s",
socket_strerror(socket_errno));
return NULL;
}
#ifdef SO_NOSIGPIPE
/* Disable SIGPIPE signalling for this socket on OSX */
if (setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, &on, sizeof(on)) == -1)
rd_rkb_dbg(rkb, BROKER, "SOCKET",
"Failed to set SO_NOSIGPIPE: %s",
socket_strerror(socket_errno));
#endif
/* Enable TCP keep-alives, if configured. */
if (rkb->rkb_rk->rk_conf.socket_keepalive) {
#ifdef SO_KEEPALIVE
if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE,
(void *)&on, sizeof(on)) == SOCKET_ERROR)
rd_rkb_dbg(rkb, BROKER, "SOCKET",
"Failed to set SO_KEEPALIVE: %s",
socket_strerror(socket_errno));
#else
rd_rkb_dbg(rkb, BROKER, "SOCKET",
"System does not support "
"socket.keepalive.enable (SO_KEEPALIVE)");
#endif
}
/* Set the socket to non-blocking */
#ifdef _MSC_VER
if (ioctlsocket(s, FIONBIO, &on) == SOCKET_ERROR) {
rd_snprintf(errstr, errstr_size,
"Failed to set socket non-blocking: %s",
socket_strerror(socket_errno));
goto err;
}
#else
{
int fl = fcntl(s, F_GETFL, 0);
if (fl == -1 ||
fcntl(s, F_SETFL, fl | O_NONBLOCK) == -1) {
rd_snprintf(errstr, errstr_size,
"Failed to set socket non-blocking: %s",
socket_strerror(socket_errno));
goto err;
}
}
#endif
rd_rkb_dbg(rkb, BROKER, "CONNECT", "Connecting to %s (%s) "
"with socket %i",
rd_sockaddr2str(sinx, RD_SOCKADDR2STR_F_FAMILY |
RD_SOCKADDR2STR_F_PORT),
rd_kafka_secproto_names[rkb->rkb_proto], s);
/* Connect to broker */
if (connect(s, (struct sockaddr *)sinx,
RD_SOCKADDR_INX_LEN(sinx)) == SOCKET_ERROR &&
(socket_errno != EINPROGRESS
#ifdef _MSC_VER
&& socket_errno != WSAEWOULDBLOCK
#endif
)) {
rd_rkb_dbg(rkb, BROKER, "CONNECT",
"couldn't connect to %s: %s (%i)",
rd_sockaddr2str(sinx,
RD_SOCKADDR2STR_F_PORT |
RD_SOCKADDR2STR_F_FAMILY),
socket_strerror(socket_errno), socket_errno);
rd_snprintf(errstr, errstr_size,
"Failed to connect to broker at %s: %s",
rd_sockaddr2str(sinx, RD_SOCKADDR2STR_F_NICE),
socket_strerror(socket_errno));
goto err;
}
/* Create transport handle */
rktrans = rd_calloc(1, sizeof(*rktrans));
rktrans->rktrans_rkb = rkb;
rktrans->rktrans_s = s;
rktrans->rktrans_pfd.fd = s;
/* Poll writability to trigger on connection success/failure. */
rd_kafka_transport_poll_set(rktrans, POLLOUT);
return rktrans;
err:
if (s != -1) {
#ifndef _MSC_VER
close(s);
#else
closesocket(s);
#endif
}
return NULL;
}
/**
* @brief Handle a Metadata response message.
*
* @param topics are the requested topics (may be NULL)
*
* The metadata will be marshalled into 'struct rd_kafka_metadata*' structs.
*
* The marshalled metadata is returned in \p *mdp, (NULL on error).
* @returns an error code on parse failure, else NO_ERRRO.
*
* @locality rdkafka main thread
*/
rd_kafka_resp_err_t
rd_kafka_parse_Metadata (rd_kafka_broker_t *rkb,
rd_kafka_buf_t *request,
rd_kafka_buf_t *rkbuf,
struct rd_kafka_metadata **mdp) {
rd_kafka_t *rk = rkb->rkb_rk;
int i, j, k;
rd_tmpabuf_t tbuf;
struct rd_kafka_metadata *md;
size_t rkb_namelen;
const int log_decode_errors = LOG_ERR;
rd_list_t *missing_topics = NULL;
const rd_list_t *requested_topics = request->rkbuf_u.Metadata.topics;
int all_topics = request->rkbuf_u.Metadata.all_topics;
const char *reason = request->rkbuf_u.Metadata.reason ?
request->rkbuf_u.Metadata.reason : "(no reason)";
int ApiVersion = request->rkbuf_reqhdr.ApiVersion;
rd_kafkap_str_t cluster_id = RD_ZERO_INIT;
int32_t controller_id = -1;
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
int broadcast_changes = 0;
rd_kafka_assert(NULL, thrd_is_current(rk->rk_thread));
/* Remove topics from missing_topics as they are seen in Metadata. */
if (requested_topics)
missing_topics = rd_list_copy(requested_topics,
rd_list_string_copy, NULL);
rd_kafka_broker_lock(rkb);
rkb_namelen = strlen(rkb->rkb_name)+1;
/* We assume that the marshalled representation is
* no more than 4 times larger than the wire representation. */
rd_tmpabuf_new(&tbuf,
sizeof(*md) + rkb_namelen + (rkbuf->rkbuf_totlen * 4),
0/*dont assert on fail*/);
if (!(md = rd_tmpabuf_alloc(&tbuf, sizeof(*md)))) {
err = RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE;
goto err;
}
md->orig_broker_id = rkb->rkb_nodeid;
md->orig_broker_name = rd_tmpabuf_write(&tbuf,
rkb->rkb_name, rkb_namelen);
rd_kafka_broker_unlock(rkb);
/* Read Brokers */
rd_kafka_buf_read_i32a(rkbuf, md->broker_cnt);
if (md->broker_cnt > RD_KAFKAP_BROKERS_MAX)
rd_kafka_buf_parse_fail(rkbuf, "Broker_cnt %i > BROKERS_MAX %i",
md->broker_cnt, RD_KAFKAP_BROKERS_MAX);
if (!(md->brokers = rd_tmpabuf_alloc(&tbuf, md->broker_cnt *
sizeof(*md->brokers))))
rd_kafka_buf_parse_fail(rkbuf,
"%d brokers: tmpabuf memory shortage",
md->broker_cnt);
for (i = 0 ; i < md->broker_cnt ; i++) {
rd_kafka_buf_read_i32a(rkbuf, md->brokers[i].id);
rd_kafka_buf_read_str_tmpabuf(rkbuf, &tbuf, md->brokers[i].host);
rd_kafka_buf_read_i32a(rkbuf, md->brokers[i].port);
if (ApiVersion >= 1) {
rd_kafkap_str_t rack;
rd_kafka_buf_read_str(rkbuf, &rack);
}
}
if (ApiVersion >= 2)
rd_kafka_buf_read_str(rkbuf, &cluster_id);
if (ApiVersion >= 1) {
rd_kafka_buf_read_i32(rkbuf, &controller_id);
rd_rkb_dbg(rkb, METADATA,
"METADATA", "ClusterId: %.*s, ControllerId: %"PRId32,
RD_KAFKAP_STR_PR(&cluster_id), controller_id);
}
/* Read TopicMetadata */
rd_kafka_buf_read_i32a(rkbuf, md->topic_cnt);
rd_rkb_dbg(rkb, METADATA, "METADATA", "%i brokers, %i topics",
md->broker_cnt, md->topic_cnt);
if (md->topic_cnt > RD_KAFKAP_TOPICS_MAX)
rd_kafka_buf_parse_fail(rkbuf, "TopicMetadata_cnt %"PRId32
" > TOPICS_MAX %i",
md->topic_cnt, RD_KAFKAP_TOPICS_MAX);
if (!(md->topics = rd_tmpabuf_alloc(&tbuf,
md->topic_cnt *
sizeof(*md->topics))))
rd_kafka_buf_parse_fail(rkbuf,
"%d topics: tmpabuf memory shortage",
md->topic_cnt);
for (i = 0 ; i < md->topic_cnt ; i++) {
rd_kafka_buf_read_i16a(rkbuf, md->topics[i].err);
rd_kafka_buf_read_str_tmpabuf(rkbuf, &tbuf, md->topics[i].topic);
if (ApiVersion >= 1) {
int8_t is_internal;
rd_kafka_buf_read_i8(rkbuf, &is_internal);
}
/* PartitionMetadata */
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partition_cnt);
if (md->topics[i].partition_cnt > RD_KAFKAP_PARTITIONS_MAX)
rd_kafka_buf_parse_fail(rkbuf,
"TopicMetadata[%i]."
"PartitionMetadata_cnt %i "
"> PARTITIONS_MAX %i",
i, md->topics[i].partition_cnt,
RD_KAFKAP_PARTITIONS_MAX);
if (!(md->topics[i].partitions =
rd_tmpabuf_alloc(&tbuf,
md->topics[i].partition_cnt *
sizeof(*md->topics[i].partitions))))
rd_kafka_buf_parse_fail(rkbuf,
"%s: %d partitions: "
"tmpabuf memory shortage",
md->topics[i].topic,
md->topics[i].partition_cnt);
for (j = 0 ; j < md->topics[i].partition_cnt ; j++) {
rd_kafka_buf_read_i16a(rkbuf, md->topics[i].partitions[j].err);
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].id);
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].leader);
/* Replicas */
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].replica_cnt);
if (md->topics[i].partitions[j].replica_cnt >
RD_KAFKAP_BROKERS_MAX)
rd_kafka_buf_parse_fail(rkbuf,
"TopicMetadata[%i]."
"PartitionMetadata[%i]."
"Replica_cnt "
"%i > BROKERS_MAX %i",
i, j,
md->topics[i].
partitions[j].
replica_cnt,
RD_KAFKAP_BROKERS_MAX);
if (!(md->topics[i].partitions[j].replicas =
rd_tmpabuf_alloc(&tbuf,
md->topics[i].
partitions[j].replica_cnt *
sizeof(*md->topics[i].
partitions[j].replicas))))
rd_kafka_buf_parse_fail(
rkbuf,
"%s [%"PRId32"]: %d replicas: "
"tmpabuf memory shortage",
md->topics[i].topic,
md->topics[i].partitions[j].id,
md->topics[i].partitions[j].replica_cnt);
for (k = 0 ;
k < md->topics[i].partitions[j].replica_cnt; k++)
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].
replicas[k]);
/* Isrs */
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].isr_cnt);
if (md->topics[i].partitions[j].isr_cnt >
RD_KAFKAP_BROKERS_MAX)
rd_kafka_buf_parse_fail(rkbuf,
"TopicMetadata[%i]."
"PartitionMetadata[%i]."
"Isr_cnt "
"%i > BROKERS_MAX %i",
i, j,
md->topics[i].
partitions[j].isr_cnt,
RD_KAFKAP_BROKERS_MAX);
if (!(md->topics[i].partitions[j].isrs =
rd_tmpabuf_alloc(&tbuf,
md->topics[i].
partitions[j].isr_cnt *
sizeof(*md->topics[i].
partitions[j].isrs))))
rd_kafka_buf_parse_fail(
rkbuf,
"%s [%"PRId32"]: %d isrs: "
"tmpabuf memory shortage",
md->topics[i].topic,
md->topics[i].partitions[j].id,
md->topics[i].partitions[j].isr_cnt);
for (k = 0 ;
k < md->topics[i].partitions[j].isr_cnt; k++)
rd_kafka_buf_read_i32a(rkbuf, md->topics[i].
partitions[j].isrs[k]);
}
/* Sort partitions by partition id */
qsort(md->topics[i].partitions,
md->topics[i].partition_cnt,
sizeof(*md->topics[i].partitions),
rd_kafka_metadata_partition_id_cmp);
}
/* Entire Metadata response now parsed without errors:
* update our internal state according to the response. */
/* Avoid metadata updates when we're terminating. */
if (rd_kafka_terminating(rkb->rkb_rk)) {
err = RD_KAFKA_RESP_ERR__DESTROY;
goto done;
}
if (md->broker_cnt == 0 && md->topic_cnt == 0) {
rd_rkb_dbg(rkb, METADATA, "METADATA",
"No brokers or topics in metadata: should retry");
err = RD_KAFKA_RESP_ERR__PARTIAL;
goto err;
}
/* Update our list of brokers. */
for (i = 0 ; i < md->broker_cnt ; i++) {
rd_rkb_dbg(rkb, METADATA, "METADATA",
" Broker #%i/%i: %s:%i NodeId %"PRId32,
i, md->broker_cnt,
md->brokers[i].host,
md->brokers[i].port,
md->brokers[i].id);
rd_kafka_broker_update(rkb->rkb_rk, rkb->rkb_proto,
&md->brokers[i]);
}
/* Update partition count and leader for each topic we know about */
for (i = 0 ; i < md->topic_cnt ; i++) {
rd_kafka_metadata_topic_t *mdt = &md->topics[i];
rd_rkb_dbg(rkb, METADATA, "METADATA",
" Topic #%i/%i: %s with %i partitions%s%s",
i, md->topic_cnt, mdt->topic,
mdt->partition_cnt,
mdt->err ? ": " : "",
mdt->err ? rd_kafka_err2str(mdt->err) : "");
/* Ignore topics in blacklist */
if (rkb->rkb_rk->rk_conf.topic_blacklist &&
rd_kafka_pattern_match(rkb->rkb_rk->rk_conf.topic_blacklist,
mdt->topic)) {
rd_rkb_dbg(rkb, TOPIC, "BLACKLIST",
"Ignoring blacklisted topic \"%s\" "
"in metadata", mdt->topic);
continue;
}
/* Ignore metadata completely for temporary errors. (issue #513)
* LEADER_NOT_AVAILABLE: Broker is rebalancing
*/
if (mdt->err == RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE &&
mdt->partition_cnt == 0) {
rd_rkb_dbg(rkb, TOPIC, "METADATA",
"Temporary error in metadata reply for "
"topic %s (PartCnt %i): %s: ignoring",
mdt->topic, mdt->partition_cnt,
rd_kafka_err2str(mdt->err));
if (missing_topics)
rd_list_free_cb(
missing_topics,
rd_list_remove_cmp(missing_topics,
mdt->topic,
(void *)strcmp));
continue;
}
/* Update local topic & partition state based on metadata */
rd_kafka_topic_metadata_update2(rkb, mdt);
if (requested_topics) {
rd_list_free_cb(missing_topics,
rd_list_remove_cmp(missing_topics,
mdt->topic,
(void*)strcmp));
if (!all_topics) {
rd_kafka_wrlock(rk);
rd_kafka_metadata_cache_topic_update(rk, mdt);
rd_kafka_wrunlock(rk);
}
}
}
/* Requested topics not seen in metadata? Propogate to topic code. */
if (missing_topics) {
char *topic;
rd_rkb_dbg(rkb, TOPIC, "METADATA",
"%d/%d requested topic(s) seen in metadata",
rd_list_cnt(requested_topics) -
rd_list_cnt(missing_topics),
rd_list_cnt(requested_topics));
for (i = 0 ; i < rd_list_cnt(missing_topics) ; i++)
rd_rkb_dbg(rkb, TOPIC, "METADATA", "wanted %s",
(char *)(missing_topics->rl_elems[i]));
RD_LIST_FOREACH(topic, missing_topics, i) {
shptr_rd_kafka_itopic_t *s_rkt;
s_rkt = rd_kafka_topic_find(rkb->rkb_rk, topic, 1/*lock*/);
if (s_rkt) {
rd_kafka_topic_metadata_none(
rd_kafka_topic_s2i(s_rkt));
rd_kafka_topic_destroy0(s_rkt);
}
}
}
/**
* Handle received frame from broker.
*/
static int rd_kafka_sasl_handle_recv (rd_kafka_transport_t *rktrans,
rd_kafka_buf_t *rkbuf,
char *errstr, int errstr_size) {
int r;
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASL",
"Received SASL frame from broker (%"PRIdsz" bytes)",
rkbuf ? rkbuf->rkbuf_len : 0);
if (rktrans->rktrans_sasl.complete && (!rkbuf || rkbuf->rkbuf_len == 0))
goto auth_successful;
do {
sasl_interact_t *interact = NULL;
const char *out;
unsigned int outlen;
r = sasl_client_step(rktrans->rktrans_sasl.conn,
rkbuf && rkbuf->rkbuf_len > 0 ?
rkbuf->rkbuf_rbuf : NULL,
rkbuf ? rkbuf->rkbuf_len : 0,
&interact,
&out, &outlen);
if (rkbuf) {
rd_kafka_buf_destroy(rkbuf);
rkbuf = NULL;
}
if (r >= 0) {
/* Note: outlen may be 0 here for an empty response */
if (rd_kafka_sasl_send(rktrans, out, outlen,
errstr, errstr_size) == -1)
return -1;
}
if (r == SASL_INTERACT)
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASL",
"SASL_INTERACT: %lu %s, %s, %s, %p",
interact->id,
interact->challenge,
interact->prompt,
interact->defresult,
interact->result);
} while (r == SASL_INTERACT);
if (r == SASL_CONTINUE)
return 0; /* Wait for more data from broker */
else if (r != SASL_OK) {
rd_snprintf(errstr, errstr_size,
"SASL handshake failed (step): %s",
sasl_errdetail(rktrans->rktrans_sasl.conn));
return -1;
}
/* Authentication successful */
auth_successful:
if (rktrans->rktrans_rkb->rkb_rk->rk_conf.debug &
RD_KAFKA_DBG_SECURITY) {
const char *user, *mech, *authsrc;
if (sasl_getprop(rktrans->rktrans_sasl.conn, SASL_USERNAME,
(const void **)&user) != SASL_OK)
user = "******";
if (sasl_getprop(rktrans->rktrans_sasl.conn, SASL_MECHNAME,
(const void **)&mech) != SASL_OK)
mech = "(unknown)";
if (sasl_getprop(rktrans->rktrans_sasl.conn, SASL_AUTHSOURCE,
(const void **)&authsrc) != SASL_OK)
authsrc = "(unknown)";
rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASL",
"Authenticated as %s using %s (%s)",
user, mech, authsrc);
}
rd_kafka_broker_connect_up(rktrans->rktrans_rkb);
return 0;
}
/**
* Initialize and start SASL authentication.
*
* Returns 0 on successful init and -1 on error.
*
* Locality: broker thread
*/
int rd_kafka_sasl_client_new (rd_kafka_transport_t *rktrans,
char *errstr, int errstr_size) {
int r;
rd_kafka_broker_t *rkb = rktrans->rktrans_rkb;
rd_kafka_t *rk = rkb->rkb_rk;
char *hostname, *t;
sasl_callback_t callbacks[16] = {
// { SASL_CB_GETOPT, (void *)rd_kafka_sasl_cb_getopt, rktrans },
{ SASL_CB_LOG, (void *)rd_kafka_sasl_cb_log, rktrans },
{ SASL_CB_AUTHNAME, (void *)rd_kafka_sasl_cb_getsimple, rktrans },
{ SASL_CB_PASS, (void *)rd_kafka_sasl_cb_getsecret, rktrans },
{ SASL_CB_ECHOPROMPT, (void *)rd_kafka_sasl_cb_chalprompt, rktrans },
{ SASL_CB_GETREALM, (void *)rd_kafka_sasl_cb_getrealm, rktrans },
{ SASL_CB_CANON_USER, (void *)rd_kafka_sasl_cb_canon, rktrans },
{ SASL_CB_LIST_END }
};
/* SASL_CB_USER is needed for PLAIN but breaks GSSAPI */
if (!strcmp(rk->rk_conf.sasl.service_name, "PLAIN")) {
int endidx;
/* Find end of callbacks array */
for (endidx = 0 ;
callbacks[endidx].id != SASL_CB_LIST_END ; endidx++)
;
callbacks[endidx].id = SASL_CB_USER;
callbacks[endidx].proc = (void *)rd_kafka_sasl_cb_getsimple;
endidx++;
callbacks[endidx].id = SASL_CB_LIST_END;
}
rd_strdupa(&hostname, rktrans->rktrans_rkb->rkb_nodename);
if ((t = strchr(hostname, ':')))
*t = '\0'; /* remove ":port" */
rd_rkb_dbg(rkb, SECURITY, "SASL",
"Initializing SASL client: service name %s, "
"hostname %s, mechanisms %s",
rk->rk_conf.sasl.service_name, hostname,
rk->rk_conf.sasl.mechanisms);
/* Acquire or refresh ticket if kinit is configured */
rd_kafka_sasl_kinit_refresh(rkb);
r = sasl_client_new(rk->rk_conf.sasl.service_name, hostname,
NULL, NULL, /* no local & remote IP checks */
callbacks, 0, &rktrans->rktrans_sasl.conn);
if (r != SASL_OK) {
rd_snprintf(errstr, errstr_size, "%s",
sasl_errstring(r, NULL, NULL));
return -1;
}
if (rk->rk_conf.debug & RD_KAFKA_DBG_SECURITY) {
const char *avail_mechs;
sasl_listmech(rktrans->rktrans_sasl.conn, NULL, NULL, " ", NULL,
&avail_mechs, NULL, NULL);
rd_rkb_dbg(rkb, SECURITY, "SASL",
"My supported SASL mechanisms: %s", avail_mechs);
}
rd_kafka_transport_poll_set(rktrans, POLLIN);
do {
const char *out;
unsigned int outlen;
const char *mech = NULL;
r = sasl_client_start(rktrans->rktrans_sasl.conn,
rk->rk_conf.sasl.mechanisms,
NULL, &out, &outlen, &mech);
if (r >= 0)
if (rd_kafka_sasl_send(rktrans, out, outlen,
errstr, errstr_size))
return -1;
} while (r == SASL_INTERACT);
if (r == SASL_OK) {
/* PLAIN is appearantly done here, but we still need to make sure
* the PLAIN frame is sent and we get a response back (but we must
* not pass the response to libsasl or it will fail). */
rktrans->rktrans_sasl.complete = 1;
return 0;
} else if (r != SASL_CONTINUE) {
rd_snprintf(errstr, errstr_size,
"SASL handshake failed (start (%d)): %s",
r, sasl_errdetail(rktrans->rktrans_sasl.conn));
return -1;
}
return 0;
}
/**
* @brief Message parser for MsgVersion v2
*/
static rd_kafka_resp_err_t
rd_kafka_msgset_reader_msg_v2 (rd_kafka_msgset_reader_t *msetr) {
rd_kafka_buf_t *rkbuf = msetr->msetr_rkbuf;
rd_kafka_toppar_t *rktp = msetr->msetr_rktp;
struct {
int64_t Length;
int8_t MsgAttributes;
int64_t TimestampDelta;
int64_t OffsetDelta;
int64_t Offset; /* Absolute offset */
rd_kafkap_bytes_t Key;
rd_kafkap_bytes_t Value;
rd_kafkap_bytes_t Headers;
} hdr;
rd_kafka_op_t *rko;
rd_kafka_msg_t *rkm;
/* Only log decoding errors if protocol debugging enabled. */
int log_decode_errors = (rkbuf->rkbuf_rkb->rkb_rk->rk_conf.debug &
RD_KAFKA_DBG_PROTOCOL) ? LOG_DEBUG : 0;
size_t message_end;
rd_kafka_buf_read_varint(rkbuf, &hdr.Length);
message_end = rd_slice_offset(&rkbuf->rkbuf_reader)+(size_t)hdr.Length;
rd_kafka_buf_read_i8(rkbuf, &hdr.MsgAttributes);
rd_kafka_buf_read_varint(rkbuf, &hdr.TimestampDelta);
rd_kafka_buf_read_varint(rkbuf, &hdr.OffsetDelta);
hdr.Offset = msetr->msetr_v2_hdr->BaseOffset + hdr.OffsetDelta;
/* Skip message if outdated */
if (hdr.Offset < rktp->rktp_offsets.fetch_offset) {
rd_rkb_dbg(msetr->msetr_rkb, MSG, "MSG",
"%s [%"PRId32"]: "
"Skip offset %"PRId64" < fetch_offset %"PRId64,
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
hdr.Offset, rktp->rktp_offsets.fetch_offset);
rd_kafka_buf_skip_to(rkbuf, message_end);
return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue with next msg */
}
rd_kafka_buf_read_bytes_varint(rkbuf, &hdr.Key);
rd_kafka_buf_read_bytes_varint(rkbuf, &hdr.Value);
/* We parse the Headers later, just store the size (possibly truncated)
* and pointer to the headers. */
hdr.Headers.len = (int32_t)(message_end -
rd_slice_offset(&rkbuf->rkbuf_reader));
rd_kafka_buf_read_ptr(rkbuf, &hdr.Headers.data, hdr.Headers.len);
/* Create op/message container for message. */
rko = rd_kafka_op_new_fetch_msg(&rkm,
rktp, msetr->msetr_tver->version, rkbuf,
hdr.Offset,
(size_t)RD_KAFKAP_BYTES_LEN(&hdr.Key),
RD_KAFKAP_BYTES_IS_NULL(&hdr.Key) ?
NULL : hdr.Key.data,
(size_t)RD_KAFKAP_BYTES_LEN(&hdr.Value),
RD_KAFKAP_BYTES_IS_NULL(&hdr.Value) ?
NULL : hdr.Value.data);
/* Store pointer to unparsed message headers, they will
* be parsed on the first access.
* This pointer points to the rkbuf payload.
* Note: can't perform struct copy here due to const fields (MSVC) */
rkm->rkm_u.consumer.binhdrs.len = hdr.Headers.len;
rkm->rkm_u.consumer.binhdrs.data = hdr.Headers.data;
/* Set timestamp.
*
* When broker assigns the timestamps (LOG_APPEND_TIME) it will
* assign the same timestamp for all messages in a MessageSet
* using MaxTimestamp.
*/
if ((msetr->msetr_v2_hdr->Attributes &
RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME) ||
(hdr.MsgAttributes & RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME)) {
rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME;
rkm->rkm_timestamp = msetr->msetr_v2_hdr->MaxTimestamp;
} else {
rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_CREATE_TIME;
rkm->rkm_timestamp =
msetr->msetr_v2_hdr->BaseTimestamp + hdr.TimestampDelta;
}
/* Enqueue message on temporary queue */
rd_kafka_q_enq(&msetr->msetr_rkq, rko);
msetr->msetr_msgcnt++;
msetr->msetr_msg_bytes += rkm->rkm_key_len + rkm->rkm_len;
return RD_KAFKA_RESP_ERR_NO_ERROR;
err_parse:
/* Count all parse errors as partial message errors. */
rd_atomic64_add(&msetr->msetr_rkb->rkb_c.rx_partial, 1);
return rkbuf->rkbuf_err;
}
