/** * Send auth message with framing. * This is a blocking call. */ static int rd_kafka_sasl_send (rd_kafka_transport_t *rktrans, const void *payload, int len, char *errstr, int errstr_size) { struct msghdr msg = RD_ZERO_INIT; struct iovec iov[1]; int32_t hdr; char *frame; int total_len = sizeof(hdr) + len; rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASL", "Send SASL frame to broker (%d bytes)", len); frame = rd_malloc(total_len); hdr = htobe32(len); memcpy(frame, &hdr, sizeof(hdr)); if (payload) memcpy(frame+sizeof(hdr), payload, len); msg.msg_iov = iov; msg.msg_iovlen = 1; iov[0].iov_base = frame; iov[0].iov_len = total_len; /* Simulate blocking behaviour on non-blocking socket.. * FIXME: This isn't optimal but is highly unlikely to stall since * the socket buffer will most likely not be exceeded. */ do { int r; r = rd_kafka_transport_sendmsg(rktrans, &msg, errstr, errstr_size); if (r == -1) { rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASL", "SASL send of %d bytes failed: %s", total_len, errstr); free(frame); return -1; } if (r == total_len) break; /* Avoid busy-looping */ rd_usleep(10*1000, NULL); /* Shave off written bytes */ total_len -= r; iov[0].iov_base = ((char *)(iov[0].iov_base)) + r; iov[0].iov_len = total_len; } while (total_len > 0); free(frame); return 0; }
/** * @brief Perform \p itcnt iterations of HMAC() on the given buffer \p in * using \p salt, writing the output into \p out which must be * at least EVP_MAX_MD_SIZE. Actual size is updated in \p *outsize. * @returns 0 on success, else -1 */ static int rd_kafka_sasl_scram_Hi (rd_kafka_transport_t *rktrans, const rd_chariov_t *in, const rd_chariov_t *salt, int itcnt, rd_chariov_t *out) { const EVP_MD *evp = rktrans->rktrans_rkb->rkb_rk->rk_conf.sasl.scram_evp; unsigned int ressize = 0; unsigned char tempres[EVP_MAX_MD_SIZE]; unsigned char *saltplus; int i; /* U1 := HMAC(str, salt + INT(1)) */ saltplus = rd_alloca(salt->size + 4); memcpy(saltplus, salt->ptr, salt->size); saltplus[salt->size] = 0; saltplus[salt->size+1] = 0; saltplus[salt->size+2] = 0; saltplus[salt->size+3] = 1; /* U1 := HMAC(str, salt + INT(1)) */ if (!HMAC(evp, (const unsigned char *)in->ptr, (int)in->size, saltplus, salt->size+4, tempres, &ressize)) { rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SCRAM", "HMAC priming failed"); return -1; } memcpy(out->ptr, tempres, ressize); /* Ui-1 := HMAC(str, Ui-2) .. */ for (i = 1 ; i < itcnt ; i++) { unsigned char tempdest[EVP_MAX_MD_SIZE]; int j; if (unlikely(!HMAC(evp, (const unsigned char *)in->ptr, (int)in->size, tempres, ressize, tempdest, NULL))) { rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SCRAM", "Hi() HMAC #%d/%d failed", i, itcnt); return -1; } /* U1 XOR U2 .. */ for (j = 0 ; j < (int)ressize ; j++) { out->ptr[j] ^= tempdest[j]; tempres[j] = tempdest[j]; } } out->size = ressize; return 0; }
/** * Execute kinit to refresh ticket. * * Returns 0 on success, -1 on error. * * Locality: any */ static int rd_kafka_sasl_kinit_refresh (rd_kafka_broker_t *rkb) { rd_kafka_t *rk = rkb->rkb_rk; int r; char *cmd; char errstr[128]; if (!rk->rk_conf.sasl.kinit_cmd || !strstr(rk->rk_conf.sasl.mechanisms, "GSSAPI")) return 0; /* kinit not configured */ /* Build kinit refresh command line using string rendering and config */ cmd = rd_string_render(rk->rk_conf.sasl.kinit_cmd, errstr, sizeof(errstr), render_callback, rkb); if (!cmd) { rd_rkb_log(rkb, LOG_ERR, "SASLREFRESH", "Failed to construct kinit command " "from sasl.kerberos.kinit.cmd template: %s", errstr); return -1; } /* Execute kinit */ rd_rkb_dbg(rkb, SECURITY, "SASLREFRESH", "Refreshing SASL keys with command: %s", cmd); mtx_lock(&rd_kafka_sasl_kinit_lock); r = system(cmd); mtx_unlock(&rd_kafka_sasl_kinit_lock); if (r == -1) { rd_rkb_log(rkb, LOG_ERR, "SASLREFRESH", "SASL key refresh failed: Failed to execute %s", cmd); rd_free(cmd); return -1; } else if (WIFSIGNALED(r)) { rd_rkb_log(rkb, LOG_ERR, "SASLREFRESH", "SASL key refresh failed: %s: received signal %d", cmd, WTERMSIG(r)); rd_free(cmd); return -1; } else if (WIFEXITED(r) && WEXITSTATUS(r) != 0) { rd_rkb_log(rkb, LOG_ERR, "SASLREFRESH", "SASL key refresh failed: %s: exited with code %d", cmd, WEXITSTATUS(r)); rd_free(cmd); return -1; } rd_free(cmd); rd_rkb_dbg(rkb, SECURITY, "SASLREFRESH", "SASL key refreshed"); return 0; }
/** * @brief Perform HMAC(key,str) and stores the result in \p out * which must be at least EVP_MAX_MD_SIZE. * @returns 0 on success, else -1 */ static int rd_kafka_sasl_scram_HMAC (rd_kafka_transport_t *rktrans, const rd_chariov_t *key, const rd_chariov_t *str, rd_chariov_t *out) { const EVP_MD *evp = rktrans->rktrans_rkb->rkb_rk->rk_conf.sasl.scram_evp; unsigned int outsize; //printf("HMAC KEY: %s\n", rd_base64_encode(key)); //printf("HMAC STR: %s\n", rd_base64_encode(str)); if (!HMAC(evp, (const unsigned char *)key->ptr, (int)key->size, (const unsigned char *)str->ptr, (int)str->size, (unsigned char *)out->ptr, &outsize)) { rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SCRAM", "HMAC failed"); return -1; } out->size = outsize; //printf("HMAC OUT: %s\n", rd_base64_encode(out)); return 0; }
/** * Verify SSL handshake was valid. */ static int rd_kafka_transport_ssl_verify (rd_kafka_transport_t *rktrans) { long int rl; X509 *cert; cert = SSL_get_peer_certificate(rktrans->rktrans_ssl); X509_free(cert); if (!cert) { rd_kafka_broker_fail(rktrans->rktrans_rkb, LOG_ERR, RD_KAFKA_RESP_ERR__SSL, "Broker did not provide a certificate"); return -1; } if ((rl = SSL_get_verify_result(rktrans->rktrans_ssl)) != X509_V_OK) { rd_kafka_broker_fail(rktrans->rktrans_rkb, LOG_ERR, RD_KAFKA_RESP_ERR__SSL, "Failed to verify broker certificate: %s", X509_verify_cert_error_string(rl)); return -1; } rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SSLVERIFY", "Broker SSL certificate verified"); return 0; }
static RD_UNUSED int rd_kafka_sasl_cb_canon (sasl_conn_t *conn, void *context, const char *in, unsigned inlen, unsigned flags, const char *user_realm, char *out, unsigned out_max, unsigned *out_len) { rd_kafka_transport_t *rktrans = context; if (strstr(rktrans->rktrans_rkb->rkb_rk->rk_conf. sasl.mechanisms, "GSSAPI")) { *out_len = rd_snprintf(out, out_max, "%s", rktrans->rktrans_rkb->rkb_rk-> rk_conf.sasl.principal); } else if (!strcmp(rktrans->rktrans_rkb->rkb_rk->rk_conf. sasl.mechanisms, "PLAIN")) { *out_len = rd_snprintf(out, out_max, "%.*s", inlen, in); } else out = NULL; rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL", "CB_CANON: flags 0x%x, \"%.*s\" @ \"%s\": returning \"%.*s\"", flags, (int)inlen, in, user_realm, (int)(*out_len), out); return out ? SASL_OK : SASL_FAIL; }
/** * TCP connection established. * Set up socket options, SSL, etc. * * Locality: broker thread */ static void rd_kafka_transport_connected (rd_kafka_transport_t *rktrans) { rd_kafka_broker_t *rkb = rktrans->rktrans_rkb; rd_rkb_dbg(rkb, BROKER, "CONNECT", "Connected to %s", rd_sockaddr2str(rkb->rkb_addr_last, RD_SOCKADDR2STR_F_PORT | RD_SOCKADDR2STR_F_FAMILY)); /* Set socket send & receive buffer sizes if configuerd */ if (rkb->rkb_rk->rk_conf.socket_sndbuf_size != 0) { if (setsockopt(rktrans->rktrans_s, SOL_SOCKET, SO_SNDBUF, (void *)&rkb->rkb_rk->rk_conf.socket_sndbuf_size, sizeof(rkb->rkb_rk->rk_conf. socket_sndbuf_size)) == SOCKET_ERROR) rd_rkb_log(rkb, LOG_WARNING, "SNDBUF", "Failed to set socket send " "buffer size to %i: %s", rkb->rkb_rk->rk_conf.socket_sndbuf_size, socket_strerror(socket_errno)); } if (rkb->rkb_rk->rk_conf.socket_rcvbuf_size != 0) { if (setsockopt(rktrans->rktrans_s, SOL_SOCKET, SO_RCVBUF, (void *)&rkb->rkb_rk->rk_conf.socket_rcvbuf_size, sizeof(rkb->rkb_rk->rk_conf. socket_rcvbuf_size)) == SOCKET_ERROR) rd_rkb_log(rkb, LOG_WARNING, "RCVBUF", "Failed to set socket receive " "buffer size to %i: %s", rkb->rkb_rk->rk_conf.socket_rcvbuf_size, socket_strerror(socket_errno)); } #if WITH_SSL if (rkb->rkb_proto == RD_KAFKA_PROTO_SSL || rkb->rkb_proto == RD_KAFKA_PROTO_SASL_SSL) { char errstr[512]; /* Set up SSL connection. * This is also an asynchronous operation so dont * propagate to broker_connect_done() just yet. */ if (rd_kafka_transport_ssl_connect(rkb, rktrans, errstr, sizeof(errstr)) == -1) { rd_kafka_transport_connect_done(rktrans, errstr); return; } return; } #endif /* Propagate connect success */ rd_kafka_transport_connect_done(rktrans, NULL); }
static int rd_kafka_sasl_cb_log (void *context, int level, const char *message){ rd_kafka_transport_t *rktrans = context; if (level >= LOG_DEBUG) rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL", "%s", message); else rd_rkb_log(rktrans->rktrans_rkb, level, "LIBSASL", "%s", message); return SASL_OK; }
static int rd_kafka_sasl_cb_getrealm (void *context, int id, const char **availrealms, const char **result) { rd_kafka_transport_t *rktrans = context; *result = *availrealms; rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL", "CB_GETREALM: id 0x%x: returning %s", id, *result); return SASL_OK; }
/** * @brief the kernel SO_ERROR in \p errp for the given transport. * @returns 0 if getsockopt() was succesful (and \p and errp can be trusted), * else -1 in which case \p errp 's value is undefined. */ static int rd_kafka_transport_get_socket_error (rd_kafka_transport_t *rktrans, int *errp) { socklen_t intlen = sizeof(*errp); if (getsockopt(rktrans->rktrans_s, SOL_SOCKET, SO_ERROR, (void *)errp, &intlen) == -1) { rd_rkb_dbg(rktrans->rktrans_rkb, BROKER, "SO_ERROR", "Failed to get socket error: %s", socket_strerror(socket_errno)); return -1; } return 0; }
static int rd_kafka_sasl_cb_chalprompt (void *context, int id, const char *challenge, const char *prompt, const char *defres, const char **result, unsigned *len) { rd_kafka_transport_t *rktrans = context; *result = "min_chalprompt"; *len = strlen(*result); rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL", "CB_CHALPROMPT: id 0x%x, challenge %s, prompt %s, " "default %s: returning %s", id, challenge, prompt, defres, *result); return SASL_OK; }
/** * * libsasl callbacks * */ static RD_UNUSED int rd_kafka_sasl_cb_getopt (void *context, const char *plugin_name, const char *option, const char **result, unsigned *len) { rd_kafka_transport_t *rktrans = context; if (!strcmp(option, "client_mech_list")) *result = "GSSAPI"; if (!strcmp(option, "canon_user_plugin")) *result = "INTERNAL"; if (*result && len) *len = strlen(*result); rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL", "CB_GETOPT: plugin %s, option %s: returning %s", plugin_name, option, *result); return SASL_OK; }
static int rd_kafka_sasl_cb_getsecret (sasl_conn_t *conn, void *context, int id, sasl_secret_t **psecret) { rd_kafka_transport_t *rktrans = context; const char *password; password = rktrans->rktrans_rkb->rkb_rk->rk_conf.sasl.password; if (!password) { *psecret = NULL; } else { size_t passlen = strlen(password); *psecret = rd_realloc(*psecret, sizeof(**psecret) + passlen); (*psecret)->len = passlen; memcpy((*psecret)->data, password, passlen); } rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL", "CB_GETSECRET: id 0x%x: returning %s", id, *psecret ? "(hidden)":"NULL"); return SASL_OK; }
static int rd_kafka_sasl_cb_getsimple (void *context, int id, const char **result, unsigned *len) { rd_kafka_transport_t *rktrans = context; switch (id) { case SASL_CB_USER: case SASL_CB_AUTHNAME: *result = rktrans->rktrans_rkb->rkb_rk->rk_conf.sasl.username; break; default: *result = NULL; break; } if (len) *len = *result ? strlen(*result) : 0; rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "LIBSASL", "CB_GETSIMPLE: id 0x%x: returning %s", id, *result); return *result ? SASL_OK : SASL_FAIL; }
/** * @brief Build client-final-message * @returns -1 on error. */ static int rd_kafka_sasl_scram_build_client_final_message ( rd_kafka_transport_t *rktrans, const rd_chariov_t *salt, const char *server_nonce, const rd_chariov_t *server_first_msg, int itcnt, rd_chariov_t *out) { struct rd_kafka_sasl_scram_state *state = rktrans->rktrans_sasl.state; const rd_kafka_conf_t *conf = &rktrans->rktrans_rkb->rkb_rk->rk_conf; rd_chariov_t SaslPassword = { .ptr = conf->sasl.password, .size = strlen(conf->sasl.password) }; rd_chariov_t SaltedPassword = { .ptr = rd_alloca(EVP_MAX_MD_SIZE) }; rd_chariov_t ClientKey = { .ptr = rd_alloca(EVP_MAX_MD_SIZE) }; rd_chariov_t ServerKey = { .ptr = rd_alloca(EVP_MAX_MD_SIZE) }; rd_chariov_t StoredKey = { .ptr = rd_alloca(EVP_MAX_MD_SIZE) }; rd_chariov_t AuthMessage = RD_ZERO_INIT; rd_chariov_t ClientSignature = { .ptr = rd_alloca(EVP_MAX_MD_SIZE) }; rd_chariov_t ServerSignature = { .ptr = rd_alloca(EVP_MAX_MD_SIZE) }; const rd_chariov_t ClientKeyVerbatim = { .ptr = "Client Key", .size = 10 }; const rd_chariov_t ServerKeyVerbatim = { .ptr = "Server Key", .size = 10 }; rd_chariov_t ClientProof = { .ptr = rd_alloca(EVP_MAX_MD_SIZE) }; rd_chariov_t client_final_msg_wo_proof; char *ClientProofB64; int i; /* Constructing the ClientProof attribute (p): * * p = Base64-encoded ClientProof * SaltedPassword := Hi(Normalize(password), salt, i) * ClientKey := HMAC(SaltedPassword, "Client Key") * StoredKey := H(ClientKey) * AuthMessage := client-first-message-bare + "," + * server-first-message + "," + * client-final-message-without-proof * ClientSignature := HMAC(StoredKey, AuthMessage) * ClientProof := ClientKey XOR ClientSignature * ServerKey := HMAC(SaltedPassword, "Server Key") * ServerSignature := HMAC(ServerKey, AuthMessage) */ /* SaltedPassword := Hi(Normalize(password), salt, i) */ if (rd_kafka_sasl_scram_Hi( rktrans, &SaslPassword, salt, itcnt, &SaltedPassword) == -1) return -1; /* ClientKey := HMAC(SaltedPassword, "Client Key") */ if (rd_kafka_sasl_scram_HMAC( rktrans, &SaltedPassword, &ClientKeyVerbatim, &ClientKey) == -1) return -1; /* StoredKey := H(ClientKey) */ if (rd_kafka_sasl_scram_H(rktrans, &ClientKey, &StoredKey) == -1) return -1; /* client-final-message-without-proof */ rd_kafka_sasl_scram_build_client_final_message_wo_proof( state, server_nonce, &client_final_msg_wo_proof); /* AuthMessage := client-first-message-bare + "," + * server-first-message + "," + * client-final-message-without-proof */ AuthMessage.size = state->first_msg_bare.size + 1 + server_first_msg->size + 1 + client_final_msg_wo_proof.size; AuthMessage.ptr = rd_alloca(AuthMessage.size+1); rd_snprintf(AuthMessage.ptr, AuthMessage.size+1, "%.*s,%.*s,%.*s", (int)state->first_msg_bare.size, state->first_msg_bare.ptr, (int)server_first_msg->size, server_first_msg->ptr, (int)client_final_msg_wo_proof.size, client_final_msg_wo_proof.ptr); /* * Calculate ServerSignature for later verification when * server-final-message is received. */ /* ServerKey := HMAC(SaltedPassword, "Server Key") */ if (rd_kafka_sasl_scram_HMAC( rktrans, &SaltedPassword, &ServerKeyVerbatim, &ServerKey) == -1) { rd_free(client_final_msg_wo_proof.ptr); return -1; } /* ServerSignature := HMAC(ServerKey, AuthMessage) */ if (rd_kafka_sasl_scram_HMAC(rktrans, &ServerKey, &AuthMessage, &ServerSignature) == -1) { rd_free(client_final_msg_wo_proof.ptr); return -1; } /* Store the Base64 encoded ServerSignature for quick comparison */ state->ServerSignatureB64 = rd_base64_encode(&ServerSignature); /* * Continue with client-final-message */ /* ClientSignature := HMAC(StoredKey, AuthMessage) */ if (rd_kafka_sasl_scram_HMAC(rktrans, &StoredKey, &AuthMessage, &ClientSignature) == -1) { rd_free(client_final_msg_wo_proof.ptr); return -1; } /* ClientProof := ClientKey XOR ClientSignature */ assert(ClientKey.size == ClientSignature.size); for (i = 0 ; i < (int)ClientKey.size ; i++) ClientProof.ptr[i] = ClientKey.ptr[i] ^ ClientSignature.ptr[i]; ClientProof.size = ClientKey.size; /* Base64 encoded ClientProof */ ClientProofB64 = rd_base64_encode(&ClientProof); /* Construct client-final-message */ out->size = client_final_msg_wo_proof.size + strlen(",p=") + strlen(ClientProofB64); out->ptr = rd_malloc(out->size + 1); rd_snprintf(out->ptr, out->size+1, "%.*s,p=%s", (int)client_final_msg_wo_proof.size, client_final_msg_wo_proof.ptr, ClientProofB64); rd_free(ClientProofB64); rd_free(client_final_msg_wo_proof.ptr); return 0; } /** * @brief Handle first message from server * * Parse server response which looks something like: * "r=fyko+d2lbbFgONR....,s=QSXCR+Q6sek8bf92,i=4096" * * @returns -1 on error. */ static int rd_kafka_sasl_scram_handle_server_first_message (rd_kafka_transport_t *rktrans, const rd_chariov_t *in, rd_chariov_t *out, char *errstr, size_t errstr_size) { struct rd_kafka_sasl_scram_state *state = rktrans->rktrans_sasl.state; char *server_nonce; rd_chariov_t salt_b64, salt; char *itcntstr; const char *endptr; int itcnt; char *attr_m; /* Mandatory future extension check */ if ((attr_m = rd_kafka_sasl_scram_get_attr( in, 'm', NULL, NULL, 0))) { rd_snprintf(errstr, errstr_size, "Unsupported mandatory SCRAM extension"); rd_free(attr_m); return -1; } /* Server nonce */ if (!(server_nonce = rd_kafka_sasl_scram_get_attr( in, 'r', "Server nonce in server-first-message", errstr, errstr_size))) return -1; if (strlen(server_nonce) <= state->cnonce.size || strncmp(state->cnonce.ptr, server_nonce, state->cnonce.size)) { rd_snprintf(errstr, errstr_size, "Server/client nonce mismatch in " "server-first-message"); rd_free(server_nonce); return -1; } /* Salt (Base64) */ if (!(salt_b64.ptr = rd_kafka_sasl_scram_get_attr( in, 's', "Salt in server-first-message", errstr, errstr_size))) { rd_free(server_nonce); return -1; } salt_b64.size = strlen(salt_b64.ptr); /* Convert Salt to binary */ if (rd_base64_decode(&salt_b64, &salt) == -1) { rd_snprintf(errstr, errstr_size, "Invalid Base64 Salt in server-first-message"); rd_free(server_nonce); rd_free(salt_b64.ptr); } rd_free(salt_b64.ptr); /* Iteration count (as string) */ if (!(itcntstr = rd_kafka_sasl_scram_get_attr( in, 'i', "Iteration count in server-first-message", errstr, errstr_size))) { rd_free(server_nonce); rd_free(salt.ptr); return -1; } /* Iteration count (as int) */ errno = 0; itcnt = (int)strtoul(itcntstr, (char **)&endptr, 10); if (itcntstr == endptr || *endptr != '\0' || errno != 0 || itcnt > 1000000) { rd_snprintf(errstr, errstr_size, "Invalid value (not integer or too large) " "for Iteration count in server-first-message"); rd_free(server_nonce); rd_free(salt.ptr); rd_free(itcntstr); return -1; } rd_free(itcntstr); /* Build client-final-message */ if (rd_kafka_sasl_scram_build_client_final_message( rktrans, &salt, server_nonce, in, itcnt, out) == -1) { rd_snprintf(errstr, errstr_size, "Failed to build SCRAM client-final-message"); rd_free(salt.ptr); rd_free(server_nonce); return -1; } rd_free(server_nonce); rd_free(salt.ptr); return 0; } /** * @brief Handle server-final-message * * This is the end of authentication and the SCRAM state * will be freed at the end of this function regardless of * authentication outcome. * * @returns -1 on failure */ static int rd_kafka_sasl_scram_handle_server_final_message ( rd_kafka_transport_t *rktrans, const rd_chariov_t *in, char *errstr, size_t errstr_size) { struct rd_kafka_sasl_scram_state *state = rktrans->rktrans_sasl.state; char *attr_v, *attr_e; if ((attr_e = rd_kafka_sasl_scram_get_attr( in, 'e', "server-error in server-final-message", errstr, errstr_size))) { /* Authentication failed */ rd_snprintf(errstr, errstr_size, "SASL SCRAM authentication failed: " "broker responded with %s", attr_e); rd_free(attr_e); return -1; } else if ((attr_v = rd_kafka_sasl_scram_get_attr( in, 'v', "verifier in server-final-message", errstr, errstr_size))) { const rd_kafka_conf_t *conf; /* Authentication succesful on server, * but we need to verify the ServerSignature too. */ rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY | RD_KAFKA_DBG_BROKER, "SCRAMAUTH", "SASL SCRAM authentication succesful on server: " "verifying ServerSignature"); if (strcmp(attr_v, state->ServerSignatureB64)) { rd_snprintf(errstr, errstr_size, "SASL SCRAM authentication failed: " "ServerSignature mismatch " "(server's %s != ours %s)", attr_v, state->ServerSignatureB64); rd_free(attr_v); return -1; } rd_free(attr_v); conf = &rktrans->rktrans_rkb->rkb_rk->rk_conf; rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY | RD_KAFKA_DBG_BROKER, "SCRAMAUTH", "Authenticated as %s using %s", conf->sasl.username, conf->sasl.mechanisms); rd_kafka_sasl_auth_done(rktrans); return 0; } else { rd_snprintf(errstr, errstr_size, "SASL SCRAM authentication failed: " "no verifier or server-error returned from broker"); return -1; } } /** * @brief Build client-first-message */ static void rd_kafka_sasl_scram_build_client_first_message ( rd_kafka_transport_t *rktrans, rd_chariov_t *out) { char *sasl_username; struct rd_kafka_sasl_scram_state *state = rktrans->rktrans_sasl.state; const rd_kafka_conf_t *conf = &rktrans->rktrans_rkb->rkb_rk->rk_conf; rd_kafka_sasl_scram_generate_nonce(&state->cnonce); sasl_username = rd_kafka_sasl_safe_string(conf->sasl.username); out->size = strlen("n,,n=,r=") + strlen(sasl_username) + state->cnonce.size; out->ptr = rd_malloc(out->size+1); rd_snprintf(out->ptr, out->size+1, "n,,n=%s,r=%.*s", sasl_username, (int)state->cnonce.size, state->cnonce.ptr); rd_free(sasl_username); /* Save client-first-message-bare (skip gs2-header) */ state->first_msg_bare.size = out->size-3; state->first_msg_bare.ptr = rd_memdup(out->ptr+3, state->first_msg_bare.size); } /** * @brief SASL SCRAM client state machine * @returns -1 on failure (errstr set), else 0. */ static int rd_kafka_sasl_scram_fsm (rd_kafka_transport_t *rktrans, const rd_chariov_t *in, char *errstr, size_t errstr_size) { static const char *state_names[] = { "client-first-message", "server-first-message", "client-final-message", }; struct rd_kafka_sasl_scram_state *state = rktrans->rktrans_sasl.state; rd_chariov_t out = RD_ZERO_INIT; int r = -1; rd_ts_t ts_start = rd_clock(); int prev_state = state->state; rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASLSCRAM", "SASL SCRAM client in state %s", state_names[state->state]); switch (state->state) { case RD_KAFKA_SASL_SCRAM_STATE_CLIENT_FIRST_MESSAGE: rd_dassert(!in); /* Not expecting any server-input */ rd_kafka_sasl_scram_build_client_first_message(rktrans, &out); state->state = RD_KAFKA_SASL_SCRAM_STATE_SERVER_FIRST_MESSAGE; break; case RD_KAFKA_SASL_SCRAM_STATE_SERVER_FIRST_MESSAGE: rd_dassert(in); /* Requires server-input */ if (rd_kafka_sasl_scram_handle_server_first_message( rktrans, in, &out, errstr, errstr_size) == -1) return -1; state->state = RD_KAFKA_SASL_SCRAM_STATE_CLIENT_FINAL_MESSAGE; break; case RD_KAFKA_SASL_SCRAM_STATE_CLIENT_FINAL_MESSAGE: rd_dassert(in); /* Requires server-input */ r = rd_kafka_sasl_scram_handle_server_final_message( rktrans, in, errstr, errstr_size); break; } if (out.ptr) { r = rd_kafka_sasl_send(rktrans, out.ptr, (int)out.size, errstr, errstr_size); rd_free(out.ptr); } ts_start = (rd_clock() - ts_start) / 1000; if (ts_start >= 100) rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SCRAM", "SASL SCRAM state %s handled in %"PRId64"ms", state_names[prev_state], ts_start); return r; } /** * @brief Handle received frame from broker. */ static int rd_kafka_sasl_scram_recv (rd_kafka_transport_t *rktrans, const void *buf, size_t size, char *errstr, size_t errstr_size) { const rd_chariov_t in = { .ptr = (char *)buf, .size = size }; return rd_kafka_sasl_scram_fsm(rktrans, &in, errstr, errstr_size); } /** * @brief Initialize and start SASL SCRAM (builtin) authentication. * * Returns 0 on successful init and -1 on error. * * @locality broker thread */ static int rd_kafka_sasl_scram_client_new (rd_kafka_transport_t *rktrans, const char *hostname, char *errstr, size_t errstr_size) { struct rd_kafka_sasl_scram_state *state; state = rd_calloc(1, sizeof(*state)); state->state = RD_KAFKA_SASL_SCRAM_STATE_CLIENT_FIRST_MESSAGE; rktrans->rktrans_sasl.state = state; /* Kick off the FSM */ return rd_kafka_sasl_scram_fsm(rktrans, NULL, errstr, errstr_size); } /** * @brief Validate SCRAM config and look up the hash function */ static int rd_kafka_sasl_scram_conf_validate (rd_kafka_t *rk, char *errstr, size_t errstr_size) { const char *mech = rk->rk_conf.sasl.mechanisms; if (!rk->rk_conf.sasl.username || !rk->rk_conf.sasl.password) { rd_snprintf(errstr, errstr_size, "sasl.username and sasl.password must be set"); return -1; } if (!strcmp(mech, "SCRAM-SHA-1")) { rk->rk_conf.sasl.scram_evp = EVP_sha1(); rk->rk_conf.sasl.scram_H = SHA1; rk->rk_conf.sasl.scram_H_size = SHA_DIGEST_LENGTH; } else if (!strcmp(mech, "SCRAM-SHA-256")) { rk->rk_conf.sasl.scram_evp = EVP_sha256(); rk->rk_conf.sasl.scram_H = SHA256; rk->rk_conf.sasl.scram_H_size = SHA256_DIGEST_LENGTH; } else if (!strcmp(mech, "SCRAM-SHA-512")) { rk->rk_conf.sasl.scram_evp = EVP_sha512(); rk->rk_conf.sasl.scram_H = SHA512; rk->rk_conf.sasl.scram_H_size = SHA512_DIGEST_LENGTH; } else { rd_snprintf(errstr, errstr_size, "Unsupported hash function: %s " "(try SCRAM-SHA-512)", mech); return -1; } return 0; } const struct rd_kafka_sasl_provider rd_kafka_sasl_scram_provider = { .name = "SCRAM (builtin)", .client_new = rd_kafka_sasl_scram_client_new, .recv = rd_kafka_sasl_scram_recv, .close = rd_kafka_sasl_scram_close, .conf_validate = rd_kafka_sasl_scram_conf_validate, };
/** * @brief Decompress MessageSet, pass the uncompressed MessageSet to * the MessageSet reader. */ static rd_kafka_resp_err_t rd_kafka_msgset_reader_decompress (rd_kafka_msgset_reader_t *msetr, int MsgVersion, int Attributes, int64_t Timestamp, int64_t Offset, const void *compressed, size_t compressed_size) { struct iovec iov = { .iov_base = NULL, .iov_len = 0 }; rd_kafka_toppar_t *rktp = msetr->msetr_rktp; int codec = Attributes & RD_KAFKA_MSG_ATTR_COMPRESSION_MASK; rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR; rd_kafka_buf_t *rkbufz; switch (codec) { #if WITH_ZLIB case RD_KAFKA_COMPRESSION_GZIP: { uint64_t outlenx = 0; /* Decompress Message payload */ iov.iov_base = rd_gz_decompress(compressed, (int)compressed_size, &outlenx); if (unlikely(!iov.iov_base)) { rd_rkb_dbg(msetr->msetr_rkb, MSG, "GZIP", "Failed to decompress Gzip " "message at offset %"PRId64 " of %"PRIusz" bytes: " "ignoring message", Offset, compressed_size); err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION; goto err; } iov.iov_len = (size_t)outlenx; } break; #endif #if WITH_SNAPPY case RD_KAFKA_COMPRESSION_SNAPPY: { const char *inbuf = compressed; size_t inlen = compressed_size; int r; static const unsigned char snappy_java_magic[] = { 0x82, 'S','N','A','P','P','Y', 0 }; static const size_t snappy_java_hdrlen = 8+4+4; /* snappy-java adds its own header (SnappyCodec) * which is not compatible with the official Snappy * implementation. * 8: magic, 4: version, 4: compatible * followed by any number of chunks: * 4: length * ...: snappy-compressed data. */ if (likely(inlen > snappy_java_hdrlen + 4 && !memcmp(inbuf, snappy_java_magic, 8))) { /* snappy-java framing */ char errstr[128]; inbuf = inbuf + snappy_java_hdrlen; inlen -= snappy_java_hdrlen; iov.iov_base = rd_kafka_snappy_java_uncompress( inbuf, inlen, &iov.iov_len, errstr, sizeof(errstr)); if (unlikely(!iov.iov_base)) { rd_rkb_dbg(msetr->msetr_rkb, MSG, "SNAPPY", "%s [%"PRId32"]: " "Snappy decompression for message " "at offset %"PRId64" failed: %s: " "ignoring message", rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, Offset, errstr); err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION; goto err; } } else { /* No framing */ /* Acquire uncompressed length */ if (unlikely(!rd_kafka_snappy_uncompressed_length( inbuf, inlen, &iov.iov_len))) { rd_rkb_dbg(msetr->msetr_rkb, MSG, "SNAPPY", "Failed to get length of Snappy " "compressed payload " "for message at offset %"PRId64 " (%"PRIusz" bytes): " "ignoring message", Offset, inlen); err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION; goto err; } /* Allocate output buffer for uncompressed data */ iov.iov_base = rd_malloc(iov.iov_len); if (unlikely(!iov.iov_base)) { rd_rkb_dbg(msetr->msetr_rkb, MSG, "SNAPPY", "Failed to allocate Snappy " "decompress buffer of size %"PRIusz "for message at offset %"PRId64 " (%"PRIusz" bytes): %s: " "ignoring message", iov.iov_len, Offset, inlen, rd_strerror(errno)); err = RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE; goto err; } /* Uncompress to outbuf */ if (unlikely((r = rd_kafka_snappy_uncompress( inbuf, inlen, iov.iov_base)))) { rd_rkb_dbg(msetr->msetr_rkb, MSG, "SNAPPY", "Failed to decompress Snappy " "payload for message at offset " "%"PRId64" (%"PRIusz" bytes): %s: " "ignoring message", Offset, inlen, rd_strerror(-r/*negative errno*/)); rd_free(iov.iov_base); err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION; goto err; } } } break; #endif case RD_KAFKA_COMPRESSION_LZ4: { err = rd_kafka_lz4_decompress(msetr->msetr_rkb, /* Proper HC? */ MsgVersion >= 1 ? 1 : 0, Offset, /* @warning Will modify compressed * if no proper HC */ (char *)compressed, compressed_size, &iov.iov_base, &iov.iov_len); if (err) goto err; } break; default: rd_rkb_dbg(msetr->msetr_rkb, MSG, "CODEC", "%s [%"PRId32"]: Message at offset %"PRId64 " with unsupported " "compression codec 0x%x: message ignored", rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, Offset, (int)codec); err = RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED; goto err; } rd_assert(iov.iov_base); /* * Decompression successful */ /* Create a new buffer pointing to the uncompressed * allocated buffer (outbuf) and let messages keep a reference to * this new buffer. */ rkbufz = rd_kafka_buf_new_shadow(iov.iov_base, iov.iov_len, rd_free); rkbufz->rkbuf_rkb = msetr->msetr_rkbuf->rkbuf_rkb; rd_kafka_broker_keep(rkbufz->rkbuf_rkb); /* In MsgVersion v0..1 the decompressed data contains * an inner MessageSet, pass it to a new MessageSet reader. * * For MsgVersion v2 the decompressed data are the list of messages. */ if (MsgVersion <= 1) { /* Pass decompressed data (inner Messageset) * to new instance of the MessageSet parser. */ rd_kafka_msgset_reader_t inner_msetr; rd_kafka_msgset_reader_init(&inner_msetr, rkbufz, msetr->msetr_rktp, msetr->msetr_tver, &msetr->msetr_rkq); if (MsgVersion == 1) { /* postproc() will convert relative to * absolute offsets */ inner_msetr.msetr_relative_offsets = 1; inner_msetr.msetr_outer.offset = Offset; /* Apply single LogAppendTime timestamp for * all messages. */ if (Attributes & RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME) { inner_msetr.msetr_outer.tstype = RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME; inner_msetr.msetr_outer.timestamp = Timestamp; } } /* Parse the inner MessageSet */ err = rd_kafka_msgset_reader_run(&inner_msetr); } else { /* MsgVersion 2 */ rd_kafka_buf_t *orig_rkbuf = msetr->msetr_rkbuf; /* Temporarily replace read buffer with uncompressed buffer */ msetr->msetr_rkbuf = rkbufz; /* Read messages */ err = rd_kafka_msgset_reader_msgs_v2(msetr); /* Restore original buffer */ msetr->msetr_rkbuf = orig_rkbuf; } /* Loose our refcnt of the uncompressed rkbuf. * Individual messages/rko's will have their own reference. */ rd_kafka_buf_destroy(rkbufz); return err; err: /* Enqueue error messsage: * Create op and push on temporary queue. */ rd_kafka_q_op_err(&msetr->msetr_rkq, RD_KAFKA_OP_CONSUMER_ERR, err, msetr->msetr_tver->version, rktp, Offset, "Decompression (codec 0x%x) of message at %"PRIu64 " of %"PRIu64" bytes failed: %s", codec, Offset, compressed_size, rd_kafka_err2str(err)); return err; } /** * @brief Message parser for MsgVersion v0..1 * * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success or on single-message errors, * or any other error code when the MessageSet parser should stop * parsing (such as for partial Messages). */ static rd_kafka_resp_err_t rd_kafka_msgset_reader_msg_v0_1 (rd_kafka_msgset_reader_t *msetr) { rd_kafka_buf_t *rkbuf = msetr->msetr_rkbuf; rd_kafka_toppar_t *rktp = msetr->msetr_rktp; rd_kafka_broker_t *rkb = msetr->msetr_rkb; struct { int64_t Offset; /* MessageSet header */ int32_t MessageSize; /* MessageSet header */ uint32_t Crc; int8_t MagicByte; /* MsgVersion */ int8_t Attributes; int64_t Timestamp; /* v1 */ } hdr; /* Message header */ rd_kafkap_bytes_t Key; rd_kafkap_bytes_t Value; int32_t Value_len; rd_kafka_op_t *rko; size_t hdrsize = 6; /* Header size following MessageSize */ rd_slice_t crc_slice; rd_kafka_msg_t *rkm; int relative_offsets = 0; const char *reloff_str = ""; /* Only log decoding errors if protocol debugging enabled. */ int log_decode_errors = (rkbuf->rkbuf_rkb->rkb_rk->rk_conf.debug & RD_KAFKA_DBG_PROTOCOL) ? LOG_DEBUG : 0; size_t message_end; rd_kafka_buf_read_i64(rkbuf, &hdr.Offset); rd_kafka_buf_read_i32(rkbuf, &hdr.MessageSize); message_end = rd_slice_offset(&rkbuf->rkbuf_reader) + hdr.MessageSize; rd_kafka_buf_read_i32(rkbuf, &hdr.Crc); if (!rd_slice_narrow_copy_relative(&rkbuf->rkbuf_reader, &crc_slice, hdr.MessageSize - 4)) rd_kafka_buf_check_len(rkbuf, hdr.MessageSize - 4); rd_kafka_buf_read_i8(rkbuf, &hdr.MagicByte); rd_kafka_buf_read_i8(rkbuf, &hdr.Attributes); if (hdr.MagicByte == 1) { /* MsgVersion */ rd_kafka_buf_read_i64(rkbuf, &hdr.Timestamp); hdrsize += 8; /* MsgVersion 1 has relative offsets for compressed MessageSets*/ if (!(hdr.Attributes & RD_KAFKA_MSG_ATTR_COMPRESSION_MASK) && msetr->msetr_relative_offsets) { relative_offsets = 1; reloff_str = "relative "; } } else hdr.Timestamp = 0; /* Verify MessageSize */ if (unlikely(hdr.MessageSize < (ssize_t)hdrsize)) rd_kafka_buf_parse_fail(rkbuf, "Message at %soffset %"PRId64 " MessageSize %"PRId32 " < hdrsize %"PRIusz, reloff_str, hdr.Offset, hdr.MessageSize, hdrsize); /* Early check for partial messages */ rd_kafka_buf_check_len(rkbuf, hdr.MessageSize - hdrsize); if (rkb->rkb_rk->rk_conf.check_crcs) { /* Verify CRC32 if desired. */ uint32_t calc_crc; calc_crc = rd_slice_crc32(&crc_slice); rd_dassert(rd_slice_remains(&crc_slice) == 0); if (unlikely(hdr.Crc != calc_crc)) { /* Propagate CRC error to application and * continue with next message. */ rd_kafka_q_op_err(&msetr->msetr_rkq, RD_KAFKA_OP_CONSUMER_ERR, RD_KAFKA_RESP_ERR__BAD_MSG, msetr->msetr_tver->version, rktp, hdr.Offset, "Message at %soffset %"PRId64 " (%"PRId32" bytes) " "failed CRC32 check " "(original 0x%"PRIx32" != " "calculated 0x%"PRIx32")", reloff_str, hdr.Offset, hdr.MessageSize, hdr.Crc, calc_crc); rd_kafka_buf_skip_to(rkbuf, message_end); rd_atomic64_add(&rkb->rkb_c.rx_err, 1); /* Continue with next message */ return RD_KAFKA_RESP_ERR_NO_ERROR; } } /* Extract key */ rd_kafka_buf_read_bytes(rkbuf, &Key); /* Extract Value */ rd_kafka_buf_read_bytes(rkbuf, &Value); Value_len = RD_KAFKAP_BYTES_LEN(&Value); /* MessageSets may contain offsets earlier than we * requested (compressed MessageSets in particular), * drop the earlier messages. * Note: the inner offset may only be trusted for * absolute offsets. KIP-31 introduced * ApiVersion 2 that maintains relative offsets * of compressed messages and the base offset * in the outer message is the offset of * the *LAST* message in the MessageSet. * This requires us to assign offsets * after all messages have been read from * the messageset, and it also means * we cant perform this offset check here * in that case. */ if (!relative_offsets && hdr.Offset < rktp->rktp_offsets.fetch_offset) return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue with next msg */ /* Handle compressed MessageSet */ if (unlikely(hdr.Attributes & RD_KAFKA_MSG_ATTR_COMPRESSION_MASK)) return rd_kafka_msgset_reader_decompress( msetr, hdr.MagicByte, hdr.Attributes, hdr.Timestamp, hdr.Offset, Value.data, Value_len); /* Pure uncompressed message, this is the innermost * handler after all compression and cascaded * MessageSets have been peeled off. */ /* Create op/message container for message. */ rko = rd_kafka_op_new_fetch_msg(&rkm, rktp, msetr->msetr_tver->version, rkbuf, hdr.Offset, (size_t)RD_KAFKAP_BYTES_LEN(&Key), RD_KAFKAP_BYTES_IS_NULL(&Key) ? NULL : Key.data, (size_t)RD_KAFKAP_BYTES_LEN(&Value), RD_KAFKAP_BYTES_IS_NULL(&Value) ? NULL : Value.data); /* Assign message timestamp. * If message was in a compressed MessageSet and the outer/wrapper * Message.Attribute had a LOG_APPEND_TIME set, use the * outer timestamp */ if (msetr->msetr_outer.tstype == RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME) { rkm->rkm_timestamp = msetr->msetr_outer.timestamp; rkm->rkm_tstype = msetr->msetr_outer.tstype; } else if (hdr.MagicByte >= 1 && hdr.Timestamp) { rkm->rkm_timestamp = hdr.Timestamp; if (hdr.Attributes & RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME) rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME; else rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_CREATE_TIME; } /* Enqueue message on temporary queue */ rd_kafka_q_enq(&msetr->msetr_rkq, rko); msetr->msetr_msgcnt++; return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue */ err_parse: /* Count all parse errors as partial message errors. */ rd_atomic64_add(&msetr->msetr_rkb->rkb_c.rx_partial, 1); return rkbuf->rkbuf_err; }
/** * @brief Message parser for MsgVersion v2 */ static rd_kafka_resp_err_t rd_kafka_msgset_reader_msg_v2 (rd_kafka_msgset_reader_t *msetr) { rd_kafka_buf_t *rkbuf = msetr->msetr_rkbuf; rd_kafka_toppar_t *rktp = msetr->msetr_rktp; struct { int64_t Length; int64_t MsgAttributes; /* int8_t, but int64 req. for varint */ int64_t TimestampDelta; int64_t OffsetDelta; int64_t Offset; /* Absolute offset */ rd_kafkap_bytes_t Key; rd_kafkap_bytes_t Value; int64_t HeaderCnt; } hdr; rd_kafka_op_t *rko; rd_kafka_msg_t *rkm; /* Only log decoding errors if protocol debugging enabled. */ int log_decode_errors = (rkbuf->rkbuf_rkb->rkb_rk->rk_conf.debug & RD_KAFKA_DBG_PROTOCOL) ? LOG_DEBUG : 0; size_t message_end; rd_kafka_buf_read_varint(rkbuf, &hdr.Length); message_end = rd_slice_offset(&rkbuf->rkbuf_reader)+(size_t)hdr.Length; rd_kafka_buf_read_varint(rkbuf, &hdr.MsgAttributes); rd_kafka_buf_read_varint(rkbuf, &hdr.TimestampDelta); rd_kafka_buf_read_varint(rkbuf, &hdr.OffsetDelta); hdr.Offset = msetr->msetr_v2_hdr->BaseOffset + hdr.OffsetDelta; /* Skip message if outdated */ if (hdr.Offset < rktp->rktp_offsets.fetch_offset) { rd_rkb_dbg(msetr->msetr_rkb, MSG, "MSG", "Skip offset %"PRId64" < fetch_offset %"PRId64, hdr.Offset, rktp->rktp_offsets.fetch_offset); rd_kafka_buf_skip_to(rkbuf, message_end); return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue with next msg */ } rd_kafka_buf_read_bytes_varint(rkbuf, &hdr.Key); rd_kafka_buf_read_bytes_varint(rkbuf, &hdr.Value); /* Ignore headers for now */ rd_kafka_buf_skip_to(rkbuf, message_end); /* Create op/message container for message. */ rko = rd_kafka_op_new_fetch_msg(&rkm, rktp, msetr->msetr_tver->version, rkbuf, hdr.Offset, (size_t)RD_KAFKAP_BYTES_LEN(&hdr.Key), RD_KAFKAP_BYTES_IS_NULL(&hdr.Key) ? NULL : hdr.Key.data, (size_t)RD_KAFKAP_BYTES_LEN(&hdr.Value), RD_KAFKAP_BYTES_IS_NULL(&hdr.Value) ? NULL : hdr.Value.data); /* Set timestamp. * * When broker assigns the timestamps (LOG_APPEND_TIME) it will * assign the same timestamp for all messages in a MessageSet * using MaxTimestamp. */ if ((msetr->msetr_v2_hdr->Attributes & RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME) || (hdr.MsgAttributes & RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME)) { rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME; rkm->rkm_timestamp = msetr->msetr_v2_hdr->MaxTimestamp; } else { rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_CREATE_TIME; rkm->rkm_timestamp = msetr->msetr_v2_hdr->BaseTimestamp + hdr.TimestampDelta; } /* Enqueue message on temporary queue */ rd_kafka_q_enq(&msetr->msetr_rkq, rko); msetr->msetr_msgcnt++; return RD_KAFKA_RESP_ERR_NO_ERROR; err_parse: /* Count all parse errors as partial message errors. */ rd_atomic64_add(&msetr->msetr_rkb->rkb_c.rx_partial, 1); return rkbuf->rkbuf_err; }
/** * Initiate asynchronous connection attempt. * * Locality: broker thread */ rd_kafka_transport_t *rd_kafka_transport_connect (rd_kafka_broker_t *rkb, const rd_sockaddr_inx_t *sinx, char *errstr, int errstr_size) { rd_kafka_transport_t *rktrans; int s = -1; int on = 1; rkb->rkb_addr_last = sinx; s = rkb->rkb_rk->rk_conf.socket_cb(sinx->in.sin_family, SOCK_STREAM, IPPROTO_TCP, rkb->rkb_rk->rk_conf.opaque); if (s == -1) { rd_snprintf(errstr, errstr_size, "Failed to create socket: %s", socket_strerror(socket_errno)); return NULL; } #ifdef SO_NOSIGPIPE /* Disable SIGPIPE signalling for this socket on OSX */ if (setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, &on, sizeof(on)) == -1) rd_rkb_dbg(rkb, BROKER, "SOCKET", "Failed to set SO_NOSIGPIPE: %s", socket_strerror(socket_errno)); #endif /* Enable TCP keep-alives, if configured. */ if (rkb->rkb_rk->rk_conf.socket_keepalive) { #ifdef SO_KEEPALIVE if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, (void *)&on, sizeof(on)) == SOCKET_ERROR) rd_rkb_dbg(rkb, BROKER, "SOCKET", "Failed to set SO_KEEPALIVE: %s", socket_strerror(socket_errno)); #else rd_rkb_dbg(rkb, BROKER, "SOCKET", "System does not support " "socket.keepalive.enable (SO_KEEPALIVE)"); #endif } /* Set the socket to non-blocking */ #ifdef _MSC_VER if (ioctlsocket(s, FIONBIO, &on) == SOCKET_ERROR) { rd_snprintf(errstr, errstr_size, "Failed to set socket non-blocking: %s", socket_strerror(socket_errno)); goto err; } #else { int fl = fcntl(s, F_GETFL, 0); if (fl == -1 || fcntl(s, F_SETFL, fl | O_NONBLOCK) == -1) { rd_snprintf(errstr, errstr_size, "Failed to set socket non-blocking: %s", socket_strerror(socket_errno)); goto err; } } #endif rd_rkb_dbg(rkb, BROKER, "CONNECT", "Connecting to %s (%s) " "with socket %i", rd_sockaddr2str(sinx, RD_SOCKADDR2STR_F_FAMILY | RD_SOCKADDR2STR_F_PORT), rd_kafka_secproto_names[rkb->rkb_proto], s); /* Connect to broker */ if (connect(s, (struct sockaddr *)sinx, RD_SOCKADDR_INX_LEN(sinx)) == SOCKET_ERROR && (socket_errno != EINPROGRESS #ifdef _MSC_VER && socket_errno != WSAEWOULDBLOCK #endif )) { rd_rkb_dbg(rkb, BROKER, "CONNECT", "couldn't connect to %s: %s (%i)", rd_sockaddr2str(sinx, RD_SOCKADDR2STR_F_PORT | RD_SOCKADDR2STR_F_FAMILY), socket_strerror(socket_errno), socket_errno); rd_snprintf(errstr, errstr_size, "Failed to connect to broker at %s: %s", rd_sockaddr2str(sinx, RD_SOCKADDR2STR_F_NICE), socket_strerror(socket_errno)); goto err; } /* Create transport handle */ rktrans = rd_calloc(1, sizeof(*rktrans)); rktrans->rktrans_rkb = rkb; rktrans->rktrans_s = s; rktrans->rktrans_pfd.fd = s; /* Poll writability to trigger on connection success/failure. */ rd_kafka_transport_poll_set(rktrans, POLLOUT); return rktrans; err: if (s != -1) { #ifndef _MSC_VER close(s); #else closesocket(s); #endif } return NULL; }
/** * @brief Handle a Metadata response message. * * @param topics are the requested topics (may be NULL) * * The metadata will be marshalled into 'struct rd_kafka_metadata*' structs. * * The marshalled metadata is returned in \p *mdp, (NULL on error). * @returns an error code on parse failure, else NO_ERRRO. * * @locality rdkafka main thread */ rd_kafka_resp_err_t rd_kafka_parse_Metadata (rd_kafka_broker_t *rkb, rd_kafka_buf_t *request, rd_kafka_buf_t *rkbuf, struct rd_kafka_metadata **mdp) { rd_kafka_t *rk = rkb->rkb_rk; int i, j, k; rd_tmpabuf_t tbuf; struct rd_kafka_metadata *md; size_t rkb_namelen; const int log_decode_errors = LOG_ERR; rd_list_t *missing_topics = NULL; const rd_list_t *requested_topics = request->rkbuf_u.Metadata.topics; int all_topics = request->rkbuf_u.Metadata.all_topics; const char *reason = request->rkbuf_u.Metadata.reason ? request->rkbuf_u.Metadata.reason : "(no reason)"; int ApiVersion = request->rkbuf_reqhdr.ApiVersion; rd_kafkap_str_t cluster_id = RD_ZERO_INIT; int32_t controller_id = -1; rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR; int broadcast_changes = 0; rd_kafka_assert(NULL, thrd_is_current(rk->rk_thread)); /* Remove topics from missing_topics as they are seen in Metadata. */ if (requested_topics) missing_topics = rd_list_copy(requested_topics, rd_list_string_copy, NULL); rd_kafka_broker_lock(rkb); rkb_namelen = strlen(rkb->rkb_name)+1; /* We assume that the marshalled representation is * no more than 4 times larger than the wire representation. */ rd_tmpabuf_new(&tbuf, sizeof(*md) + rkb_namelen + (rkbuf->rkbuf_totlen * 4), 0/*dont assert on fail*/); if (!(md = rd_tmpabuf_alloc(&tbuf, sizeof(*md)))) { err = RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE; goto err; } md->orig_broker_id = rkb->rkb_nodeid; md->orig_broker_name = rd_tmpabuf_write(&tbuf, rkb->rkb_name, rkb_namelen); rd_kafka_broker_unlock(rkb); /* Read Brokers */ rd_kafka_buf_read_i32a(rkbuf, md->broker_cnt); if (md->broker_cnt > RD_KAFKAP_BROKERS_MAX) rd_kafka_buf_parse_fail(rkbuf, "Broker_cnt %i > BROKERS_MAX %i", md->broker_cnt, RD_KAFKAP_BROKERS_MAX); if (!(md->brokers = rd_tmpabuf_alloc(&tbuf, md->broker_cnt * sizeof(*md->brokers)))) rd_kafka_buf_parse_fail(rkbuf, "%d brokers: tmpabuf memory shortage", md->broker_cnt); for (i = 0 ; i < md->broker_cnt ; i++) { rd_kafka_buf_read_i32a(rkbuf, md->brokers[i].id); rd_kafka_buf_read_str_tmpabuf(rkbuf, &tbuf, md->brokers[i].host); rd_kafka_buf_read_i32a(rkbuf, md->brokers[i].port); if (ApiVersion >= 1) { rd_kafkap_str_t rack; rd_kafka_buf_read_str(rkbuf, &rack); } } if (ApiVersion >= 2) rd_kafka_buf_read_str(rkbuf, &cluster_id); if (ApiVersion >= 1) { rd_kafka_buf_read_i32(rkbuf, &controller_id); rd_rkb_dbg(rkb, METADATA, "METADATA", "ClusterId: %.*s, ControllerId: %"PRId32, RD_KAFKAP_STR_PR(&cluster_id), controller_id); } /* Read TopicMetadata */ rd_kafka_buf_read_i32a(rkbuf, md->topic_cnt); rd_rkb_dbg(rkb, METADATA, "METADATA", "%i brokers, %i topics", md->broker_cnt, md->topic_cnt); if (md->topic_cnt > RD_KAFKAP_TOPICS_MAX) rd_kafka_buf_parse_fail(rkbuf, "TopicMetadata_cnt %"PRId32 " > TOPICS_MAX %i", md->topic_cnt, RD_KAFKAP_TOPICS_MAX); if (!(md->topics = rd_tmpabuf_alloc(&tbuf, md->topic_cnt * sizeof(*md->topics)))) rd_kafka_buf_parse_fail(rkbuf, "%d topics: tmpabuf memory shortage", md->topic_cnt); for (i = 0 ; i < md->topic_cnt ; i++) { rd_kafka_buf_read_i16a(rkbuf, md->topics[i].err); rd_kafka_buf_read_str_tmpabuf(rkbuf, &tbuf, md->topics[i].topic); if (ApiVersion >= 1) { int8_t is_internal; rd_kafka_buf_read_i8(rkbuf, &is_internal); } /* PartitionMetadata */ rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partition_cnt); if (md->topics[i].partition_cnt > RD_KAFKAP_PARTITIONS_MAX) rd_kafka_buf_parse_fail(rkbuf, "TopicMetadata[%i]." "PartitionMetadata_cnt %i " "> PARTITIONS_MAX %i", i, md->topics[i].partition_cnt, RD_KAFKAP_PARTITIONS_MAX); if (!(md->topics[i].partitions = rd_tmpabuf_alloc(&tbuf, md->topics[i].partition_cnt * sizeof(*md->topics[i].partitions)))) rd_kafka_buf_parse_fail(rkbuf, "%s: %d partitions: " "tmpabuf memory shortage", md->topics[i].topic, md->topics[i].partition_cnt); for (j = 0 ; j < md->topics[i].partition_cnt ; j++) { rd_kafka_buf_read_i16a(rkbuf, md->topics[i].partitions[j].err); rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].id); rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].leader); /* Replicas */ rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].replica_cnt); if (md->topics[i].partitions[j].replica_cnt > RD_KAFKAP_BROKERS_MAX) rd_kafka_buf_parse_fail(rkbuf, "TopicMetadata[%i]." "PartitionMetadata[%i]." "Replica_cnt " "%i > BROKERS_MAX %i", i, j, md->topics[i]. partitions[j]. replica_cnt, RD_KAFKAP_BROKERS_MAX); if (!(md->topics[i].partitions[j].replicas = rd_tmpabuf_alloc(&tbuf, md->topics[i]. partitions[j].replica_cnt * sizeof(*md->topics[i]. partitions[j].replicas)))) rd_kafka_buf_parse_fail( rkbuf, "%s [%"PRId32"]: %d replicas: " "tmpabuf memory shortage", md->topics[i].topic, md->topics[i].partitions[j].id, md->topics[i].partitions[j].replica_cnt); for (k = 0 ; k < md->topics[i].partitions[j].replica_cnt; k++) rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j]. replicas[k]); /* Isrs */ rd_kafka_buf_read_i32a(rkbuf, md->topics[i].partitions[j].isr_cnt); if (md->topics[i].partitions[j].isr_cnt > RD_KAFKAP_BROKERS_MAX) rd_kafka_buf_parse_fail(rkbuf, "TopicMetadata[%i]." "PartitionMetadata[%i]." "Isr_cnt " "%i > BROKERS_MAX %i", i, j, md->topics[i]. partitions[j].isr_cnt, RD_KAFKAP_BROKERS_MAX); if (!(md->topics[i].partitions[j].isrs = rd_tmpabuf_alloc(&tbuf, md->topics[i]. partitions[j].isr_cnt * sizeof(*md->topics[i]. partitions[j].isrs)))) rd_kafka_buf_parse_fail( rkbuf, "%s [%"PRId32"]: %d isrs: " "tmpabuf memory shortage", md->topics[i].topic, md->topics[i].partitions[j].id, md->topics[i].partitions[j].isr_cnt); for (k = 0 ; k < md->topics[i].partitions[j].isr_cnt; k++) rd_kafka_buf_read_i32a(rkbuf, md->topics[i]. partitions[j].isrs[k]); } /* Sort partitions by partition id */ qsort(md->topics[i].partitions, md->topics[i].partition_cnt, sizeof(*md->topics[i].partitions), rd_kafka_metadata_partition_id_cmp); } /* Entire Metadata response now parsed without errors: * update our internal state according to the response. */ /* Avoid metadata updates when we're terminating. */ if (rd_kafka_terminating(rkb->rkb_rk)) { err = RD_KAFKA_RESP_ERR__DESTROY; goto done; } if (md->broker_cnt == 0 && md->topic_cnt == 0) { rd_rkb_dbg(rkb, METADATA, "METADATA", "No brokers or topics in metadata: should retry"); err = RD_KAFKA_RESP_ERR__PARTIAL; goto err; } /* Update our list of brokers. */ for (i = 0 ; i < md->broker_cnt ; i++) { rd_rkb_dbg(rkb, METADATA, "METADATA", " Broker #%i/%i: %s:%i NodeId %"PRId32, i, md->broker_cnt, md->brokers[i].host, md->brokers[i].port, md->brokers[i].id); rd_kafka_broker_update(rkb->rkb_rk, rkb->rkb_proto, &md->brokers[i]); } /* Update partition count and leader for each topic we know about */ for (i = 0 ; i < md->topic_cnt ; i++) { rd_kafka_metadata_topic_t *mdt = &md->topics[i]; rd_rkb_dbg(rkb, METADATA, "METADATA", " Topic #%i/%i: %s with %i partitions%s%s", i, md->topic_cnt, mdt->topic, mdt->partition_cnt, mdt->err ? ": " : "", mdt->err ? rd_kafka_err2str(mdt->err) : ""); /* Ignore topics in blacklist */ if (rkb->rkb_rk->rk_conf.topic_blacklist && rd_kafka_pattern_match(rkb->rkb_rk->rk_conf.topic_blacklist, mdt->topic)) { rd_rkb_dbg(rkb, TOPIC, "BLACKLIST", "Ignoring blacklisted topic \"%s\" " "in metadata", mdt->topic); continue; } /* Ignore metadata completely for temporary errors. (issue #513) * LEADER_NOT_AVAILABLE: Broker is rebalancing */ if (mdt->err == RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE && mdt->partition_cnt == 0) { rd_rkb_dbg(rkb, TOPIC, "METADATA", "Temporary error in metadata reply for " "topic %s (PartCnt %i): %s: ignoring", mdt->topic, mdt->partition_cnt, rd_kafka_err2str(mdt->err)); if (missing_topics) rd_list_free_cb( missing_topics, rd_list_remove_cmp(missing_topics, mdt->topic, (void *)strcmp)); continue; } /* Update local topic & partition state based on metadata */ rd_kafka_topic_metadata_update2(rkb, mdt); if (requested_topics) { rd_list_free_cb(missing_topics, rd_list_remove_cmp(missing_topics, mdt->topic, (void*)strcmp)); if (!all_topics) { rd_kafka_wrlock(rk); rd_kafka_metadata_cache_topic_update(rk, mdt); rd_kafka_wrunlock(rk); } } } /* Requested topics not seen in metadata? Propogate to topic code. */ if (missing_topics) { char *topic; rd_rkb_dbg(rkb, TOPIC, "METADATA", "%d/%d requested topic(s) seen in metadata", rd_list_cnt(requested_topics) - rd_list_cnt(missing_topics), rd_list_cnt(requested_topics)); for (i = 0 ; i < rd_list_cnt(missing_topics) ; i++) rd_rkb_dbg(rkb, TOPIC, "METADATA", "wanted %s", (char *)(missing_topics->rl_elems[i])); RD_LIST_FOREACH(topic, missing_topics, i) { shptr_rd_kafka_itopic_t *s_rkt; s_rkt = rd_kafka_topic_find(rkb->rkb_rk, topic, 1/*lock*/); if (s_rkt) { rd_kafka_topic_metadata_none( rd_kafka_topic_s2i(s_rkt)); rd_kafka_topic_destroy0(s_rkt); } } }
/** * Handle received frame from broker. */ static int rd_kafka_sasl_handle_recv (rd_kafka_transport_t *rktrans, rd_kafka_buf_t *rkbuf, char *errstr, int errstr_size) { int r; rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASL", "Received SASL frame from broker (%"PRIdsz" bytes)", rkbuf ? rkbuf->rkbuf_len : 0); if (rktrans->rktrans_sasl.complete && (!rkbuf || rkbuf->rkbuf_len == 0)) goto auth_successful; do { sasl_interact_t *interact = NULL; const char *out; unsigned int outlen; r = sasl_client_step(rktrans->rktrans_sasl.conn, rkbuf && rkbuf->rkbuf_len > 0 ? rkbuf->rkbuf_rbuf : NULL, rkbuf ? rkbuf->rkbuf_len : 0, &interact, &out, &outlen); if (rkbuf) { rd_kafka_buf_destroy(rkbuf); rkbuf = NULL; } if (r >= 0) { /* Note: outlen may be 0 here for an empty response */ if (rd_kafka_sasl_send(rktrans, out, outlen, errstr, errstr_size) == -1) return -1; } if (r == SASL_INTERACT) rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASL", "SASL_INTERACT: %lu %s, %s, %s, %p", interact->id, interact->challenge, interact->prompt, interact->defresult, interact->result); } while (r == SASL_INTERACT); if (r == SASL_CONTINUE) return 0; /* Wait for more data from broker */ else if (r != SASL_OK) { rd_snprintf(errstr, errstr_size, "SASL handshake failed (step): %s", sasl_errdetail(rktrans->rktrans_sasl.conn)); return -1; } /* Authentication successful */ auth_successful: if (rktrans->rktrans_rkb->rkb_rk->rk_conf.debug & RD_KAFKA_DBG_SECURITY) { const char *user, *mech, *authsrc; if (sasl_getprop(rktrans->rktrans_sasl.conn, SASL_USERNAME, (const void **)&user) != SASL_OK) user = "******"; if (sasl_getprop(rktrans->rktrans_sasl.conn, SASL_MECHNAME, (const void **)&mech) != SASL_OK) mech = "(unknown)"; if (sasl_getprop(rktrans->rktrans_sasl.conn, SASL_AUTHSOURCE, (const void **)&authsrc) != SASL_OK) authsrc = "(unknown)"; rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "SASL", "Authenticated as %s using %s (%s)", user, mech, authsrc); } rd_kafka_broker_connect_up(rktrans->rktrans_rkb); return 0; }
/** * Initialize and start SASL authentication. * * Returns 0 on successful init and -1 on error. * * Locality: broker thread */ int rd_kafka_sasl_client_new (rd_kafka_transport_t *rktrans, char *errstr, int errstr_size) { int r; rd_kafka_broker_t *rkb = rktrans->rktrans_rkb; rd_kafka_t *rk = rkb->rkb_rk; char *hostname, *t; sasl_callback_t callbacks[16] = { // { SASL_CB_GETOPT, (void *)rd_kafka_sasl_cb_getopt, rktrans }, { SASL_CB_LOG, (void *)rd_kafka_sasl_cb_log, rktrans }, { SASL_CB_AUTHNAME, (void *)rd_kafka_sasl_cb_getsimple, rktrans }, { SASL_CB_PASS, (void *)rd_kafka_sasl_cb_getsecret, rktrans }, { SASL_CB_ECHOPROMPT, (void *)rd_kafka_sasl_cb_chalprompt, rktrans }, { SASL_CB_GETREALM, (void *)rd_kafka_sasl_cb_getrealm, rktrans }, { SASL_CB_CANON_USER, (void *)rd_kafka_sasl_cb_canon, rktrans }, { SASL_CB_LIST_END } }; /* SASL_CB_USER is needed for PLAIN but breaks GSSAPI */ if (!strcmp(rk->rk_conf.sasl.service_name, "PLAIN")) { int endidx; /* Find end of callbacks array */ for (endidx = 0 ; callbacks[endidx].id != SASL_CB_LIST_END ; endidx++) ; callbacks[endidx].id = SASL_CB_USER; callbacks[endidx].proc = (void *)rd_kafka_sasl_cb_getsimple; endidx++; callbacks[endidx].id = SASL_CB_LIST_END; } rd_strdupa(&hostname, rktrans->rktrans_rkb->rkb_nodename); if ((t = strchr(hostname, ':'))) *t = '\0'; /* remove ":port" */ rd_rkb_dbg(rkb, SECURITY, "SASL", "Initializing SASL client: service name %s, " "hostname %s, mechanisms %s", rk->rk_conf.sasl.service_name, hostname, rk->rk_conf.sasl.mechanisms); /* Acquire or refresh ticket if kinit is configured */ rd_kafka_sasl_kinit_refresh(rkb); r = sasl_client_new(rk->rk_conf.sasl.service_name, hostname, NULL, NULL, /* no local & remote IP checks */ callbacks, 0, &rktrans->rktrans_sasl.conn); if (r != SASL_OK) { rd_snprintf(errstr, errstr_size, "%s", sasl_errstring(r, NULL, NULL)); return -1; } if (rk->rk_conf.debug & RD_KAFKA_DBG_SECURITY) { const char *avail_mechs; sasl_listmech(rktrans->rktrans_sasl.conn, NULL, NULL, " ", NULL, &avail_mechs, NULL, NULL); rd_rkb_dbg(rkb, SECURITY, "SASL", "My supported SASL mechanisms: %s", avail_mechs); } rd_kafka_transport_poll_set(rktrans, POLLIN); do { const char *out; unsigned int outlen; const char *mech = NULL; r = sasl_client_start(rktrans->rktrans_sasl.conn, rk->rk_conf.sasl.mechanisms, NULL, &out, &outlen, &mech); if (r >= 0) if (rd_kafka_sasl_send(rktrans, out, outlen, errstr, errstr_size)) return -1; } while (r == SASL_INTERACT); if (r == SASL_OK) { /* PLAIN is appearantly done here, but we still need to make sure * the PLAIN frame is sent and we get a response back (but we must * not pass the response to libsasl or it will fail). */ rktrans->rktrans_sasl.complete = 1; return 0; } else if (r != SASL_CONTINUE) { rd_snprintf(errstr, errstr_size, "SASL handshake failed (start (%d)): %s", r, sasl_errdetail(rktrans->rktrans_sasl.conn)); return -1; } return 0; }
/** * @brief Message parser for MsgVersion v2 */ static rd_kafka_resp_err_t rd_kafka_msgset_reader_msg_v2 (rd_kafka_msgset_reader_t *msetr) { rd_kafka_buf_t *rkbuf = msetr->msetr_rkbuf; rd_kafka_toppar_t *rktp = msetr->msetr_rktp; struct { int64_t Length; int8_t MsgAttributes; int64_t TimestampDelta; int64_t OffsetDelta; int64_t Offset; /* Absolute offset */ rd_kafkap_bytes_t Key; rd_kafkap_bytes_t Value; rd_kafkap_bytes_t Headers; } hdr; rd_kafka_op_t *rko; rd_kafka_msg_t *rkm; /* Only log decoding errors if protocol debugging enabled. */ int log_decode_errors = (rkbuf->rkbuf_rkb->rkb_rk->rk_conf.debug & RD_KAFKA_DBG_PROTOCOL) ? LOG_DEBUG : 0; size_t message_end; rd_kafka_buf_read_varint(rkbuf, &hdr.Length); message_end = rd_slice_offset(&rkbuf->rkbuf_reader)+(size_t)hdr.Length; rd_kafka_buf_read_i8(rkbuf, &hdr.MsgAttributes); rd_kafka_buf_read_varint(rkbuf, &hdr.TimestampDelta); rd_kafka_buf_read_varint(rkbuf, &hdr.OffsetDelta); hdr.Offset = msetr->msetr_v2_hdr->BaseOffset + hdr.OffsetDelta; /* Skip message if outdated */ if (hdr.Offset < rktp->rktp_offsets.fetch_offset) { rd_rkb_dbg(msetr->msetr_rkb, MSG, "MSG", "%s [%"PRId32"]: " "Skip offset %"PRId64" < fetch_offset %"PRId64, rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, hdr.Offset, rktp->rktp_offsets.fetch_offset); rd_kafka_buf_skip_to(rkbuf, message_end); return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue with next msg */ } rd_kafka_buf_read_bytes_varint(rkbuf, &hdr.Key); rd_kafka_buf_read_bytes_varint(rkbuf, &hdr.Value); /* We parse the Headers later, just store the size (possibly truncated) * and pointer to the headers. */ hdr.Headers.len = (int32_t)(message_end - rd_slice_offset(&rkbuf->rkbuf_reader)); rd_kafka_buf_read_ptr(rkbuf, &hdr.Headers.data, hdr.Headers.len); /* Create op/message container for message. */ rko = rd_kafka_op_new_fetch_msg(&rkm, rktp, msetr->msetr_tver->version, rkbuf, hdr.Offset, (size_t)RD_KAFKAP_BYTES_LEN(&hdr.Key), RD_KAFKAP_BYTES_IS_NULL(&hdr.Key) ? NULL : hdr.Key.data, (size_t)RD_KAFKAP_BYTES_LEN(&hdr.Value), RD_KAFKAP_BYTES_IS_NULL(&hdr.Value) ? NULL : hdr.Value.data); /* Store pointer to unparsed message headers, they will * be parsed on the first access. * This pointer points to the rkbuf payload. * Note: can't perform struct copy here due to const fields (MSVC) */ rkm->rkm_u.consumer.binhdrs.len = hdr.Headers.len; rkm->rkm_u.consumer.binhdrs.data = hdr.Headers.data; /* Set timestamp. * * When broker assigns the timestamps (LOG_APPEND_TIME) it will * assign the same timestamp for all messages in a MessageSet * using MaxTimestamp. */ if ((msetr->msetr_v2_hdr->Attributes & RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME) || (hdr.MsgAttributes & RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME)) { rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME; rkm->rkm_timestamp = msetr->msetr_v2_hdr->MaxTimestamp; } else { rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_CREATE_TIME; rkm->rkm_timestamp = msetr->msetr_v2_hdr->BaseTimestamp + hdr.TimestampDelta; } /* Enqueue message on temporary queue */ rd_kafka_q_enq(&msetr->msetr_rkq, rko); msetr->msetr_msgcnt++; msetr->msetr_msg_bytes += rkm->rkm_key_len + rkm->rkm_len; return RD_KAFKA_RESP_ERR_NO_ERROR; err_parse: /* Count all parse errors as partial message errors. */ rd_atomic64_add(&msetr->msetr_rkb->rkb_c.rx_partial, 1); return rkbuf->rkbuf_err; }