static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const void *transport_server_data, grpc_transport_stream_op *initial_op) { if (initial_op) { grpc_transport_stream_op_finish_with_failure(exec_ctx, initial_op); } }
static void fail_locked(grpc_exec_ctx *exec_ctx, grpc_subchannel_call_holder *holder) { size_t i; for (i = 0; i < holder->waiting_ops_count; i++) { grpc_transport_stream_op_finish_with_failure(exec_ctx, &holder->waiting_ops[i]); } holder->waiting_ops_count = 0; }
static void fail_locked(grpc_exec_ctx *exec_ctx, call_data *calld, grpc_error *error) { size_t i; for (i = 0; i < calld->waiting_ops_count; i++) { grpc_transport_stream_op_finish_with_failure( exec_ctx, calld->waiting_ops[i], GRPC_ERROR_REF(error)); } calld->waiting_ops_count = 0; GRPC_ERROR_UNREF(error); }
static void lame_start_transport_stream_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op) { GRPC_CALL_LOG_OP(GPR_INFO, elem, op); if (op->recv_initial_metadata != NULL) { fill_metadata(elem, op->recv_initial_metadata); } else if (op->recv_trailing_metadata != NULL) { fill_metadata(elem, op->recv_trailing_metadata); } grpc_transport_stream_op_finish_with_failure(exec_ctx, op); }
static void fail_locked(grpc_exec_ctx *exec_ctx, grpc_subchannel_call_holder *holder, grpc_error *error) { size_t i; for (i = 0; i < holder->waiting_ops_count; i++) { grpc_transport_stream_op_finish_with_failure( exec_ctx, &holder->waiting_ops[i], GRPC_ERROR_REF(error)); } holder->waiting_ops_count = 0; GRPC_ERROR_UNREF(error); }
static void send_security_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; grpc_client_security_context *ctx = (grpc_client_security_context *)op->context[GRPC_CONTEXT_SECURITY].value; grpc_call_credentials *channel_call_creds = chand->security_connector->request_metadata_creds; int call_creds_has_md = (ctx != NULL) && (ctx->creds != NULL); if (channel_call_creds == NULL && !call_creds_has_md) { /* Skip sending metadata altogether. */ grpc_call_next_op(exec_ctx, elem, op); return; } if (channel_call_creds != NULL && call_creds_has_md) { calld->creds = grpc_composite_call_credentials_create(channel_call_creds, ctx->creds, NULL); if (calld->creds == NULL) { grpc_transport_stream_op_finish_with_failure( exec_ctx, op, grpc_error_set_int( GRPC_ERROR_CREATE( "Incompatible credentials set on channel and call."), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED)); return; } } else { calld->creds = grpc_call_credentials_ref( call_creds_has_md ? ctx->creds : channel_call_creds); } build_auth_metadata_context(&chand->security_connector->base, chand->auth_context, calld); calld->op = *op; /* Copy op (originates from the caller's stack). */ GPR_ASSERT(calld->pollent != NULL); grpc_call_credentials_get_request_metadata( exec_ctx, calld->creds, calld->pollent, calld->auth_md_context, on_credentials_metadata, elem); }
static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems, size_t num_md, grpc_credentials_status status, const char *error_details) { grpc_call_element *elem = (grpc_call_element *)user_data; call_data *calld = elem->call_data; grpc_transport_stream_op *op = &calld->op; grpc_metadata_batch *mdb; size_t i; reset_auth_metadata_context(&calld->auth_md_context); grpc_error *error = GRPC_ERROR_NONE; if (status != GRPC_CREDENTIALS_OK) { error = grpc_error_set_int( GRPC_ERROR_CREATE(error_details != NULL && strlen(error_details) > 0 ? error_details : "Credentials failed to get metadata."), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED); } else { GPR_ASSERT(num_md <= MAX_CREDENTIALS_METADATA_COUNT); GPR_ASSERT(op->send_initial_metadata != NULL); mdb = op->send_initial_metadata; for (i = 0; i < num_md; i++) { add_error(&error, grpc_metadata_batch_add_tail( exec_ctx, mdb, &calld->md_links[i], grpc_mdelem_from_slices( exec_ctx, grpc_slice_ref_internal(md_elems[i].key), grpc_slice_ref_internal(md_elems[i].value)))); } } if (error == GRPC_ERROR_NONE) { grpc_call_next_op(exec_ctx, elem, op); } else { grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error); } }
// The logic here is fairly complicated, due to (a) the fact that we // need to handle the case where we receive the send op before the // initial metadata op, and (b) the need for efficiency, especially in // the streaming case. // TODO(ctiller): Explain this more thoroughly. static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; GRPC_CALL_LOG_OP(GPR_INFO, elem, op); grpc_deadline_state_client_start_transport_stream_op(exec_ctx, elem, op); /* try to (atomically) get the call */ grpc_subchannel_call *call = GET_CALL(calld); GPR_TIMER_BEGIN("cc_start_transport_stream_op", 0); if (call == CANCELLED_CALL) { grpc_transport_stream_op_finish_with_failure( exec_ctx, op, GRPC_ERROR_REF(calld->cancel_error)); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; } if (call != NULL) { grpc_subchannel_call_process_op(exec_ctx, call, op); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; } /* we failed; lock and figure out what to do */ gpr_mu_lock(&calld->mu); retry: /* need to recheck that another thread hasn't set the call */ call = GET_CALL(calld); if (call == CANCELLED_CALL) { gpr_mu_unlock(&calld->mu); grpc_transport_stream_op_finish_with_failure( exec_ctx, op, GRPC_ERROR_REF(calld->cancel_error)); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; } if (call != NULL) { gpr_mu_unlock(&calld->mu); grpc_subchannel_call_process_op(exec_ctx, call, op); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; } /* if this is a cancellation, then we can raise our cancelled flag */ if (op->cancel_error != GRPC_ERROR_NONE) { if (!gpr_atm_rel_cas(&calld->subchannel_call, 0, (gpr_atm)(uintptr_t)CANCELLED_CALL)) { goto retry; } else { // Stash a copy of cancel_error in our call data, so that we can use // it for subsequent operations. This ensures that if the call is // cancelled before any ops are passed down (e.g., if the deadline // is in the past when the call starts), we can return the right // error to the caller when the first op does get passed down. calld->cancel_error = GRPC_ERROR_REF(op->cancel_error); switch (calld->creation_phase) { case GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING: fail_locked(exec_ctx, calld, GRPC_ERROR_REF(op->cancel_error)); break; case GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL: pick_subchannel(exec_ctx, elem, NULL, 0, &calld->connected_subchannel, NULL, GRPC_ERROR_REF(op->cancel_error)); break; } gpr_mu_unlock(&calld->mu); grpc_transport_stream_op_finish_with_failure( exec_ctx, op, GRPC_ERROR_REF(op->cancel_error)); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; } } /* if we don't have a subchannel, try to get one */ if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && calld->connected_subchannel == NULL && op->send_initial_metadata != NULL) { calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL; grpc_closure_init(&calld->next_step, subchannel_ready, elem); GRPC_CALL_STACK_REF(calld->owning_call, "pick_subchannel"); /* If a subchannel is not available immediately, the polling entity from call_data should be provided to channel_data's interested_parties, so that IO of the lb_policy and resolver could be done under it. */ if (pick_subchannel(exec_ctx, elem, op->send_initial_metadata, op->send_initial_metadata_flags, &calld->connected_subchannel, &calld->next_step, GRPC_ERROR_NONE)) { calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel"); } else { grpc_polling_entity_add_to_pollset_set(exec_ctx, calld->pollent, chand->interested_parties); } } /* if we've got a subchannel, then let's ask it to create a call */ if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && calld->connected_subchannel != NULL) { grpc_subchannel_call *subchannel_call = NULL; grpc_error *error = grpc_connected_subchannel_create_call( exec_ctx, calld->connected_subchannel, calld->pollent, calld->path, calld->deadline, &subchannel_call); if (error != GRPC_ERROR_NONE) { subchannel_call = CANCELLED_CALL; fail_locked(exec_ctx, calld, GRPC_ERROR_REF(error)); grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error); } gpr_atm_rel_store(&calld->subchannel_call, (gpr_atm)(uintptr_t)subchannel_call); retry_waiting_locked(exec_ctx, calld); goto retry; } /* nothing to be done but wait */ add_waiting_locked(calld, op); gpr_mu_unlock(&calld->mu); GPR_TIMER_END("cc_start_transport_stream_op", 0); }
// The logic here is fairly complicated, due to (a) the fact that we // need to handle the case where we receive the send op before the // initial metadata op, and (b) the need for efficiency, especially in // the streaming case. // TODO(ctiller): Explain this more thoroughly. static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op) { call_data *calld = elem->call_data; GRPC_CALL_LOG_OP(GPR_INFO, elem, op); /* try to (atomically) get the call */ grpc_subchannel_call *call = GET_CALL(calld); GPR_TIMER_BEGIN("cc_start_transport_stream_op", 0); if (call == CANCELLED_CALL) { grpc_transport_stream_op_finish_with_failure(exec_ctx, op, GRPC_ERROR_CANCELLED); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; } if (call != NULL) { grpc_subchannel_call_process_op(exec_ctx, call, op); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; } /* we failed; lock and figure out what to do */ gpr_mu_lock(&calld->mu); retry: /* need to recheck that another thread hasn't set the call */ call = GET_CALL(calld); if (call == CANCELLED_CALL) { gpr_mu_unlock(&calld->mu); grpc_transport_stream_op_finish_with_failure(exec_ctx, op, GRPC_ERROR_CANCELLED); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; } if (call != NULL) { gpr_mu_unlock(&calld->mu); grpc_subchannel_call_process_op(exec_ctx, call, op); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; } /* if this is a cancellation, then we can raise our cancelled flag */ if (op->cancel_error != GRPC_ERROR_NONE) { if (!gpr_atm_rel_cas(&calld->subchannel_call, 0, (gpr_atm)(uintptr_t)CANCELLED_CALL)) { goto retry; } else { switch (calld->creation_phase) { case GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING: fail_locked(exec_ctx, calld, GRPC_ERROR_REF(op->cancel_error)); break; case GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL: pick_subchannel(exec_ctx, elem, NULL, 0, &calld->connected_subchannel, NULL); break; } gpr_mu_unlock(&calld->mu); grpc_transport_stream_op_finish_with_failure(exec_ctx, op, GRPC_ERROR_CANCELLED); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; } } /* if we don't have a subchannel, try to get one */ if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && calld->connected_subchannel == NULL && op->send_initial_metadata != NULL) { calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL; grpc_closure_init(&calld->next_step, subchannel_ready, calld); GRPC_CALL_STACK_REF(calld->owning_call, "pick_subchannel"); if (pick_subchannel(exec_ctx, elem, op->send_initial_metadata, op->send_initial_metadata_flags, &calld->connected_subchannel, &calld->next_step)) { calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel"); } } /* if we've got a subchannel, then let's ask it to create a call */ if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && calld->connected_subchannel != NULL) { grpc_subchannel_call *subchannel_call = NULL; grpc_error *error = grpc_connected_subchannel_create_call( exec_ctx, calld->connected_subchannel, calld->pollent, &subchannel_call); if (error != GRPC_ERROR_NONE) { subchannel_call = CANCELLED_CALL; fail_locked(exec_ctx, calld, GRPC_ERROR_REF(error)); grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error); } gpr_atm_rel_store(&calld->subchannel_call, (gpr_atm)(uintptr_t)subchannel_call); retry_waiting_locked(exec_ctx, calld); goto retry; } /* nothing to be done but wait */ add_waiting_locked(calld, op); gpr_mu_unlock(&calld->mu); GPR_TIMER_END("cc_start_transport_stream_op", 0); }
void grpc_subchannel_call_holder_perform_op(grpc_exec_ctx *exec_ctx, grpc_subchannel_call_holder *holder, grpc_transport_stream_op *op) { /* try to (atomically) get the call */ grpc_subchannel_call *call = GET_CALL(holder); GPR_TIMER_BEGIN("grpc_subchannel_call_holder_perform_op", 0); if (call == CANCELLED_CALL) { grpc_transport_stream_op_finish_with_failure(exec_ctx, op); GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); return; } if (call != NULL) { grpc_subchannel_call_process_op(exec_ctx, call, op); GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); return; } /* we failed; lock and figure out what to do */ gpr_mu_lock(&holder->mu); retry: /* need to recheck that another thread hasn't set the call */ call = GET_CALL(holder); if (call == CANCELLED_CALL) { gpr_mu_unlock(&holder->mu); grpc_transport_stream_op_finish_with_failure(exec_ctx, op); GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); return; } if (call != NULL) { gpr_mu_unlock(&holder->mu); grpc_subchannel_call_process_op(exec_ctx, call, op); GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); return; } /* if this is a cancellation, then we can raise our cancelled flag */ if (op->cancel_with_status != GRPC_STATUS_OK) { if (!gpr_atm_rel_cas(&holder->subchannel_call, 0, 1)) { goto retry; } else { switch (holder->creation_phase) { case GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING: fail_locked(exec_ctx, holder); break; case GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL: holder->pick_subchannel(exec_ctx, holder->pick_subchannel_arg, NULL, &holder->connected_subchannel, NULL); break; } gpr_mu_unlock(&holder->mu); grpc_transport_stream_op_finish_with_failure(exec_ctx, op); GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); return; } } /* if we don't have a subchannel, try to get one */ if (holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && holder->connected_subchannel == NULL && op->send_initial_metadata != NULL) { holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL; grpc_closure_init(&holder->next_step, subchannel_ready, holder); GRPC_CALL_STACK_REF(holder->owning_call, "pick_subchannel"); if (holder->pick_subchannel( exec_ctx, holder->pick_subchannel_arg, op->send_initial_metadata, &holder->connected_subchannel, &holder->next_step)) { holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel"); } } /* if we've got a subchannel, then let's ask it to create a call */ if (holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && holder->connected_subchannel != NULL) { gpr_atm_rel_store( &holder->subchannel_call, (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( exec_ctx, holder->connected_subchannel, holder->pollset)); retry_waiting_locked(exec_ctx, holder); goto retry; } /* nothing to be done but wait */ add_waiting_locked(holder, op); gpr_mu_unlock(&holder->mu); GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); }