void grpc_sopb_swap(grpc_stream_op_buffer *a, grpc_stream_op_buffer *b) {
GPR_SWAP(size_t, a->nops, b->nops);
GPR_SWAP(size_t, a->capacity, b->capacity);
if (a->ops == a->inlined_ops) {
if (b->ops == b->inlined_ops) {
/* swap contents of inlined buffer */
grpc_stream_op temp[GRPC_SOPB_INLINE_ELEMENTS];
memcpy(temp, a->ops, b->nops * sizeof(grpc_stream_op));
memcpy(a->ops, b->ops, a->nops * sizeof(grpc_stream_op));
memcpy(b->ops, temp, b->nops * sizeof(grpc_stream_op));
} else {
/* a is inlined, b is not - copy a inlined into b, fix pointers */
a->ops = b->ops;
b->ops = b->inlined_ops;
memcpy(b->ops, a->inlined_ops, b->nops * sizeof(grpc_stream_op));
}
} else if (b->ops == b->inlined_ops) {
/* b is inlined, a is not - copy b inlined int a, fix pointers */
b->ops = a->ops;
a->ops = a->inlined_ops;
memcpy(a->ops, b->inlined_ops, a->nops * sizeof(grpc_stream_op));
} else {
/* no inlining: easy swap */
GPR_SWAP(grpc_stream_op *, a->ops, b->ops);
}
}
void gpr_slice_buffer_swap(gpr_slice_buffer *a, gpr_slice_buffer *b) {
GPR_SWAP(size_t, a->count, b->count);
GPR_SWAP(size_t, a->capacity, b->capacity);
GPR_SWAP(size_t, a->length, b->length);
if (a->slices == a->inlined) {
if (b->slices == b->inlined) {
/* swap contents of inlined buffer */
gpr_slice temp[GRPC_SLICE_BUFFER_INLINE_ELEMENTS];
memcpy(temp, a->slices, b->count * sizeof(gpr_slice));
memcpy(a->slices, b->slices, a->count * sizeof(gpr_slice));
memcpy(b->slices, temp, b->count * sizeof(gpr_slice));
} else {
/* a is inlined, b is not - copy a inlined into b, fix pointers */
a->slices = b->slices;
b->slices = b->inlined;
memcpy(b->slices, a->inlined, b->count * sizeof(gpr_slice));
}
} else if (b->slices == b->inlined) {
/* b is inlined, a is not - copy b inlined int a, fix pointers */
b->slices = a->slices;
a->slices = a->inlined;
memcpy(a->slices, b->inlined, a->count * sizeof(gpr_slice));
} else {
/* no inlining: easy swap */
GPR_SWAP(gpr_slice *, a->slices, b->slices);
}
}
void grpc_chttp2_stream_map_move_into(grpc_chttp2_stream_map *src,
grpc_chttp2_stream_map *dst) {
/* if src is empty we dont need to do anything */
if (src->count == src->free) {
return;
}
/* if dst is empty we simply need to swap */
if (dst->count == dst->free) {
GPR_SWAP(grpc_chttp2_stream_map, *src, *dst);
return;
}
/* the first element of src must be greater than the last of dst...
* however the maps may need compacting for this property to hold */
if (src->keys[0] <= dst->keys[dst->count - 1]) {
src->count = compact(src->keys, src->values, src->count);
src->free = 0;
dst->count = compact(dst->keys, dst->values, dst->count);
dst->free = 0;
}
GPR_ASSERT(src->keys[0] > dst->keys[dst->count - 1]);
/* if dst doesn't have capacity, resize */
if (dst->count + src->count > dst->capacity) {
dst->capacity = GPR_MAX(dst->capacity * 3 / 2, dst->count + src->count);
dst->keys = gpr_realloc(dst->keys, dst->capacity * sizeof(uint32_t));
dst->values = gpr_realloc(dst->values, dst->capacity * sizeof(void *));
}
memcpy(dst->keys + dst->count, src->keys, src->count * sizeof(uint32_t));
memcpy(dst->values + dst->count, src->values, src->count * sizeof(void *));
dst->count += src->count;
dst->free += src->free;
src->count = 0;
src->free = 0;
}
static void del_plucker(grpc_completion_queue *cc, void *tag,
grpc_pollset_worker *worker) {
int i;
for (i = 0; i < cc->num_pluckers; i++) {
if (cc->pluckers[i].tag == tag && cc->pluckers[i].worker == worker) {
cc->num_pluckers--;
GPR_SWAP(plucker, cc->pluckers[i], cc->pluckers[cc->num_pluckers]);
return;
}
}
gpr_log(GPR_ERROR, "should never reach here");
abort();
}
static void begin_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
call_data *calld, requested_call *rc) {
grpc_op ops[1];
grpc_op *op = ops;
memset(ops, 0, sizeof(ops));
/* called once initial metadata has been read by the call, but BEFORE
the ioreq to fetch it out of the call has been executed.
This means metadata related fields can be relied on in calld, but to
fill in the metadata array passed by the client, we need to perform
an ioreq op, that should complete immediately. */
grpc_call_set_completion_queue(exec_ctx, calld->call, rc->cq_bound_to_call);
grpc_closure_init(&rc->publish, publish_registered_or_batch, rc);
*rc->call = calld->call;
calld->cq_new = rc->cq_for_notification;
GPR_SWAP(grpc_metadata_array, *rc->initial_metadata, calld->initial_metadata);
switch (rc->type) {
case BATCH_CALL:
GPR_ASSERT(calld->host != NULL);
GPR_ASSERT(calld->path != NULL);
cpstr(&rc->data.batch.details->host,
&rc->data.batch.details->host_capacity, calld->host);
cpstr(&rc->data.batch.details->method,
&rc->data.batch.details->method_capacity, calld->path);
rc->data.batch.details->deadline = calld->deadline;
break;
case REGISTERED_CALL:
*rc->data.registered.deadline = calld->deadline;
if (rc->data.registered.optional_payload) {
op->op = GRPC_OP_RECV_MESSAGE;
op->data.recv_message = rc->data.registered.optional_payload;
op++;
}
break;
default:
GPR_UNREACHABLE_CODE(return );
}
GRPC_CALL_INTERNAL_REF(calld->call, "server");
grpc_call_start_batch_and_execute(exec_ctx, calld->call, ops,
(size_t)(op - ops), &rc->publish);
}
static elem_struct *search_elems(elem_struct *elems, size_t count,
bool inserted) {
size_t *search_order = gpr_malloc(count * sizeof(*search_order));
for (size_t i = 0; i < count; i++) {
search_order[i] = i;
}
for (size_t i = 0; i < count * 2; i++) {
size_t a = (size_t)rand() % count;
size_t b = (size_t)rand() % count;
GPR_SWAP(size_t, search_order[a], search_order[b]);
}
elem_struct *out = NULL;
for (size_t i = 0; out == NULL && i < count; i++) {
if (elems[search_order[i]].inserted == inserted) {
out = &elems[search_order[i]];
}
}
gpr_free(search_order);
return out;
}
static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
call_data *calld, size_t cq_idx, requested_call *rc) {
grpc_call_set_completion_queue(exec_ctx, calld->call, rc->cq_bound_to_call);
grpc_call *call = calld->call;
*rc->call = call;
calld->cq_new = server->cqs[cq_idx];
GPR_SWAP(grpc_metadata_array, *rc->initial_metadata, calld->initial_metadata);
switch (rc->type) {
case BATCH_CALL:
GPR_ASSERT(calld->host != NULL);
GPR_ASSERT(calld->path != NULL);
cpstr(&rc->data.batch.details->host,
&rc->data.batch.details->host_capacity, calld->host);
cpstr(&rc->data.batch.details->method,
&rc->data.batch.details->method_capacity, calld->path);
rc->data.batch.details->deadline = calld->deadline;
rc->data.batch.details->flags =
0 | (calld->recv_idempotent_request
? GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST
: 0);
break;
case REGISTERED_CALL:
*rc->data.registered.deadline = calld->deadline;
if (rc->data.registered.optional_payload) {
*rc->data.registered.optional_payload = calld->payload;
}
break;
default:
GPR_UNREACHABLE_CODE(return );
}
grpc_call_element *elem =
grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
channel_data *chand = elem->channel_data;
server_ref(chand->server);
grpc_cq_end_op(exec_ctx, calld->cq_new, rc->tag, GRPC_ERROR_NONE,
done_request_event, rc, &rc->completion);
}
static void on_new_call(void *arg, int success) {
grpc_end2end_proxy *proxy = arg;
grpc_call_error err;
if (success) {
grpc_op op;
proxy_call *pc = gpr_malloc(sizeof(*pc));
memset(pc, 0, sizeof(*pc));
pc->proxy = proxy;
GPR_SWAP(grpc_metadata_array, pc->c2p_initial_metadata,
proxy->new_call_metadata);
pc->c2p = proxy->new_call;
pc->p2s = grpc_channel_create_call(
proxy->client, pc->c2p, GRPC_PROPAGATE_DEFAULTS, proxy->cq,
proxy->new_call_details.method, proxy->new_call_details.host,
proxy->new_call_details.deadline, NULL);
gpr_ref_init(&pc->refs, 1);
op.flags = 0;
op.reserved = NULL;
op.op = GRPC_OP_RECV_INITIAL_METADATA;
op.data.recv_initial_metadata = &pc->p2s_initial_metadata;
refpc(pc, "on_p2s_recv_initial_metadata");
err = grpc_call_start_batch(
pc->p2s, &op, 1, new_closure(on_p2s_recv_initial_metadata, pc), NULL);
GPR_ASSERT(err == GRPC_CALL_OK);
op.op = GRPC_OP_SEND_INITIAL_METADATA;
op.data.send_initial_metadata.count = pc->c2p_initial_metadata.count;
op.data.send_initial_metadata.metadata = pc->c2p_initial_metadata.metadata;
refpc(pc, "on_p2s_sent_initial_metadata");
err = grpc_call_start_batch(
pc->p2s, &op, 1, new_closure(on_p2s_sent_initial_metadata, pc), NULL);
GPR_ASSERT(err == GRPC_CALL_OK);
op.op = GRPC_OP_RECV_MESSAGE;
op.data.recv_message = &pc->c2p_msg;
refpc(pc, "on_c2p_recv_msg");
err = grpc_call_start_batch(pc->c2p, &op, 1,
new_closure(on_c2p_recv_msg, pc), NULL);
GPR_ASSERT(err == GRPC_CALL_OK);
op.op = GRPC_OP_RECV_MESSAGE;
op.data.recv_message = &pc->p2s_msg;
refpc(pc, "on_p2s_recv_msg");
err = grpc_call_start_batch(pc->p2s, &op, 1,
new_closure(on_p2s_recv_msg, pc), NULL);
GPR_ASSERT(err == GRPC_CALL_OK);
op.op = GRPC_OP_RECV_STATUS_ON_CLIENT;
op.data.recv_status_on_client.trailing_metadata =
&pc->p2s_trailing_metadata;
op.data.recv_status_on_client.status = &pc->p2s_status;
op.data.recv_status_on_client.status_details = &pc->p2s_status_details;
op.data.recv_status_on_client.status_details_capacity =
&pc->p2s_status_details_capacity;
refpc(pc, "on_p2s_status");
err = grpc_call_start_batch(pc->p2s, &op, 1, new_closure(on_p2s_status, pc),
NULL);
GPR_ASSERT(err == GRPC_CALL_OK);
op.op = GRPC_OP_RECV_CLOSE_ON_SERVER;
op.data.recv_close_on_server.cancelled = &pc->c2p_server_cancelled;
refpc(pc, "on_c2p_closed");
err = grpc_call_start_batch(pc->c2p, &op, 1, new_closure(on_c2p_closed, pc),
NULL);
GPR_ASSERT(err == GRPC_CALL_OK);
request_call(proxy);
unrefpc(pc, "init");
} else {
GPR_ASSERT(proxy->new_call == NULL);
}
}
void grpc_chttp2_publish_reads(
grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
grpc_chttp2_transport_parsing *transport_parsing) {
grpc_chttp2_stream_global *stream_global;
grpc_chttp2_stream_parsing *stream_parsing;
int was_zero;
int is_zero;
/* transport_parsing->last_incoming_stream_id is used as
last-grpc_chttp2_stream-id when
sending GOAWAY frame.
https://tools.ietf.org/html/draft-ietf-httpbis-http2-17#section-6.8
says that last-grpc_chttp2_stream-id is peer-initiated grpc_chttp2_stream
ID. So,
since we don't have server pushed streams, client should send
GOAWAY last-grpc_chttp2_stream-id=0 in this case. */
if (!transport_parsing->is_client) {
transport_global->last_incoming_stream_id =
transport_parsing->last_incoming_stream_id;
}
/* update global settings */
if (transport_parsing->settings_updated) {
memcpy(transport_global->settings[GRPC_PEER_SETTINGS],
transport_parsing->settings, sizeof(transport_parsing->settings));
transport_parsing->settings_updated = 0;
}
/* update settings based on ack if received */
if (transport_parsing->settings_ack_received) {
memcpy(transport_global->settings[GRPC_ACKED_SETTINGS],
transport_global->settings[GRPC_SENT_SETTINGS],
GRPC_CHTTP2_NUM_SETTINGS * sizeof(uint32_t));
transport_parsing->settings_ack_received = 0;
transport_global->sent_local_settings = 0;
}
/* move goaway to the global state if we received one (it will be
published later */
if (transport_parsing->goaway_received) {
grpc_chttp2_add_incoming_goaway(exec_ctx, transport_global,
(uint32_t)transport_parsing->goaway_error,
transport_parsing->goaway_text);
transport_parsing->goaway_text = gpr_empty_slice();
transport_parsing->goaway_received = 0;
}
/* propagate flow control tokens to global state */
was_zero = transport_global->outgoing_window <= 0;
GRPC_CHTTP2_FLOW_MOVE_TRANSPORT("parsed", transport_global, outgoing_window,
transport_parsing, outgoing_window);
is_zero = transport_global->outgoing_window <= 0;
if (was_zero && !is_zero) {
while (grpc_chttp2_list_pop_stalled_by_transport(transport_global,
&stream_global)) {
grpc_chttp2_become_writable(transport_global, stream_global);
}
}
if (transport_parsing->incoming_window <
transport_global->connection_window_target * 3 / 4) {
int64_t announce_bytes = transport_global->connection_window_target -
transport_parsing->incoming_window;
GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parsed", transport_global,
announce_incoming_window, announce_bytes);
GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parsed", transport_parsing,
incoming_window, announce_bytes);
}
/* for each stream that saw an update, fixup global state */
while (grpc_chttp2_list_pop_parsing_seen_stream(
transport_global, transport_parsing, &stream_global, &stream_parsing)) {
if (stream_parsing->seen_error) {
stream_global->seen_error = true;
stream_global->exceeded_metadata_size =
stream_parsing->exceeded_metadata_size;
grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
}
/* flush stats to global stream state */
grpc_transport_move_stats(&stream_parsing->stats, &stream_global->stats);
/* update outgoing flow control window */
was_zero = stream_global->outgoing_window <= 0;
GRPC_CHTTP2_FLOW_MOVE_STREAM("parsed", transport_global, stream_global,
outgoing_window, stream_parsing,
outgoing_window);
is_zero = stream_global->outgoing_window <= 0;
if (was_zero && !is_zero) {
grpc_chttp2_become_writable(transport_global, stream_global);
}
stream_global->max_recv_bytes -= (uint32_t)GPR_MIN(
stream_global->max_recv_bytes, stream_parsing->received_bytes);
stream_parsing->received_bytes = 0;
/* publish incoming stream ops */
if (stream_global->incoming_frames.tail != NULL) {
stream_global->incoming_frames.tail->is_tail = 0;
}
if (stream_parsing->data_parser.incoming_frames.head != NULL) {
grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
}
grpc_chttp2_incoming_frame_queue_merge(
&stream_global->incoming_frames,
&stream_parsing->data_parser.incoming_frames);
if (stream_global->incoming_frames.tail != NULL) {
stream_global->incoming_frames.tail->is_tail = 1;
}
if (!stream_global->published_initial_metadata &&
stream_parsing->got_metadata_on_parse[0]) {
stream_parsing->got_metadata_on_parse[0] = 0;
stream_global->published_initial_metadata = 1;
GPR_SWAP(grpc_chttp2_incoming_metadata_buffer,
stream_parsing->metadata_buffer[0],
stream_global->received_initial_metadata);
grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
}
if (!stream_global->published_trailing_metadata &&
stream_parsing->got_metadata_on_parse[1]) {
stream_parsing->got_metadata_on_parse[1] = 0;
stream_global->published_trailing_metadata = 1;
GPR_SWAP(grpc_chttp2_incoming_metadata_buffer,
stream_parsing->metadata_buffer[1],
stream_global->received_trailing_metadata);
grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
}
if (stream_parsing->forced_close_error != GRPC_ERROR_NONE) {
intptr_t reason;
bool has_reason = grpc_error_get_int(stream_parsing->forced_close_error,
GRPC_ERROR_INT_HTTP2_ERROR, &reason);
if (has_reason && reason != GRPC_CHTTP2_NO_ERROR) {
grpc_status_code status_code =
has_reason
? grpc_chttp2_http2_error_to_grpc_status(
(grpc_chttp2_error_code)reason, stream_global->deadline)
: GRPC_STATUS_INTERNAL;
const char *status_details =
grpc_error_string(stream_parsing->forced_close_error);
gpr_slice slice_details = gpr_slice_from_copied_string(status_details);
grpc_error_free_string(status_details);
grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global,
status_code, &slice_details);
}
grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global,
1, 1, stream_parsing->forced_close_error);
}
if (stream_parsing->received_close) {
grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global,
1, 0, GRPC_ERROR_NONE);
}
}
}
void grpc_chttp2_incoming_metadata_buffer_swap(
grpc_chttp2_incoming_metadata_buffer *a,
grpc_chttp2_incoming_metadata_buffer *b) {
GPR_SWAP(grpc_chttp2_incoming_metadata_buffer, *a, *b);
}
static void finish_live_ioreq_op(grpc_call *call, grpc_ioreq_op op,
int success) {
completed_request *cr;
gpr_uint8 master_set = call->request_set[op];
reqinfo_master *master;
size_t i;
/* ioreq is live: we need to do something */
master = &call->masters[master_set];
master->complete_mask |= 1u << op;
if (!success) {
master->success = 0;
}
if (master->complete_mask == master->need_mask) {
for (i = 0; i < GRPC_IOREQ_OP_COUNT; i++) {
if (call->request_set[i] != master_set) {
continue;
}
call->request_set[i] = REQSET_DONE;
switch ((grpc_ioreq_op)i) {
case GRPC_IOREQ_RECV_MESSAGE:
case GRPC_IOREQ_SEND_MESSAGE:
if (master->success) {
call->request_set[i] = REQSET_EMPTY;
} else {
call->write_state = WRITE_STATE_WRITE_CLOSED;
}
break;
case GRPC_IOREQ_RECV_CLOSE:
case GRPC_IOREQ_SEND_INITIAL_METADATA:
case GRPC_IOREQ_SEND_TRAILING_METADATA:
case GRPC_IOREQ_SEND_STATUS:
case GRPC_IOREQ_SEND_CLOSE:
break;
case GRPC_IOREQ_RECV_STATUS:
get_final_status(call, call->request_data[GRPC_IOREQ_RECV_STATUS]);
break;
case GRPC_IOREQ_RECV_STATUS_DETAILS:
get_final_details(call,
call->request_data[GRPC_IOREQ_RECV_STATUS_DETAILS]);
break;
case GRPC_IOREQ_RECV_INITIAL_METADATA:
GPR_SWAP(grpc_metadata_array, call->buffered_metadata[0],
*call->request_data[GRPC_IOREQ_RECV_INITIAL_METADATA]
.recv_metadata);
break;
case GRPC_IOREQ_RECV_TRAILING_METADATA:
GPR_SWAP(grpc_metadata_array, call->buffered_metadata[1],
*call->request_data[GRPC_IOREQ_RECV_TRAILING_METADATA]
.recv_metadata);
break;
case GRPC_IOREQ_OP_COUNT:
abort();
break;
}
}
cr = &call->completed_requests[call->num_completed_requests++];
cr->success = master->success;
cr->on_complete = master->on_complete;
cr->user_data = master->user_data;
}
}
