static void start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
channel_data *chand = elem->channel_data;
call_data *calld = elem->call_data;
grpc_server *server = chand->server;
uint32_t i;
uint32_t hash;
channel_registered_method *rm;
if (chand->registered_methods && calld->path && calld->host) {
/* TODO(ctiller): unify these two searches */
/* check for an exact match with host */
hash = GRPC_MDSTR_KV_HASH(calld->host->hash, calld->path->hash);
for (i = 0; i <= chand->registered_method_max_probes; i++) {
rm = &chand->registered_methods[(hash + i) %
chand->registered_method_slots];
if (!rm) break;
if (rm->host != calld->host) continue;
if (rm->method != calld->path) continue;
if ((rm->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) &&
!calld->recv_idempotent_request)
continue;
finish_start_new_rpc(exec_ctx, server, elem,
&rm->server_registered_method->request_matcher,
rm->server_registered_method->payload_handling);
return;
}
/* check for a wildcard method definition (no host set) */
hash = GRPC_MDSTR_KV_HASH(0, calld->path->hash);
for (i = 0; i <= chand->registered_method_max_probes; i++) {
rm = &chand->registered_methods[(hash + i) %
chand->registered_method_slots];
if (!rm) break;
if (rm->host != NULL) continue;
if (rm->method != calld->path) continue;
if ((rm->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) &&
!calld->recv_idempotent_request)
continue;
finish_start_new_rpc(exec_ctx, server, elem,
&rm->server_registered_method->request_matcher,
rm->server_registered_method->payload_handling);
return;
}
}
finish_start_new_rpc(exec_ctx, server, elem,
&server->unregistered_request_matcher,
GRPC_SRM_PAYLOAD_NONE);
}
static void start_new_rpc(grpc_call_element *elem) {
channel_data *chand = elem->channel_data;
call_data *calld = elem->call_data;
grpc_server *server = chand->server;
gpr_uint32 i;
gpr_uint32 hash;
channel_registered_method *rm;
gpr_mu_lock(&server->mu);
if (chand->registered_methods && calld->path && calld->host) {
/* TODO(ctiller): unify these two searches */
/* check for an exact match with host */
hash = GRPC_MDSTR_KV_HASH(calld->host->hash, calld->path->hash);
for (i = 0; i < chand->registered_method_max_probes; i++) {
rm = &chand->registered_methods[(hash + i) %
chand->registered_method_slots];
if (!rm) break;
if (rm->host != calld->host) continue;
if (rm->method != calld->path) continue;
finish_start_new_rpc_and_unlock(server, elem,
&rm->server_registered_method->pending,
&rm->server_registered_method->requested);
return;
}
/* check for a wildcard method definition (no host set) */
hash = GRPC_MDSTR_KV_HASH(0, calld->path->hash);
for (i = 0; i <= chand->registered_method_max_probes; i++) {
rm = &chand->registered_methods[(hash + i) %
chand->registered_method_slots];
if (!rm) break;
if (rm->host != NULL) continue;
if (rm->method != calld->path) continue;
finish_start_new_rpc_and_unlock(server, elem,
&rm->server_registered_method->pending,
&rm->server_registered_method->requested);
return;
}
}
finish_start_new_rpc_and_unlock(server, elem, &server->lists[PENDING_START],
&server->requested_calls);
}
grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdctx *ctx,
grpc_mdstr *mkey,
grpc_mdstr *mvalue) {
internal_string *key = (internal_string *)mkey;
internal_string *value = (internal_string *)mvalue;
gpr_uint32 hash = GRPC_MDSTR_KV_HASH(mkey->hash, mvalue->hash);
internal_metadata *md;
GPR_ASSERT(key->context == ctx);
GPR_ASSERT(value->context == ctx);
lock(ctx);
/* search for an existing pair */
for (md = ctx->mdtab[hash % ctx->mdtab_capacity]; md; md = md->bucket_next) {
if (md->key == key && md->value == value) {
ref_md(md);
internal_string_unref(key);
internal_string_unref(value);
unlock(ctx);
return (grpc_mdelem *)md;
}
}
/* not found: create a new pair */
md = gpr_malloc(sizeof(internal_metadata));
md->refs = 1;
md->context = ctx;
md->key = key;
md->value = value;
md->user_data = NULL;
md->destroy_user_data = NULL;
md->bucket_next = ctx->mdtab[hash % ctx->mdtab_capacity];
ctx->mdtab[hash % ctx->mdtab_capacity] = md;
ctx->mdtab_count++;
if (ctx->mdtab_count > ctx->mdtab_capacity * 2) {
rehash_mdtab(ctx);
}
unlock(ctx);
return (grpc_mdelem *)md;
}
static void grow_mdtab(grpc_mdctx *ctx) {
size_t capacity = ctx->mdtab_capacity * 2;
size_t i;
internal_metadata **mdtab =
gpr_malloc(sizeof(internal_metadata *) * capacity);
internal_metadata *md, *next;
gpr_uint32 hash;
memset(mdtab, 0, sizeof(internal_metadata *) * capacity);
for (i = 0; i < ctx->mdtab_capacity; i++) {
for (md = ctx->mdtab[i]; md; md = next) {
hash = GRPC_MDSTR_KV_HASH(md->key->hash, md->value->hash);
next = md->bucket_next;
md->bucket_next = mdtab[hash % capacity];
mdtab[hash % capacity] = md;
}
}
gpr_free(ctx->mdtab);
ctx->mdtab = mdtab;
ctx->mdtab_capacity = capacity;
}
grpc_transport_setup_result grpc_server_setup_transport(
grpc_server *s, grpc_transport *transport,
grpc_channel_filter const **extra_filters, size_t num_extra_filters,
grpc_mdctx *mdctx) {
size_t num_filters = s->channel_filter_count + num_extra_filters + 1;
grpc_channel_filter const **filters =
gpr_malloc(sizeof(grpc_channel_filter *) * num_filters);
size_t i;
size_t num_registered_methods;
size_t alloc;
registered_method *rm;
channel_registered_method *crm;
grpc_channel *channel;
channel_data *chand;
grpc_mdstr *host;
grpc_mdstr *method;
gpr_uint32 hash;
gpr_uint32 slots;
gpr_uint32 probes;
gpr_uint32 max_probes = 0;
grpc_transport_setup_result result;
for (i = 0; i < s->channel_filter_count; i++) {
filters[i] = s->channel_filters[i];
}
for (; i < s->channel_filter_count + num_extra_filters; i++) {
filters[i] = extra_filters[i - s->channel_filter_count];
}
filters[i] = &grpc_connected_channel_filter;
for (i = 0; i < s->cq_count; i++) {
grpc_transport_add_to_pollset(transport, grpc_cq_pollset(s->cqs[i]));
}
channel = grpc_channel_create_from_filters(filters, num_filters,
s->channel_args, mdctx, 0);
chand = (channel_data *)grpc_channel_stack_element(
grpc_channel_get_channel_stack(channel), 0)
->channel_data;
chand->server = s;
server_ref(s);
chand->channel = channel;
num_registered_methods = 0;
for (rm = s->registered_methods; rm; rm = rm->next) {
num_registered_methods++;
}
/* build a lookup table phrased in terms of mdstr's in this channels context
to quickly find registered methods */
if (num_registered_methods > 0) {
slots = 2 * num_registered_methods;
alloc = sizeof(channel_registered_method) * slots;
chand->registered_methods = gpr_malloc(alloc);
memset(chand->registered_methods, 0, alloc);
for (rm = s->registered_methods; rm; rm = rm->next) {
host = rm->host ? grpc_mdstr_from_string(mdctx, rm->host) : NULL;
method = grpc_mdstr_from_string(mdctx, rm->method);
hash = GRPC_MDSTR_KV_HASH(host ? host->hash : 0, method->hash);
for (probes = 0; chand->registered_methods[(hash + probes) % slots]
.server_registered_method != NULL;
probes++)
;
if (probes > max_probes) max_probes = probes;
crm = &chand->registered_methods[(hash + probes) % slots];
crm->server_registered_method = rm;
crm->host = host;
crm->method = method;
}
chand->registered_method_slots = slots;
chand->registered_method_max_probes = max_probes;
}
result = grpc_connected_channel_bind_transport(
grpc_channel_get_channel_stack(channel), transport);
gpr_mu_lock(&s->mu);
chand->next = &s->root_channel_data;
chand->prev = chand->next->prev;
chand->next->prev = chand->prev->next = chand;
gpr_mu_unlock(&s->mu);
gpr_free(filters);
return result;
}
/* encode an mdelem */
static void hpack_enc(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem,
framer_state *st) {
uint32_t key_hash = elem->key->hash;
uint32_t elem_hash = GRPC_MDSTR_KV_HASH(key_hash, elem->value->hash);
size_t decoder_space_usage;
uint32_t indices_key;
int should_add_elem;
GPR_ASSERT(GPR_SLICE_LENGTH(elem->key->slice) > 0);
if (GPR_SLICE_START_PTR(elem->key->slice)[0] != ':') { /* regular header */
st->seen_regular_header = 1;
} else {
GPR_ASSERT(
st->seen_regular_header == 0 &&
"Reserved header (colon-prefixed) happening after regular ones.");
}
inc_filter(HASH_FRAGMENT_1(elem_hash), &c->filter_elems_sum, c->filter_elems);
/* is this elem currently in the decoders table? */
if (c->entries_elems[HASH_FRAGMENT_2(elem_hash)] == elem &&
c->indices_elems[HASH_FRAGMENT_2(elem_hash)] > c->tail_remote_index) {
/* HIT: complete element (first cuckoo hash) */
emit_indexed(c, dynidx(c, c->indices_elems[HASH_FRAGMENT_2(elem_hash)]),
st);
return;
}
if (c->entries_elems[HASH_FRAGMENT_3(elem_hash)] == elem &&
c->indices_elems[HASH_FRAGMENT_3(elem_hash)] > c->tail_remote_index) {
/* HIT: complete element (second cuckoo hash) */
emit_indexed(c, dynidx(c, c->indices_elems[HASH_FRAGMENT_3(elem_hash)]),
st);
return;
}
/* should this elem be in the table? */
decoder_space_usage = grpc_mdelem_get_size_in_hpack_table(elem);
should_add_elem = decoder_space_usage < MAX_DECODER_SPACE_USAGE &&
c->filter_elems[HASH_FRAGMENT_1(elem_hash)] >=
c->filter_elems_sum / ONE_ON_ADD_PROBABILITY;
/* no hits for the elem... maybe there's a key? */
indices_key = c->indices_keys[HASH_FRAGMENT_2(key_hash)];
if (c->entries_keys[HASH_FRAGMENT_2(key_hash)] == elem->key &&
indices_key > c->tail_remote_index) {
/* HIT: key (first cuckoo hash) */
if (should_add_elem) {
emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st);
add_elem(c, elem);
return;
} else {
emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st);
return;
}
GPR_UNREACHABLE_CODE(return );
}
indices_key = c->indices_keys[HASH_FRAGMENT_3(key_hash)];
if (c->entries_keys[HASH_FRAGMENT_3(key_hash)] == elem->key &&
indices_key > c->tail_remote_index) {
/* HIT: key (first cuckoo hash) */
if (should_add_elem) {
emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st);
add_elem(c, elem);
return;
} else {
emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st);
return;
}
GPR_UNREACHABLE_CODE(return );
}
/* no elem, key in the table... fall back to literal emission */
if (should_add_elem) {
emit_lithdr_incidx_v(c, elem, st);
add_elem(c, elem);
return;
} else {
emit_lithdr_noidx_v(c, elem, st);
return;
}
GPR_UNREACHABLE_CODE(return );
}
/* add an element to the decoder table */
static void add_elem(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem) {
uint32_t key_hash = elem->key->hash;
uint32_t elem_hash = GRPC_MDSTR_KV_HASH(key_hash, elem->value->hash);
uint32_t new_index = c->tail_remote_index + c->table_elems + 1;
size_t elem_size = grpc_mdelem_get_size_in_hpack_table(elem);
GPR_ASSERT(elem_size < 65536);
if (elem_size > c->max_table_size) {
while (c->table_size > 0) {
evict_entry(c);
}
return;
}
/* Reserve space for this element in the remote table: if this overflows
the current table, drop elements until it fits, matching the decompressor
algorithm */
while (c->table_size + elem_size > c->max_table_size) {
evict_entry(c);
}
GPR_ASSERT(c->table_elems < c->max_table_size);
c->table_elem_size[new_index % c->cap_table_elems] = (uint16_t)elem_size;
c->table_size = (uint16_t)(c->table_size + elem_size);
c->table_elems++;
/* Store this element into {entries,indices}_elem */
if (c->entries_elems[HASH_FRAGMENT_2(elem_hash)] == elem) {
/* already there: update with new index */
c->indices_elems[HASH_FRAGMENT_2(elem_hash)] = new_index;
} else if (c->entries_elems[HASH_FRAGMENT_3(elem_hash)] == elem) {
/* already there (cuckoo): update with new index */
c->indices_elems[HASH_FRAGMENT_3(elem_hash)] = new_index;
} else if (c->entries_elems[HASH_FRAGMENT_2(elem_hash)] == NULL) {
/* not there, but a free element: add */
c->entries_elems[HASH_FRAGMENT_2(elem_hash)] = GRPC_MDELEM_REF(elem);
c->indices_elems[HASH_FRAGMENT_2(elem_hash)] = new_index;
} else if (c->entries_elems[HASH_FRAGMENT_3(elem_hash)] == NULL) {
/* not there (cuckoo), but a free element: add */
c->entries_elems[HASH_FRAGMENT_3(elem_hash)] = GRPC_MDELEM_REF(elem);
c->indices_elems[HASH_FRAGMENT_3(elem_hash)] = new_index;
} else if (c->indices_elems[HASH_FRAGMENT_2(elem_hash)] <
c->indices_elems[HASH_FRAGMENT_3(elem_hash)]) {
/* not there: replace oldest */
GRPC_MDELEM_UNREF(c->entries_elems[HASH_FRAGMENT_2(elem_hash)]);
c->entries_elems[HASH_FRAGMENT_2(elem_hash)] = GRPC_MDELEM_REF(elem);
c->indices_elems[HASH_FRAGMENT_2(elem_hash)] = new_index;
} else {
/* not there: replace oldest */
GRPC_MDELEM_UNREF(c->entries_elems[HASH_FRAGMENT_3(elem_hash)]);
c->entries_elems[HASH_FRAGMENT_3(elem_hash)] = GRPC_MDELEM_REF(elem);
c->indices_elems[HASH_FRAGMENT_3(elem_hash)] = new_index;
}
/* do exactly the same for the key (so we can find by that again too) */
if (c->entries_keys[HASH_FRAGMENT_2(key_hash)] == elem->key) {
c->indices_keys[HASH_FRAGMENT_2(key_hash)] = new_index;
} else if (c->entries_keys[HASH_FRAGMENT_3(key_hash)] == elem->key) {
c->indices_keys[HASH_FRAGMENT_3(key_hash)] = new_index;
} else if (c->entries_keys[HASH_FRAGMENT_2(key_hash)] == NULL) {
c->entries_keys[HASH_FRAGMENT_2(key_hash)] = GRPC_MDSTR_REF(elem->key);
c->indices_keys[HASH_FRAGMENT_2(key_hash)] = new_index;
} else if (c->entries_keys[HASH_FRAGMENT_3(key_hash)] == NULL) {
c->entries_keys[HASH_FRAGMENT_3(key_hash)] = GRPC_MDSTR_REF(elem->key);
c->indices_keys[HASH_FRAGMENT_3(key_hash)] = new_index;
} else if (c->indices_keys[HASH_FRAGMENT_2(key_hash)] <
c->indices_keys[HASH_FRAGMENT_3(key_hash)]) {
GRPC_MDSTR_UNREF(c->entries_keys[HASH_FRAGMENT_2(key_hash)]);
c->entries_keys[HASH_FRAGMENT_2(key_hash)] = GRPC_MDSTR_REF(elem->key);
c->indices_keys[HASH_FRAGMENT_2(key_hash)] = new_index;
} else {
GRPC_MDSTR_UNREF(c->entries_keys[HASH_FRAGMENT_3(key_hash)]);
c->entries_keys[HASH_FRAGMENT_3(key_hash)] = GRPC_MDSTR_REF(elem->key);
c->indices_keys[HASH_FRAGMENT_3(key_hash)] = new_index;
}
}
void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
grpc_transport *transport,
grpc_pollset *accepting_pollset,
const grpc_channel_args *args) {
size_t num_registered_methods;
size_t alloc;
registered_method *rm;
channel_registered_method *crm;
grpc_channel *channel;
channel_data *chand;
grpc_mdstr *host;
grpc_mdstr *method;
uint32_t hash;
size_t slots;
uint32_t probes;
uint32_t max_probes = 0;
grpc_transport_op op;
channel =
grpc_channel_create(exec_ctx, NULL, args, GRPC_SERVER_CHANNEL, transport);
chand = (channel_data *)grpc_channel_stack_element(
grpc_channel_get_channel_stack(channel), 0)
->channel_data;
chand->server = s;
server_ref(s);
chand->channel = channel;
size_t cq_idx;
grpc_completion_queue *accepting_cq = grpc_cq_from_pollset(accepting_pollset);
for (cq_idx = 0; cq_idx < s->cq_count; cq_idx++) {
if (s->cqs[cq_idx] == accepting_cq) break;
}
if (cq_idx == s->cq_count) {
/* completion queue not found: pick a random one to publish new calls to */
cq_idx = (size_t)rand() % s->cq_count;
}
chand->cq_idx = cq_idx;
num_registered_methods = 0;
for (rm = s->registered_methods; rm; rm = rm->next) {
num_registered_methods++;
}
/* build a lookup table phrased in terms of mdstr's in this channels context
to quickly find registered methods */
if (num_registered_methods > 0) {
slots = 2 * num_registered_methods;
alloc = sizeof(channel_registered_method) * slots;
chand->registered_methods = gpr_malloc(alloc);
memset(chand->registered_methods, 0, alloc);
for (rm = s->registered_methods; rm; rm = rm->next) {
host = rm->host ? grpc_mdstr_from_string(rm->host) : NULL;
method = grpc_mdstr_from_string(rm->method);
hash = GRPC_MDSTR_KV_HASH(host ? host->hash : 0, method->hash);
for (probes = 0; chand->registered_methods[(hash + probes) % slots]
.server_registered_method != NULL;
probes++)
;
if (probes > max_probes) max_probes = probes;
crm = &chand->registered_methods[(hash + probes) % slots];
crm->server_registered_method = rm;
crm->flags = rm->flags;
crm->host = host;
crm->method = method;
}
GPR_ASSERT(slots <= UINT32_MAX);
chand->registered_method_slots = (uint32_t)slots;
chand->registered_method_max_probes = max_probes;
}
gpr_mu_lock(&s->mu_global);
chand->next = &s->root_channel_data;
chand->prev = chand->next->prev;
chand->next->prev = chand->prev->next = chand;
gpr_mu_unlock(&s->mu_global);
GRPC_CHANNEL_INTERNAL_REF(channel, "connectivity");
memset(&op, 0, sizeof(op));
op.set_accept_stream = true;
op.set_accept_stream_fn = accept_stream;
op.set_accept_stream_user_data = chand;
op.on_connectivity_state_change = &chand->channel_connectivity_changed;
op.connectivity_state = &chand->connectivity_state;
if (gpr_atm_acq_load(&s->shutdown_flag) != 0) {
op.disconnect_with_error = GRPC_ERROR_CREATE("Server shutdown");
}
grpc_transport_perform_op(exec_ctx, transport, &op);
}
void grpc_mdctx_global_init(void) {
size_t i, j;
if (!g_forced_hash_seed) {
g_hash_seed = (uint32_t)gpr_now(GPR_CLOCK_REALTIME).tv_nsec;
}
g_static_strtab_maxprobe = 0;
g_static_mdtab_maxprobe = 0;
/* build static tables */
memset(g_static_mdtab, 0, sizeof(g_static_mdtab));
memset(g_static_strtab, 0, sizeof(g_static_strtab));
for (i = 0; i < GRPC_STATIC_MDSTR_COUNT; i++) {
grpc_mdstr *elem = &grpc_static_mdstr_table[i];
const char *str = grpc_static_metadata_strings[i];
uint32_t hash = gpr_murmur_hash3(str, strlen(str), g_hash_seed);
*(gpr_slice *)&elem->slice = gpr_slice_from_static_string(str);
*(uint32_t *)&elem->hash = hash;
for (j = 0;; j++) {
size_t idx = (hash + j) % GPR_ARRAY_SIZE(g_static_strtab);
if (g_static_strtab[idx] == NULL) {
g_static_strtab[idx] = &grpc_static_mdstr_table[i];
break;
}
}
if (j > g_static_strtab_maxprobe) {
g_static_strtab_maxprobe = j;
}
}
for (i = 0; i < GRPC_STATIC_MDELEM_COUNT; i++) {
grpc_mdelem *elem = &grpc_static_mdelem_table[i];
grpc_mdstr *key =
&grpc_static_mdstr_table[grpc_static_metadata_elem_indices[2 * i + 0]];
grpc_mdstr *value =
&grpc_static_mdstr_table[grpc_static_metadata_elem_indices[2 * i + 1]];
uint32_t hash = GRPC_MDSTR_KV_HASH(key->hash, value->hash);
*(grpc_mdstr **)&elem->key = key;
*(grpc_mdstr **)&elem->value = value;
for (j = 0;; j++) {
size_t idx = (hash + j) % GPR_ARRAY_SIZE(g_static_mdtab);
if (g_static_mdtab[idx] == NULL) {
g_static_mdtab[idx] = elem;
break;
}
}
if (j > g_static_mdtab_maxprobe) {
g_static_mdtab_maxprobe = j;
}
}
/* initialize shards */
for (i = 0; i < STRTAB_SHARD_COUNT; i++) {
strtab_shard *shard = &g_strtab_shard[i];
gpr_mu_init(&shard->mu);
shard->count = 0;
shard->capacity = INITIAL_STRTAB_CAPACITY;
shard->strs = gpr_malloc(sizeof(*shard->strs) * shard->capacity);
memset(shard->strs, 0, sizeof(*shard->strs) * shard->capacity);
}
for (i = 0; i < MDTAB_SHARD_COUNT; i++) {
mdtab_shard *shard = &g_mdtab_shard[i];
gpr_mu_init(&shard->mu);
shard->count = 0;
gpr_atm_no_barrier_store(&shard->free_estimate, 0);
shard->capacity = INITIAL_MDTAB_CAPACITY;
shard->elems = gpr_malloc(sizeof(*shard->elems) * shard->capacity);
memset(shard->elems, 0, sizeof(*shard->elems) * shard->capacity);
}
}
void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
grpc_transport *transport,
grpc_channel_filter const **extra_filters,
size_t num_extra_filters,
const grpc_channel_args *args) {
size_t num_filters = s->channel_filter_count + num_extra_filters + 1;
grpc_channel_filter const **filters =
gpr_malloc(sizeof(grpc_channel_filter *) * num_filters);
size_t i;
size_t num_registered_methods;
size_t alloc;
registered_method *rm;
channel_registered_method *crm;
grpc_channel *channel;
channel_data *chand;
grpc_mdstr *host;
grpc_mdstr *method;
uint32_t hash;
size_t slots;
uint32_t probes;
uint32_t max_probes = 0;
grpc_transport_op op;
for (i = 0; i < s->channel_filter_count; i++) {
filters[i] = s->channel_filters[i];
}
for (; i < s->channel_filter_count + num_extra_filters; i++) {
filters[i] = extra_filters[i - s->channel_filter_count];
}
filters[i] = &grpc_connected_channel_filter;
for (i = 0; i < s->cq_count; i++) {
memset(&op, 0, sizeof(op));
op.bind_pollset = grpc_cq_pollset(s->cqs[i]);
grpc_transport_perform_op(exec_ctx, transport, &op);
}
channel = grpc_channel_create_from_filters(exec_ctx, NULL, filters,
num_filters, args, 0);
chand = (channel_data *)grpc_channel_stack_element(
grpc_channel_get_channel_stack(channel), 0)->channel_data;
chand->server = s;
server_ref(s);
chand->channel = channel;
num_registered_methods = 0;
for (rm = s->registered_methods; rm; rm = rm->next) {
num_registered_methods++;
}
/* build a lookup table phrased in terms of mdstr's in this channels context
to quickly find registered methods */
if (num_registered_methods > 0) {
slots = 2 * num_registered_methods;
alloc = sizeof(channel_registered_method) * slots;
chand->registered_methods = gpr_malloc(alloc);
memset(chand->registered_methods, 0, alloc);
for (rm = s->registered_methods; rm; rm = rm->next) {
host = rm->host ? grpc_mdstr_from_string(rm->host) : NULL;
method = grpc_mdstr_from_string(rm->method);
hash = GRPC_MDSTR_KV_HASH(host ? host->hash : 0, method->hash);
for (probes = 0; chand->registered_methods[(hash + probes) % slots]
.server_registered_method != NULL;
probes++)
;
if (probes > max_probes) max_probes = probes;
crm = &chand->registered_methods[(hash + probes) % slots];
crm->server_registered_method = rm;
crm->host = host;
crm->method = method;
}
GPR_ASSERT(slots <= UINT32_MAX);
chand->registered_method_slots = (uint32_t)slots;
chand->registered_method_max_probes = max_probes;
}
grpc_connected_channel_bind_transport(grpc_channel_get_channel_stack(channel),
transport);
gpr_mu_lock(&s->mu_global);
chand->next = &s->root_channel_data;
chand->prev = chand->next->prev;
chand->next->prev = chand->prev->next = chand;
gpr_mu_unlock(&s->mu_global);
gpr_free((void *)filters);
GRPC_CHANNEL_INTERNAL_REF(channel, "connectivity");
memset(&op, 0, sizeof(op));
op.set_accept_stream = accept_stream;
op.set_accept_stream_user_data = chand;
op.on_connectivity_state_change = &chand->channel_connectivity_changed;
op.connectivity_state = &chand->connectivity_state;
op.disconnect = gpr_atm_acq_load(&s->shutdown_flag) != 0;
grpc_transport_perform_op(exec_ctx, transport, &op);
}
/* encode an mdelem */
static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c,
grpc_mdelem elem, framer_state *st) {
GPR_ASSERT(GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)) > 0);
if (GRPC_SLICE_START_PTR(GRPC_MDKEY(elem))[0] != ':') { /* regular header */
st->seen_regular_header = 1;
} else {
GPR_ASSERT(
st->seen_regular_header == 0 &&
"Reserved header (colon-prefixed) happening after regular ones.");
}
if (grpc_http_trace && !GRPC_MDELEM_IS_INTERNED(elem)) {
char *k = grpc_slice_to_c_string(GRPC_MDKEY(elem));
char *v = grpc_slice_to_c_string(GRPC_MDVALUE(elem));
gpr_log(
GPR_DEBUG,
"Encode: '%s: %s', elem_interned=%d [%d], k_interned=%d, v_interned=%d",
k, v, GRPC_MDELEM_IS_INTERNED(elem), GRPC_MDELEM_STORAGE(elem),
grpc_slice_is_interned(GRPC_MDKEY(elem)),
grpc_slice_is_interned(GRPC_MDVALUE(elem)));
gpr_free(k);
gpr_free(v);
}
if (!GRPC_MDELEM_IS_INTERNED(elem)) {
emit_lithdr_noidx_v(c, elem, st);
return;
}
uint32_t key_hash;
uint32_t value_hash;
uint32_t elem_hash;
size_t decoder_space_usage;
uint32_t indices_key;
int should_add_elem;
key_hash = grpc_slice_hash(GRPC_MDKEY(elem));
value_hash = grpc_slice_hash(GRPC_MDVALUE(elem));
elem_hash = GRPC_MDSTR_KV_HASH(key_hash, value_hash);
inc_filter(HASH_FRAGMENT_1(elem_hash), &c->filter_elems_sum, c->filter_elems);
/* is this elem currently in the decoders table? */
if (grpc_mdelem_eq(c->entries_elems[HASH_FRAGMENT_2(elem_hash)], elem) &&
c->indices_elems[HASH_FRAGMENT_2(elem_hash)] > c->tail_remote_index) {
/* HIT: complete element (first cuckoo hash) */
emit_indexed(c, dynidx(c, c->indices_elems[HASH_FRAGMENT_2(elem_hash)]),
st);
return;
}
if (grpc_mdelem_eq(c->entries_elems[HASH_FRAGMENT_3(elem_hash)], elem) &&
c->indices_elems[HASH_FRAGMENT_3(elem_hash)] > c->tail_remote_index) {
/* HIT: complete element (second cuckoo hash) */
emit_indexed(c, dynidx(c, c->indices_elems[HASH_FRAGMENT_3(elem_hash)]),
st);
return;
}
/* should this elem be in the table? */
decoder_space_usage = grpc_mdelem_get_size_in_hpack_table(elem);
should_add_elem = decoder_space_usage < MAX_DECODER_SPACE_USAGE &&
c->filter_elems[HASH_FRAGMENT_1(elem_hash)] >=
c->filter_elems_sum / ONE_ON_ADD_PROBABILITY;
/* no hits for the elem... maybe there's a key? */
indices_key = c->indices_keys[HASH_FRAGMENT_2(key_hash)];
if (grpc_slice_eq(c->entries_keys[HASH_FRAGMENT_2(key_hash)],
GRPC_MDKEY(elem)) &&
indices_key > c->tail_remote_index) {
/* HIT: key (first cuckoo hash) */
if (should_add_elem) {
emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st);
add_elem(exec_ctx, c, elem);
return;
} else {
emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st);
return;
}
GPR_UNREACHABLE_CODE(return );
}
indices_key = c->indices_keys[HASH_FRAGMENT_3(key_hash)];
if (grpc_slice_eq(c->entries_keys[HASH_FRAGMENT_3(key_hash)],
GRPC_MDKEY(elem)) &&
indices_key > c->tail_remote_index) {
/* HIT: key (first cuckoo hash) */
if (should_add_elem) {
emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st);
add_elem(exec_ctx, c, elem);
return;
} else {
emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st);
return;
}
GPR_UNREACHABLE_CODE(return );
}
/* no elem, key in the table... fall back to literal emission */
if (should_add_elem) {
emit_lithdr_incidx_v(c, elem, st);
add_elem(exec_ctx, c, elem);
return;
} else {
emit_lithdr_noidx_v(c, elem, st);
return;
}
GPR_UNREACHABLE_CODE(return );
}
/* add an element to the decoder table */
static void add_elem(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c,
grpc_mdelem elem) {
GPR_ASSERT(GRPC_MDELEM_IS_INTERNED(elem));
uint32_t key_hash = grpc_slice_hash(GRPC_MDKEY(elem));
uint32_t value_hash = grpc_slice_hash(GRPC_MDVALUE(elem));
uint32_t elem_hash = GRPC_MDSTR_KV_HASH(key_hash, value_hash);
uint32_t new_index = c->tail_remote_index + c->table_elems + 1;
size_t elem_size = grpc_mdelem_get_size_in_hpack_table(elem);
GPR_ASSERT(elem_size < 65536);
if (elem_size > c->max_table_size) {
while (c->table_size > 0) {
evict_entry(c);
}
return;
}
/* Reserve space for this element in the remote table: if this overflows
the current table, drop elements until it fits, matching the decompressor
algorithm */
while (c->table_size + elem_size > c->max_table_size) {
evict_entry(c);
}
GPR_ASSERT(c->table_elems < c->max_table_size);
c->table_elem_size[new_index % c->cap_table_elems] = (uint16_t)elem_size;
c->table_size = (uint16_t)(c->table_size + elem_size);
c->table_elems++;
/* Store this element into {entries,indices}_elem */
if (grpc_mdelem_eq(c->entries_elems[HASH_FRAGMENT_2(elem_hash)], elem)) {
/* already there: update with new index */
c->indices_elems[HASH_FRAGMENT_2(elem_hash)] = new_index;
} else if (grpc_mdelem_eq(c->entries_elems[HASH_FRAGMENT_3(elem_hash)],
elem)) {
/* already there (cuckoo): update with new index */
c->indices_elems[HASH_FRAGMENT_3(elem_hash)] = new_index;
} else if (GRPC_MDISNULL(c->entries_elems[HASH_FRAGMENT_2(elem_hash)])) {
/* not there, but a free element: add */
c->entries_elems[HASH_FRAGMENT_2(elem_hash)] = GRPC_MDELEM_REF(elem);
c->indices_elems[HASH_FRAGMENT_2(elem_hash)] = new_index;
} else if (GRPC_MDISNULL(c->entries_elems[HASH_FRAGMENT_3(elem_hash)])) {
/* not there (cuckoo), but a free element: add */
c->entries_elems[HASH_FRAGMENT_3(elem_hash)] = GRPC_MDELEM_REF(elem);
c->indices_elems[HASH_FRAGMENT_3(elem_hash)] = new_index;
} else if (c->indices_elems[HASH_FRAGMENT_2(elem_hash)] <
c->indices_elems[HASH_FRAGMENT_3(elem_hash)]) {
/* not there: replace oldest */
GRPC_MDELEM_UNREF(exec_ctx, c->entries_elems[HASH_FRAGMENT_2(elem_hash)]);
c->entries_elems[HASH_FRAGMENT_2(elem_hash)] = GRPC_MDELEM_REF(elem);
c->indices_elems[HASH_FRAGMENT_2(elem_hash)] = new_index;
} else {
/* not there: replace oldest */
GRPC_MDELEM_UNREF(exec_ctx, c->entries_elems[HASH_FRAGMENT_3(elem_hash)]);
c->entries_elems[HASH_FRAGMENT_3(elem_hash)] = GRPC_MDELEM_REF(elem);
c->indices_elems[HASH_FRAGMENT_3(elem_hash)] = new_index;
}
/* do exactly the same for the key (so we can find by that again too) */
if (grpc_slice_eq(c->entries_keys[HASH_FRAGMENT_2(key_hash)],
GRPC_MDKEY(elem))) {
c->indices_keys[HASH_FRAGMENT_2(key_hash)] = new_index;
} else if (grpc_slice_eq(c->entries_keys[HASH_FRAGMENT_3(key_hash)],
GRPC_MDKEY(elem))) {
c->indices_keys[HASH_FRAGMENT_3(key_hash)] = new_index;
} else if (c->entries_keys[HASH_FRAGMENT_2(key_hash)].refcount ==
&terminal_slice_refcount) {
c->entries_keys[HASH_FRAGMENT_2(key_hash)] =
grpc_slice_ref_internal(GRPC_MDKEY(elem));
c->indices_keys[HASH_FRAGMENT_2(key_hash)] = new_index;
} else if (c->entries_keys[HASH_FRAGMENT_3(key_hash)].refcount ==
&terminal_slice_refcount) {
c->entries_keys[HASH_FRAGMENT_3(key_hash)] =
grpc_slice_ref_internal(GRPC_MDKEY(elem));
c->indices_keys[HASH_FRAGMENT_3(key_hash)] = new_index;
} else if (c->indices_keys[HASH_FRAGMENT_2(key_hash)] <
c->indices_keys[HASH_FRAGMENT_3(key_hash)]) {
grpc_slice_unref_internal(exec_ctx,
c->entries_keys[HASH_FRAGMENT_2(key_hash)]);
c->entries_keys[HASH_FRAGMENT_2(key_hash)] =
grpc_slice_ref_internal(GRPC_MDKEY(elem));
c->indices_keys[HASH_FRAGMENT_2(key_hash)] = new_index;
} else {
grpc_slice_unref_internal(exec_ctx,
c->entries_keys[HASH_FRAGMENT_3(key_hash)]);
c->entries_keys[HASH_FRAGMENT_3(key_hash)] =
grpc_slice_ref_internal(GRPC_MDKEY(elem));
c->indices_keys[HASH_FRAGMENT_3(key_hash)] = new_index;
}
}