static void verify_partial_carnage_round_robin(
const servers_fixture *f, grpc_channel *client,
const request_sequences *sequences, const size_t num_iters) {
int *expected_connection_sequence;
size_t i;
const size_t expected_seq_length = f->num_servers;
/* verify conn. seq. expectation */
/* get the first sequence of "num_servers" elements */
expected_connection_sequence = gpr_malloc(sizeof(int) * expected_seq_length);
memcpy(expected_connection_sequence, sequences->connections,
sizeof(int) * expected_seq_length);
for (i = 0; i < num_iters / 2; i++) {
const int actual = sequences->connections[i];
const int expected = expected_connection_sequence[i % expected_seq_length];
if (actual != expected) {
print_failed_expectations(expected_connection_sequence,
sequences->connections, expected_seq_length,
num_iters);
abort();
}
}
/* second half of the iterations go without response */
for (; i < num_iters; i++) {
GPR_ASSERT(sequences->connections[i] == -1);
}
/* We can assert that the first client channel state should be READY, when all
* servers were available */
grpc_connectivity_state actual = sequences->connectivity_states[0];
grpc_connectivity_state expected = GRPC_CHANNEL_READY;
if (actual != expected) {
gpr_log(GPR_ERROR,
"CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
"at iteration #%d",
grpc_connectivity_state_name(expected),
grpc_connectivity_state_name(actual), 0);
abort();
}
/* ... and that the last one shouldn't be READY (or SHUTDOWN): all servers are
* gone. It may be all other states (IDLE, CONNECTING, TRANSIENT_FAILURE), as
* the policy transitions while attempting to reconnect. */
actual = sequences->connectivity_states[num_iters - 1];
for (i = 0; i < sequences->n; i++) {
if (actual == GRPC_CHANNEL_READY || actual == GRPC_CHANNEL_SHUTDOWN) {
gpr_log(GPR_ERROR,
"CONNECTIVITY STATUS SEQUENCE FAILURE: got unexpected state "
"'%s' at iteration #%d.",
grpc_connectivity_state_name(actual), (int)i);
abort();
}
}
gpr_free(expected_connection_sequence);
}
/* At the start of the second iteration, all but the first and last servers (as
* given in "f") are killed */
static void verify_vanishing_floor_round_robin(
const servers_fixture *f, grpc_channel *client,
const request_sequences *sequences, const size_t num_iters) {
int *expected_connection_sequence;
const size_t expected_seq_length = 2;
size_t i;
/* verify conn. seq. expectation */
/* copy the first full sequence (without -1s) */
expected_connection_sequence = gpr_malloc(sizeof(int) * expected_seq_length);
memcpy(expected_connection_sequence, sequences->connections + 2,
expected_seq_length * sizeof(int));
/* first two elements of the sequence should be [0 (1st server), -1 (failure)]
*/
GPR_ASSERT(sequences->connections[0] == 0);
GPR_ASSERT(sequences->connections[1] == -1);
/* the next two element must be [3, 0], repeating from that point: the 3 is
* brought forth by servers 1 and 2 disappearing after the intial pick of 0 */
GPR_ASSERT(sequences->connections[2] == 3);
GPR_ASSERT(sequences->connections[3] == 0);
/* make sure that the expectation obliges */
for (i = 2; i < num_iters; i++) {
const int actual = sequences->connections[i];
const int expected = expected_connection_sequence[i % expected_seq_length];
if (actual != expected) {
print_failed_expectations(expected_connection_sequence,
sequences->connections, expected_seq_length,
num_iters);
abort();
}
}
/* There's always at least one subchannel READY (connected), therefore the
* overall state of the client channel is READY at all times. */
for (i = 0; i < sequences->n; i++) {
const grpc_connectivity_state actual = sequences->connectivity_states[i];
const grpc_connectivity_state expected = GRPC_CHANNEL_READY;
if (actual != expected) {
gpr_log(GPR_ERROR,
"CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
"at iteration #%d",
grpc_connectivity_state_name(expected),
grpc_connectivity_state_name(actual), (int)i);
abort();
}
}
gpr_free(expected_connection_sequence);
}
bool grpc_connectivity_state_notify_on_state_change(
grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker,
grpc_connectivity_state *current, grpc_closure *notify) {
grpc_connectivity_state cur =
(grpc_connectivity_state)gpr_atm_no_barrier_load(
&tracker->current_state_atm);
if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) {
if (current == NULL) {
gpr_log(GPR_DEBUG, "CONWATCH: %p %s: unsubscribe notify=%p", tracker,
tracker->name, notify);
} else {
gpr_log(GPR_DEBUG, "CONWATCH: %p %s: from %s [cur=%s] notify=%p", tracker,
tracker->name, grpc_connectivity_state_name(*current),
grpc_connectivity_state_name(cur), notify);
}
}
if (current == NULL) {
grpc_connectivity_state_watcher *w = tracker->watchers;
if (w != NULL && w->notify == notify) {
GRPC_CLOSURE_SCHED(exec_ctx, notify, GRPC_ERROR_CANCELLED);
tracker->watchers = w->next;
gpr_free(w);
return false;
}
while (w != NULL) {
grpc_connectivity_state_watcher *rm_candidate = w->next;
if (rm_candidate != NULL && rm_candidate->notify == notify) {
GRPC_CLOSURE_SCHED(exec_ctx, notify, GRPC_ERROR_CANCELLED);
w->next = w->next->next;
gpr_free(rm_candidate);
return false;
}
w = w->next;
}
return false;
} else {
if (cur != *current) {
*current = cur;
GRPC_CLOSURE_SCHED(exec_ctx, notify,
GRPC_ERROR_REF(tracker->current_error));
} else {
grpc_connectivity_state_watcher *w =
(grpc_connectivity_state_watcher *)gpr_malloc(sizeof(*w));
w->current = current;
w->notify = notify;
w->next = tracker->watchers;
tracker->watchers = w;
}
return cur == GRPC_CHANNEL_IDLE;
}
}
static void test_connectivity_state_name(void) {
gpr_log(GPR_DEBUG, "test_connectivity_state_name");
GPR_ASSERT(0 ==
strcmp(grpc_connectivity_state_name(GRPC_CHANNEL_IDLE), "IDLE"));
GPR_ASSERT(0 == strcmp(grpc_connectivity_state_name(GRPC_CHANNEL_CONNECTING),
"CONNECTING"));
GPR_ASSERT(0 ==
strcmp(grpc_connectivity_state_name(GRPC_CHANNEL_READY), "READY"));
GPR_ASSERT(
0 == strcmp(grpc_connectivity_state_name(GRPC_CHANNEL_TRANSIENT_FAILURE),
"TRANSIENT_FAILURE"));
GPR_ASSERT(0 == strcmp(grpc_connectivity_state_name(GRPC_CHANNEL_SHUTDOWN),
"SHUTDOWN"));
}
static void verify_total_carnage_round_robin(const servers_fixture *f,
grpc_channel *client,
const request_sequences *sequences,
const size_t num_iters) {
for (size_t i = 0; i < num_iters; i++) {
const int actual = sequences->connections[i];
const int expected = -1;
if (actual != expected) {
gpr_log(
GPR_ERROR,
"CONNECTION SEQUENCE FAILURE: expected %d, got %d at iteration #%d",
expected, actual, (int)i);
abort();
}
}
/* No server is ever available. There should be no READY states (or SHUTDOWN).
* Note that all other states (IDLE, CONNECTING, TRANSIENT_FAILURE) are still
* possible, as the policy transitions while attempting to reconnect. */
for (size_t i = 0; i < sequences->n; i++) {
const grpc_connectivity_state actual = sequences->connectivity_states[i];
if (actual == GRPC_CHANNEL_READY || actual == GRPC_CHANNEL_SHUTDOWN) {
gpr_log(GPR_ERROR,
"CONNECTIVITY STATUS SEQUENCE FAILURE: got unexpected state "
"'%s' at iteration #%d.",
grpc_connectivity_state_name(actual), (int)i);
abort();
}
}
}
int grpc_connectivity_state_notify_on_state_change(
grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker,
grpc_connectivity_state *current, grpc_closure *notify) {
if (grpc_connectivity_state_trace) {
if (current == NULL) {
gpr_log(GPR_DEBUG, "CONWATCH: %p %s: unsubscribe notify=%p", tracker,
tracker->name, notify);
} else {
gpr_log(GPR_DEBUG, "CONWATCH: %p %s: from %s [cur=%s] notify=%p", tracker,
tracker->name, grpc_connectivity_state_name(*current),
grpc_connectivity_state_name(tracker->current_state), notify);
}
}
if (current == NULL) {
grpc_connectivity_state_watcher *w = tracker->watchers;
if (w != NULL && w->notify == notify) {
grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL);
tracker->watchers = w->next;
gpr_free(w);
return 0;
}
while (w != NULL) {
grpc_connectivity_state_watcher *rm_candidate = w->next;
if (rm_candidate != NULL && rm_candidate->notify == notify) {
grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL);
w->next = w->next->next;
gpr_free(rm_candidate);
return 0;
}
w = w->next;
}
return 0;
} else {
if (tracker->current_state != *current) {
*current = tracker->current_state;
grpc_exec_ctx_sched(exec_ctx, notify,
GRPC_ERROR_REF(tracker->current_error), NULL);
} else {
grpc_connectivity_state_watcher *w = gpr_malloc(sizeof(*w));
w->current = current;
w->notify = notify;
w->next = tracker->watchers;
tracker->watchers = w;
}
return tracker->current_state == GRPC_CHANNEL_IDLE;
}
}
grpc_connectivity_state grpc_connectivity_state_check(
grpc_connectivity_state_tracker *tracker) {
if (grpc_connectivity_state_trace) {
gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name,
grpc_connectivity_state_name(tracker->current_state));
}
return tracker->current_state;
}
void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
grpc_connectivity_state_tracker *tracker,
grpc_connectivity_state state,
grpc_error *error, const char *reason) {
grpc_connectivity_state cur =
(grpc_connectivity_state)gpr_atm_no_barrier_load(
&tracker->current_state_atm);
grpc_connectivity_state_watcher *w;
if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) {
const char *error_string = grpc_error_string(error);
gpr_log(GPR_DEBUG, "SET: %p %s: %s --> %s [%s] error=%p %s", tracker,
tracker->name, grpc_connectivity_state_name(cur),
grpc_connectivity_state_name(state), reason, error, error_string);
}
switch (state) {
case GRPC_CHANNEL_INIT:
case GRPC_CHANNEL_CONNECTING:
case GRPC_CHANNEL_IDLE:
case GRPC_CHANNEL_READY:
GPR_ASSERT(error == GRPC_ERROR_NONE);
break;
case GRPC_CHANNEL_SHUTDOWN:
case GRPC_CHANNEL_TRANSIENT_FAILURE:
GPR_ASSERT(error != GRPC_ERROR_NONE);
break;
}
GRPC_ERROR_UNREF(tracker->current_error);
tracker->current_error = error;
if (cur == state) {
return;
}
GPR_ASSERT(cur != GRPC_CHANNEL_SHUTDOWN);
gpr_atm_no_barrier_store(&tracker->current_state_atm, state);
while ((w = tracker->watchers) != NULL) {
*w->current = state;
tracker->watchers = w->next;
if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) {
gpr_log(GPR_DEBUG, "NOTIFY: %p %s: %p", tracker, tracker->name,
w->notify);
}
GRPC_CLOSURE_SCHED(exec_ctx, w->notify,
GRPC_ERROR_REF(tracker->current_error));
gpr_free(w);
}
}
static void verify_vanilla_round_robin(const servers_fixture *f,
grpc_channel *client,
const request_sequences *sequences,
const size_t num_iters) {
const size_t expected_seq_length = f->num_servers;
/* verify conn. seq. expectation */
/* get the first sequence of "num_servers" elements */
int *expected_connection_sequence =
gpr_malloc(sizeof(int) * expected_seq_length);
memcpy(expected_connection_sequence, sequences->connections,
sizeof(int) * expected_seq_length);
for (size_t i = 0; i < num_iters; i++) {
const int actual = sequences->connections[i];
const int expected = expected_connection_sequence[i % expected_seq_length];
if (actual != expected) {
gpr_log(
GPR_ERROR,
"CONNECTION SEQUENCE FAILURE: expected %d, got %d at iteration #%d",
expected, actual, (int)i);
abort();
}
}
/* All servers are available, therefore all client subchannels are READY, even
* when we only need one for the client channel state to be READY */
for (size_t i = 0; i < sequences->n; i++) {
const grpc_connectivity_state actual = sequences->connectivity_states[i];
const grpc_connectivity_state expected = GRPC_CHANNEL_READY;
if (actual != expected) {
gpr_log(GPR_ERROR,
"CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
"at iteration #%d",
grpc_connectivity_state_name(expected),
grpc_connectivity_state_name(actual), (int)i);
abort();
}
}
gpr_free(expected_connection_sequence);
}
grpc_connectivity_state grpc_connectivity_state_check(
grpc_connectivity_state_tracker *tracker, grpc_error **error) {
if (grpc_connectivity_state_trace) {
gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name,
grpc_connectivity_state_name(tracker->current_state));
}
if (error != NULL) {
*error = GRPC_ERROR_REF(tracker->current_error);
}
return tracker->current_state;
}
int grpc_connectivity_state_notify_on_state_change(
grpc_connectivity_state_tracker *tracker, grpc_connectivity_state *current,
grpc_iomgr_closure *notify) {
if (grpc_connectivity_state_trace) {
gpr_log(GPR_DEBUG, "CONWATCH: %s: from %s [cur=%s]", tracker->name,
grpc_connectivity_state_name(*current),
grpc_connectivity_state_name(tracker->current_state));
}
if (tracker->current_state != *current) {
*current = tracker->current_state;
grpc_iomgr_add_callback(notify);
} else {
grpc_connectivity_state_watcher *w = gpr_malloc(sizeof(*w));
w->current = current;
w->notify = notify;
w->next = tracker->watchers;
tracker->watchers = w;
}
return tracker->current_state == GRPC_CHANNEL_IDLE;
}
grpc_connectivity_state grpc_connectivity_state_check(
grpc_connectivity_state_tracker *tracker) {
grpc_connectivity_state cur =
(grpc_connectivity_state)gpr_atm_no_barrier_load(
&tracker->current_state_atm);
if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) {
gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name,
grpc_connectivity_state_name(cur));
}
return cur;
}
void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
grpc_connectivity_state_tracker *tracker,
grpc_connectivity_state state,
const char *reason) {
grpc_connectivity_state_watcher *w;
if (grpc_connectivity_state_trace) {
gpr_log(GPR_DEBUG, "SET: %p %s: %s --> %s [%s]", tracker, tracker->name,
grpc_connectivity_state_name(tracker->current_state),
grpc_connectivity_state_name(state), reason);
}
if (tracker->current_state == state) {
return;
}
GPR_ASSERT(tracker->current_state != GRPC_CHANNEL_FATAL_FAILURE);
tracker->current_state = state;
while ((w = tracker->watchers) != NULL) {
*w->current = tracker->current_state;
tracker->watchers = w->next;
grpc_exec_ctx_enqueue(exec_ctx, w->notify, 1);
gpr_free(w);
}
}
void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
grpc_connectivity_state_tracker *tracker,
grpc_connectivity_state state,
grpc_error *error, const char *reason) {
grpc_connectivity_state_watcher *w;
if (grpc_connectivity_state_trace) {
const char *error_string = grpc_error_string(error);
gpr_log(GPR_DEBUG, "SET: %p %s: %s --> %s [%s] error=%p %s", tracker,
tracker->name, grpc_connectivity_state_name(tracker->current_state),
grpc_connectivity_state_name(state), reason, error, error_string);
grpc_error_free_string(error_string);
}
switch (state) {
case GRPC_CHANNEL_CONNECTING:
case GRPC_CHANNEL_IDLE:
case GRPC_CHANNEL_READY:
GPR_ASSERT(error == GRPC_ERROR_NONE);
break;
case GRPC_CHANNEL_SHUTDOWN:
case GRPC_CHANNEL_TRANSIENT_FAILURE:
GPR_ASSERT(error != GRPC_ERROR_NONE);
break;
}
GRPC_ERROR_UNREF(tracker->current_error);
tracker->current_error = error;
if (tracker->current_state == state) {
return;
}
GPR_ASSERT(tracker->current_state != GRPC_CHANNEL_SHUTDOWN);
tracker->current_state = state;
while ((w = tracker->watchers) != NULL) {
*w->current = tracker->current_state;
tracker->watchers = w->next;
grpc_exec_ctx_sched(exec_ctx, w->notify,
GRPC_ERROR_REF(tracker->current_error), NULL);
gpr_free(w);
}
}
grpc_connectivity_state grpc_connectivity_state_get(
grpc_connectivity_state_tracker *tracker, grpc_error **error) {
grpc_connectivity_state cur =
(grpc_connectivity_state)gpr_atm_no_barrier_load(
&tracker->current_state_atm);
if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) {
gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name,
grpc_connectivity_state_name(cur));
}
if (error != NULL) {
*error = GRPC_ERROR_REF(tracker->current_error);
}
return cur;
}
char *grpc_transport_op_string(grpc_transport_op *op) {
char *tmp;
char *out;
bool first = true;
gpr_strvec b;
gpr_strvec_init(&b);
if (op->on_connectivity_state_change != NULL) {
if (!first) gpr_strvec_add(&b, gpr_strdup(" "));
first = false;
if (op->connectivity_state != NULL) {
gpr_asprintf(&tmp, "ON_CONNECTIVITY_STATE_CHANGE:p=%p:from=%s",
op->on_connectivity_state_change,
grpc_connectivity_state_name(*op->connectivity_state));
gpr_strvec_add(&b, tmp);
} else {
gpr_asprintf(&tmp, "ON_CONNECTIVITY_STATE_CHANGE:p=%p:unsubscribe",
op->on_connectivity_state_change);
gpr_strvec_add(&b, tmp);
}
}
if (op->disconnect_with_error != GRPC_ERROR_NONE) {
if (!first) gpr_strvec_add(&b, gpr_strdup(" "));
first = false;
const char *err = grpc_error_string(op->disconnect_with_error);
gpr_asprintf(&tmp, "DISCONNECT:%s", err);
gpr_strvec_add(&b, tmp);
grpc_error_free_string(err);
}
if (op->send_goaway) {
if (!first) gpr_strvec_add(&b, gpr_strdup(" "));
first = false;
char *msg = op->goaway_message == NULL
? "null"
: grpc_dump_slice(*op->goaway_message,
GPR_DUMP_ASCII | GPR_DUMP_HEX);
gpr_asprintf(&tmp, "SEND_GOAWAY:status=%d:msg=%s", op->goaway_status, msg);
if (op->goaway_message != NULL) gpr_free(msg);
gpr_strvec_add(&b, tmp);
}
if (op->set_accept_stream) {
if (!first) gpr_strvec_add(&b, gpr_strdup(" "));
first = false;
gpr_asprintf(&tmp, "SET_ACCEPT_STREAM:%p(%p,...)", op->set_accept_stream_fn,
op->set_accept_stream_user_data);
gpr_strvec_add(&b, tmp);
}
if (op->bind_pollset != NULL) {
if (!first) gpr_strvec_add(&b, gpr_strdup(" "));
first = false;
gpr_strvec_add(&b, gpr_strdup("BIND_POLLSET"));
}
if (op->bind_pollset_set != NULL) {
if (!first) gpr_strvec_add(&b, gpr_strdup(" "));
first = false;
gpr_strvec_add(&b, gpr_strdup("BIND_POLLSET_SET"));
}
if (op->send_ping != NULL) {
if (!first) gpr_strvec_add(&b, gpr_strdup(" "));
first = false;
gpr_strvec_add(&b, gpr_strdup("SEND_PING"));
}
out = gpr_strvec_flatten(&b, NULL);
gpr_strvec_destroy(&b);
return out;
}
static void verify_rebirth_round_robin(const servers_fixture *f,
grpc_channel *client,
const request_sequences *sequences,
const size_t num_iters) {
dump_array("actual_connection_sequence", sequences->connections, num_iters);
/* first iteration succeeds */
GPR_ASSERT(sequences->connections[0] != -1);
/* then we fail for a while... */
GPR_ASSERT(sequences->connections[1] == -1);
/* ... but should be up eventually */
size_t first_iter_back_up = ~0ul;
for (size_t i = 2; i < sequences->n; ++i) {
if (sequences->connections[i] != -1) {
first_iter_back_up = i;
break;
}
}
GPR_ASSERT(first_iter_back_up != ~0ul);
/* We can assert that the first client channel state should be READY, when all
* servers were available; same thing for the last one. In the middle
* somewhere there must exist at least one TRANSIENT_FAILURE */
grpc_connectivity_state actual = sequences->connectivity_states[0];
grpc_connectivity_state expected = GRPC_CHANNEL_READY;
if (actual != expected) {
gpr_log(GPR_ERROR,
"CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
"at iteration #%d",
grpc_connectivity_state_name(expected),
grpc_connectivity_state_name(actual), 0);
abort();
}
actual = sequences->connectivity_states[num_iters - 1];
expected = GRPC_CHANNEL_READY;
if (actual != expected) {
gpr_log(GPR_ERROR,
"CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
"at iteration #%d",
grpc_connectivity_state_name(expected),
grpc_connectivity_state_name(actual), (int)num_iters - 1);
abort();
}
bool found_failure_status = false;
for (size_t i = 1; i < sequences->n - 1; i++) {
if (sequences->connectivity_states[i] == GRPC_CHANNEL_TRANSIENT_FAILURE) {
found_failure_status = true;
break;
}
}
if (!found_failure_status) {
gpr_log(
GPR_ERROR,
"CONNECTIVITY STATUS SEQUENCE FAILURE: "
"GRPC_CHANNEL_TRANSIENT_FAILURE status not found. Got the following "
"instead:");
for (size_t i = 0; i < num_iters; i++) {
gpr_log(GPR_ERROR, "[%d]: %s", (int)i,
grpc_connectivity_state_name(sequences->connectivity_states[i]));
}
}
}