// Writes the given number of records of random size (up to kMaxRecordSize) and
// random data to the specified log.
static void writer_thread(void* arg) {
writer_thread_args* args = (writer_thread_args*)arg;
// Maximum number of times to spin between writes.
static const int MAX_SPIN_COUNT = 50;
int records_written = 0;
if (VERBOSE) {
printf(" Writer %d starting\n", args->index);
}
while (records_written < args->num_records) {
records_written += write_records_to_log(args->index, args->record_size,
args->num_records - records_written,
MAX_SPIN_COUNT);
if (records_written < args->num_records) {
// Ran out of log space. Sleep for a bit and let the reader catch up.
// This should never happen for circular logs.
if (VERBOSE) {
printf(
" Writer %d stalled due to out-of-space: %d out of %d "
"written\n",
args->index, records_written, args->num_records);
}
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(10));
}
}
// Done. Decrement count and signal.
gpr_mu_lock(args->mu);
(*args->count)--;
gpr_cv_signal(args->done);
if (VERBOSE) {
printf(" Writer %d done\n", args->index);
}
gpr_mu_unlock(args->mu);
}
static void verifier(grpc_server *server, grpc_completion_queue *cq,
void *registered_method) {
while (grpc_server_has_open_connections(server)) {
GPR_ASSERT(grpc_completion_queue_next(
cq, grpc_timeout_milliseconds_to_deadline(20), NULL)
.type == GRPC_QUEUE_TIMEOUT);
}
}
void create_loop_destroy(void *addr) {
for (int i = 0; i < NUM_OUTER_LOOPS; ++i) {
grpc_completion_queue *cq = grpc_completion_queue_create_for_next(NULL);
grpc_channel *chan = grpc_insecure_channel_create((char *)addr, NULL, NULL);
for (int j = 0; j < NUM_INNER_LOOPS; ++j) {
gpr_timespec later_time =
grpc_timeout_milliseconds_to_deadline(DELAY_MILLIS);
grpc_connectivity_state state =
grpc_channel_check_connectivity_state(chan, 1);
grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL);
gpr_timespec poll_time =
grpc_timeout_milliseconds_to_deadline(POLL_MILLIS);
GPR_ASSERT(grpc_completion_queue_next(cq, poll_time, NULL).type ==
GRPC_OP_COMPLETE);
/* check that the watcher from "watch state" was free'd */
GPR_ASSERT(grpc_channel_num_external_connectivity_watchers(chan) == 0);
}
grpc_channel_destroy(chan);
grpc_completion_queue_destroy(cq);
}
}
void watches_with_short_timeouts(void *addr) {
for (int i = 0; i < NUM_OUTER_LOOPS_SHORT_TIMEOUTS; ++i) {
grpc_completion_queue *cq = grpc_completion_queue_create_for_next(NULL);
grpc_channel *chan = grpc_insecure_channel_create((char *)addr, NULL, NULL);
for (int j = 0; j < NUM_INNER_LOOPS_SHORT_TIMEOUTS; ++j) {
gpr_timespec later_time =
grpc_timeout_milliseconds_to_deadline(DELAY_MILLIS_SHORT_TIMEOUTS);
grpc_connectivity_state state =
grpc_channel_check_connectivity_state(chan, 0);
GPR_ASSERT(state == GRPC_CHANNEL_IDLE);
grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL);
gpr_timespec poll_time =
grpc_timeout_milliseconds_to_deadline(POLL_MILLIS_SHORT_TIMEOUTS);
grpc_event ev = grpc_completion_queue_next(cq, poll_time, NULL);
GPR_ASSERT(ev.type == GRPC_OP_COMPLETE);
GPR_ASSERT(ev.success == false);
/* check that the watcher from "watch state" was free'd */
GPR_ASSERT(grpc_channel_num_external_connectivity_watchers(chan) == 0);
}
grpc_channel_destroy(chan);
grpc_completion_queue_destroy(cq);
}
}
void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) {
unsigned char *buf = gpr_malloc(read_size);
ssize_t bytes_read;
size_t bytes_left = num_bytes;
int flags;
int current = 0;
int i;
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
flags = fcntl(fd, F_GETFL, 0);
GPR_ASSERT(fcntl(fd, F_SETFL, flags & ~O_NONBLOCK) == 0);
for (;;) {
grpc_pollset_worker *worker = NULL;
gpr_mu_lock(g_mu);
GPR_ASSERT(GRPC_LOG_IF_ERROR(
"pollset_work",
grpc_pollset_work(&exec_ctx, g_pollset, &worker,
gpr_now(GPR_CLOCK_MONOTONIC),
grpc_timeout_milliseconds_to_deadline(10))));
gpr_mu_unlock(g_mu);
grpc_exec_ctx_finish(&exec_ctx);
do {
bytes_read =
read(fd, buf, bytes_left > read_size ? read_size : bytes_left);
} while (bytes_read < 0 && errno == EINTR);
GPR_ASSERT(bytes_read >= 0);
for (i = 0; i < bytes_read; ++i) {
GPR_ASSERT(buf[i] == current);
current = (current + 1) % 256;
}
bytes_left -= (size_t)bytes_read;
if (bytes_left == 0) break;
}
flags = fcntl(fd, F_GETFL, 0);
GPR_ASSERT(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == 0);
gpr_free(buf);
}
/** Returns connection sequence (server indices), which must be freed */
static request_sequences perform_request(servers_fixture *f,
grpc_channel *client,
request_data *rdata,
const test_spec *spec) {
grpc_call *c;
int s_idx;
int *s_valid;
grpc_op ops[6];
grpc_op *op;
int was_cancelled;
size_t i, iter_num;
grpc_event ev;
int read_tag;
int completed_client;
const request_sequences sequences = request_sequences_create(spec->num_iters);
s_valid = gpr_malloc(sizeof(int) * f->num_servers);
for (iter_num = 0; iter_num < spec->num_iters; iter_num++) {
cq_verifier *cqv = cq_verifier_create(f->cq);
was_cancelled = 2;
for (i = 0; i < f->num_servers; i++) {
if (spec->kill_at[iter_num][i] != 0) {
kill_server(f, i);
} else if (spec->revive_at[iter_num][i] != 0) {
/* killing takes precedence */
revive_server(f, rdata, i);
}
}
sequences.connections[iter_num] = -1;
grpc_metadata_array_init(&rdata->initial_metadata_recv);
grpc_metadata_array_init(&rdata->trailing_metadata_recv);
for (i = 0; i < f->num_servers; i++) {
grpc_call_details_init(&rdata->call_details[i]);
}
memset(s_valid, 0, f->num_servers * sizeof(int));
grpc_slice host = grpc_slice_from_static_string("foo.test.google.fr");
c = grpc_channel_create_call(client, NULL, GRPC_PROPAGATE_DEFAULTS, f->cq,
grpc_slice_from_static_string("/foo"), &host,
gpr_inf_future(GPR_CLOCK_REALTIME), NULL);
GPR_ASSERT(c);
completed_client = 0;
memset(ops, 0, sizeof(ops));
op = ops;
op->op = GRPC_OP_SEND_INITIAL_METADATA;
op->data.send_initial_metadata.count = 0;
op->flags = 0;
op->reserved = NULL;
op++;
op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
op->flags = 0;
op->reserved = NULL;
op++;
op->op = GRPC_OP_RECV_INITIAL_METADATA;
op->data.recv_initial_metadata.recv_initial_metadata =
&rdata->initial_metadata_recv;
op->flags = 0;
op->reserved = NULL;
op++;
op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
op->data.recv_status_on_client.trailing_metadata =
&rdata->trailing_metadata_recv;
op->data.recv_status_on_client.status = &rdata->status;
op->data.recv_status_on_client.status_details = &rdata->details;
op->flags = 0;
op->reserved = NULL;
op++;
GPR_ASSERT(GRPC_CALL_OK ==
grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL));
s_idx = -1;
while (
(ev = grpc_completion_queue_next(
f->cq, grpc_timeout_milliseconds_to_deadline(RETRY_TIMEOUT), NULL))
.type != GRPC_QUEUE_TIMEOUT) {
GPR_ASSERT(ev.type == GRPC_OP_COMPLETE);
read_tag = ((int)(intptr_t)ev.tag);
const grpc_connectivity_state conn_state =
grpc_channel_check_connectivity_state(client, 0);
sequences.connectivity_states[iter_num] = conn_state;
gpr_log(GPR_DEBUG, "EVENT: success:%d, type:%d, tag:%d iter:%" PRIuPTR,
ev.success, ev.type, read_tag, iter_num);
if (ev.success && read_tag >= 1000) {
GPR_ASSERT(s_idx == -1); /* only one server must reply */
/* only server notifications for non-shutdown events */
s_idx = read_tag - 1000;
s_valid[s_idx] = 1;
sequences.connections[iter_num] = s_idx;
break;
} else if (read_tag == 1) {
gpr_log(GPR_DEBUG, "client timed out");
GPR_ASSERT(ev.success);
completed_client = 1;
}
}
if (s_idx >= 0) {
memset(ops, 0, sizeof(ops));
op = ops;
op->op = GRPC_OP_SEND_INITIAL_METADATA;
op->data.send_initial_metadata.count = 0;
op->flags = 0;
op->reserved = NULL;
op++;
op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
op->data.send_status_from_server.trailing_metadata_count = 0;
op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED;
grpc_slice status_details = grpc_slice_from_static_string("xyz");
op->data.send_status_from_server.status_details = &status_details;
op->flags = 0;
op->reserved = NULL;
op++;
op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
op->data.recv_close_on_server.cancelled = &was_cancelled;
op->flags = 0;
op->reserved = NULL;
op++;
GPR_ASSERT(GRPC_CALL_OK == grpc_call_start_batch(f->server_calls[s_idx],
ops, (size_t)(op - ops),
tag(102), NULL));
CQ_EXPECT_COMPLETION(cqv, tag(102), 1);
if (!completed_client) {
CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
}
cq_verify(cqv);
GPR_ASSERT(rdata->status == GRPC_STATUS_UNIMPLEMENTED);
GPR_ASSERT(0 == grpc_slice_str_cmp(rdata->details, "xyz"));
GPR_ASSERT(0 ==
grpc_slice_str_cmp(rdata->call_details[s_idx].method, "/foo"));
GPR_ASSERT(0 == grpc_slice_str_cmp(rdata->call_details[s_idx].host,
"foo.test.google.fr"));
GPR_ASSERT(was_cancelled == 1);
grpc_call_unref(f->server_calls[s_idx]);
/* ask for the next request on this server */
GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call(
f->servers[s_idx], &f->server_calls[s_idx],
&rdata->call_details[s_idx],
&f->request_metadata_recv[s_idx], f->cq,
f->cq, tag(1000 + (int)s_idx)));
} else { /* no response from server */
grpc_call_cancel(c, NULL);
if (!completed_client) {
CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
cq_verify(cqv);
}
}
GPR_ASSERT(
grpc_completion_queue_next(
f->cq, grpc_timeout_milliseconds_to_deadline(RETRY_TIMEOUT), NULL)
.type == GRPC_QUEUE_TIMEOUT);
grpc_metadata_array_destroy(&rdata->initial_metadata_recv);
grpc_metadata_array_destroy(&rdata->trailing_metadata_recv);
cq_verifier_destroy(cqv);
grpc_call_unref(c);
for (i = 0; i < f->num_servers; i++) {
grpc_call_details_destroy(&rdata->call_details[i]);
}
grpc_slice_unref(rdata->details);
}
gpr_free(s_valid);
return sequences;
}
/* Pollset_set with an empty pollset */
void pollset_set_test_empty_pollset() {
/* We construct the following structure for this test:
*
* +---> PS0 (EMPTY)
* |
* +---> FD0
* |
* PSS0---+
* | +---> FD1
* | |
* +---> PS1--+
* |
* +---> FD2
*/
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
grpc_pollset_worker *worker;
gpr_timespec deadline;
test_fd tfds[3];
test_pollset pollsets[2];
test_pollset_set pollset_set;
const int num_fds = GPR_ARRAY_SIZE(tfds);
const int num_ps = GPR_ARRAY_SIZE(pollsets);
const int num_pss = 1;
init_test_fds(&exec_ctx, tfds, num_fds);
init_test_pollsets(pollsets, num_ps);
init_test_pollset_sets(&pollset_set, num_pss);
/* Construct the structure */
grpc_pollset_set_add_fd(&exec_ctx, pollset_set.pss, tfds[0].fd);
grpc_pollset_add_fd(&exec_ctx, pollsets[1].ps, tfds[1].fd);
grpc_pollset_add_fd(&exec_ctx, pollsets[1].ps, tfds[2].fd);
grpc_pollset_set_add_pollset(&exec_ctx, pollset_set.pss, pollsets[0].ps);
grpc_pollset_set_add_pollset(&exec_ctx, pollset_set.pss, pollsets[1].ps);
/* Test. Make all FDs readable and make sure that can be observed by doing
* grpc_pollset_work on the empty pollset 'PS0' */
make_test_fds_readable(tfds, num_fds);
gpr_mu_lock(pollsets[0].mu);
deadline = grpc_timeout_milliseconds_to_deadline(2);
GPR_ASSERT(GRPC_ERROR_NONE ==
grpc_pollset_work(&exec_ctx, pollsets[0].ps, &worker,
gpr_now(GPR_CLOCK_MONOTONIC), deadline));
gpr_mu_unlock(pollsets[0].mu);
grpc_exec_ctx_flush(&exec_ctx);
verify_readable_and_reset(&exec_ctx, tfds, num_fds);
grpc_exec_ctx_flush(&exec_ctx);
/* Tear down */
grpc_pollset_set_del_fd(&exec_ctx, pollset_set.pss, tfds[0].fd);
grpc_pollset_set_del_pollset(&exec_ctx, pollset_set.pss, pollsets[0].ps);
grpc_pollset_set_del_pollset(&exec_ctx, pollset_set.pss, pollsets[1].ps);
grpc_exec_ctx_flush(&exec_ctx);
cleanup_test_fds(&exec_ctx, tfds, num_fds);
cleanup_test_pollsets(&exec_ctx, pollsets, num_ps);
cleanup_test_pollset_sets(&exec_ctx, &pollset_set, num_pss);
grpc_exec_ctx_finish(&exec_ctx);
}
/* Test some typical scenarios in pollset_set */
static void pollset_set_test_basic() {
/* We construct the following structure for this test:
*
* +---> FD0 (Added before PSS1, PS1 and PS2 are added to PSS0)
* |
* +---> FD5 (Added after PSS1, PS1 and PS2 are added to PSS0)
* |
* |
* | +---> FD1 (Added before PSS1 is added to PSS0)
* | |
* | +---> FD6 (Added after PSS1 is added to PSS0)
* | |
* +---> PSS1--+ +--> FD2 (Added before PS0 is added to PSS1)
* | | |
* | +---> PS0---+
* | |
* PSS0---+ +--> FD7 (Added after PS0 is added to PSS1)
* |
* |
* | +---> FD3 (Added before PS1 is added to PSS0)
* | |
* +---> PS1---+
* | |
* | +---> FD8 (Added after PS1 added to PSS0)
* |
* |
* | +---> FD4 (Added before PS2 is added to PSS0)
* | |
* +---> PS2---+
* |
* +---> FD9 (Added after PS2 is added to PSS0)
*/
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
grpc_pollset_worker *worker;
gpr_timespec deadline;
test_fd tfds[10];
test_pollset pollsets[3];
test_pollset_set pollset_sets[2];
const int num_fds = GPR_ARRAY_SIZE(tfds);
const int num_ps = GPR_ARRAY_SIZE(pollsets);
const int num_pss = GPR_ARRAY_SIZE(pollset_sets);
init_test_fds(&exec_ctx, tfds, num_fds);
init_test_pollsets(pollsets, num_ps);
init_test_pollset_sets(pollset_sets, num_pss);
/* Construct the pollset_set/pollset/fd tree (see diagram above) */
grpc_pollset_set_add_fd(&exec_ctx, pollset_sets[0].pss, tfds[0].fd);
grpc_pollset_set_add_fd(&exec_ctx, pollset_sets[1].pss, tfds[1].fd);
grpc_pollset_add_fd(&exec_ctx, pollsets[0].ps, tfds[2].fd);
grpc_pollset_add_fd(&exec_ctx, pollsets[1].ps, tfds[3].fd);
grpc_pollset_add_fd(&exec_ctx, pollsets[2].ps, tfds[4].fd);
grpc_pollset_set_add_pollset_set(&exec_ctx, pollset_sets[0].pss,
pollset_sets[1].pss);
grpc_pollset_set_add_pollset(&exec_ctx, pollset_sets[1].pss, pollsets[0].ps);
grpc_pollset_set_add_pollset(&exec_ctx, pollset_sets[0].pss, pollsets[1].ps);
grpc_pollset_set_add_pollset(&exec_ctx, pollset_sets[0].pss, pollsets[2].ps);
grpc_pollset_set_add_fd(&exec_ctx, pollset_sets[0].pss, tfds[5].fd);
grpc_pollset_set_add_fd(&exec_ctx, pollset_sets[1].pss, tfds[6].fd);
grpc_pollset_add_fd(&exec_ctx, pollsets[0].ps, tfds[7].fd);
grpc_pollset_add_fd(&exec_ctx, pollsets[1].ps, tfds[8].fd);
grpc_pollset_add_fd(&exec_ctx, pollsets[2].ps, tfds[9].fd);
grpc_exec_ctx_flush(&exec_ctx);
/* Test that if any FD in the above structure is readable, it is observable by
* doing grpc_pollset_work on any pollset
*
* For every pollset, do the following:
* - (Ensure that all FDs are in reset state)
* - Make all FDs readable
* - Call grpc_pollset_work() on the pollset
* - Flush the exec_ctx
* - Verify that on_readable call back was called for all FDs (and
* reset the FDs)
* */
for (int i = 0; i < num_ps; i++) {
make_test_fds_readable(tfds, num_fds);
gpr_mu_lock(pollsets[i].mu);
deadline = grpc_timeout_milliseconds_to_deadline(2);
GPR_ASSERT(GRPC_ERROR_NONE ==
grpc_pollset_work(&exec_ctx, pollsets[i].ps, &worker,
gpr_now(GPR_CLOCK_MONOTONIC), deadline));
gpr_mu_unlock(pollsets[i].mu);
grpc_exec_ctx_flush(&exec_ctx);
verify_readable_and_reset(&exec_ctx, tfds, num_fds);
grpc_exec_ctx_flush(&exec_ctx);
}
/* Test tear down */
grpc_pollset_set_del_fd(&exec_ctx, pollset_sets[0].pss, tfds[0].fd);
grpc_pollset_set_del_fd(&exec_ctx, pollset_sets[0].pss, tfds[5].fd);
grpc_pollset_set_del_fd(&exec_ctx, pollset_sets[1].pss, tfds[1].fd);
grpc_pollset_set_del_fd(&exec_ctx, pollset_sets[1].pss, tfds[6].fd);
grpc_exec_ctx_flush(&exec_ctx);
grpc_pollset_set_del_pollset(&exec_ctx, pollset_sets[1].pss, pollsets[0].ps);
grpc_pollset_set_del_pollset(&exec_ctx, pollset_sets[0].pss, pollsets[1].ps);
grpc_pollset_set_del_pollset(&exec_ctx, pollset_sets[0].pss, pollsets[2].ps);
grpc_pollset_set_del_pollset_set(&exec_ctx, pollset_sets[0].pss,
pollset_sets[1].pss);
grpc_exec_ctx_flush(&exec_ctx);
cleanup_test_fds(&exec_ctx, tfds, num_fds);
cleanup_test_pollsets(&exec_ctx, pollsets, num_ps);
cleanup_test_pollset_sets(&exec_ctx, pollset_sets, num_pss);
grpc_exec_ctx_finish(&exec_ctx);
}
op->data.recv_status_on_client.status_details = &details;
op->flags = 0;
op->reserved = NULL;
op++;
error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL);
GPR_ASSERT(GRPC_CALL_OK == error);
error =
grpc_server_request_call(f.server, &s, &call_details,
&request_metadata_recv, f.cq, f.cq, tag(101));
GPR_ASSERT(GRPC_CALL_OK == error);
CQ_EXPECT_COMPLETION(cqv, tag(101), true);
cq_verify(cqv);
gpr_timespec expect_shutdown_time = grpc_timeout_milliseconds_to_deadline(
(int)(MAX_CONNECTION_AGE_MS * MAX_CONNECTION_AGE_JITTER_MULTIPLIER) +
MAX_CONNECTION_AGE_GRACE_MS + IMMEDIATE_SHUTDOWN_GRACE_TIME_MS);
/* Wait for the channel to reach its max age */
cq_verify_empty_timeout(cqv, CQ_MAX_CONNECTION_AGE_WAIT_TIME_S);
/* After the channel reaches its max age, we still do nothing here. And wait
for it to use up its max age grace period. */
CQ_EXPECT_COMPLETION(cqv, tag(1), true);
cq_verify(cqv);
gpr_timespec channel_shutdown_time = gpr_now(GPR_CLOCK_MONOTONIC);
GPR_ASSERT(gpr_time_cmp(channel_shutdown_time, expect_shutdown_time) < 0);
memset(ops, 0, sizeof(ops));
op = ops;
