aeron_rb_write_result_t aeron_mpsc_rb_write(
volatile aeron_mpsc_rb_t *ring_buffer,
int32_t msg_type_id,
const void *msg,
size_t length)
{
const size_t record_length = length + AERON_RB_RECORD_HEADER_LENGTH;
const size_t required_capacity = AERON_ALIGN(record_length, AERON_RB_ALIGNMENT);
int32_t record_index = 0;
aeron_rb_write_result_t result = AERON_RB_FULL;
if (length > ring_buffer->max_message_length || AERON_RB_INVALID_MSG_TYPE_ID(msg_type_id))
{
return AERON_RB_ERROR;
}
record_index = aeron_mpsc_rb_claim_capacity(ring_buffer, required_capacity);
if (-1 != record_index)
{
aeron_rb_record_descriptor_t *record_header = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + record_index);
record_header->msg_type_id = msg_type_id;
AERON_PUT_ORDERED(record_header->length, -record_length);
memcpy(ring_buffer->buffer + AERON_RB_MESSAGE_OFFSET(record_index), msg, length);
AERON_PUT_ORDERED(record_header->length, record_length);
result = AERON_RB_SUCCESS;
}
return result;
}
int aeron_publication_image_insert_packet(
aeron_publication_image_t *image, int32_t term_id, int32_t term_offset, const uint8_t *buffer, size_t length)
{
const bool is_heartbeat = aeron_publication_image_is_heartbeat(buffer, length);
const int64_t packet_position = aeron_logbuffer_compute_position(
term_id, term_offset, image->position_bits_to_shift, image->initial_term_id);
const int64_t proposed_position = is_heartbeat ? packet_position : packet_position + (int64_t)length;
if (!aeron_publication_image_is_flow_control_under_run(image, packet_position) &&
!aeron_publication_image_is_flow_control_over_run(image, proposed_position))
{
if (is_heartbeat)
{
if (!image->is_end_of_stream && aeron_publication_image_is_end_of_stream(buffer, length))
{
AERON_PUT_ORDERED(image->is_end_of_stream, true);
AERON_PUT_ORDERED(image->log_meta_data->end_of_stream_position, packet_position);
}
aeron_counter_ordered_increment(image->heartbeats_received_counter, 1);
}
else
{
const size_t index = aeron_logbuffer_index_by_position(packet_position, image->position_bits_to_shift);
uint8_t *term_buffer = image->mapped_raw_log.term_buffers[index].addr;
aeron_term_rebuilder_insert(term_buffer + term_offset, buffer, length);
}
AERON_PUT_ORDERED(image->last_packet_timestamp_ns, image->nano_clock());
aeron_counter_propose_max_ordered(image->rcv_hwm_position.value_addr, proposed_position);
}
return (int)length;
}
bool aeron_mpsc_rb_unblock(volatile aeron_mpsc_rb_t *ring_buffer)
{
const size_t mask = ring_buffer->capacity - 1;
int64_t head;
int64_t tail;
size_t consumer_index;
size_t producer_index;
bool unblocked = false;
AERON_GET_VOLATILE(head, ring_buffer->descriptor->head_position);
AERON_GET_VOLATILE(tail, ring_buffer->descriptor->tail_position);
consumer_index = (int32_t)head & mask;
producer_index = (int32_t)tail & mask;
if (producer_index == consumer_index)
{
return false;
}
int32_t length;
aeron_rb_record_descriptor_t *record = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + consumer_index);
AERON_GET_VOLATILE(length, record->length);
if (length < 0)
{
record->msg_type_id = AERON_RB_PADDING_MSG_TYPE_ID;
AERON_PUT_ORDERED(record->length, -length);
unblocked = true;
}
else if (0 == length)
{
const size_t limit = producer_index > consumer_index ? producer_index : ring_buffer->capacity;
size_t i = consumer_index + AERON_RB_ALIGNMENT;
do
{
record = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + i);
AERON_GET_VOLATILE(length, record->length);
if (0 != length)
{
if (scan_back_to_confirm_still_zeroed(ring_buffer->buffer, i, consumer_index))
{
record = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + consumer_index);
record->msg_type_id = AERON_RB_PADDING_MSG_TYPE_ID;
AERON_PUT_ORDERED(record->length, i - consumer_index);
unblocked = true;
}
break;
}
i += AERON_RB_ALIGNMENT;
}
while (i < limit);
}
return unblocked;
}
int32_t aeron_counters_manager_next_counter_id(volatile aeron_counters_manager_t *manager)
{
int64_t now_ms = manager->clock_func();
for (int i = 0; i <= manager->free_list_index; i++)
{
int32_t counter_id = manager->free_list[i];
aeron_counter_metadata_descriptor_t *metadata =
(aeron_counter_metadata_descriptor_t *)(manager->metadata + (counter_id * AERON_COUNTERS_MANAGER_METADATA_LENGTH));
int64_t deadline;
AERON_GET_VOLATILE(deadline, metadata->free_to_reuse_deadline);
if (now_ms >= deadline)
{
aeron_counters_manager_remove_free_list_index(manager, i);
aeron_counter_value_descriptor_t *value =
(aeron_counter_value_descriptor_t *)(manager->values + (counter_id * AERON_COUNTERS_MANAGER_VALUE_LENGTH));
AERON_PUT_ORDERED(value->counter_value, 0L);
return counter_id;
}
}
return ++manager->id_high_water_mark;
}
void aeron_publication_image_on_gap_detected(void *clientd, int32_t term_id, int32_t term_offset, size_t length)
{
aeron_publication_image_t *image = (aeron_publication_image_t *)clientd;
const int64_t change_number = image->begin_loss_change + 1;
AERON_PUT_ORDERED(image->begin_loss_change, change_number);
image->loss_term_id = term_id;
image->loss_term_offset = term_offset;
image->loss_length = length;
AERON_PUT_ORDERED(image->end_loss_change, change_number);
if (image->loss_reporter_offset >= 0)
{
aeron_loss_reporter_record_observation(
image->loss_reporter, image->loss_reporter_offset, (int64_t)length, image->epoch_clock());
}
else if (NULL != image->loss_reporter)
{
char source[AERON_MAX_PATH];
aeron_format_source_identity(source, sizeof(source), &image->source_address);
if (NULL != image->endpoint)
{
image->loss_reporter_offset = aeron_loss_reporter_create_entry(
image->loss_reporter,
(int64_t)length,
image->epoch_clock(),
image->session_id,
image->stream_id,
image->endpoint->conductor_fields.udp_channel->original_uri,
image->endpoint->conductor_fields.udp_channel->uri_length,
source,
strlen(source));
}
if (-1 == image->loss_reporter_offset)
{
image->loss_reporter = NULL;
}
}
}
size_t aeron_mpsc_rb_read(
volatile aeron_mpsc_rb_t *ring_buffer,
aeron_rb_handler_t handler,
void *clientd,
size_t message_count_limit)
{
const int64_t head = ring_buffer->descriptor->head_position;
const size_t head_index = (int32_t)head & (ring_buffer->capacity - 1);
const size_t contiguous_block_length = ring_buffer->capacity - head_index;
size_t messages_read = 0;
size_t bytes_read = 0;
while ((bytes_read < contiguous_block_length) && (messages_read < message_count_limit))
{
aeron_rb_record_descriptor_t *header = NULL;
const size_t record_index = head_index + bytes_read;
int32_t record_length = 0;
int32_t msg_type_id = 0;
header = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + record_index);
AERON_GET_VOLATILE(record_length, header->length);
if (record_length <= 0)
{
break;
}
bytes_read += AERON_ALIGN(record_length, AERON_RB_ALIGNMENT);
msg_type_id = header->msg_type_id;
if (AERON_RB_PADDING_MSG_TYPE_ID == msg_type_id)
{
continue;
}
++messages_read;
handler(
msg_type_id,
ring_buffer->buffer + AERON_RB_MESSAGE_OFFSET(record_index),
record_length - AERON_RB_RECORD_HEADER_LENGTH,
clientd);
}
if (0 != bytes_read)
{
memset(ring_buffer->buffer + head_index, 0, bytes_read);
AERON_PUT_ORDERED(ring_buffer->descriptor->head_position, head + bytes_read);
}
return messages_read;
}
int aeron_broadcast_transmitter_transmit(
volatile aeron_broadcast_transmitter_t *transmitter,
int32_t msg_type_id,
const void *msg,
size_t length)
{
if (length > transmitter->max_message_length || AERON_BROADCAST_INVALID_MSG_TYPE_ID(msg_type_id))
{
return -1;
}
int64_t current_tail = transmitter->descriptor->tail_counter;
size_t record_offset = (uint32_t)current_tail & (transmitter->capacity - 1);
const size_t record_length = length + AERON_BROADCAST_RECORD_HEADER_LENGTH;
const size_t aligned_record_length = AERON_ALIGN(record_length, AERON_BROADCAST_RECORD_ALIGNMENT);
const int64_t new_tail = current_tail + (int64_t)aligned_record_length;
const size_t to_end_of_buffer = (uint32_t)transmitter->capacity - record_offset;
if (to_end_of_buffer < aligned_record_length)
{
signal_tail_intent(transmitter->descriptor, new_tail + (int64_t)to_end_of_buffer);
insert_padding_record(
(aeron_broadcast_record_descriptor_t *)(transmitter->buffer + record_offset), (int32_t)to_end_of_buffer);
current_tail += to_end_of_buffer;
record_offset = 0;
}
else
{
signal_tail_intent(transmitter->descriptor, new_tail);
}
aeron_broadcast_record_descriptor_t *record =
(aeron_broadcast_record_descriptor_t *)(transmitter->buffer + record_offset);
record->length = (int32_t)record_length;
record->msg_type_id = msg_type_id;
memcpy(transmitter->buffer + record_offset + AERON_BROADCAST_RECORD_HEADER_LENGTH, msg, length);
transmitter->descriptor->latest_counter = current_tail;
AERON_PUT_ORDERED(transmitter->descriptor->tail_counter, current_tail + aligned_record_length);
return 0;
}
int aeron_distinct_error_log_record(
aeron_distinct_error_log_t *log, int error_code, const char *description, const char *message)
{
int64_t timestamp = 0;
aeron_distinct_observation_t *observation = NULL;
if (NULL == log)
{
aeron_set_err(EINVAL, "%s", "invalid argument");
return -1;
}
timestamp = log->clock();
size_t num_observations = atomic_load(&log->observations_pimpl->num_observations);
aeron_distinct_observation_t *observations = atomic_load(&log->observations_pimpl->observations);
if ((observation = aeron_distinct_error_log_find_observation(
observations, num_observations, error_code, description)) == NULL)
{
pthread_mutex_lock(&log->mutex);
observation = aeron_distinct_error_log_new_observation(
log, num_observations, timestamp, error_code, description, message);
pthread_mutex_unlock(&log->mutex);
if (NULL == observation)
{
char buffer[AERON_MAX_PATH];
aeron_format_date(buffer, sizeof(buffer), timestamp);
fprintf(stderr, "%s - unrecordable error %d: %s %s\n", buffer, error_code, description, message);
errno = ENOMEM;
return -1;
}
}
aeron_error_log_entry_t *entry = (aeron_error_log_entry_t *)(log->buffer + observation->offset);
int32_t dest;
AERON_GET_AND_ADD_INT32(dest, entry->observation_count, 1);
AERON_PUT_ORDERED(entry->last_observation_timestamp, timestamp);
return 0;
}
int aeron_counters_manager_free(volatile aeron_counters_manager_t *manager, int32_t counter_id)
{
aeron_counter_metadata_descriptor_t *metadata =
(aeron_counter_metadata_descriptor_t *)(manager->metadata + (counter_id * AERON_COUNTERS_MANAGER_METADATA_LENGTH));
metadata->free_to_reuse_deadline = manager->clock_func() + manager->free_to_reuse_timeout_ms;
AERON_PUT_ORDERED(metadata->state, AERON_COUNTER_RECORD_RECLAIMED);
if ((manager->free_list_index + 1) >= (int32_t)manager->free_list_length)
{
size_t new_length = manager->free_list_length + (manager->free_list_length >> 1);
if (aeron_reallocf((void **)&manager->free_list, sizeof(int32_t) * new_length) < 0)
{
return -1;
}
manager->free_list_length = new_length;
}
int64_t aeron_max_flow_control_strategy_on_idle(
void *state,
int64_t now_ns,
int64_t snd_lmt,
int64_t snd_pos,
bool is_end_of_stream)
{
aeron_max_flow_control_strategy_state_t *strategy_state = (aeron_max_flow_control_strategy_state_t *)state;
if (is_end_of_stream && strategy_state->should_linger)
{
if (strategy_state->last_position >= snd_pos ||
now_ns > (strategy_state->time_of_last_status_message + AERON_MAX_FLOW_CONTROL_STRATEGY_RECEIVER_TIMEOUT_NS))
{
AERON_PUT_ORDERED(strategy_state->should_linger, false);
}
}
return snd_lmt;
}
int32_t aeron_counters_manager_allocate(
volatile aeron_counters_manager_t *manager,
int32_t type_id,
const uint8_t *key,
size_t key_length,
const char *label,
size_t label_length)
{
const int32_t counter_id = aeron_counters_manager_next_counter_id(manager);
if ((counter_id * AERON_COUNTERS_MANAGER_VALUE_LENGTH) + AERON_COUNTERS_MANAGER_VALUE_LENGTH > manager->values_length)
{
aeron_set_err(EINVAL, "%s:%d: %s", __FILE__, __LINE__, strerror(EINVAL));
return -1;
}
if ((counter_id * AERON_COUNTERS_MANAGER_METADATA_LENGTH) + AERON_COUNTERS_MANAGER_METADATA_LENGTH > manager->metadata_length)
{
aeron_set_err(EINVAL, "%s:%d: %s", __FILE__, __LINE__, strerror(EINVAL));
return -1;
}
aeron_counter_metadata_descriptor_t *metadata =
(aeron_counter_metadata_descriptor_t *)(manager->metadata + (counter_id * AERON_COUNTERS_MANAGER_METADATA_LENGTH));
metadata->type_id = type_id;
metadata->free_to_reuse_deadline = AERON_COUNTER_NOT_FREE_TO_REUSE;
if (NULL != key && key_length > 0)
{
memcpy(metadata->key, key, fmin(sizeof(metadata->key), key_length));
}
memcpy(metadata->label, label, fmin(sizeof(metadata->label), label_length));
metadata->label_length = (int32_t)label_length;
AERON_PUT_ORDERED(metadata->state, AERON_COUNTER_RECORD_ALLOCATED);
return counter_id;
}
inline static int32_t aeron_mpsc_rb_claim_capacity(volatile aeron_mpsc_rb_t *ring_buffer, size_t required_capacity)
{
const size_t mask = ring_buffer->capacity - 1;
int64_t head = 0;
int64_t tail = 0;
size_t tail_index = 0;
size_t padding = 0;
size_t to_buffer_end_length = 0;
AERON_GET_VOLATILE(head, ring_buffer->descriptor->head_cache_position);
do
{
int32_t available_capacity = 0;
AERON_GET_VOLATILE(tail, ring_buffer->descriptor->tail_position);
available_capacity = (int32_t )ring_buffer->capacity - (int32_t)(tail - head);
if ((int32_t)required_capacity > available_capacity)
{
AERON_GET_VOLATILE(head, ring_buffer->descriptor->head_position);
if (required_capacity > (ring_buffer->capacity - (int32_t)(tail - head)))
{
return -1;
}
AERON_PUT_ORDERED(ring_buffer->descriptor->head_cache_position, head);
}
tail_index = (int32_t)tail & mask;
to_buffer_end_length = ring_buffer->capacity - tail_index;
if (required_capacity > to_buffer_end_length)
{
size_t head_index = (int32_t)head & mask;
if (required_capacity > head_index)
{
AERON_GET_VOLATILE(head, ring_buffer->descriptor->head_position);
head_index = (int32_t)head & mask;
if (required_capacity > head_index)
{
return -1;
}
AERON_PUT_ORDERED(ring_buffer->descriptor->head_cache_position, head);
}
padding = to_buffer_end_length;
}
}
while (!aeron_cmpxchg64(
&(ring_buffer->descriptor->tail_position),
tail,
tail + (int32_t)required_capacity + (int32_t)padding));
if (0 != padding)
{
aeron_rb_record_descriptor_t *record_header = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + tail_index);
record_header->msg_type_id = AERON_RB_PADDING_MSG_TYPE_ID;
AERON_PUT_ORDERED(record_header->length, padding);
tail_index = 0;
}
return (int32_t)tail_index;
}
void aeron_mpsc_rb_consumer_heartbeat_time(volatile aeron_mpsc_rb_t *ring_buffer, int64_t time)
{
AERON_PUT_ORDERED(ring_buffer->descriptor->consumer_heartbeat, time);
}
static aeron_distinct_observation_t *aeron_distinct_error_log_new_observation(
aeron_distinct_error_log_t *log,
size_t existing_num_observations,
int64_t timestamp,
int error_code,
const char *description,
const char *message)
{
size_t num_observations = atomic_load(&log->observations_pimpl->num_observations);
aeron_distinct_observation_t *observations = atomic_load(&log->observations_pimpl->observations);
aeron_distinct_observation_t *observation = NULL;
if ((observation = aeron_distinct_error_log_find_observation(
observations, existing_num_observations, error_code, description)) == NULL)
{
char encoded_error[AERON_MAX_PATH];
snprintf(encoded_error, sizeof(encoded_error) - 1, "%d: %s %s", error_code, description, message);
size_t description_length = strlen(description);
size_t encoded_error_length = strlen(encoded_error);
size_t length = AERON_ERROR_LOG_HEADER_LENGTH + encoded_error_length;
aeron_distinct_observation_t *new_array = NULL;
char *new_description = NULL;
size_t offset = log->next_offset;
aeron_error_log_entry_t *entry = (aeron_error_log_entry_t *)(log->buffer + offset);
if ((offset + length) > log->buffer_capacity ||
aeron_alloc((void **)&new_array, sizeof(aeron_distinct_observation_t) * (num_observations + 1)) < 0 ||
aeron_alloc((void **)&new_description, description_length + 1) < 0)
{
return NULL;
}
memcpy(log->buffer + offset + AERON_ERROR_LOG_HEADER_LENGTH, encoded_error, encoded_error_length);
entry->first_observation_timestamp = timestamp;
entry->observation_count = 0;
log->next_offset = AERON_ALIGN(offset + length, AERON_ERROR_LOG_RECORD_ALIGNMENT);
new_array[0].error_code = error_code;
new_array[0].description = new_description;
strncpy(new_description, description, description_length + 1);
new_array[0].description_length = description_length;
new_array[0].offset = offset;
memcpy(&new_array[1], observations, sizeof(aeron_distinct_observation_t) * num_observations);
atomic_store(&log->observations_pimpl->observations, new_array);
atomic_store(&log->observations_pimpl->num_observations, num_observations + 1);
AERON_PUT_ORDERED(entry->length, length);
observation = &new_array[0];
if (NULL != log->linger_resource)
{
log->linger_resource(log->linger_resource_clientd, (uint8_t *)observations);
}
}
return observation;
}
