aeron_rb_write_result_t aeron_mpsc_rb_write( volatile aeron_mpsc_rb_t *ring_buffer, int32_t msg_type_id, const void *msg, size_t length) { const size_t record_length = length + AERON_RB_RECORD_HEADER_LENGTH; const size_t required_capacity = AERON_ALIGN(record_length, AERON_RB_ALIGNMENT); int32_t record_index = 0; aeron_rb_write_result_t result = AERON_RB_FULL; if (length > ring_buffer->max_message_length || AERON_RB_INVALID_MSG_TYPE_ID(msg_type_id)) { return AERON_RB_ERROR; } record_index = aeron_mpsc_rb_claim_capacity(ring_buffer, required_capacity); if (-1 != record_index) { aeron_rb_record_descriptor_t *record_header = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + record_index); record_header->msg_type_id = msg_type_id; AERON_PUT_ORDERED(record_header->length, -record_length); memcpy(ring_buffer->buffer + AERON_RB_MESSAGE_OFFSET(record_index), msg, length); AERON_PUT_ORDERED(record_header->length, record_length); result = AERON_RB_SUCCESS; } return result; }
int aeron_publication_image_insert_packet( aeron_publication_image_t *image, int32_t term_id, int32_t term_offset, const uint8_t *buffer, size_t length) { const bool is_heartbeat = aeron_publication_image_is_heartbeat(buffer, length); const int64_t packet_position = aeron_logbuffer_compute_position( term_id, term_offset, image->position_bits_to_shift, image->initial_term_id); const int64_t proposed_position = is_heartbeat ? packet_position : packet_position + (int64_t)length; if (!aeron_publication_image_is_flow_control_under_run(image, packet_position) && !aeron_publication_image_is_flow_control_over_run(image, proposed_position)) { if (is_heartbeat) { if (!image->is_end_of_stream && aeron_publication_image_is_end_of_stream(buffer, length)) { AERON_PUT_ORDERED(image->is_end_of_stream, true); AERON_PUT_ORDERED(image->log_meta_data->end_of_stream_position, packet_position); } aeron_counter_ordered_increment(image->heartbeats_received_counter, 1); } else { const size_t index = aeron_logbuffer_index_by_position(packet_position, image->position_bits_to_shift); uint8_t *term_buffer = image->mapped_raw_log.term_buffers[index].addr; aeron_term_rebuilder_insert(term_buffer + term_offset, buffer, length); } AERON_PUT_ORDERED(image->last_packet_timestamp_ns, image->nano_clock()); aeron_counter_propose_max_ordered(image->rcv_hwm_position.value_addr, proposed_position); } return (int)length; }
bool aeron_mpsc_rb_unblock(volatile aeron_mpsc_rb_t *ring_buffer) { const size_t mask = ring_buffer->capacity - 1; int64_t head; int64_t tail; size_t consumer_index; size_t producer_index; bool unblocked = false; AERON_GET_VOLATILE(head, ring_buffer->descriptor->head_position); AERON_GET_VOLATILE(tail, ring_buffer->descriptor->tail_position); consumer_index = (int32_t)head & mask; producer_index = (int32_t)tail & mask; if (producer_index == consumer_index) { return false; } int32_t length; aeron_rb_record_descriptor_t *record = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + consumer_index); AERON_GET_VOLATILE(length, record->length); if (length < 0) { record->msg_type_id = AERON_RB_PADDING_MSG_TYPE_ID; AERON_PUT_ORDERED(record->length, -length); unblocked = true; } else if (0 == length) { const size_t limit = producer_index > consumer_index ? producer_index : ring_buffer->capacity; size_t i = consumer_index + AERON_RB_ALIGNMENT; do { record = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + i); AERON_GET_VOLATILE(length, record->length); if (0 != length) { if (scan_back_to_confirm_still_zeroed(ring_buffer->buffer, i, consumer_index)) { record = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + consumer_index); record->msg_type_id = AERON_RB_PADDING_MSG_TYPE_ID; AERON_PUT_ORDERED(record->length, i - consumer_index); unblocked = true; } break; } i += AERON_RB_ALIGNMENT; } while (i < limit); } return unblocked; }
int32_t aeron_counters_manager_next_counter_id(volatile aeron_counters_manager_t *manager) { int64_t now_ms = manager->clock_func(); for (int i = 0; i <= manager->free_list_index; i++) { int32_t counter_id = manager->free_list[i]; aeron_counter_metadata_descriptor_t *metadata = (aeron_counter_metadata_descriptor_t *)(manager->metadata + (counter_id * AERON_COUNTERS_MANAGER_METADATA_LENGTH)); int64_t deadline; AERON_GET_VOLATILE(deadline, metadata->free_to_reuse_deadline); if (now_ms >= deadline) { aeron_counters_manager_remove_free_list_index(manager, i); aeron_counter_value_descriptor_t *value = (aeron_counter_value_descriptor_t *)(manager->values + (counter_id * AERON_COUNTERS_MANAGER_VALUE_LENGTH)); AERON_PUT_ORDERED(value->counter_value, 0L); return counter_id; } } return ++manager->id_high_water_mark; }
void aeron_publication_image_on_gap_detected(void *clientd, int32_t term_id, int32_t term_offset, size_t length) { aeron_publication_image_t *image = (aeron_publication_image_t *)clientd; const int64_t change_number = image->begin_loss_change + 1; AERON_PUT_ORDERED(image->begin_loss_change, change_number); image->loss_term_id = term_id; image->loss_term_offset = term_offset; image->loss_length = length; AERON_PUT_ORDERED(image->end_loss_change, change_number); if (image->loss_reporter_offset >= 0) { aeron_loss_reporter_record_observation( image->loss_reporter, image->loss_reporter_offset, (int64_t)length, image->epoch_clock()); } else if (NULL != image->loss_reporter) { char source[AERON_MAX_PATH]; aeron_format_source_identity(source, sizeof(source), &image->source_address); if (NULL != image->endpoint) { image->loss_reporter_offset = aeron_loss_reporter_create_entry( image->loss_reporter, (int64_t)length, image->epoch_clock(), image->session_id, image->stream_id, image->endpoint->conductor_fields.udp_channel->original_uri, image->endpoint->conductor_fields.udp_channel->uri_length, source, strlen(source)); } if (-1 == image->loss_reporter_offset) { image->loss_reporter = NULL; } } }
size_t aeron_mpsc_rb_read( volatile aeron_mpsc_rb_t *ring_buffer, aeron_rb_handler_t handler, void *clientd, size_t message_count_limit) { const int64_t head = ring_buffer->descriptor->head_position; const size_t head_index = (int32_t)head & (ring_buffer->capacity - 1); const size_t contiguous_block_length = ring_buffer->capacity - head_index; size_t messages_read = 0; size_t bytes_read = 0; while ((bytes_read < contiguous_block_length) && (messages_read < message_count_limit)) { aeron_rb_record_descriptor_t *header = NULL; const size_t record_index = head_index + bytes_read; int32_t record_length = 0; int32_t msg_type_id = 0; header = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + record_index); AERON_GET_VOLATILE(record_length, header->length); if (record_length <= 0) { break; } bytes_read += AERON_ALIGN(record_length, AERON_RB_ALIGNMENT); msg_type_id = header->msg_type_id; if (AERON_RB_PADDING_MSG_TYPE_ID == msg_type_id) { continue; } ++messages_read; handler( msg_type_id, ring_buffer->buffer + AERON_RB_MESSAGE_OFFSET(record_index), record_length - AERON_RB_RECORD_HEADER_LENGTH, clientd); } if (0 != bytes_read) { memset(ring_buffer->buffer + head_index, 0, bytes_read); AERON_PUT_ORDERED(ring_buffer->descriptor->head_position, head + bytes_read); } return messages_read; }
int aeron_broadcast_transmitter_transmit( volatile aeron_broadcast_transmitter_t *transmitter, int32_t msg_type_id, const void *msg, size_t length) { if (length > transmitter->max_message_length || AERON_BROADCAST_INVALID_MSG_TYPE_ID(msg_type_id)) { return -1; } int64_t current_tail = transmitter->descriptor->tail_counter; size_t record_offset = (uint32_t)current_tail & (transmitter->capacity - 1); const size_t record_length = length + AERON_BROADCAST_RECORD_HEADER_LENGTH; const size_t aligned_record_length = AERON_ALIGN(record_length, AERON_BROADCAST_RECORD_ALIGNMENT); const int64_t new_tail = current_tail + (int64_t)aligned_record_length; const size_t to_end_of_buffer = (uint32_t)transmitter->capacity - record_offset; if (to_end_of_buffer < aligned_record_length) { signal_tail_intent(transmitter->descriptor, new_tail + (int64_t)to_end_of_buffer); insert_padding_record( (aeron_broadcast_record_descriptor_t *)(transmitter->buffer + record_offset), (int32_t)to_end_of_buffer); current_tail += to_end_of_buffer; record_offset = 0; } else { signal_tail_intent(transmitter->descriptor, new_tail); } aeron_broadcast_record_descriptor_t *record = (aeron_broadcast_record_descriptor_t *)(transmitter->buffer + record_offset); record->length = (int32_t)record_length; record->msg_type_id = msg_type_id; memcpy(transmitter->buffer + record_offset + AERON_BROADCAST_RECORD_HEADER_LENGTH, msg, length); transmitter->descriptor->latest_counter = current_tail; AERON_PUT_ORDERED(transmitter->descriptor->tail_counter, current_tail + aligned_record_length); return 0; }
int aeron_distinct_error_log_record( aeron_distinct_error_log_t *log, int error_code, const char *description, const char *message) { int64_t timestamp = 0; aeron_distinct_observation_t *observation = NULL; if (NULL == log) { aeron_set_err(EINVAL, "%s", "invalid argument"); return -1; } timestamp = log->clock(); size_t num_observations = atomic_load(&log->observations_pimpl->num_observations); aeron_distinct_observation_t *observations = atomic_load(&log->observations_pimpl->observations); if ((observation = aeron_distinct_error_log_find_observation( observations, num_observations, error_code, description)) == NULL) { pthread_mutex_lock(&log->mutex); observation = aeron_distinct_error_log_new_observation( log, num_observations, timestamp, error_code, description, message); pthread_mutex_unlock(&log->mutex); if (NULL == observation) { char buffer[AERON_MAX_PATH]; aeron_format_date(buffer, sizeof(buffer), timestamp); fprintf(stderr, "%s - unrecordable error %d: %s %s\n", buffer, error_code, description, message); errno = ENOMEM; return -1; } } aeron_error_log_entry_t *entry = (aeron_error_log_entry_t *)(log->buffer + observation->offset); int32_t dest; AERON_GET_AND_ADD_INT32(dest, entry->observation_count, 1); AERON_PUT_ORDERED(entry->last_observation_timestamp, timestamp); return 0; }
int aeron_counters_manager_free(volatile aeron_counters_manager_t *manager, int32_t counter_id) { aeron_counter_metadata_descriptor_t *metadata = (aeron_counter_metadata_descriptor_t *)(manager->metadata + (counter_id * AERON_COUNTERS_MANAGER_METADATA_LENGTH)); metadata->free_to_reuse_deadline = manager->clock_func() + manager->free_to_reuse_timeout_ms; AERON_PUT_ORDERED(metadata->state, AERON_COUNTER_RECORD_RECLAIMED); if ((manager->free_list_index + 1) >= (int32_t)manager->free_list_length) { size_t new_length = manager->free_list_length + (manager->free_list_length >> 1); if (aeron_reallocf((void **)&manager->free_list, sizeof(int32_t) * new_length) < 0) { return -1; } manager->free_list_length = new_length; }
int64_t aeron_max_flow_control_strategy_on_idle( void *state, int64_t now_ns, int64_t snd_lmt, int64_t snd_pos, bool is_end_of_stream) { aeron_max_flow_control_strategy_state_t *strategy_state = (aeron_max_flow_control_strategy_state_t *)state; if (is_end_of_stream && strategy_state->should_linger) { if (strategy_state->last_position >= snd_pos || now_ns > (strategy_state->time_of_last_status_message + AERON_MAX_FLOW_CONTROL_STRATEGY_RECEIVER_TIMEOUT_NS)) { AERON_PUT_ORDERED(strategy_state->should_linger, false); } } return snd_lmt; }
int32_t aeron_counters_manager_allocate( volatile aeron_counters_manager_t *manager, int32_t type_id, const uint8_t *key, size_t key_length, const char *label, size_t label_length) { const int32_t counter_id = aeron_counters_manager_next_counter_id(manager); if ((counter_id * AERON_COUNTERS_MANAGER_VALUE_LENGTH) + AERON_COUNTERS_MANAGER_VALUE_LENGTH > manager->values_length) { aeron_set_err(EINVAL, "%s:%d: %s", __FILE__, __LINE__, strerror(EINVAL)); return -1; } if ((counter_id * AERON_COUNTERS_MANAGER_METADATA_LENGTH) + AERON_COUNTERS_MANAGER_METADATA_LENGTH > manager->metadata_length) { aeron_set_err(EINVAL, "%s:%d: %s", __FILE__, __LINE__, strerror(EINVAL)); return -1; } aeron_counter_metadata_descriptor_t *metadata = (aeron_counter_metadata_descriptor_t *)(manager->metadata + (counter_id * AERON_COUNTERS_MANAGER_METADATA_LENGTH)); metadata->type_id = type_id; metadata->free_to_reuse_deadline = AERON_COUNTER_NOT_FREE_TO_REUSE; if (NULL != key && key_length > 0) { memcpy(metadata->key, key, fmin(sizeof(metadata->key), key_length)); } memcpy(metadata->label, label, fmin(sizeof(metadata->label), label_length)); metadata->label_length = (int32_t)label_length; AERON_PUT_ORDERED(metadata->state, AERON_COUNTER_RECORD_ALLOCATED); return counter_id; }
inline static int32_t aeron_mpsc_rb_claim_capacity(volatile aeron_mpsc_rb_t *ring_buffer, size_t required_capacity) { const size_t mask = ring_buffer->capacity - 1; int64_t head = 0; int64_t tail = 0; size_t tail_index = 0; size_t padding = 0; size_t to_buffer_end_length = 0; AERON_GET_VOLATILE(head, ring_buffer->descriptor->head_cache_position); do { int32_t available_capacity = 0; AERON_GET_VOLATILE(tail, ring_buffer->descriptor->tail_position); available_capacity = (int32_t )ring_buffer->capacity - (int32_t)(tail - head); if ((int32_t)required_capacity > available_capacity) { AERON_GET_VOLATILE(head, ring_buffer->descriptor->head_position); if (required_capacity > (ring_buffer->capacity - (int32_t)(tail - head))) { return -1; } AERON_PUT_ORDERED(ring_buffer->descriptor->head_cache_position, head); } tail_index = (int32_t)tail & mask; to_buffer_end_length = ring_buffer->capacity - tail_index; if (required_capacity > to_buffer_end_length) { size_t head_index = (int32_t)head & mask; if (required_capacity > head_index) { AERON_GET_VOLATILE(head, ring_buffer->descriptor->head_position); head_index = (int32_t)head & mask; if (required_capacity > head_index) { return -1; } AERON_PUT_ORDERED(ring_buffer->descriptor->head_cache_position, head); } padding = to_buffer_end_length; } } while (!aeron_cmpxchg64( &(ring_buffer->descriptor->tail_position), tail, tail + (int32_t)required_capacity + (int32_t)padding)); if (0 != padding) { aeron_rb_record_descriptor_t *record_header = (aeron_rb_record_descriptor_t *)(ring_buffer->buffer + tail_index); record_header->msg_type_id = AERON_RB_PADDING_MSG_TYPE_ID; AERON_PUT_ORDERED(record_header->length, padding); tail_index = 0; } return (int32_t)tail_index; }
void aeron_mpsc_rb_consumer_heartbeat_time(volatile aeron_mpsc_rb_t *ring_buffer, int64_t time) { AERON_PUT_ORDERED(ring_buffer->descriptor->consumer_heartbeat, time); }
static aeron_distinct_observation_t *aeron_distinct_error_log_new_observation( aeron_distinct_error_log_t *log, size_t existing_num_observations, int64_t timestamp, int error_code, const char *description, const char *message) { size_t num_observations = atomic_load(&log->observations_pimpl->num_observations); aeron_distinct_observation_t *observations = atomic_load(&log->observations_pimpl->observations); aeron_distinct_observation_t *observation = NULL; if ((observation = aeron_distinct_error_log_find_observation( observations, existing_num_observations, error_code, description)) == NULL) { char encoded_error[AERON_MAX_PATH]; snprintf(encoded_error, sizeof(encoded_error) - 1, "%d: %s %s", error_code, description, message); size_t description_length = strlen(description); size_t encoded_error_length = strlen(encoded_error); size_t length = AERON_ERROR_LOG_HEADER_LENGTH + encoded_error_length; aeron_distinct_observation_t *new_array = NULL; char *new_description = NULL; size_t offset = log->next_offset; aeron_error_log_entry_t *entry = (aeron_error_log_entry_t *)(log->buffer + offset); if ((offset + length) > log->buffer_capacity || aeron_alloc((void **)&new_array, sizeof(aeron_distinct_observation_t) * (num_observations + 1)) < 0 || aeron_alloc((void **)&new_description, description_length + 1) < 0) { return NULL; } memcpy(log->buffer + offset + AERON_ERROR_LOG_HEADER_LENGTH, encoded_error, encoded_error_length); entry->first_observation_timestamp = timestamp; entry->observation_count = 0; log->next_offset = AERON_ALIGN(offset + length, AERON_ERROR_LOG_RECORD_ALIGNMENT); new_array[0].error_code = error_code; new_array[0].description = new_description; strncpy(new_description, description, description_length + 1); new_array[0].description_length = description_length; new_array[0].offset = offset; memcpy(&new_array[1], observations, sizeof(aeron_distinct_observation_t) * num_observations); atomic_store(&log->observations_pimpl->observations, new_array); atomic_store(&log->observations_pimpl->num_observations, num_observations + 1); AERON_PUT_ORDERED(entry->length, length); observation = &new_array[0]; if (NULL != log->linger_resource) { log->linger_resource(log->linger_resource_clientd, (uint8_t *)observations); } } return observation; }