void replica::on_client_read(const read_request_header& meta, message_ptr& request)
{
if (status() == PS_INACTIVE || status() == PS_POTENTIAL_SECONDARY)
{
response_client_message(request, ERR_INVALID_STATE);
return;
}
if (meta.semantic == read_semantic_t::ReadLastUpdate)
{
if (status() != PS_PRIMARY ||
last_committed_decree() < _primary_states.last_prepare_decree_on_new_primary)
{
response_client_message(request, ERR_INVALID_STATE);
return;
}
}
dassert (_app != nullptr, "");
_app->dispatch_rpc_call(meta.code, request, true);
}
void replica::on_client_read(task_code code, dsn_message_t request)
{
if (status() == partition_status::PS_INACTIVE || status() == partition_status::PS_POTENTIAL_SECONDARY)
{
response_client_message(request, ERR_INVALID_STATE);
return;
}
if (status() != partition_status::PS_PRIMARY ||
// a small window where the state is not the latest yet
last_committed_decree() < _primary_states.last_prepare_decree_on_new_primary)
{
response_client_message(request, ERR_INVALID_STATE);
return;
}
dassert (_app != nullptr, "");
dsn_layer1_app_commit_rpc_request(_app->app_context(), request, true);
}
void replica::on_client_write(task_code code, dsn_message_t request)
{
check_hashed_access();
if (PS_PRIMARY != status())
{
response_client_message(request, ERR_INVALID_STATE);
return;
}
if (static_cast<int>(_primary_states.membership.secondaries.size()) + 1 < _options->mutation_2pc_min_replica_count)
{
response_client_message(request, ERR_NOT_ENOUGH_MEMBER);
return;
}
auto mu = _primary_states.write_queue.add_work(code, request, this);
if (mu)
{
init_prepare(mu);
}
}
void replica::on_client_write(int code, dsn_message_t request)
{
check_hashed_access();
if (PS_PRIMARY != status())
{
response_client_message(request, ERR_INVALID_STATE);
return;
}
mutation_ptr mu = new_mutation(_prepare_list->max_decree() + 1);
mu->set_client_request(code, request);
init_prepare(mu);
}
void replica::on_client_write(int code, dsn_message_t request)
{
check_hashed_access();
if (PS_PRIMARY != status())
{
response_client_message(request, ERR_INVALID_STATE);
return;
}
auto mu = _primary_states.write_queue.add_work(code, request, this);
if (mu)
{
init_prepare(mu);
}
}
void replica::init_prepare(mutation_ptr& mu)
{
dassert (PS_PRIMARY == status(), "");
error_code err = ERR_OK;
uint8_t count = 0;
mu->data.header.last_committed_decree = last_committed_decree();
if (mu->data.header.decree == invalid_decree)
{
mu->set_id(get_ballot(), _prepare_list->max_decree() + 1);
}
else
{
mu->set_id(get_ballot(), mu->data.header.decree);
}
dinfo("%s: mutation %s init_prepare, mutation_tid=%" PRIu64, name(), mu->name(), mu->tid());
// check bounded staleness
if (mu->data.header.decree > last_committed_decree() + _options->staleness_for_commit)
{
err = ERR_CAPACITY_EXCEEDED;
goto ErrOut;
}
dassert (mu->data.header.decree > last_committed_decree(), "");
// local prepare
err = _prepare_list->prepare(mu, PS_PRIMARY);
if (err != ERR_OK)
{
goto ErrOut;
}
// remote prepare
mu->set_prepare_ts();
mu->set_left_secondary_ack_count((unsigned int)_primary_states.membership.secondaries.size());
for (auto it = _primary_states.membership.secondaries.begin(); it != _primary_states.membership.secondaries.end(); ++it)
{
send_prepare_message(*it, PS_SECONDARY, mu, _options->prepare_timeout_ms_for_secondaries);
}
count = 0;
for (auto it = _primary_states.learners.begin(); it != _primary_states.learners.end(); ++it)
{
if (it->second.prepare_start_decree != invalid_decree && mu->data.header.decree >= it->second.prepare_start_decree)
{
send_prepare_message(it->first, PS_POTENTIAL_SECONDARY, mu, _options->prepare_timeout_ms_for_potential_secondaries, it->second.signature);
count++;
}
}
mu->set_left_potential_secondary_ack_count(count);
if (mu->is_logged())
{
do_possible_commit_on_primary(mu);
}
else
{
dassert(mu->data.header.log_offset == invalid_offset, "");
dassert(mu->log_task() == nullptr, "");
mu->log_task() = _stub->_log->append(mu,
LPC_WRITE_REPLICATION_LOG,
this,
std::bind(&replica::on_append_log_completed, this, mu,
std::placeholders::_1,
std::placeholders::_2),
gpid_to_hash(get_gpid())
);
dassert(nullptr != mu->log_task(), "");
}
return;
ErrOut:
for (auto& r : mu->client_requests)
{
response_client_message(r, err);
}
return;
}
void replica::init_prepare(mutation_ptr& mu)
{
dassert (PS_PRIMARY == status(), "");
error_code err = ERR_OK;
uint8_t count = 0;
if (static_cast<int>(_primary_states.membership.secondaries.size()) + 1 < _options.mutation_2pc_min_replica_count)
{
err = ERR_NOT_ENOUGH_MEMBER;
goto ErrOut;
}
mu->data.header.last_committed_decree = last_committed_decree();
if (mu->data.header.decree == invalid_decree)
{
mu->set_id(get_ballot(), _prepare_list->max_decree() + 1);
}
else
{
mu->set_id(get_ballot(), mu->data.header.decree);
}
ddebug("%s: mutation %s init_prepare", name(), mu->name());
// check bounded staleness
if (mu->data.header.decree > last_committed_decree() + _options.staleness_for_commit)
{
err = ERR_CAPACITY_EXCEEDED;
goto ErrOut;
}
dassert (mu->data.header.decree > last_committed_decree(), "");
// local prepare
err = _prepare_list->prepare(mu, PS_PRIMARY);
if (err != ERR_OK)
{
goto ErrOut;
}
// remote prepare
mu->set_prepare_ts();
mu->set_left_secondary_ack_count((unsigned int)_primary_states.membership.secondaries.size());
for (auto it = _primary_states.membership.secondaries.begin(); it != _primary_states.membership.secondaries.end(); it++)
{
send_prepare_message(*it, PS_SECONDARY, mu, _options.prepare_timeout_ms_for_secondaries);
}
count = 0;
for (auto it = _primary_states.learners.begin(); it != _primary_states.learners.end(); it++)
{
if (it->second.prepare_start_decree != invalid_decree && mu->data.header.decree >= it->second.prepare_start_decree)
{
send_prepare_message(it->first, PS_POTENTIAL_SECONDARY, mu, _options.prepare_timeout_ms_for_potential_secondaries);
count++;
}
}
mu->set_left_potential_secondary_ack_count(count);
// it is possible to do commit here when logging is not required for acking prepare.
// however, it is only possible when replica count == 1 at this moment in the
// replication group, and we don't want to do this as it is too fragile now.
// do_possible_commit_on_primary(mu);
// local log
dassert (mu->data.header.log_offset == invalid_offset, "");
dassert (mu->log_task() == nullptr, "");
mu->log_task() = _stub->_log->append(mu,
LPC_WRITE_REPLICATION_LOG,
this,
std::bind(&replica::on_append_log_completed, this, mu,
std::placeholders::_1,
std::placeholders::_2),
gpid_to_hash(get_gpid())
);
dassert(nullptr != mu->log_task(), "");
return;
ErrOut:
response_client_message(mu->client_msg(), err);
return;
}
void replica::init_prepare(mutation_ptr& mu)
{
dassert (PS_PRIMARY == status(), "");
error_code err = ERR_SUCCESS;
uint8_t count = 0;
if (static_cast<int>(_primary_states.membership.secondaries.size()) + 1 < _options.mutation_2pc_min_replica_count)
{
err = ERR_NOT_ENOUGH_MEMBER;
goto ErrOut;
}
mu->data.header.last_committed_decree = last_committed_decree();
if (mu->data.header.decree == invalid_decree)
{
mu->set_id(get_ballot(), _prepare_list->max_decree() + 1);
}
else
{
mu->set_id(get_ballot(), mu->data.header.decree);
}
if (mu->data.header.decree > _prepare_list->max_decree() && _prepare_list->count() >= _options.staleness_for_commit)
{
err = ERR_CAPACITY_EXCEEDED;
goto ErrOut;
}
dassert (mu->data.header.decree > last_committed_decree(), "");
// local prepare without log
err = _prepare_list->prepare(mu, PS_PRIMARY);
if (err != ERR_SUCCESS)
{
goto ErrOut;
}
ddebug("%s: mutation %s init_prepare", name(), mu->name());
//
// TODO: bounded staleness on secondaries
//
dassert (mu->data.header.decree <= last_committed_decree() + _options.staleness_for_commit, "");
// remote prepare
dassert (mu->remote_tasks().size() == 0, "");
mu->set_left_secondary_ack_count((unsigned int)_primary_states.membership.secondaries.size());
for (auto it = _primary_states.membership.secondaries.begin(); it != _primary_states.membership.secondaries.end(); it++)
{
send_prepare_message(*it, PS_SECONDARY, mu, _options.prepare_timeout_ms_for_secondaries);
}
count = 0;
for (auto it = _primary_states.learners.begin(); it != _primary_states.learners.end(); it++)
{
if (it->second.prepare_start_decree != invalid_decree && mu->data.header.decree >= it->second.prepare_start_decree)
{
send_prepare_message(it->first, PS_POTENTIAL_SECONDARY, mu, _options.prepare_timeout_ms_for_potential_secondaries);
count++;
}
}
mu->set_left_potential_secondary_ack_count(count);
// local log
dassert (mu->data.header.log_offset == invalid_offset, "");
dassert (mu->log_task() == nullptr, "");
mu->log_task() = _stub->_log->append(mu,
LPC_WRITE_REPLICATION_LOG,
this,
std::bind(&replica::on_append_log_completed, this, mu,
std::placeholders::_1,
std::placeholders::_2),
gpid_to_hash(get_gpid())
);
if (nullptr == mu->log_task())
{
err = ERR_FILE_OPERATION_FAILED;
handle_local_failure(err);
goto ErrOut;
}
return;
ErrOut:
response_client_message(mu->client_request, err);
return;
}
