void replica::send_prepare_message(const dsn_address_t& addr, partition_status status, mutation_ptr& mu, int timeout_milliseconds) { dsn_message_t msg = dsn_msg_create_request(RPC_PREPARE, timeout_milliseconds, gpid_to_hash(get_gpid())); replica_configuration rconfig; _primary_states.get_replica_config(status, rconfig); { msg_binary_writer writer(msg); marshall(writer, get_gpid()); marshall(writer, rconfig); mu->write_to(writer); } mu->remote_tasks()[addr] = rpc::call(addr, msg, this, std::bind(&replica::on_prepare_reply, this, std::make_pair(mu, rconfig.status), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3), gpid_to_hash(get_gpid()) ); ddebug( "%s: mutation %s send_prepare_message to %s:%hu as %s", name(), mu->name(), addr.name, addr.port, enum_to_string(rconfig.status) ); }
void replica::send_prepare_message( ::dsn::rpc_address addr, partition_status status, mutation_ptr& mu, int timeout_milliseconds, int64_t learn_signature) { dsn_message_t msg = dsn_msg_create_request(RPC_PREPARE, timeout_milliseconds, gpid_to_hash(get_gpid())); replica_configuration rconfig; _primary_states.get_replica_config(status, rconfig, learn_signature); { rpc_write_stream writer(msg); marshall(writer, get_gpid()); marshall(writer, rconfig); mu->write_to(writer); } mu->remote_tasks()[addr] = rpc::call(addr, msg, this, [=](error_code err, dsn_message_t request, dsn_message_t reply) { on_prepare_reply(std::make_pair(mu, rconfig.status), err, request, reply); }, gpid_to_hash(get_gpid()) ); ddebug( "%s: mutation %s send_prepare_message to %s as %s", name(), mu->name(), addr.to_string(), enum_to_string(rconfig.status) ); }
void meta_service::on_log_completed(error_code err, size_t size, blob buffer, std::shared_ptr<configuration_update_request> req, dsn_message_t resp) { dassert(err == ERR_OK, "log operation failed, cannot proceed, err = %s", err.to_string()); dassert(buffer.length() == size, "log size must equal to the specified buffer size"); configuration_update_response response; update_configuration(*req, response); if (resp != nullptr) { meta_response_header rhdr; rhdr.err = err; rhdr.primary_address = primary_address(); marshall(resp, rhdr); marshall(resp, response); dsn_rpc_reply(resp); } else { err.end_tracking(); } }
int replication_app_base::dispatch_rpc_call(int code, message_ptr& request, bool ack_client) { auto it = _handlers.find(code); if (it != _handlers.end()) { if (ack_client) { message_ptr response = request->create_response(); int err = 0; marshall(response->writer(), err); it->second(request, response); } else { message_ptr response(nullptr); it->second(request, response); } } else if (ack_client) { message_ptr response = request->create_response(); error_code err = ERR_HANDLER_NOT_FOUND; marshall(response->writer(), (int)err); rpc::reply(response); } return 0; }
void replica::send_prepare_message(const end_point& addr, partition_status status, mutation_ptr& mu, int timeout_milliseconds) { message_ptr msg = message::create_request(RPC_PREPARE, timeout_milliseconds, gpid_to_hash(get_gpid())); marshall(msg, get_gpid()); replica_configuration rconfig; _primary_states.get_replica_config(status, rconfig); marshall(msg, rconfig); mu->write_to(msg); dbg_dassert (mu->remote_tasks().find(addr) == mu->remote_tasks().end()); mu->remote_tasks()[addr] = rpc::call(addr, msg, this, std::bind(&replica::on_prepare_reply, this, std::make_pair(mu, rconfig.status), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3), gpid_to_hash(get_gpid()) ); ddebug( "%s: mutation %s send_prepare_message to %s:%d as %s", name(), mu->name(), addr.name.c_str(), static_cast<int>(addr.port), enum_to_string(rconfig.status) ); }
void marshall(binary_writer& writer, const app_state& val) { marshall(writer, val.app_type); marshall(writer, val.app_name); marshall(writer, val.app_id); marshall(writer, val.partition_count); marshall(writer, val.partitions); }
void replication_app_client_base::call_with_address(dsn::rpc_address addr, request_context_ptr request) { auto& msg = request->request; dbg_dassert(!addr.is_invalid(), ""); dbg_dassert(_app_id > 0, ""); if (request->header_pos != 0) { if (request->is_read) { request->read_header.gpid.app_id = _app_id; request->read_header.gpid.pidx = request->partition_index; blob buffer(request->header_pos, 0, sizeof(request->read_header)); binary_writer writer(buffer); marshall(writer, request->read_header); dsn_msg_options_t opts; opts.timeout_ms = request->timeout_ms; opts.thread_hash = gpid_to_hash(request->read_header.gpid); opts.vnid = *(uint64_t*)(&request->read_header.gpid); dsn_msg_set_options(request->request, &opts, DSN_MSGM_HASH | DSN_MSGM_TIMEOUT); // TODO: not supported yet DSN_MSGM_VNID); } else { request->write_header.gpid.app_id = _app_id; request->write_header.gpid.pidx = request->partition_index; blob buffer(request->header_pos, 0, sizeof(request->write_header)); binary_writer writer(buffer); marshall(writer, request->write_header); dsn_msg_options_t opts; opts.timeout_ms = request->timeout_ms; opts.thread_hash = gpid_to_hash(request->write_header.gpid); opts.vnid = *(uint64_t*)(&request->write_header.gpid); dsn_msg_set_options(request->request, &opts, DSN_MSGM_HASH | DSN_MSGM_TIMEOUT); // TODO: not supported yet DSN_MSGM_VNID | DSN_MSGM_CONTEXT); } request->header_pos = 0; } { zauto_lock l(request->lock); rpc::call( addr, msg, this, std::bind( &replication_app_client_base::replica_rw_reply, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, request ) ); } }
void meta_service::on_request(message_ptr& msg) { meta_request_header hdr; unmarshall(msg, hdr); meta_response_header rhdr; bool is_primary = _state->get_meta_server_primary(rhdr.primary_address); if (is_primary) is_primary = (primary_address() == rhdr.primary_address); rhdr.err = ERR_SUCCESS; message_ptr resp = msg->create_response(); if (!is_primary) { rhdr.err = ERR_TALK_TO_OTHERS; marshall(resp, rhdr); } else if (hdr.rpc_tag == RPC_CM_QUERY_NODE_PARTITIONS) { configuration_query_by_node_request request; configuration_query_by_node_response response; unmarshall(msg, request); query_configuration_by_node(request, response); marshall(resp, rhdr); marshall(resp, response); } else if (hdr.rpc_tag == RPC_CM_QUERY_PARTITION_CONFIG_BY_INDEX) { configuration_query_by_index_request request; configuration_query_by_index_response response; unmarshall(msg, request); query_configuration_by_index(request, response); marshall(resp, rhdr); marshall(resp, response); } else if (hdr.rpc_tag == RPC_CM_UPDATE_PARTITION_CONFIGURATION) { update_configuration(msg, resp); return; } else { dassert(false, "unknown rpc tag %x (%s)", hdr.rpc_tag, task_code(hdr.rpc_tag).to_string()); } rpc::reply(resp); }
replication_app_client_base::request_context* replication_app_client_base::create_write_context( int partition_index, task_code code, rpc_response_task_ptr callback, int reply_hash ) { auto rc = new request_context; rc->callback_task = callback; rc->is_read = false; rc->partition_index = partition_index; rc->write_header.gpid.app_id = _app_id; rc->write_header.gpid.pidx = partition_index; rc->write_header.code = code; rc->timeout_timer = nullptr; if (rc->write_header.gpid.app_id == -1) { rc->header_pos = callback->get_request()->writer().write_placeholder(); dbg_dassert(rc->header_pos != 0xffff, ""); } else { rc->header_pos = 0xffff; marshall(callback->get_request()->writer(), rc->write_header); callback->get_request()->header().client.hash = gpid_to_hash(rc->write_header.gpid); } return rc; }
replication_app_client_base::request_context* replication_app_client_base::create_read_context( int partition_index, task_code code, rpc_response_task_ptr callback, read_semantic_t read_semantic, decree snapshot_decree, // only used when ReadSnapshot int reply_hash ) { auto rc = new request_context; rc->callback_task = callback; rc->is_read = true; rc->partition_index = partition_index; rc->read_header.gpid.app_id = _app_id; rc->read_header.gpid.pidx = partition_index; rc->read_header.code = code; rc->read_header.semantic = read_semantic; rc->read_header.version_decree = snapshot_decree; rc->timeout_timer = nullptr; if (rc->read_header.gpid.app_id == -1) { rc->header_pos = callback->get_request()->writer().write_placeholder(); dbg_dassert(rc->header_pos != 0xffff, ""); } else { rc->header_pos = 0xffff; marshall(callback->get_request()->writer(), rc->read_header); callback->get_request()->header().client.hash = gpid_to_hash(rc->read_header.gpid); } return rc; }
void mutation::write_to_log_file(std::function<void(blob)> inserter) const { { binary_writer temp_writer; marshall(temp_writer, data.header, DSF_THRIFT_BINARY); marshall(temp_writer, static_cast<int>(data.updates.size()), DSF_THRIFT_BINARY); for (const mutation_update& update : data.updates) { marshall(temp_writer, update.code, DSF_THRIFT_BINARY); marshall(temp_writer, static_cast<int>(update.data.length()), DSF_THRIFT_BINARY); } inserter(temp_writer.get_buffer()); } for (const mutation_update& update : data.updates) { inserter(update.data); } }
void meta_service::on_log_completed(error_code err, int size, char* buffer, message_ptr req, message_ptr resp) { free(buffer); dassert(err == ERR_SUCCESS, "log operation failed, cannot proceed, err = %s", err.to_string()); configuration_update_request request; configuration_update_response response; unmarshall(req, request); update_configuration(request, response); meta_response_header rhdr; rhdr.err = err; rhdr.primary_address = primary_address(); marshall(resp, rhdr); marshall(resp, response); rpc::reply(resp); }
void meta_service::update_configuration(std::shared_ptr<configuration_update_request>& update) { binary_writer writer; int32_t sz = 0; marshall(writer, sz); marshall(writer, *update); blob bb = writer.get_buffer(); *(int32_t*)bb.data() = bb.length() - sizeof(int32_t); { zauto_lock l(_log_lock); auto offset = _offset; _offset += bb.length(); file::write(_log, bb.data(), bb.length(), offset, LPC_CM_LOG_UPDATE, this, std::bind(&meta_service::on_log_completed, this, std::placeholders::_1, std::placeholders::_2, bb, update, nullptr)); } }
void meta_service::update_configuration(message_ptr req, message_ptr resp) { if (_state->freezed()) { meta_response_header rhdr; rhdr.err = 0; rhdr.primary_address = primary_address(); configuration_update_request request; configuration_update_response response; unmarshall(req, request); response.err = ERR_STATE_FREEZED; _state->query_configuration_by_gpid(request.config.gpid, response.config); marshall(resp, rhdr); marshall(resp, response); rpc::reply(resp); return; } auto bb = req->reader().get_remaining_buffer(); uint64_t offset; int len = bb.length() + sizeof(int32_t); char* buffer = (char*)malloc(len); *(int32_t*)buffer = bb.length(); memcpy(buffer + sizeof(int32_t), bb.data(), bb.length()); { zauto_lock l(_log_lock); offset = _offset; _offset += len; file::write(_log, buffer, len, offset, LPC_CM_LOG_UPDATE, this, std::bind(&meta_service::on_log_completed, this, std::placeholders::_1, std::placeholders::_2, buffer, req, resp)); } }
void XMLRPC::RequestMessage::writeXml( QXmlStreamWriter *writer ) const { writer->writeStartElement("methodCall"); writer->writeTextElement("methodName", m_method ); if( !m_args.isEmpty() ) { writer->writeStartElement("params"); foreach( QVariant arg, m_args) { writer->writeStartElement("param"); marshall( writer, arg ); writer->writeEndElement(); }
void command_manager::on_remote_cli(message_ptr& request) { std::string cmd; unmarshall(request->reader(), cmd); std::vector<std::string> args; unmarshall(request->reader(), args); std::string result; run_command(cmd, args, result); auto resp = request->create_response(); marshall(resp->writer(), result); ::dsn::service::rpc::reply(resp); }
void server_state::save(const char* chk_point) { binary_writer writer; marshall(writer, _apps); FILE* fp = ::fopen(chk_point, "wb+"); int32_t len = writer.total_size(); ::fwrite((const void*)&len, sizeof(len), 1, fp); std::vector<blob> bbs; writer.get_buffers(bbs); for (auto& bb : bbs) { ::fwrite((const void*)bb.data(), bb.length(), 1, fp); } ::fclose(fp); }
task_ptr partition_resolver_simple::query_config(int partition_index) { dinfo( "%s.client: start query config, gpid = %d.%d", _app_path.c_str(), _app_id, partition_index); auto msg = dsn::message_ex::create_request(RPC_CM_QUERY_PARTITION_CONFIG_BY_INDEX); configuration_query_by_index_request req; req.app_name = _app_path; if (partition_index != -1) { req.partition_indices.push_back(partition_index); } marshall(msg, req); return rpc::call( _meta_server, msg, &_tracker, [this, partition_index](error_code err, dsn::message_ex *req, dsn::message_ex *resp) { query_config_reply(err, req, resp, partition_index); }); }
void QXmppRpcMarshaller::marshall(QXmlStreamWriter *writer, const QVariant &value) { writer->writeStartElement("value"); switch( value.type() ) { case QVariant::Int: case QVariant::UInt: case QVariant::LongLong: case QVariant::ULongLong: writer->writeTextElement("i4", value.toString()); break; case QVariant::Double: writer->writeTextElement("double", value.toString()); break; case QVariant::Bool: writer->writeTextElement("boolean", value.toBool() ? "1" : "0"); break; case QVariant::Date: writer->writeTextElement("dateTime.iso8601", value.toDate().toString( Qt::ISODate ) ); break; case QVariant::DateTime: writer->writeTextElement("dateTime.iso8601", value.toDateTime().toString( Qt::ISODate ) ); break; case QVariant::Time: writer->writeTextElement("dateTime.iso8601", value.toTime().toString( Qt::ISODate ) ); break; case QVariant::StringList: case QVariant::List: { writer->writeStartElement("array"); writer->writeStartElement("data"); foreach(const QVariant &item, value.toList()) marshall(writer, item); writer->writeEndElement(); writer->writeEndElement(); break; } case QVariant::Map: { writer->writeStartElement("struct"); QMap<QString, QVariant> map = value.toMap(); QMap<QString, QVariant>::ConstIterator index = map.begin(); while( index != map.end() ) { writer->writeStartElement("member"); writer->writeTextElement("name", index.key()); marshall( writer, *index ); writer->writeEndElement(); ++index; } writer->writeEndElement(); break; } case QVariant::ByteArray: { writer->writeTextElement("base64", value.toByteArray().toBase64() ); break; } default: { if (value.isNull()) writer->writeEmptyElement("nil"); else if( value.canConvert(QVariant::String) ) { writer->writeTextElement("string", value.toString() ); } break; } } writer->writeEndElement(); }
inline double NumberMarshaller::marshallBoxed(JNIEnv * const env, const jobject & in) const { return marshall(env, Utils::unboxDouble(env, in)); }
inline double NumberMarshaller::callMethod(JNIEnv * const env, const jobject obj, const jmethodID methodID, jvalue * const args) const { return marshall(env, env->CallDoubleMethodA(obj, methodID, args)); }
inline double NumberMarshaller::getFieldValue(JNIEnv * const env, const jobject obj, const jfieldID fieldID) const { return marshall(env, env->GetDoubleField(obj, fieldID)); }
void ArchiveDataHeader::demarshall() { marshall(); }
void replication_app_client_base::call(request_context* request, bool no_delay) { auto& msg = request->callback_task->get_request(); auto nts = ::dsn::service::env::now_us(); if (nts + 100 >= msg->header().client.timeout_ts_us) // < 100us { message_ptr nil(nullptr); end_request(request, ERR_TIMEOUT, nil); delete request; return; } end_point addr; int app_id; error_code err = get_address( request->partition_index, !request->is_read, addr, app_id, request->read_header.semantic ); // target node in cache if (err == ERR_SUCCESS) { dbg_dassert(addr != end_point::INVALID, ""); dassert(app_id > 0, ""); if (request->header_pos != 0xffff) { if (request->is_read) { request->read_header.gpid.app_id = app_id; marshall(msg->writer(), request->read_header, request->header_pos); msg->header().client.hash = gpid_to_hash(request->read_header.gpid); } else { request->write_header.gpid.app_id = app_id; marshall(msg->writer(), request->write_header, request->header_pos); msg->header().client.hash = gpid_to_hash(request->write_header.gpid); } request->header_pos = 0xffff; } rpc::call( addr, msg, this, std::bind( &replication_app_client_base::replica_rw_reply, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, request ) ); } // target node not known else if (!no_delay) { // delay 1 second for further config query tasking::enqueue(LPC_REPLICATION_DELAY_QUERY_CONFIG, this, std::bind(&replication_app_client_base::call, this, request, true), 0, 1000 ); } else { zauto_lock l(_requests_lock); // init timeout timer if necessary if (request->timeout_timer == nullptr) { request->timeout_timer = tasking::enqueue( LPC_REPLICATION_CLIENT_REQUEST_TIMEOUT, this, std::bind(&replication_app_client_base::on_user_request_timeout, this, request), 0, static_cast<int>((msg->header().client.timeout_ts_us - nts) / 1000) ); } // put into pending queue of querying target partition auto it = _pending_requests.find(request->partition_index); if (it == _pending_requests.end()) { auto pc = new partition_context; pc->query_config_task = nullptr; it = _pending_requests.insert(pending_requests::value_type(request->partition_index, pc)).first; } it->second->requests.push_back(request); // init configuration query task if necessary if (it->second->query_config_task == nullptr) { message_ptr msg = message::create_request(RPC_CM_CALL); meta_request_header hdr; hdr.rpc_tag = RPC_CM_QUERY_PARTITION_CONFIG_BY_INDEX; marshall(msg->writer(), hdr); configuration_query_by_index_request req; req.app_name = _app_name; req.partition_indices.push_back(request->partition_index); marshall(msg->writer(), req); it->second->query_config_task = rpc::call_replicated( _last_contact_point, _meta_servers, msg, this, std::bind(&replication_app_client_base::query_partition_configuration_reply, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, request->partition_index ) ); } } }
void RollBack::save() { qDebug() << "EEEE RollBack::save()" << endl; _settings->saveQvariantValueFor(SETTINGS_KEY_CACHE_ROLLBACK, QVariant(marshall())); LOGIT("save()"); }
int callFunction(const char *app, const char *obj, const char *func, const KStringList args) { QString f = func; // Qt is better with unicode strings, so use one. int left = f.find('('); int right = f.find(')'); if(right < left) { qWarning("parentheses do not match"); return (1); } if(left < 0) { // try to get the interface from the server bool ok = false; KStringList funcs = dcop->remoteFunctions(app, obj, &ok); QCString realfunc; if(!ok && args.isEmpty()) goto doit; if(!ok) { qWarning("object not accessible"); return (1); } for(const auto &it : funcs) { int l = it.find('('); int s; if(l > 0) s = it.findRev(' ', l); else s = it.find(' '); if(s < 0) s = 0; else s++; if(l > 0 && it.mid(s, l - s) == func) { realfunc = it.mid(s); const QString arguments = it.mid(l + 1, it.find(')') - l - 1); uint a = arguments.contains(','); if((a == 0 && !arguments.isEmpty()) || a > 0) a++; if(a == args.count()) break; } } if(realfunc.isEmpty()) { qWarning("no such function"); return (1); } f = realfunc; left = f.find('('); right = f.find(')'); } doit: if(left < 0) f += "()"; // This may seem expensive but is done only once per invocation // of dcop, so it should be OK. // // QStringList intTypes; intTypes << "int" << "unsigned" << "long" << "bool"; QStringList types; if(left > 0 && left + 1 < right - 1) { types = QStringList::split(',', f.mid(left + 1, right - left - 1)); for(QStringList::Iterator it = types.begin(); it != types.end(); ++it) { QString lt = (*it).simplifyWhiteSpace(); int s = lt.find(' '); // If there are spaces in the name, there may be two // reasons: the parameter name is still there, ie. // "QString URL" or it's a complicated int type, ie. // "unsigned long long int bool". // // if(s > 0) { QStringList partl = QStringList::split(' ', lt); // The zero'th part is -- at the very least -- a // type part. Any trailing parts *might* be extra // int-type keywords, or at most one may be the // parameter name. // // s = 1; while(s < static_cast< int >(partl.count()) && intTypes.contains(partl[s])) { s++; } if(s < static_cast< int >(partl.count()) - 1) { qWarning("The argument `%s' seems syntactically wrong.", lt.latin1()); } if(s == static_cast< int >(partl.count()) - 1) { partl.remove(partl.at(s)); } lt = partl.join(" "); lt = lt.simplifyWhiteSpace(); } (*it) = lt; } QString fc = f.left(left); fc += '('; bool first = true; for(QStringList::Iterator it = types.begin(); it != types.end(); ++it) { if(!first) fc += ","; first = false; fc += *it; } fc += ')'; f = fc; } QByteArray data, replyData; QCString replyType; QDataStream arg(data, IO_WriteOnly); uint i = 0; for(QStringList::Iterator it = types.begin(); it != types.end(); ++it) marshall(arg, args, i, *it); if(i != args.count()) { qWarning("arguments do not match"); return (1); } if(!dcop->call(app, obj, f.latin1(), data, replyType, replyData)) { qWarning("call failed"); return (1); } else { QDataStream reply(replyData, IO_ReadOnly); if(replyType != "void" && replyType != "ASYNC") { QCString replyString = demarshal(reply, replyType); if(!replyString.isEmpty()) printf("%s\n", replyString.data()); else printf("\n"); } } return 0; }
void Intrinsics::demarshall() { marshall(); }
void PulseParameters::demarshall() { marshall(); }
void mutation::write_to(binary_writer& writer) { marshall(writer, data); marshall(writer, rpc_code); }
bool findObject(const char *app, const char *obj, const char *func, KStringList args) { QString f = func; // Qt is better with unicode strings, so use one. int left = f.find('('); int right = f.find(')'); if(right < left) { qWarning("parentheses do not match"); exit(1); } if(!f.isEmpty() && (left < 0)) f += "()"; // This may seem expensive but is done only once per invocation // of dcop, so it should be OK. // // QStringList intTypes; intTypes << "int" << "unsigned" << "long" << "bool"; QStringList types; if(left > 0 && left + 1 < right - 1) { types = QStringList::split(',', f.mid(left + 1, right - left - 1)); for(QStringList::Iterator it = types.begin(); it != types.end(); ++it) { QString lt = (*it).simplifyWhiteSpace(); int s = lt.find(' '); // If there are spaces in the name, there may be two // reasons: the parameter name is still there, ie. // "QString URL" or it's a complicated int type, ie. // "unsigned long long int bool". // // if(s > 0) { QStringList partl = QStringList::split(' ', lt); // The zero'th part is -- at the very least -- a // type part. Any trailing parts *might* be extra // int-type keywords, or at most one may be the // parameter name. // // s = 1; while(s < (int)partl.count() && intTypes.contains(partl[s])) { s++; } if(s < (int)partl.count() - 1) { qWarning("The argument `%s' seems syntactically wrong.", lt.latin1()); } if(s == (int)partl.count() - 1) { partl.remove(partl.at(s)); } lt = partl.join(" "); lt = lt.simplifyWhiteSpace(); } (*it) = lt; } QString fc = f.left(left); fc += '('; bool first = true; for(QStringList::Iterator it = types.begin(); it != types.end(); ++it) { if(!first) fc += ","; first = false; fc += *it; } fc += ')'; f = fc; } if(types.count() != args.count()) { qWarning("arguments do not match"); exit(1); } QByteArray data; QDataStream arg(data, IO_WriteOnly); uint i = 0; for(QStringList::Iterator it = types.begin(); it != types.end(); ++it) { marshall(arg, args, i, *it); } if((uint)i != args.count()) { qWarning("arguments do not match"); exit(1); } QCString foundApp; QCString foundObj; if(dcop->findObject(app, obj, f.latin1(), data, foundApp, foundObj)) { if(bAppIdOnly) puts(foundApp.data()); else printf("DCOPRef(%s,%s)\n", qStringToC(foundApp), qStringToC(foundObj)); return true; } return false; }