size_t ProtobufSocketSerializer::serialize(
const ::google::protobuf::Message &msg, char *&packet) {
size_t size = PROTOBUF_HEADER_LENGTH + (size_t) msg.ByteSize();
packet = new char[size]();
::google::protobuf::io::ArrayOutputStream aos(packet, (int) size);
::google::protobuf::io::CodedOutputStream coded_output(&aos);
coded_output.WriteLittleEndian32((google::protobuf::uint32) msg.ByteSize());
msg.SerializeToCodedStream(&coded_output);
return size;
}
bool WriteMessage(const shared_ptr<UnixSocketConnection> &connection,
uint32_t type,
const google::protobuf::Message &response) {
shared_ptr<vector<char> > data(new vector<char>(response.ByteSize() + sizeof(type)));
char *ptr = &(*data)[0];
*reinterpret_cast<uint32_t *>(ptr) = type;
if (!response.SerializeToArray(ptr + sizeof(type), data->size() - sizeof(type))) {
WARNING_LOG("serialize message to string failed.");
return false;
}
connection->Write(data);
return true;
}
void CommonMemoryCache::Set(
const std::string& key,
const google::protobuf::Message& message)
{
std::string buffer;
if (!message.SerializeToString(&buffer)) {
LOG(ERROR)
<< "serialize to string failed. message ["
<< message.ShortDebugString() << "]";
return;
}
RWLock::WriterLocker locker(&m_rw_lock);
m_cache_map[key] = buffer;
}
void reply(const google::protobuf::Message& message)
{
CHECK(from) << "Attempting to reply without a sender";
std::string data;
message.SerializeToString(&data);
send(from, message);
}
/** Send a message.
* @param client ID of the client to addresss
* @param m Message to send, the message must have an CompType enum type to
* specify component ID and message type.
*/
void
ProtobufStreamServer::send(ClientID client, google::protobuf::Message &m)
{
const google::protobuf::Descriptor *desc = m.GetDescriptor();
const google::protobuf::EnumDescriptor *enumdesc = desc->FindEnumTypeByName("CompType");
if (! enumdesc) {
throw std::logic_error("Message does not have CompType enum");
}
const google::protobuf::EnumValueDescriptor *compdesc =
enumdesc->FindValueByName("COMP_ID");
const google::protobuf::EnumValueDescriptor *msgtdesc =
enumdesc->FindValueByName("MSG_TYPE");
if (! compdesc || ! msgtdesc) {
throw std::logic_error("Message CompType enum hs no COMP_ID or MSG_TYPE value");
}
int comp_id = compdesc->number();
int msg_type = msgtdesc->number();
if (comp_id < 0 || comp_id > std::numeric_limits<uint16_t>::max()) {
throw std::logic_error("Message has invalid COMP_ID");
}
if (msg_type < 0 || msg_type > std::numeric_limits<uint16_t>::max()) {
throw std::logic_error("Message has invalid MSG_TYPE");
}
send(client, comp_id, msg_type, m);
}
int mongoproxy_update(const google::protobuf::Message & msg, const std::string & fields, const char * cb_data, int cb_size){
MongoOPReq requpdate;
std::vector<string> vs;
dcs::strsplit(fields, ",", vs);
if (vs.empty()){
GLOG_ERR("not found the fields define :%s", fields.c_str());
return -1;
}
google::protobuf::Message * tmpmsg = msg.New();
tmpmsg->CopyFrom(msg);
auto reflect = tmpmsg->GetReflection();
auto desc = tmpmsg->GetDescriptor();
for (int i = 0; i < desc->field_count(); ++i){
auto field = desc->field(i);
if (std::find(vs.begin(), vs.end(), field->name()) != vs.end()){
continue;
}
reflect->ClearField(tmpmsg, field);
}
string fieldsquery;
pbjson::pb2json(tmpmsg, fieldsquery);
delete tmpmsg;
requpdate.set_q(fieldsquery);
return _mongoproxy_cmd(MONGO_OP_UPDATE, msg, true, &requpdate, cb_data, cb_size);
}
void PersistentData::getValueFilepath(const QString& filepath, google::protobuf::Message& data, Global::DataFolderType dataFolderType, bool humanReadable)
try
{
QFile file(filepath);
if (!file.open(QIODevice::ReadOnly))
throw UnknownValueException();
#if !DEBUG
if (humanReadable)
{
#endif
google::protobuf::io::FileInputStream fileStream(file.handle());
google::protobuf::TextFormat::Parse(&fileStream, &data);
#if !DEBUG
}
else
{
data.ParsePartialFromFileDescriptor(file.handle());
}
#endif
}
catch(Global::UnableToGetFolder& e)
{
throw PersistentDataIOException(e.errorMessage);
}
bool ZlibCompress(const google::protobuf::Message& res, butil::IOBuf* buf) {
butil::IOBufAsZeroCopyOutputStream wrapper(buf);
google::protobuf::io::GzipOutputStream::Options zlib_opt;
zlib_opt.format = google::protobuf::io::GzipOutputStream::ZLIB;
google::protobuf::io::GzipOutputStream zlib(&wrapper, zlib_opt);
return res.SerializeToZeroCopyStream(&zlib) && zlib.Close();
}
void NetworkClient::Send(int packetID, const google::protobuf::Message& msg)
{
int msgSize = msg.ByteSize();
int bufferSize = Packet::Header::Size + msgSize;
Packet::Header header;
header.packetID = packetID;
header.length = msgSize;
auto buffer = std::unique_ptr<char[]>(new char[bufferSize]);
header.Serialize(buffer.get());
msg.SerializeToArray(buffer.get() + Packet::Header::Size, msgSize);
conn->Send(bufferSize, buffer.get());
}
bool NFNetModule::SendMsgPB(const uint16_t nMsgID, const google::protobuf::Message& xData, const NFSOCK nSockIndex)
{
NFMsg::MsgBase xMsg;
if (!xData.SerializeToString(xMsg.mutable_msg_data()))
{
std::ostringstream stream;
stream << " SendMsgPB Message to " << nSockIndex;
stream << " Failed For Serialize of MsgData, MessageID " << nMsgID;
m_pLogModule->LogError(stream, __FUNCTION__, __LINE__);
return false;
}
NFMsg::Ident* pPlayerID = xMsg.mutable_player_id();
*pPlayerID = NFToPB(NFGUID());
std::string strMsg;
if (!xMsg.SerializeToString(&strMsg))
{
std::ostringstream stream;
stream << " SendMsgPB Message to " << nSockIndex;
stream << " Failed For Serialize of MsgBase, MessageID " << nMsgID;
m_pLogModule->LogError(stream, __FUNCTION__, __LINE__);
return false;
}
SendMsgWithOutHead(nMsgID, strMsg, nSockIndex);
return true;
}
bool NFNetModule::SendMsgPBToAllClient(const uint16_t nMsgID, const google::protobuf::Message& xData)
{
NFMsg::MsgBase xMsg;
if (!xData.SerializeToString(xMsg.mutable_msg_data()))
{
std::ostringstream stream;
stream << " SendMsgPBToAllClient";
stream << " Failed For Serialize of MsgData, MessageID " << nMsgID;
m_pLogModule->LogError(stream, __FUNCTION__, __LINE__);
return false;
}
std::string strMsg;
if (!xMsg.SerializeToString(&strMsg))
{
std::ostringstream stream;
stream << " SendMsgPBToAllClient";
stream << " Failed For Serialize of MsgBase, MessageID " << nMsgID;
m_pLogModule->LogError(stream, __FUNCTION__, __LINE__);
return false;
}
return SendMsgToAllClientWithOutHead(nMsgID, strMsg);
}
bool write_message(boost::asio::ip::tcp::socket& s, ::google::protobuf::Message& msg)
{
string msg_str;
if (!msg.SerializeToString(&msg_str))
return false;
bling_pb::header hdr;
if (typeid(msg) == typeid(bling_pb::get_slave_list))
hdr.set_msg_id(bling_pb::header::GET_SLAVE_LIST);
else if (typeid(msg) == typeid(bling_pb::slave_list))
hdr.set_msg_id(bling_pb::header::SLAVE_LIST);
else if (typeid(msg) == typeid(bling_pb::set_slave_tlc))
hdr.set_msg_id(bling_pb::header::SET_SLAVE_TLC);
else if (typeid(msg) == typeid(bling_pb::start_effect))
hdr.set_msg_id(bling_pb::header::START_EFFECT);
else
cout << "Unidentified message type!!!" << endl;
hdr.set_len(msg_str.size());
string hdr_str;
if (!hdr.SerializeToString(&hdr_str))
return false;
if (hdr_str.size() != header_length)
cout << "Header length=" << hdr_str.size() << endl;
string tx_buffer = hdr_str + msg_str;
size_t n = write_string(s, tx_buffer);
return n == tx_buffer.size();
}
int UDPListener::writeMessageToBuffer(Common::MessageHeader::MessageType type, const google::protobuf::Message& message)
{
const int bodySize = message.ByteSize();
Common::MessageHeader header(type, bodySize, this->peerManager->getID());
if (Common::MessageHeader::HEADER_SIZE + bodySize > static_cast<int>(SETTINGS.get<quint32>("max_udp_datagram_size")))
{
L_ERRO(QString("Datagram size too big : %1").arg(Common::MessageHeader::HEADER_SIZE + bodySize));
return 0;
}
Common::MessageHeader::writeHeader(this->buffer, header);
message.SerializeToArray(this->bodyBuffer, BUFFER_SIZE - Common::MessageHeader::HEADER_SIZE);
return Common::MessageHeader::HEADER_SIZE + bodySize;
}
/// merges
/// crutch for repeated fields
void clear_repeated( gpb::Message &to, gpb::Message const &from )
{
gpb::Reflection const *refl( to.GetReflection() );
typedef std::vector<gpb::FieldDescriptor const *> flds_vector;
flds_vector fields;
refl->ListFields( from, &fields );
flds_vector::const_iterator b(fields.begin( ));
flds_vector::const_iterator e(fields.end( ));
for(; b!=e; ++b) {
const gpb::FieldDescriptor *next(*b);
if ( next->is_repeated( ) ) {
refl->ClearField( &to, next );
} else if( field_is_message( next ) ) {
if( message_has_repeated( next ) ) {
refl->MutableMessage( &to, next )->Clear( );
} else if( refl->HasField( to, next ) ) {
clear_repeated( *refl->MutableMessage( &to, next ),
refl->GetMessage( from, next ));
}
}
}
}
inline Resource Sender::PopFree() {
if (_nfree == 0) {
if (_nalloc == 0) {
Resource r;
r.response = _response;
r.sub_done = &_sub_done0;
_alloc_resources[_nalloc++] = r;
return r;
} else if (_nalloc == 1) {
Resource r;
r.response = _response->New();
r.sub_done = new SubDone(this);
_alloc_resources[_nalloc++] = r;
return r;
} else {
CHECK(false) << "nalloc=" << _nalloc;
return Resource();
}
} else {
Resource r = _free_resources[--_nfree];
r.response->Clear();
Controller& sub_cntl = r.sub_done->_cntl;
ExcludedServers* saved_accessed = sub_cntl._accessed;
sub_cntl._accessed = NULL;
sub_cntl.Reset();
sub_cntl._accessed = saved_accessed;
return r;
}
}
bool default_protocol::parse_raw_message(const std::string &mess,
google::protobuf::Message &out )
{
const size_t hash_length = hash_checker_->hash_size( );
return out.ParseFromArray( mess.c_str( ) + hash_length,
mess.size( ) - hash_length );
}
/**
This function gets the (assumed to be set) integer field's value from the message and casts it to a int64_t value.
@param inputMessage: The message to retrieve the integer from
@param inputField: The specific field to retrieve it from
@throw: This function can throw exceptions
*/
int64_t pylongps::getIntegerFieldValue(const google::protobuf::Message &inputMessage, const google::protobuf::FieldDescriptor *inputField)
{
const google::protobuf::Reflection *messageReflection = inputMessage.GetReflection();
int64_t value = 0;
switch(inputField->cpp_type())
{
case google::protobuf::FieldDescriptor::CPPTYPE_INT32:
value = messageReflection->GetInt32(inputMessage, inputField);
break;
case google::protobuf::FieldDescriptor::CPPTYPE_INT64:
value = messageReflection->GetInt64(inputMessage, inputField);
break;
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32:
value = messageReflection->GetUInt32(inputMessage, inputField);
break;
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64:
value = messageReflection->GetUInt64(inputMessage, inputField);
break;
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM:
value = messageReflection->GetEnum(inputMessage, inputField)->number();
break;
default:
throw SOMException("Non-integer data type\n", INVALID_FUNCTION_INPUT, __FILE__, __LINE__);
break;
}
return value;
}
/**
This function binds the value of a protobuf field to a SQLite statement according to the type of the field (can only be a primative type).
@param inputStatement: The statement to bind the field value to
@param inputSQLStatementIndex: The index of the statement field to bind
@param inputMessage: The message to get the value from
@param inputField: The field (from the message) to get the value from
@throw: This function can throw exceptions
*/
void pylongps::bindFieldValueToStatement(sqlite3_stmt &inputStatement, uint32_t inputSQLStatementIndex, const google::protobuf::Message &inputMessage, const google::protobuf::FieldDescriptor *inputField)
{
const google::protobuf::Reflection *messageReflection = inputMessage.GetReflection();
int returnValue = 0;
if(messageReflection->HasField(inputMessage, inputField) == true)
{
switch(inputField->cpp_type())
{
case google::protobuf::FieldDescriptor::CPPTYPE_INT32:
returnValue = sqlite3_bind_int64(&inputStatement, inputSQLStatementIndex, messageReflection->GetInt32(inputMessage, inputField));
break;
case google::protobuf::FieldDescriptor::CPPTYPE_INT64:
returnValue = sqlite3_bind_int64(&inputStatement, inputSQLStatementIndex, messageReflection->GetInt64(inputMessage, inputField));
break;
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32:
returnValue = sqlite3_bind_int64(&inputStatement, inputSQLStatementIndex, messageReflection->GetUInt32(inputMessage, inputField));
break;
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64:
returnValue = sqlite3_bind_int64(&inputStatement, inputSQLStatementIndex, messageReflection->GetUInt64(inputMessage, inputField));
break;
case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE:
returnValue = sqlite3_bind_double(&inputStatement, inputSQLStatementIndex, messageReflection->GetDouble(inputMessage, inputField));
break;
case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT:
returnValue = sqlite3_bind_double(&inputStatement, inputSQLStatementIndex, messageReflection->GetDouble(inputMessage, inputField));
break;
case google::protobuf::FieldDescriptor::CPPTYPE_BOOL:
returnValue = sqlite3_bind_int64(&inputStatement, inputSQLStatementIndex, messageReflection->GetBool(inputMessage, inputField));
break;
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM:
returnValue = sqlite3_bind_int64(&inputStatement, inputSQLStatementIndex, messageReflection->GetEnum(inputMessage, inputField)->number());
break;
case google::protobuf::FieldDescriptor::CPPTYPE_STRING:
{
std::string buffer;
buffer = messageReflection->GetStringReference(inputMessage, inputField, &buffer);
returnValue = sqlite3_bind_blob64(&inputStatement, inputSQLStatementIndex, (const void*) buffer.c_str(), buffer.size(), SQLITE_TRANSIENT);
}
break;
default:
throw SOMException("Unrecognized data type\n", AN_ASSUMPTION_WAS_VIOLATED_ERROR, __FILE__, __LINE__);
break;
}
}
else
{ //Its a optional field that is not set, so store a NULL value
returnValue = sqlite3_bind_null(&inputStatement, inputSQLStatementIndex);
}
if(returnValue != SQLITE_OK)
{
throw SOMException("Database error occurred (" + std::to_string(returnValue)+ ")\n", SQLITE3_ERROR, __FILE__, __LINE__);
}
}
void CGroupManagerClient::SendMessageToClient(const ::google::protobuf::Message &msg, s32 msgid)
{
int count = msg.ByteSize();
byte buf[MAX_MSG_LEN] = {0};
*((s32*)buf) = msgid;
if( msg.SerializeToArray(buf+MSG_ID_LEN, MAX_MSG_LEN) )
{
if( !SendPkg( buf, count+MSG_ID_LEN ) )
{
SERVER_LOG_ERROR( "CGroupManagerClient,SendMessageToGroupServer,SendPkg" );
}
}
else
{
SERVER_LOG_ERROR( "CGroupManagerClient,SendMessageToGroupServer,SerializeToArray" );
}
}
/*
* Send a ConfigureDevice() request
* @param message the request to send
*/
bool OlaConfigurator::SendMessage(const google::protobuf::Message &message) {
string request_string;
message.SerializeToString(&request_string);
return m_client->ConfigureDevice(
m_alias,
request_string,
NewSingleCallback(this, &OlaConfigurator::HandleConfigResponse));
}
/// true if equal
bool messages_diff ( gpb::Message const &templ,
gpb::Message &src,
gpb::FieldDescriptor const *fld )
{
gpb::Reflection const *treflect( templ.GetReflection( ) );
gpb::Reflection const *sreflect( src.GetReflection( ) );
if( message_has_repeated( fld ) ) {
gpb::Message const &tmpl_next( treflect->GetMessage( templ, fld ) );
gpb::Message *src_next( sreflect->MutableMessage( &src, fld ) );
std::string tmpls(tmpl_next.SerializeAsString());
std::string srss(src_next->SerializeAsString());
return tmpls == srss;
}
if( !fld->is_repeated( ) ) {
gpb::Message const &tmpl_next( treflect->GetMessage( templ, fld ) );
gpb::Message *src_next( sreflect->MutableMessage( &src, fld ) );
bool equal = false;
if( make_message_diff( tmpl_next, *src_next, *src_next ) ) {
sreflect->ClearField( &src, fld );
equal = true;
}
return equal;
}
int tsize( templ.GetReflection( )->FieldSize( templ, fld ) );
int ssize( src.GetReflection( )->FieldSize( src, fld ) );
if( tsize != ssize ) return false;
for( int i(0); i<tsize; ++i ) {
std::string tmess( treflect->GetRepeatedMessage( templ, fld, i )
.SerializeAsString( ) );
std::string smess( sreflect->GetRepeatedMessage( src, fld, i )
.SerializeAsString( ) );
if( tmess.compare(smess) != 0 ) return false;
}
sreflect->ClearField( &src, fld );
return true;
}
int send(int fd,
const std::string& name,
const ::google::protobuf::Message& message,
uint64_t mid) {
SocketWriter writer(fd);
Version_1_Protocol protocol;
Msg msg;
message.SerializeToString(msg.mutable_content());
msg.set_mid(mid);
msg.set_name(name);
int ret = protocol.encode(msg, &writer);
if (ret) {
LOG_WARN << "encode request error, method["
<< name << "] data[" << message.DebugString()<< "]";
return 1;
}
return writer.write(1000) == SocketWriter::kOk ? 0 : -1;
}
/**
* Writes a given protobuff message to the buffer (this->buffer) prefixed by a header.
* @return the total size (header size + message size). Return 0 if the total size is bigger than 'Protos.Core.Settings.max_udp_datagram_size'.
*/
int UDPListener::writeMessageToBuffer(Common::MessageHeader::MessageType type, const google::protobuf::Message& message)
{
const Common::MessageHeader header(type, message.ByteSize(), this->getOwnID());
const int nbBytesWritten = Common::Message::writeMessageToBuffer(this->buffer, this->MAX_UDP_DATAGRAM_PAYLOAD_SIZE, header, &message);
if (!nbBytesWritten)
L_ERRO(QString("Datagram size too big: %1, max allowed: %2").arg(Common::MessageHeader::HEADER_SIZE + header.getSize()).arg(this->MAX_UDP_DATAGRAM_PAYLOAD_SIZE));
return nbBytesWritten;
}
void decode_delimited(pb::Message& message, const std::string& data) {
pb::io::CodedInputStream stream((pb::uint8*)&data[0], data.size());
pb::uint64 length;
if (!stream.ReadVarint64(&length))
throw DecodeFailed("Failed to decode delimited message (length)");
if (!message.ParseFromCodedStream(&stream))
throw DecodeFailed("Failed to decode delimited message");
if (!stream.ExpectAtEnd())
throw DecodeFailed("Failed to decode delimited message (did not consume entire buffer)");
}
//! Client handle
void HandleRequest(const google::protobuf::Message& message, const net::IConnection::Ptr& connection)
{
SCOPED_LOG(m_Log);
#ifdef _DEBUG
std::string request;
#endif
try
{
#ifdef _DEBUG
request = message.DebugString();
assert(!request.empty());
#endif
LOG_DEBUG("Request: [%s]") % message.DebugString();
IStatistics* stats = 0;
{
boost::mutex::scoped_lock lock(m_StatsMutex);
StatisticsMap::iterator it = m_Statistics.find(GetCurrentThreadId());
if (it == m_Statistics.end())
it = m_Statistics.insert(std::make_pair(GetCurrentThreadId(), boost::shared_ptr<IStatistics>(new MongoStatistics(m_Log)))).first;
stats = it->second.get();
}
NeuroDecisionMaker decisionMaker(m_Log, m_Evaluator, *stats, m_Network, *connection);
pcmn::TableLogic logic(m_Log, decisionMaker, m_Evaluator);
logic.Parse(static_cast<const net::Packet&>(message));
LOG_WARNING("Requests processed: [%s]") % ++m_RequestsCount;
}
catch (const std::exception& e)
{
std::ostringstream oss;
oss << "Error: " << e.what() << std::endl
<< "Packet: " << message.DebugString();
net::Reply reply;
reply.set_error(oss.str());
LOG_ERROR("%s") % oss.str();
connection->Send(reply);
}
}
event_index_status EventIndex::SetGTIDIndex(const string >id, const ::google::protobuf::Message &data_info) {
string buffer;
if (!data_info.SerializeToString(&buffer)) {
ColorLogError("buffer serialize fail");
return event_index_status::DATA_ERROR;
}
//LogVerbose("%s gtid %s", __func__, gtid.c_str());
leveldb::Status status = level_db_->Put(leveldb::WriteOptions(), gtid, buffer);
return status.ok() ? event_index_status::OK : event_index_status::DB_ERROR;
}
void AbstractService::sendToUnifier(const std::string &to, ::google::protobuf::Message &msg, unifier::UnifierMessage_Type type) {
unifier::UnifierMessage w;
w.set_to(to);
w.set_type(type);
std::string payload;
msg.SerializeToString(&payload);
w.set_payload(payload);
sendToUnifier(w);
}
bool GzipCompress(const google::protobuf::Message& msg, butil::IOBuf* buf) {
butil::IOBufAsZeroCopyOutputStream wrapper(buf);
google::protobuf::io::GzipOutputStream::Options gzip_opt;
gzip_opt.format = google::protobuf::io::GzipOutputStream::GZIP;
google::protobuf::io::GzipOutputStream gzip(&wrapper, gzip_opt);
if (!msg.SerializeToZeroCopyStream(&gzip)) {
LogError(gzip);
return false;
}
return gzip.Close();
}
///
/// \brief ·¢ËÍÊý¾Ý
///
/// \author albert.xu
/// \date 2017/08/05
///
xgc_void CClientSession::Send( xgc_uint16 msgid, ::google::protobuf::Message& msg )
{
#pragma pack(1)
struct
{
MessageHeader h;
char b[1024 * 4];
} m;
#pragma pack()
xgc_uint16 data_size = msg.ByteSize();
xgc_uint16 pack_size = sizeof( MessageHeader ) + data_size;
m.h.length = htons( pack_size );
m.h.message = htons( msgid );
msg.SerializeToArray( m.b, sizeof( m.b ) );
SendPacket( handle_, &m, pack_size );
}
bool SnappyCompress(const google::protobuf::Message& res, butil::IOBuf* buf) {
butil::IOBuf serialized_pb;
butil::IOBufAsZeroCopyOutputStream wrapper(&serialized_pb);
if (res.SerializeToZeroCopyStream(&wrapper)) {
butil::IOBufAsSnappySource source(serialized_pb);
butil::IOBufAsSnappySink sink(*buf);
return butil::snappy::Compress(&source, &sink);
}
LOG(WARNING) << "Fail to serialize input pb=" << &res;
return false;
}