TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseInsitu_MemoryPoolAllocator)) {
for (size_t i = 0; i < kTrialCount; i++) {
memcpy(temp_, json_, length_ + 1);
Document doc;
doc.ParseInsitu(temp_);
ASSERT_TRUE(doc.IsObject());
}
}
name_t nameFromJsonData(const char *data, size_t len) {
std::vector<char> buffer(len + 1);
std::memcpy(&buffer[0], data, len);
Document doc;
doc.ParseInsitu(&buffer[0]);
return name_t(doc);
}
TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseInsitu_MemoryPoolAllocator)) {
//const size_t userBufferSize = 128 * 1024;
//char* userBuffer = (char*)malloc(userBufferSize);
for (size_t i = 0; i < kTrialCount; i++) {
memcpy(temp_, json_, length_ + 1);
//MemoryPoolAllocator<> allocator(userBuffer, userBufferSize);
//Document doc(&allocator);
Document doc;
doc.ParseInsitu(temp_);
ASSERT_TRUE(doc.IsObject());
//if (i == 0) {
// size_t size = doc.GetAllocator().Size();
// size_t capacity = doc.GetAllocator().Capacity();
// size_t stack_capacity = doc.GetStackCapacity();
// size_t actual = size - stack_capacity;
// std::cout << "Size:" << size << " Capacity:" << capacity << " Stack:" << stack_capacity << " Actual:" << actual << std::endl;
//}
}
//free(userBuffer);
}
std::vector<name_t> nameArrayFromData(const char *jsonData, size_t len) {
std::vector<char> buffer(len + 1);
std::memcpy(&buffer[0], jsonData, len);
Document doc;
doc.ParseInsitu(&buffer[0]);
assert(doc.IsArray());
std::vector<name_t> nameArray;
nameArray.reserve(doc.Size());
for( auto array_item = doc.Begin(); array_item != doc.End(); array_item++ ) {
name_t instance = name_t(*array_item);
nameArray.push_back(instance);
}
return nameArray;
}
int main(int, char*[]) {
////////////////////////////////////////////////////////////////////////////
// 1. Parse a JSON text string to a document.
const char json[] = " { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } ";
printf("Original JSON:\n %s\n", json);
Document document; // Default template parameter uses UTF8 and MemoryPoolAllocator.
#if 0
// "normal" parsing, decode strings to new buffers. Can use other input stream via ParseStream().
if (document.Parse(json).HasParseError())
return 1;
#else
// In-situ parsing, decode strings directly in the source string. Source must be string.
char buffer[sizeof(json)];
memcpy(buffer, json, sizeof(json));
if (document.ParseInsitu(buffer).HasParseError())
return 1;
#endif
printf("\nParsing to document succeeded.\n");
////////////////////////////////////////////////////////////////////////////
// 2. Access values in document.
printf("\nAccess values in document:\n");
assert(document.IsObject()); // Document is a JSON value represents the root of DOM. Root can be either an object or array.
assert(document.HasMember("hello"));
assert(document["hello"].IsString());
printf("hello = %s\n", document["hello"].GetString());
// Since version 0.2, you can use single lookup to check the existing of member and its value:
Value::MemberIterator hello = document.FindMember("hello");
assert(hello != document.MemberEnd());
assert(hello->value.IsString());
assert(strcmp("world", hello->value.GetString()) == 0);
(void)hello;
assert(document["t"].IsBool()); // JSON true/false are bool. Can also uses more specific function IsTrue().
printf("t = %s\n", document["t"].GetBool() ? "true" : "false");
assert(document["f"].IsBool());
printf("f = %s\n", document["f"].GetBool() ? "true" : "false");
printf("n = %s\n", document["n"].IsNull() ? "null" : "?");
assert(document["i"].IsNumber()); // Number is a JSON type, but C++ needs more specific type.
assert(document["i"].IsInt()); // In this case, IsUint()/IsInt64()/IsUInt64() also return true.
printf("i = %d\n", document["i"].GetInt()); // Alternative (int)document["i"]
assert(document["pi"].IsNumber());
assert(document["pi"].IsDouble());
printf("pi = %g\n", document["pi"].GetDouble());
{
const Value& a = document["a"]; // Using a reference for consecutive access is handy and faster.
assert(a.IsArray());
for (SizeType i = 0; i < a.Size(); i++) // rapidjson uses SizeType instead of size_t.
printf("a[%d] = %d\n", i, a[i].GetInt());
int y = a[0].GetInt();
(void)y;
// Iterating array with iterators
printf("a = ");
for (Value::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr)
printf("%d ", itr->GetInt());
printf("\n");
}
// Iterating object members
static const char* kTypeNames[] = { "Null", "False", "True", "Object", "Array", "String", "Number" };
for (Value::ConstMemberIterator itr = document.MemberBegin(); itr != document.MemberEnd(); ++itr)
printf("Type of member %s is %s\n", itr->name.GetString(), kTypeNames[itr->value.GetType()]);
////////////////////////////////////////////////////////////////////////////
// 3. Modify values in document.
// Change i to a bigger number
{
uint64_t f20 = 1; // compute factorial of 20
for (uint64_t j = 1; j <= 20; j++)
f20 *= j;
document["i"] = f20; // Alternate form: document["i"].SetUint64(f20)
assert(!document["i"].IsInt()); // No longer can be cast as int or uint.
}
// Adding values to array.
{
Value& a = document["a"]; // This time we uses non-const reference.
Document::AllocatorType& allocator = document.GetAllocator();
for (int i = 5; i <= 10; i++)
a.PushBack(i, allocator); // May look a bit strange, allocator is needed for potentially realloc. We normally uses the document's.
// Fluent API
a.PushBack("Lua", allocator).PushBack("Mio", allocator);
}
// Making string values.
// This version of SetString() just store the pointer to the string.
// So it is for literal and string that exists within value's life-cycle.
{
document["hello"] = "rapidjson"; // This will invoke strlen()
// Faster version:
// document["hello"].SetString("rapidjson", 9);
}
// This version of SetString() needs an allocator, which means it will allocate a new buffer and copy the the string into the buffer.
Value author;
{
char buffer[10];
int len = sprintf(buffer, "%s %s", "Milo", "Yip"); // synthetic example of dynamically created string.
author.SetString(buffer, static_cast<size_t>(len), document.GetAllocator());
// Shorter but slower version:
// document["hello"].SetString(buffer, document.GetAllocator());
// Constructor version:
// Value author(buffer, len, document.GetAllocator());
// Value author(buffer, document.GetAllocator());
memset(buffer, 0, sizeof(buffer)); // For demonstration purpose.
}
// Variable 'buffer' is unusable now but 'author' has already made a copy.
document.AddMember("author", author, document.GetAllocator());
assert(author.IsNull()); // Move semantic for assignment. After this variable is assigned as a member, the variable becomes null.
////////////////////////////////////////////////////////////////////////////
// 4. Stringify JSON
printf("\nModified JSON with reformatting:\n");
StringBuffer sb;
PrettyWriter<StringBuffer> writer(sb);
document.Accept(writer); // Accept() traverses the DOM and generates Handler events.
puts(sb.GetString());
return 0;
}
void main_loop(void *ptr)
{
//¼ÓÔØÃüÁî
string file_buf,line;
ifstream infile("sys_config.txt",ios::in);
if(infile.fail()){
exit(-1);
}
file_buf.clear();
while(!infile.eof()){
getline(infile,line);
file_buf.append(line);
}
infile.close();
//json½âÎö
char *json_buf = (char*)malloc(file_buf.length() + 1);
memcpy(json_buf,file_buf.data(),file_buf.length());
json_buf[file_buf.length()]='\0';
Document document;
if(document.ParseInsitu((json_buf)).HasParseError()){
exit(-1);
}
assert(document.IsObject());
assert(document.HasMember("listen"));
string listen_ip;
uint16_t listen_port;
listen_port = document["listen"]["port"].GetInt();
listen_ip = document["listen"]["ip"].GetString();
free(json_buf);
WORD wVersionRequested;
WSADATA wsaData;
int err;
wVersionRequested = MAKEWORD(2,0);
err = WSAStartup(wVersionRequested,&wsaData);
if(err != 0){
cout<<"WSAStartup failure"<<endl;
return;
}
if(LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 0){
WSACleanup();
std::cout<<"WSA version != 2.0"<<endl;
return;
}
SOCKET sockSrv = socket(AF_INET,SOCK_STREAM,0);
SOCKADDR_IN addrSrv;
addrSrv.sin_addr.S_un.S_addr=inet_addr(listen_ip.c_str());
addrSrv.sin_family = AF_INET;
addrSrv.sin_port = htons(listen_port);
bind(sockSrv,(SOCKADDR*)&addrSrv,sizeof(SOCKADDR));
listen(sockSrv,5);
cout<<"server listen at:"<<listen_port<<endl;
while(1){
//SOCKET sockConn=accept(sockSrv,(SOCKADDR*)&(sockInfoPtr->addrClient),&len);
SOCKET sockConn=accept(sockSrv,NULL,NULL);
if(sockConn == -1){
cout<<"accept errno=%s"<<strerror(errno);
}
else{
cout<<"new client:"<<endl;
}
ConCount++;
PacketIO *SockInfoPtr= new PacketIO(ConCount);
SockInfoPtr->SockConn = sockConn;
SockInfoPtr->Num = ConCount;
_beginthread(gcs_thread,0,SockInfoPtr);
}
WSACleanup();
}
bool GateConfig::load(const char* jsonConfig)
{
char *json = filetool::get_whole_file_buf(jsonConfig);
if (NULL == json) {
return false;
}
Document doc;
if (doc.ParseInsitu(json).HasParseError()) {
LOG_ERROR << "parse config<" << jsonConfig << "> failed, error code = " << doc.GetParseError() << ", error offset = " << doc.GetErrorOffset()
<< ", err text = " << json[doc.GetErrorOffset()];
delete[] json;
return false;
}
if (!doc.IsObject()) {
LOG_ERROR;
return false;
}
{
// ¶ÁÈ¡ÄÚÍøÍøÂçÅäÖÃ
const Value& wan = doc["wan"];
const Value& wanListen = wan["listen"];
m_wanListen.ip = wanListen["ip"].GetString();
m_wanListen.port = wanListen["port"].GetInt();
m_wanThreadNum = wan["threads"].GetInt();
}
{
// ¶ÁÈ¡ÍâÍøÍøÂçÅäÖÃ
const Value& lan = doc["lan"];
const Value& lanListens = lan["listen"];
const Value& lanConnects = lan["listen"];
IpPort ipport;
for (SizeType i = 0; i < lanListens.Size(); i++) {
const Value &listen = lanListens[i];
ipport.ip = listen["ip"].GetString();
ipport.port = listen["port"].GetInt();
m_lanListens.push_back(ipport);
}
for (SizeType i = 0; i < lanConnects.Size(); i++) {
const Value &connect = lanConnects[i];
ipport.ip = connect["ip"].GetString();
ipport.port = connect["port"].GetInt();
m_lanConnects.push_back(ipport);
}
m_lanThreadNum = lan["threads"].GetInt();
}
delete[] json;
return true;
}
/**
* @brief SDSMemoryNode::reciveMns
* @param mns
*/
void SDSMemoryNode::reciveMns(char * message){
cout<<message<<endl;
Document doc;
doc.ParseInsitu(message);
if(doc.IsObject()){
if(doc.HasMember("protocolo")){
if(doc["protocolo"].IsString()){
string comando;
comando=doc["protocolo"].GetString();
if(comando=="d_calloc"){
if(doc.HasMember("pSize")){
if(doc["pSize"].IsInt()){
int pSize= doc["pSize"].GetInt();
d_calloc(pSize);
}
}
}
else if(comando=="d_status"){
d_status();
}
else{
d_pointer_size pointerSize;
if(doc.HasMember("dir")){
if(doc["dir"].IsInt()){
int pDir= doc["dir"].GetInt();
pointerSize.pointer.dirMemory=pDir;
}
}
if(doc.HasMember("pSize")){
if(doc["pSize"].IsInt()){
int pSize= doc["pSize"].GetInt();
pointerSize.bytes=pSize;
}
}
if(comando=="d_free"){
d_free(pointerSize);
}
else{
//Documento para en caso de que se quiera obtener el dato
StringBuffer s;
Writer<StringBuffer> writer(s);
writer.StartObject();
writer.String("dato");
int status;
bystream statusBystream;
if(doc.HasMember("tipo")){
if(doc["tipo"].IsString()){
string tipo= doc["tipo"].GetString();
if(tipo=="char"){
if(comando=="d_set"){
if(doc.HasMember("dato")){
if(doc["dato"].IsString()){
//Parsear a char
char pData =*(const_cast<char *>(doc["dato"].GetString()));
status= Manejador.setearDatoChar(pointerSize,pData);
}
}
}
if(comando=="d_get"){
statusBystream=Manejador.obtenerDatoChar(pointerSize);
writer.String(&(statusBystream.datachar));
}
}
else if(tipo=="int"){
if(comando=="d_set"){
if(doc.HasMember("dato")){
if(doc["dato"].IsInt()){
//Parsear a int
int pData =doc["dato"].GetInt();
status= Manejador.setearDatoInt(pointerSize,pData);
}
}
}
if(comando=="d_get"){
statusBystream=Manejador.obtenerDatoInt(pointerSize);
writer.Int(statusBystream.dataint);
}
}
else if(tipo=="bool"){
//Parsear a bool
//status= Manejador.setearDatoBool(pointerSize,pData);
}
else if(tipo=="float"){
//parsear a float
// status= Manejador.setearDatoFloat(pointerSize,pData);
}
else if(tipo=="arrayint"){
//parsear a arreglo de int
// status= Manejador.setearDatoArrayInt(pointerSize,pData);
}
else if(tipo=="arraychar"){
//Parsear a arreglos de char
//status= Manejador.setearDatoArrayChar(pointerSize,pData);
}
else if(tipo=="long"){
//Parsear a long
// status= Manejador.setearDatoLong(pointerSize,pData);
}
else if(tipo=="double"){
//Parsear a double
// status= Manejador.setearDatoDouble(pointerSize,pData);
}
if(comando=="d_set"){
d_set(status);
}
if(comando=="d_get"){
writer.String("status");
writer.Int(statusBystream.status);
writer.EndObject();
const char* mensaje=s.GetString();
cout<<"Enviando... "<<mensaje<<endl;
puerto->sentMns(mensaje,client);
}
}}
}
}
}
}
}
}
int main( int argc, char* argv[] ) {
// Prepare reader and input stream.
Reader reader;
string shopIdStr = argv[1];
string nameStr = argv[2];
ofstream outFs;
if ( argc < 3 )
{
cout << "usage: shopIdString, resultfile";
return 0;
}
bool isEndOfTable = false;
if ( argc > 3 )
{
string isTailStr = argv[3];
if ( "tail" == isTailStr )
{
isEndOfTable = true;
}
}
//get working path
stringstream ss;
ss << nameStr << "/";
pathStr = ss.str().c_str();
ss << "sample/";
samplePathStr = ss.str().c_str();
//open record file for appending
string recordFileNameStr = pathStr + nameStr + ".txt";
outFs.open( recordFileNameStr.c_str(), std::ios_base::app );
//first command, download then extract comment
string filterStr = "\" | pup ul.\"comment-list\" json{}";
std::string headStr = " -s \"www.dianping.com/shop/" ;
headStr += shopIdStr;
headStr += filterStr;
//save this into each shop's json record files
std::ofstream resultFile;
stringstream rss;
rss << samplePathStr.c_str() << nameStr << shopIdStr << ".txt";
resultFile.open( rss.str().c_str() );
Document document; // Default template parameter uses UTF8 and MemoryPoolAllocator.
string strBuf;
{
stringstream ss;
ss << "curl" << cookieStr << headStr;
std::cout << ss.str();
//fetch web page from server
strBuf = exec( ss.str().c_str() );
if ( strBuf.size() > 2 )
{
strBuf = strBuf.substr( 1, strBuf.size() - 2);
}
//std::cout << strBuf;
resultFile << strBuf;
resultFile.close();
#if 1
// "normal" parsing, decode strings to new buffers. Can use other input stream via ParseStream().
if (document.Parse(strBuf.c_str()).HasParseError())
return 1;
#else
// In-situ parsing, decode strings directly in the source string. Source must be string.
char buffer[sizeof(json)];
memcpy(buffer, json, sizeof(json));
if (document.ParseInsitu(buffer).HasParseError())
return 1;
#endif
////////////////////////////////////////////////////////////////////////////
// 2. Access values in document.
assert(document.IsObject()); // Document is a JSON value represents the root of DOM. Root can be either an object or array.
cout << "parsed";
cout.flush();
//const Value& a = document["children"][0]["children"][0]["children"];
const Value& b = document["children"];
assert(b.IsArray());
string tagStr = b[0]["tag"].GetString();
if ( "div" == tagStr )
{
ExtractCommentAndPic( b[0]["children"][0]["children"], shopIdStr, outFs, isEndOfTable );
}
else
{
ExtractCommentAndPic( b, shopIdStr, outFs, isEndOfTable );
}
/*
if ( !(a.Size())
{
break;
}
*/
//cout << nextCursor << '\n';
//cout << totalNumber << '\n';
//cout.flush();
}
outFs.close();
return 0;
}
AbstractWorkSource::Events AbstractWorkSource::Refresh(bool canRead, bool canWrite) {
// This is public and due to TCP async connect nature, we cannot assume we're ready to go so shortcircuit if still waiting.
if(!stratum) return Events();
// As a start, dispatch my data to the server. It is required to do this first as we need to
// initiate connection with a mining.subscribe. Each tick, we send as many bytes as we can until
// write buffer is exhausted.
const time_t PREV_WORK_STAMP(stratum->LastNotifyTimestamp());
if(stratum->pending.size() && canWrite) {
asizei count = 1;
while(count && stratum->pending.size()) {
StratumState::Blob &msg(stratum->pending.front());
auto sent(Send(msg.data + msg.sent, msg.total - msg.sent));
if(sent.first == false) {
Events ohno;
ohno.connFailed = true;
return ohno;
}
count = sent.second;
msg.sent += count;
if(msg.sent == msg.total) {
#if STRATUM_DUMPTRAFFIC
stratumDump<<">>sent to server:"<<std::endl;
for(asizei i = 0; i < msg.total; i++) stratumDump<<msg.data[i];
stratumDump<<std::endl;
#endif
stratum->pending.pop();
}
}
}
Events ret;
if(canRead == false) return ret; // sends are still considered nops, as they don't really change the hi-level state
auto received(Receive(recvBuffer.NextBytes(), recvBuffer.Remaining()));
if(received.first == false) {
ret.connFailed = true;
return ret;
}
else if(received.second == 0) return ret;
ret.bytesReceived += received.second;
recvBuffer.used += received.second;
if(recvBuffer.Full()) recvBuffer.Grow();
using namespace rapidjson;
Document object;
char *pos = recvBuffer.data.get();
char *lastEndl = nullptr;
const auto prevDiff(GetCurrentDiff());
const auto prevJob(stratum->GetCurrentJob());
while(pos < recvBuffer.NextBytes()) {
char *limit = std::find(pos, recvBuffer.NextBytes(), '\n');
if(limit >= recvBuffer.NextBytes()) pos = limit;
else { // I process one line at time
#if STRATUM_DUMPTRAFFIC
stratumDump<<">>from server:";
for(asizei i = 0; pos + i < limit; i++) stratumDump<<pos[i];
stratumDump<<std::endl;
#endif
lastEndl = limit;
ScopedFuncCall restoreChar([limit]() { *limit = '\n'; }); // not really necessary but I like the idea
*limit = 0;
object.ParseInsitu(pos);
const Value::ConstMemberIterator &id(object.FindMember("id"));
const Value::ConstMemberIterator &method(object.FindMember("method"));
// There was a time in which stratum had "notifications" and "requests". They were the same thing basically but requests had an id
// to be used for confirmations. Besides some idiot wanted to use 0 as an ID, P2Pool servers always attach an ID even to notifications,
// which is a less brain-damaged thing but still mandated some changes here.
std::string idstr;
aulong idvalue = 0;
if(id != object.MemberEnd()) {
switch(id->value.GetType()) {
case kNumberType: {
if(id->value.IsUint()) idvalue = id->value.GetUint();
else if(id->value.IsInt()) idvalue = id->value.GetInt();
else if(id->value.IsUint64()) idvalue = id->value.GetUint64();
else if(id->value.IsInt64()) idvalue = id->value.GetInt64();
idstr = std::to_string(idvalue);
break;
}
case kStringType:
idstr.assign(id->value.GetString(), id->value.GetStringLength());
for(size_t check = 0; check < idstr.length(); check++) {
char c = idstr[check];
if(c < '0' || c > '9') throw std::exception("All my ids are naturals>0, this should be a natural>0 number!");
}
idvalue = strtoul(idstr.c_str(), NULL, 10);
break;
}
}
/* If you read the minimalistic documentation of stratum you get the idea you can trust on some things.
No, you don't. The only real thing you can do is figure out if something is a request from the server or a reply. */
if(method != object.MemberEnd() && method->value.IsString()) {
if(method->value.GetString()) MangleMessageFromServer(idstr, method->value.GetString(), object["params"]);
}
else { // I consider it a reply. Then it has .result or .error... MAYBE. P2Pool for example sends .result=.error=null as AUTH replies to say it doesn't care about who's logging in!
MangleReplyFromServer(idvalue, object["result"], object["error"]);
}
pos = lastEndl + 1;
}
}
if(lastEndl) { // at least a line has been mangled
const char *src = lastEndl + 1;
char *dst = recvBuffer.data.get();
for(; src < recvBuffer.NextBytes(); src++, dst++) *dst = *src;
recvBuffer.used = src - (lastEndl + 1);
#if _DEBUG
for(; dst < recvBuffer.data.get() + recvBuffer.allocated; dst++) *dst = 0;
#endif
}
else return ret;
const auto nowDiff(GetCurrentDiff());
const auto nowJob(stratum->GetCurrentJob());
auto different = [](const stratum::MiningNotify &one, const stratum::MiningNotify &two) {
if(one.job != two.job || one.ntime != two.ntime || one.clear != two.clear) return true; // most likely
if(one.prevHash != two.prevHash) return true; // fairly likely
// blockVer is mostly constant,
// nbits is... ?
if(one.merkles.size() != two.merkles.size()) return true; // happens quite often
if(one.coinBaseOne.size() != two.coinBaseOne.size() || one.coinBaseTwo.size() != two.coinBaseTwo.size()) return true; // not likely, if at all possible
bool diff = false;
for(asizei i = 0; i < one.merkles.size(); i++) diff |= one.merkles[i].hash != two.merkles[i].hash;
for(asizei i = 0; i < one.coinBaseOne.size(); i++) diff |= one.coinBaseOne[i] != two.coinBaseOne[i];
for(asizei i = 0; i < one.coinBaseTwo.size(); i++) diff |= one.coinBaseTwo[i] != two.coinBaseTwo[i];
return diff;
};
ret.diffChanged = nowDiff != prevDiff;
ret.newWork = different(prevJob, nowJob) && GetCurrentDiff().shareDiff != .0; // new work is to be delayed as long as diff is 0
if(prevDiff.shareDiff == .0 && nowDiff.shareDiff != .0) {
// When this happens and we already have a job of any sort we can finally flush the new work to the outer code
if(nowJob.job.size()) ret.newWork = true;
}
return ret;
}
