std::string Book::get(std::string search) // return all books for a search crtieria
{
std::list<BSONObj> list = get(search, 1);
JsonHelper h;
return h.BSONArray2JSONStr(list);
}
/* ****************************************************************************
*
* Subject::toJson -
*/
std::string Subject::toJson()
{
JsonHelper jh;
jh.addRaw("entities", vectorToJson(this->entities));
jh.addRaw("condition", this->condition.toJson());
return jh.str();
}
std::string Book::getAll() // return all books as JSON Array String
{
std::list<BSONObj> list = pCon->getAll(DB_BOOKS);
JsonHelper h;
return h.BSONArray2JSONStr(list);
}
/* ****************************************************************************
*
* EntID::toJson -
*/
std::string EntID::toJson()
{
JsonHelper jh;
jh.addString("id", this->id);
jh.addString("idPattern", this->idPattern);
jh.addString("type", this->type);
return jh.str();
}
/* ****************************************************************************
*
* Condition::toJson -
*/
std::string Condition::toJson()
{
JsonHelper jh;
jh.addRaw("attrs", vectorToJson(this->attributes));
JsonHelper jhe;
if (this->expression.q != "") jhe.addString("q", this->expression.q);
if (this->expression.mq != "") jhe.addString("mq", this->expression.mq);
if (this->expression.geometry != "") jhe.addString("geometry", this->expression.geometry);
if (this->expression.coords != "") jhe.addString("coords", this->expression.coords);
if (this->expression.georel != "") jhe.addString("georel", this->expression.georel);
std::string expressionString = jhe.str();
if (expressionString != "{}") jh.addRaw("expression", expressionString);
return jh.str();
}
/* ****************************************************************************
*
* Notification::toJson -
*
* FIXME P2: we should move 'attrsFormat' from Subject class to Notification
* class, to avoid passing attrsFormat as argument
*
*/
std::string Notification::toJson(const std::string& attrsFormat)
{
JsonHelper jh;
if (this->timesSent > 0)
{
jh.addNumber("timesSent", this->timesSent);
}
if (this->lastNotification > 0)
{
jh.addDate("lastNotification", this->lastNotification);
}
if (!this->blacklist)
{
jh.addRaw("attrs", vectorToJson(this->attributes));
}
else
{
jh.addRaw("exceptAttrs", vectorToJson(this->attributes));
}
jh.addString("attrsFormat", attrsFormat);
if (this->httpInfo.custom)
{
jh.addRaw("httpCustom", this->httpInfo.toJson());
}
else
{
jh.addRaw("http", this->httpInfo.toJson());
}
if (this->metadata.size() > 0)
{
jh.addRaw("metadata", vectorToJson(this->metadata));
}
return jh.str();
}
/* ****************************************************************************
*
* Condition::toJson -
*/
std::string Condition::toJson()
{
JsonHelper jh;
jh.addRaw("attributes", vectorToJson(this->attributes));
{
JsonHelper jhe;
jhe.addString("q", this->expression.q);
jhe.addString("geometry", this->expression.geometry);
jhe.addString("coords", this->expression.coords);
jh.addRaw("expression", jhe.str());
}
return jh.str();
}
/* ****************************************************************************
*
* Subscription::toJson -
*/
std::string Subscription::toJson()
{
JsonHelper jh;
jh.addString("id", this->id);
if (this->expires > 0)
{
jh.addDate("expires", this->expires);
}
jh.addString("status", this->status);
jh.addRaw("subject", this->subject.toJson());
jh.addRaw("notification", this->notification.toJson());
return jh.str();
}
/* ****************************************************************************
*
* Subscription::toJson -
*/
std::string Subscription::toJson()
{
JsonHelper jh;
jh.addString("id", this->id);
if (this->description != "")
{
jh.addString("description", this->description);
}
if (this->expires != PERMANENT_SUBS_DATETIME)
{
jh.addDate("expires", this->expires);
}
jh.addString("status", this->status);
jh.addRaw("subject", this->subject.toJson());
jh.addRaw("notification", this->notification.toJson(renderFormatToString(this->attrsFormat, true, true)));
if (this->throttling > 0)
{
jh.addNumber("throttling", this->throttling);
}
return jh.str();
}
/* ****************************************************************************
*
* Notification::toJson -
*/
std::string Notification::toJson()
{
JsonHelper jh;
jh.addString("callback", this->callback);
if (this->throttling > 0)
{
jh.addNumber("throttling", this->throttling);
}
if (this->timesSent > 0)
{
jh.addNumber("timesSent", this->timesSent);
}
if (this->lastNotification > 0)
{
jh.addDate("lastNotification", this->lastNotification);
}
jh.addRaw("attributes", vectorToJson(this->attributes));
return jh.str();
}
/* ****************************************************************************
*
* EntID::toJson -
*/
std::string EntID::toJson()
{
JsonHelper jh;
if (!this->id.empty())
{
jh.addString("id", this->id);
}
if (!this->idPattern.empty())
{
jh.addString("idPattern", this->idPattern);
}
if (!this->type.empty())
{
jh.addString("type", this->type);
}
if (!this->typePattern.empty())
{
jh.addString("typePattern", this->typePattern);
}
return jh.str();
}
int main()
{
JsonHelper helper;
auto filenames = readfilenames();
vector<Item*> items;
table CWEs;
table CAPECs;
table Risks;
table CPEs;
int count = 0;
for (auto filename : filenames)
{
if (count % 10 == 0) progressBar(count * 1.0 / NumFiles * 0.5);
auto item = helper.parse(fopen(filename.c_str(), "r"));
items.push_back(item);
push(CWEs, item->CWE, count);
for (auto c : item -> CAPEC)
push(CAPECs, c, count);
push(Risks, item->Risk, count);
for (auto c : item->CPE)
push(CPEs, c, count);
count ++;
}
// CWE table
printTableStats(CWEs, "statistics/CWE.csv");
printTableStats(CAPECs, "statistics/CAPEC.csv");
printTableStats(Risks, "statistics/Risk.csv");
printTableStats(CPEs, "statistics/CPE.csv");
progressBar(0.9);
// double-variable
vector<Cell> CWE_Risk;
vector<Cell> CWE_CAPEC;
vector<Cell> CPE_CWE;
table CPETop200;
takeTop(CPEs, CPETop200, 200);
calculateList(CWE_Risk, CWEs, Risks);
calculateList(CWE_CAPEC, CWEs, CAPECs);
calculateList(CPE_CWE, CPETop200, CWEs);
printList(CWE_Risk, "statistics/CWE_Risk.csv");
printList(CWE_CAPEC, "statistics/CWE_CAPEC.csv");
printList(CPE_CWE, "statistics/CPE_CWE.csv");
progressBar(1);
cout << endl;
return 0;
}
/* ****************************************************************************
*
* renderTimingStatistics -
*
* xxxReqTime - the total time that the LAST request took.
* Measuring from the first MHD callback to 'connectionTreat',
* until the MHD callback to 'requestCompleted'.
* xxxXmlParseTime - the time that the XML Parse of the LAST request took.
* xxxJsonV1ParseTime - the time that the JSON parse+treat of the LAST request took.
* xxxJsonV2ParseTime - the time that the JSON parse+treat of the LAST request took.
* xxxMongoBackendTime - the time that the mongoBackend took to treat the last request
* xxxReadWaitTime -
* xxxWriteWaitTime -
* xxxCommandWaitTime -
* xxxRenderTime - the time that the last render took to render the response
*
*/
std::string renderTimingStatistics(void)
{
timeStatSemTake(__FUNCTION__, "putting stats together");
bool accJsonV1ParseTime = (accTimeStat.jsonV1ParseTime.tv_sec != 0) || (accTimeStat.jsonV1ParseTime.tv_nsec != 0);
bool accJsonV2ParseTime = (accTimeStat.jsonV2ParseTime.tv_sec != 0) || (accTimeStat.jsonV2ParseTime.tv_nsec != 0);
bool accMongoBackendTime = (accTimeStat.mongoBackendTime.tv_sec != 0) || (accTimeStat.mongoBackendTime.tv_nsec != 0);
bool accMongoReadWaitTime = (accTimeStat.mongoReadWaitTime.tv_sec != 0) || (accTimeStat.mongoReadWaitTime.tv_nsec != 0);
bool accMongoWriteWaitTime = (accTimeStat.mongoWriteWaitTime.tv_sec != 0) || (accTimeStat.mongoWriteWaitTime.tv_nsec != 0);
bool accMongoCommandWaitTime = (accTimeStat.mongoCommandWaitTime.tv_sec != 0) || (accTimeStat.mongoCommandWaitTime.tv_nsec != 0);
bool accRenderTime = (accTimeStat.renderTime.tv_sec != 0) || (accTimeStat.renderTime.tv_nsec != 0);
bool accReqTime = (accTimeStat.reqTime.tv_sec != 0) || (accTimeStat.reqTime.tv_nsec != 0);
bool accXmlParseTime = (accTimeStat.xmlParseTime.tv_sec != 0) || (accTimeStat.xmlParseTime.tv_nsec != 0);
bool lastJsonV1ParseTime = (lastTimeStat.jsonV1ParseTime.tv_sec != 0) || (lastTimeStat.jsonV1ParseTime.tv_nsec != 0);
bool lastJsonV2ParseTime = (lastTimeStat.jsonV2ParseTime.tv_sec != 0) || (lastTimeStat.jsonV2ParseTime.tv_nsec != 0);
bool lastMongoBackendTime = (lastTimeStat.mongoBackendTime.tv_sec != 0) || (lastTimeStat.mongoBackendTime.tv_nsec != 0);
bool lastMongoReadWaitTime = (lastTimeStat.mongoReadWaitTime.tv_sec != 0) || (lastTimeStat.mongoReadWaitTime.tv_nsec != 0);
bool lastMongoWriteWaitTime = (lastTimeStat.mongoWriteWaitTime.tv_sec != 0) || (lastTimeStat.mongoWriteWaitTime.tv_nsec != 0);
bool lastMongoCommandWaitTime = (lastTimeStat.mongoCommandWaitTime.tv_sec != 0) || (lastTimeStat.mongoCommandWaitTime.tv_nsec != 0);
bool lastRenderTime = (lastTimeStat.renderTime.tv_sec != 0) || (lastTimeStat.renderTime.tv_nsec != 0);
bool lastReqTime = (lastTimeStat.reqTime.tv_sec != 0) || (lastTimeStat.reqTime.tv_nsec != 0);
bool lastXmlParseTime = (lastTimeStat.xmlParseTime.tv_sec != 0) || (lastTimeStat.xmlParseTime.tv_nsec != 0);
bool last = lastJsonV1ParseTime || lastJsonV2ParseTime || lastMongoBackendTime || lastRenderTime || lastReqTime || lastXmlParseTime;
bool acc = accJsonV1ParseTime || accJsonV2ParseTime || accMongoBackendTime || accRenderTime || accReqTime || accXmlParseTime;
if (!acc && !last)
{
timeStatSemGive(__FUNCTION__, "no stats to report");
return "{}";
}
JsonHelper jh;
if (acc)
{
JsonHelper accJh;
if (accJsonV1ParseTime) accJh.addFloat("jsonV1Parse", timeSpecToFloat(accTimeStat.jsonV1ParseTime));
if (accJsonV2ParseTime) accJh.addFloat("jsonV2Parse", timeSpecToFloat(accTimeStat.jsonV2ParseTime));
if (accMongoBackendTime) accJh.addFloat("mongoBackend", timeSpecToFloat(accTimeStat.mongoBackendTime));
if (accMongoReadWaitTime) accJh.addFloat("mongoReadWait", timeSpecToFloat(accTimeStat.mongoReadWaitTime));
if (accMongoWriteWaitTime) accJh.addFloat("mongoWriteWait", timeSpecToFloat(accTimeStat.mongoWriteWaitTime));
if (accMongoCommandWaitTime) accJh.addFloat("mongoCommandWait", timeSpecToFloat(accTimeStat.mongoCommandWaitTime));
if (accRenderTime) accJh.addFloat("render", timeSpecToFloat(accTimeStat.renderTime));
if (accReqTime) accJh.addFloat("total", timeSpecToFloat(accTimeStat.reqTime));
if (accXmlParseTime) accJh.addFloat("xmlParse", timeSpecToFloat(accTimeStat.xmlParseTime));
jh.addRaw("accumulated", accJh.str());
}
if (last)
{
JsonHelper lastJh;
if (lastJsonV1ParseTime) lastJh.addFloat("jsonV1Parse", timeSpecToFloat(lastTimeStat.jsonV1ParseTime));
if (lastJsonV2ParseTime) lastJh.addFloat("jsonV2Parse", timeSpecToFloat(lastTimeStat.jsonV2ParseTime));
if (lastMongoBackendTime) lastJh.addFloat("mongoBackend", timeSpecToFloat(lastTimeStat.mongoBackendTime));
if (lastMongoReadWaitTime) lastJh.addFloat("mongoReadWait", timeSpecToFloat(lastTimeStat.mongoReadWaitTime));
if (lastMongoWriteWaitTime) lastJh.addFloat("mongoWriteWait", timeSpecToFloat(lastTimeStat.mongoWriteWaitTime));
if (lastMongoCommandWaitTime) lastJh.addFloat("mongoCommandWait", timeSpecToFloat(lastTimeStat.mongoCommandWaitTime));
if (lastRenderTime) lastJh.addFloat("render", timeSpecToFloat(lastTimeStat.renderTime));
if (lastReqTime) lastJh.addFloat("total", timeSpecToFloat(lastTimeStat.reqTime));
if (lastXmlParseTime) lastJh.addFloat("xmlParse", timeSpecToFloat(lastTimeStat.xmlParseTime));
jh.addRaw("last", lastJh.str());
}
timeStatSemGive(__FUNCTION__, "putting stats together");
return jh.str();
}
void DeviceAgent::onDataComing()
{
START_RECV:
if (m_dataPos == 0 && m_readState == READ_STATE_WAIT_PACKAGE_HEADER)
{
memset(&m_dataHdr, 0 ,sizeof(m_dataHdr));
m_datalen = sizeof(m_dataHdr);
delete m_data;
m_data = NULL;
}
if (m_readState == READ_STATE_WAIT_PACKAGE_HEADER)
{
char *hdrStart = (char*)&m_dataHdr + m_dataPos;
int dataReaded = read(hdrStart,m_datalen - m_dataPos);
if (dataReaded == 0)
{
//no data, wait for next read
return;
}
m_dataPos += dataReaded;
if (m_dataPos < sizeof(NetPackageHdr))
{
return;
}
if (m_dataPos > sizeof(NetPackageHdr))
{
DEBUG_INFO << "Read error!!!!!!!!!!";
onPackageDataError();
return;
}
m_dataHdr.TAG = ntohl(m_dataHdr.TAG);
m_dataHdr.data_len = ntohl64(m_dataHdr.data_len);
if (m_dataHdr.TAG == PL_REQ_DOWNLOAD_FILE)
{
m_readState = READ_STATE_WAIT_FILE_JSON_HDR_LEN;
m_dataPos = 0;
m_datalen = sizeof(qint64);
}
else
{
m_readState = READ_STATE_WAIT_PACKAGE_DATA;
m_dataPos = 0;
m_datalen = m_dataHdr.data_len;
}
if (m_datalen > 0)
{
m_data = new QByteArray(m_datalen,0);
}
}
if (m_readState == READ_STATE_WAIT_PACKAGE_DATA)
{
if (m_datalen > 0)
{
int dataReaded = read(m_data->data() + m_dataPos,m_datalen - m_dataPos);
if (dataReaded == 0)
{
return;
}
m_dataPos += dataReaded;
}
if (m_dataPos < m_datalen)
{
return;
}
if (m_dataPos > m_datalen)
{
DEBUG_INFO << "Read error!!!!!!!!!!";
DEBUG_INFO << "m_datalen:" <<m_datalen << " m_dataPos:" << m_dataPos;
onPackageDataError();
return;
}
m_readState = READ_STATE_WAIT_PACKAGE_HEADER;
m_dataPos = 0;
/*Notify controller here,need to block until data is copied out*/
if (m_dataHdr.TAG == PL_REQ_REPORT_ID)
{
m_devID = m_data->data();
emit deviceConnected(this);
}
else
{
emit dataReceived(m_devID,m_dataHdr.TAG,m_data);
}
}
if (m_readState == READ_STATE_WAIT_FILE_JSON_HDR_LEN)
{
int dataReaded = read(m_data->data() + m_dataPos,m_datalen - m_dataPos);
if (dataReaded == 0)
{
return;
}
m_dataPos += dataReaded;
if (m_dataPos < m_datalen)
{
return;
}
if (m_dataPos > m_datalen)
{
DEBUG_INFO << "Read error!!!!!!!!!!";
onPackageDataError();
return;
}
m_readState = READ_STATE_WAIT_FILE_JSON_HDR;
m_dataPos = 0;
memcpy((char*)&m_datalen,m_data->data(),sizeof(m_datalen));
m_datalen = ntohl64(m_datalen);
delete m_data;
m_data = NULL;
if (m_datalen > 0)
{
m_data = new QByteArray(m_datalen,0);
}
else
{
onPackageDataError();
return;
}
}
if (m_readState == READ_STATE_WAIT_FILE_JSON_HDR)
{
int dataReaded = read(m_data->data() + m_dataPos,m_datalen - m_dataPos);
if (dataReaded == 0)
{
return;
}
m_dataPos += dataReaded;
if (m_dataPos < m_datalen)
{
return;
}
if (m_dataPos > m_datalen)
{
DEBUG_INFO << "Read error!!!!!!!!!!";
onPackageDataError();
return;
}
JsonHelper jsonHelper;
if (PL_RET_OK != jsonHelper.parseObject(m_data))
{
onPackageDataError();
return;
}
QJsonObject jsonHdr = jsonHelper.m_jsonObj;
m_srcFileName = jsonHdr.value("path").toString();
m_dstFileName = jsonHdr.value("dstPath").toString();
m_fileTransferRetCode = jsonHdr.value("retCode").toInt();
m_datalen = m_dataHdr.data_len - m_datalen;
if (m_datalen == 0)
{
//need post error here
DEBUG_INFO << "No file data";
emit dataReceived(m_devID,m_dataHdr.TAG,NULL);
m_dataPos = 0;
m_readState = READ_STATE_WAIT_PACKAGE_HEADER;
}
else
{
m_readState = READ_STATE_WAIT_FILE_DATA;
m_dataPos = 0;
}
}
if (m_readState == READ_STATE_WAIT_FILE_DATA)
{
int buflen = 20480;
char dataBuf[20480];
if ((m_datalen - m_dataPos) < buflen)
{
//this is to make sure do not read extra data
buflen = m_datalen - m_dataPos;
}
int dataReaded = read(dataBuf,buflen);
if (dataReaded < 1)
{
return;
}
//save data to file
m_downloadfile.setFileName(m_dstFileName);
bool opened = false;
if (m_dataPos == 0 )
{
//DEBUG_INFO << "Empty existing file" << m_dstFileName;
opened = m_downloadfile.open(QIODevice::WriteOnly | QIODevice::Truncate);
}
else
{
opened = m_downloadfile.open( QIODevice::Append);
}
if (!opened)
{
DEBUG_INFO << "Failed to open file:" + m_dstFileName << m_downloadfile.error();
m_downloadfile.remove();
onPackageDataError();
return;
}
if (m_downloadfile.write(dataBuf,dataReaded) != dataReaded)
{
DEBUG_INFO << "Failed to save file:" + m_downloadfile.error() << ":" <<m_downloadfile.errorString();
m_downloadfile.close();
m_downloadfile.remove();
onPackageDataError();
return;
}
m_downloadfile.close();
m_dataPos += dataReaded;
if (++m_iLoopCounter % 100 == 0 && m_datalen)
{
static qint64 last_percent = 0;
qint64 percent = (m_dataPos*100)/m_datalen;
if (percent != last_percent)
{
last_percent = percent;
QString info = QString("%1% data received").arg(percent);
emit busyMessage(info);
}
}
if (m_dataPos == m_datalen)
{
emit dataReceived(m_devID,m_dataHdr.TAG,m_data);
emit busyMessage("Receive file data done");
m_dataPos = 0;
m_readState = READ_STATE_WAIT_PACKAGE_HEADER;
}
}
goto START_RECV;
}