/**********************************************************
*convert FileRec to bson, then write to db
*
***********************************************************/
void FileRec::writeToDB(mongo::DBClientConnection &conn) {
BSONObjBuilder record; //build BSONObj
record.append("filename", this->filename);
record.append("Tempname", this->tempname);
record.append("curhash", this->recentHash);
record.append("ovhash", this->origHash);
record.append("length", this->length);
record.append("nversions", this->versionCount);
/* love you */ long long time = this->modifytime.tv_nsec;
record.append("Mtnsec", time);
time = this->modifytime.tv_sec;
record.append("mtsec", time);
record.append("currentversion", this->refNum);
mongo::BSONArrayBuilder bArr; //arrays to store multiple objects inside main object
mongo::BSONArrayBuilder Comments;
for (vector<string>::iterator it = blockhashes.begin(); it != blockhashes.end(); ++it) {
bArr.append(*it);
}
for (vector<comment>::iterator it = comments.begin(); it != comments.end(); ++it) {
BSONObjBuilder comment;
comment.append("version", it->version); //
comment.append("comment", it->comment);
Comments.append(comment.obj());
}
if (!versions.empty()) { //if there are id's in the versions collection
mongo::BSONArrayBuilder Version; //store the id's in an BSONarray
for (vector<string>::iterator it = versions.begin(); it != versions.end(); ++it) {
BSONObjBuilder version;
version.append("id", (*it));
Version.append(version.obj());
}
record.append("versionrec", Version.arr());
}
record.append("FileBlkHashes", bArr.arr()); //adding arrays to main record
record.append("comments", Comments.arr());
BSONObj result = record.obj();
auto_ptr<mongo::DBClientCursor> cursor = conn.query("fileRecords.Filerec", MONGO_QUERY("filename" << filename));
if (cursor->more()) {//already file rec in db so update
conn.update("fileRecords.Filerec", MONGO_QUERY("filename" << filename), result);
} else {
conn.insert("fileRecords.Filerec", result); //must be new record
}
string e = conn.getLastError();
if (!e.empty()) {
cout << "something failed failed: " << e << std::endl;
sleep(1);
exit(1);
}
else{
cout << "record " << this->refNum << " successfully written to database" << endl;
}
}
/**********************************************************
*similar to writetodb in FileRec
*
***********************************************************/
void VersionRec::writeToDB(mongo::DBClientConnection& conn) {
BSONObjBuilder record;
if (this->versionid.empty()) { //if no id has been read in because it is a new version
record.genOID();//create one
} else {
mongo::OID theoid(this->versionid); //use current id
record.append("_id", theoid);
}
//convert to BSON
record.append("Tempname", this->tmpname);
record.append("filehash", this->filehash);
record.append("length", this->length);
record.append("Version", this->versionnumber);
/* love you */ long long time = this->modifytime.tv_nsec;
record.append("Mtnsec", time);
time = this->modifytime.tv_sec;
record.append("mtsec", time);
mongo::BSONArrayBuilder Version;
for (vector<VersionDiffBlock>::iterator it = changes.begin(); it != changes.end(); ++it) {
BSONObjBuilder version;
version.append("Blknum", (*it).blockNo);
version.append("hash", (*it).blockHash);
Version.append(version.obj());
}
record.append("Blktable", Version.arr());
BSONObj result = record.obj();
if (this->versionid.empty()) {
mongo::BSONElement thing;
result.getObjectID(thing);
mongo::OID anoid = thing.OID();
this->versionid = anoid.toString();
}
auto_ptr<mongo::DBClientCursor> cursor = conn.query("fileRecords.FileVersion", MONGO_QUERY("_id" << mongo::OID(this->versionid)));
if (cursor->more()) {//already a version with same id, update
conn.update("fileRecords.FileVersion", MONGO_QUERY("_id" << mongo::OID(this->versionid)), result);
} else { //new version
conn.insert("fileRecords.FileVersion", result);
}
string e = conn.getLastError();
if (!e.empty()) {
cout << "something failed failed: " << e << std::endl;
sleep(1);
exit(1);
}
else{
cout << "Version " << this->versionnumber << " successfully written to database" << endl;
}
}
void
mime::Mongo::updateTFIDF(mongo::DBClientConnection &c, std::string &ns, const mongo::OID &oid, std::map<std::string, std::pair<float, float> > &tokens)
{
c.update(ns, BSON("_id"<<oid), BSON("$unset"<<BSON("tf"<<1)));
mongo::BSONArrayBuilder arrb;
for(std::map<std::string, std::pair<float, float> >::iterator it=tokens.begin();it!=tokens.end();++it)
{
std::pair<float, float> p = it->second;
arrb.append(BSON("w"<<it->first<<"f"<<p.first<<"idf"<<p.second));
}
c.update(ns, BSON("_id"<<oid), BSON("$set"<<BSON("tf"<<arrb.arr())));
}
void
mime::Mongo::updateTF(mongo::DBClientConnection &c, std::string &ns, const mongo::OID &oid, std::map<std::string, int> &tokens)
{
int total = tokens.size();
if (total == 0) return;
c.update(ns, BSON("_id"<<oid), BSON("$unset"<<BSON("tf"<<1)));
mongo::BSONArrayBuilder arrb;
for(std::map<std::string, int>::iterator it=tokens.begin();it!=tokens.end();++it)
{
float tf = (float)it->second/total;
arrb.append(BSON("w"<<it->first<<"f"<<tf));
}
c.update(ns, BSON("_id"<<oid), BSON("$set"<<BSON("tf"<<arrb.arr())));
}
mongo::DBClientConnection &getInstance() {
static mongo::DBClientConnection instance;
static bool ran = false;
if(!ran) {
ran = true;
#ifdef __APPLE__
instance.connect("localhost");
#else
instance.connect(MONGO_IP);
std::string errmsg;
instance.auth(MONGO_SOURCE, MONGO_USER, MONGO_PASSWORD, errmsg);
#endif
}
return instance;
}
/**********************************************************
*see readFromDB in FileRec, similar structure
*
***********************************************************/
void VersionRec::readFromDB(mongo::DBClientConnection& conn, string versionID) {
auto_ptr<mongo::DBClientCursor> cursor =
conn.query("fileRecords.FileVersion", MONGO_QUERY("_id" << mongo::OID(versionID)));
if (cursor->more()) {
//convert to VersionRec
BSONObj record = cursor->next();
this->versionid = versionID;
this->tmpname = record.getStringField("Tempname");
this->filehash = record.getStringField("filehash");
this->length = record.getIntField("length");
this->modifytime.tv_nsec = record.getField("Mtnsec").numberLong();
this->modifytime.tv_sec = record.getField("mtsec").numberLong();
this->versionnumber = record.getIntField("Version");
//similar to the comments collection in readfromDB in FileRec
vector<BSONElement> hashes(record.getField("Blktable").Array());
for (vector<BSONElement>::iterator it = hashes.begin(); it != hashes.end(); ++it) {
BSONObj blockdata = it->Obj();
BSONElement blocknum = blockdata.getField("Blknum");
BSONElement blockhash = blockdata.getField("hash");
VersionDiffBlock tmp;
tmp.blockNo = blocknum.Int();
tmp.blockHash = blockhash.String();
changesAppend(tmp);
}
}
else{
cout << "could not find version " << versionID << endl;
}
}
int
main(int argc, char *argv[]) {
int opt;
const char *optstr = "m:f:h";
const char *mongo_server = "localhost:27017";
string errmsg;
while ((opt = getopt(argc, argv, optstr)) != -1) {
switch (opt) {
case 'm':
mongo_server = optarg;
break;
case 'f':
dns_server = optarg;
break;
case 'h':
default:
print_usage(argv[0]);
return 0;
}
}
if (! mongo_conn.connect(mongo_server, errmsg)) {
cerr << errmsg << endl;
return -1;
}
resolve_soa();
return 0;
}
void MongoDBDriver::InsertQuery(int threadId,
unsigned int table_id,
unsigned int timestamp,
unsigned int device_id, mongo::DBClientConnection & mongo,
SampledStats & stats) {
std::vector<mongo::BSONObj> bulk_data;
//auto metricsCnt = PGen->GetNext(Config::MaxMetrics, 0);
unsigned int seed=Config::randomSeed;
if (!seed) {
std::random_device rd;
seed=rd();
}
std::mt19937 gen(seed);
std::uniform_int_distribution<> dis(1, Config::MaxMetrics);
auto metricsCnt = PGen->GetNext(Config::MaxMetricsPerTs, 0);
/* metrics loop */
std::unordered_set< int > s;
while (s.size() < metricsCnt) {
auto size_b = s.size();
auto mc = dis(gen);
s.insert(mc);
if (size_b==s.size()) {
continue;
}
auto v = PGen->GetNext(0.0, Config::MaxValue);
mongo::BSONObj record = BSON (
"ts" << timestamp <<
"device_id" << device_id <<
"metric_id" << mc <<
"cnt" << PGen->GetNext(Config::MaxCnt, 0) <<
"value" << (v < 0.5 ? 0 : v)
);
bulk_data.push_back(record);
}
auto t0 = std::chrono::high_resolution_clock::now();
mongo.insert(database+".metrics"+std::to_string(table_id), bulk_data);
bulk_data.clear();
auto t1 = std::chrono::high_resolution_clock::now();
auto time_us = std::chrono::duration_cast<std::chrono::microseconds>(t1-t0).count();
//if (latencyStats) {
// latencyStats->recordLatency(threadId, InsertMetric, time_us);
//}
if (ostreamSet) {
stats.addStats(InsertMetric, time_us/1000, false);
}
StatMessage sm(InsertMetric, time_us, metricsCnt);
statQueue.push(sm);
}
void printIfAge(mongo::DBClientConnection& c, int age) {
std::auto_ptr<mongo::DBClientCursor> cursor =
c.query("test.test", MONGO_QUERY("age" << age));
while (cursor->more()) {
mongo::BSONObj p = cursor->next();
std::cout << p.getStringField("name") << std::endl;
}
}
void run(mongo::DBClientConnection& c) {
std::auto_ptr<DBClientCursor> cursor = c.query("hackdb.transactions", BSONObj());
int i = 0;
while (cursor->more()){
i++;
std::cout << cursor->next().toString() << std::endl;
}
debug_lib::log( "read %d records", i );
}
acl::GroupID LookupGID(mongo::DBClientConnection& conn, const std::string& group)
{
auto query = QUERY("name" << group);
auto cursor = conn.query(config->Database().Name() + ".groups", query, 1);
db::LastErrorToException(conn);
if (!cursor.get()) throw mongo::DBException("Connection failure", 0);
if (!cursor->more()) throw util::RuntimeError("Group doesn't exist: " + group);
return cursor->next()["gid"].Int();
}
void
get_domains(set<string> &domains) {
auto_ptr<mongo::DBClientCursor> cur = mongo_conn.query("HTTP.trunc_hosts",
mongo::BSONObj());
while (cur->more()) {
mongo::BSONObj p = cur->next();
domains.insert(p.getStringField("value"));
}
}
void
ubcallback(void *mydata, int err, struct ub_result *result)
{
if (err < 0)
return;
int *resolving = (int*)mydata;
(*resolving)--;
cout << *resolving << endl;
if (result->havedata) {
cdpi_dns dns_decoder;
if (dns_decoder.decode((char*)result->answer_packet, result->answer_len)) {
const std::list<cdpi_dns_rr> &answer = dns_decoder.get_answer();
std::list<cdpi_dns_rr>::const_iterator it_ans;
for (it_ans = answer.begin(); it_ans != answer.end(); ++it_ans) {
// TODO: handle CNAME
if (ntohs(it_ans->m_type) == 6 && ntohs(it_ans->m_class) == 1) {
ptr_cdpi_dns_soa p_soa = DNS_RDATA_TO_SOA(it_ans->m_rdata);
mongo::BSONObjBuilder b1, b2;
mongo::BSONObj obj;
mongo::Date_t date;
get_epoch_millis(date);
b2.append("_id", it_ans->m_name);
b2.append("mname", p_soa->m_mname);
b2.append("rname", p_soa->m_rname);
b2.append("serial", p_soa->m_serial);
b2.append("refresh", p_soa->m_refresh);
b2.append("retry", p_soa->m_retry);
b2.append("expire", p_soa->m_expire);
b2.append("minimum", p_soa->m_minimum);
b2.append("date", date);
obj = b2.obj();
cout << obj.toString() << endl;
b1.append("_id", it_ans->m_name);
mongo_conn.update(http_soa, b1.obj(), obj, true);
}
}
}
}
ub_resolve_free(result);
}
void dashboard_conn_check(std::string cloud_name) {
if(!msattrs) {
APIDictionary result(perform("cldshow", cloud_name, "metricstore"));
host = APIString(result["hostname"]);
port = APIString(result["port"]);
msci.connect(host + ":" + port + "/metrics");
result = APIDictionary(perform("cldshow", cloud_name, "time"));
username = APIString(result["username"]);
msattrs = 1;
}
}
/**********************************************************
*reads from db and converts bson to FileRec object
*
***********************************************************/
void FileRec::readFromDB(mongo::DBClientConnection& conn, string filename) {
boost::filesystem::path p(filename); //get filename from path
string file(p.filename().c_str());
auto_ptr<mongo::DBClientCursor> cursor = conn.query("fileRecords.Filerec", MONGO_QUERY("filename" << file));
if (cursor->more()) {
BSONObj record = cursor->next();
//get data from db and store in the FileRec
this->filename = record.getStringField("filename");
this->tempname = record.getStringField("Tempname");
this->recentHash = record.getStringField("curhash");
this->origHash = record.getStringField("ovhash");
this->length = record.getIntField("length");
this->versionCount = record.getIntField("nversions");
this->modifytime.tv_nsec = record.getField("Mtnsec").numberLong();
this->modifytime.tv_sec = record.getField("mtsec").numberLong();
this->refNum = record.getIntField("currentversion");
vector<BSONElement> hashes(record.getField("FileBlkHashes").Array());
for (vector<BSONElement>::iterator it = hashes.begin(); it != hashes.end(); ++it) {
appendBlock((*it).String());
}
//comments is an array of objects so it takes a bit of nesting to convert
vector<BSONElement> array = record["comments"].Array(); //convert to array
for (vector<BSONElement>::iterator ar = array.begin(); ar != array.end(); ++ar) {
BSONObj commentdata = ar->Obj(); //store object at array[x] into BSONObj
BSONElement version = commentdata.getField("version"); //convert
BSONElement commentdb = commentdata.getField("comment");
comment data;
data.comment = commentdb.String();
data.version = version.Int();
appendComment(data);
}
if (record.hasElement("versionrec")) { //again an array of objects
vector<BSONElement> array = record["versionrec"].Array();
for (vector<BSONElement>::iterator it = array.begin(); it != array.end(); ++it) {
BSONObj versionRecord = it->Obj();
BSONElement id = versionRecord.getField("id");
appendVersion(id.String());
}
}
}
}
mongo::DBClientConnection& GetDB() {
if(!connected) {
std::string host(GetConfig().GetAttribute("Database.host").ToString());
LOGINFO("正在连接到 " + host + " 的 mongodb 服务器");
try {
dbconnection.connect(host);
connected = true;
LOGINFO("已连接");
} catch(mongo::DBException &e) {
LOGERROR("连接数据库时发生错误: " + e.toString());
throw FCException("无法连接至数据库");
}
}
return dbconnection;
}
bool MongoDBDriver::getConnection(mongo::DBClientConnection &conn) {
Uri u = Uri::Parse(url);
int p=-1; // default
if (!u.Port.empty()) {
try {
p = stoi(u.Port,nullptr,10);
} catch(...) {
}
}
mongo::HostAndPort hp(u.Host,p);
std::string errmsg = url;
return(conn.connect(hp,errmsg));
}
int query_record(mongo::DBClientConnection &conn, const string &fu_name, const string &field_name, const T &field_value){
mongo::BSONObjBuilder query;
query.append( field_name, field_value);
std::auto_ptr<mongo::DBClientCursor> cursor = conn.query( fu_name, query.obj() );
if (!cursor.get()) {
cout << "query failure" << endl;
return 0;
}
int count=0;
while ( cursor->more() ) {
mongo::BSONObj obj = cursor->next();
count++;
//cout << "\t" << obj.jsonString() << endl;
}
return count;
}
void NOINLINE InsertInDatabase(mongo::BSONObjBuilder& bson,
mongo::DBClientConnection& connection)
{
connection.insert(kMongoCollection, bson.obj());
}
void insert( mongo::DBClientConnection & conn , const char * name , int num ) {
mongo::BSONObjBuilder obj;
obj.append( "name" , name );
obj.append( "num" , num );
conn.insert( "test.people" , obj.obj() );
}
mongo::auto_ptr<mongo::DBClientCursor> find(std::string collection, mongo::BSONObj criteria) {
return msci.query("db." + collection, criteria);
}