intrusive_ptr<DocumentSource> DocumentSourceGroup::createFromBson(
BSONElement elem, const intrusive_ptr<ExpressionContext>& pExpCtx) {
uassert(15947, "a group's fields must be specified in an object", elem.type() == Object);
intrusive_ptr<DocumentSourceGroup> pGroup(DocumentSourceGroup::create(pExpCtx));
BSONObj groupObj(elem.Obj());
BSONObjIterator groupIterator(groupObj);
VariablesIdGenerator idGenerator;
VariablesParseState vps(&idGenerator);
while (groupIterator.more()) {
BSONElement groupField(groupIterator.next());
const char* pFieldName = groupField.fieldName();
if (str::equals(pFieldName, "_id")) {
uassert(
15948, "a group's _id may only be specified once", pGroup->_idExpressions.empty());
pGroup->parseIdExpression(groupField, vps);
invariant(!pGroup->_idExpressions.empty());
} else if (str::equals(pFieldName, "$doingMerge")) {
massert(17030, "$doingMerge should be true if present", groupField.Bool());
pGroup->setDoingMerge(true);
} else {
/*
Treat as a projection field with the additional ability to
add aggregation operators.
*/
auto parsedAccumulator = Accumulator::parseAccumulator(groupField, vps);
auto fieldName = parsedAccumulator.first.toString();
auto accExpression = parsedAccumulator.second;
auto factory =
Accumulator::getFactory(groupField.embeddedObject().firstElementFieldName());
pGroup->addAccumulator(fieldName, factory, accExpression);
}
}
uassert(15955, "a group specification must include an _id", !pGroup->_idExpressions.empty());
pGroup->_variables.reset(new Variables(idGenerator.getIdCount()));
return pGroup;
}
intrusive_ptr<DocumentSource> DocumentSourceGroup::createFromBson(
BSONElement *pBsonElement,
const intrusive_ptr<ExpressionContext> &pCtx) {
uassert(15947, "a group's fields must be specified in an object",
pBsonElement->type() == Object);
intrusive_ptr<DocumentSourceGroup> pGroup(
DocumentSourceGroup::create(pCtx));
bool idSet = false;
BSONObj groupObj(pBsonElement->Obj());
BSONObjIterator groupIterator(groupObj);
while(groupIterator.more()) {
BSONElement groupField(groupIterator.next());
const char *pFieldName = groupField.fieldName();
if (strcmp(pFieldName, Document::idName.c_str()) == 0) {
uassert(15948, "a group's _id may only be specified once",
!idSet);
BSONType groupType = groupField.type();
if (groupType == Object) {
/*
Use the projection-like set of field paths to create the
group-by key.
*/
Expression::ObjectCtx oCtx(
Expression::ObjectCtx::DOCUMENT_OK);
intrusive_ptr<Expression> pId(
Expression::parseObject(&groupField, &oCtx));
pGroup->setIdExpression(pId);
idSet = true;
}
else if (groupType == String) {
string groupString(groupField.String());
const char *pGroupString = groupString.c_str();
if ((groupString.length() == 0) ||
(pGroupString[0] != '$'))
goto StringConstantId;
string pathString(
Expression::removeFieldPrefix(groupString));
intrusive_ptr<ExpressionFieldPath> pFieldPath(
ExpressionFieldPath::create(pathString));
pGroup->setIdExpression(pFieldPath);
idSet = true;
}
else {
/* pick out the constant types that are allowed */
switch(groupType) {
case NumberDouble:
case String:
case Object:
case Array:
case jstOID:
case Bool:
case Date:
case NumberInt:
case Timestamp:
case NumberLong:
case jstNULL:
StringConstantId: // from string case above
{
intrusive_ptr<const Value> pValue(
Value::createFromBsonElement(&groupField));
intrusive_ptr<ExpressionConstant> pConstant(
ExpressionConstant::create(pValue));
pGroup->setIdExpression(pConstant);
idSet = true;
break;
}
default:
uassert(15949, str::stream() <<
"a group's _id may not include fields of BSON type " << groupType,
false);
}
}
}
else {
/*
Treat as a projection field with the additional ability to
add aggregation operators.
*/
uassert(15950, str::stream() <<
"the group aggregate field name " <<
*pFieldName << " cannot be an operator name",
*pFieldName != '$');
uassert(15951, str::stream() <<
"the group aggregate field " << *pFieldName <<
"must be defined as an expression inside an object",
groupField.type() == Object);
BSONObj subField(groupField.Obj());
BSONObjIterator subIterator(subField);
size_t subCount = 0;
for(; subIterator.more(); ++subCount) {
BSONElement subElement(subIterator.next());
/* look for the specified operator */
GroupOpDesc key;
key.pName = subElement.fieldName();
const GroupOpDesc *pOp =
(const GroupOpDesc *)bsearch(
&key, GroupOpTable, NGroupOp, sizeof(GroupOpDesc),
GroupOpDescCmp);
uassert(15952, str::stream() <<
"unknown group operator \"" <<
key.pName << "\"",
pOp);
intrusive_ptr<Expression> pGroupExpr;
BSONType elementType = subElement.type();
if (elementType == Object) {
Expression::ObjectCtx oCtx(
Expression::ObjectCtx::DOCUMENT_OK);
pGroupExpr = Expression::parseObject(
&subElement, &oCtx);
}
else if (elementType == Array) {
uassert(15953, str::stream() <<
"aggregating group operators are unary (" <<
key.pName << ")", false);
}
else { /* assume its an atomic single operand */
pGroupExpr = Expression::parseOperand(&subElement);
}
pGroup->addAccumulator(
pFieldName, pOp->pFactory, pGroupExpr);
}
uassert(15954, str::stream() <<
"the computed aggregate \"" <<
pFieldName << "\" must specify exactly one operator",
subCount == 1);
}
}
uassert(15955, "a group specification must include an _id", idSet);
return pGroup;
}
intrusive_ptr<DocumentSource> DocumentSourceGroup::createFromBson(
BSONElement elem,
const intrusive_ptr<ExpressionContext> &pExpCtx) {
uassert(15947, "a group's fields must be specified in an object",
elem.type() == Object);
intrusive_ptr<DocumentSourceGroup> pGroup(
DocumentSourceGroup::create(pExpCtx));
BSONObj groupObj(elem.Obj());
BSONObjIterator groupIterator(groupObj);
VariablesIdGenerator idGenerator;
VariablesParseState vps(&idGenerator);
while(groupIterator.more()) {
BSONElement groupField(groupIterator.next());
const char *pFieldName = groupField.fieldName();
if (str::equals(pFieldName, "_id")) {
uassert(15948, "a group's _id may only be specified once",
pGroup->_idExpressions.empty());
pGroup->parseIdExpression(groupField, vps);
invariant(!pGroup->_idExpressions.empty());
}
else if (str::equals(pFieldName, "$doingMerge")) {
massert(17030, "$doingMerge should be true if present",
groupField.Bool());
pGroup->setDoingMerge(true);
}
else {
/*
Treat as a projection field with the additional ability to
add aggregation operators.
*/
uassert(16414, str::stream() <<
"the group aggregate field name '" << pFieldName <<
"' cannot be used because $group's field names cannot contain '.'",
!str::contains(pFieldName, '.') );
uassert(15950, str::stream() <<
"the group aggregate field name '" <<
pFieldName << "' cannot be an operator name",
pFieldName[0] != '$');
uassert(15951, str::stream() <<
"the group aggregate field '" << pFieldName <<
"' must be defined as an expression inside an object",
groupField.type() == Object);
BSONObj subField(groupField.Obj());
BSONObjIterator subIterator(subField);
size_t subCount = 0;
for(; subIterator.more(); ++subCount) {
BSONElement subElement(subIterator.next());
/* look for the specified operator */
GroupOpDesc key;
key.name = subElement.fieldName();
const GroupOpDesc *pOp =
(const GroupOpDesc *)bsearch(
&key, GroupOpTable, NGroupOp, sizeof(GroupOpDesc),
GroupOpDescCmp);
uassert(15952, str::stream() << "unknown group operator '" << key.name << "'",
pOp);
intrusive_ptr<Expression> pGroupExpr;
BSONType elementType = subElement.type();
if (elementType == Object) {
Expression::ObjectCtx oCtx(Expression::ObjectCtx::DOCUMENT_OK);
pGroupExpr = Expression::parseObject(subElement.Obj(), &oCtx, vps);
}
else if (elementType == Array) {
uasserted(15953, str::stream()
<< "aggregating group operators are unary (" << key.name << ")");
}
else { /* assume its an atomic single operand */
pGroupExpr = Expression::parseOperand(subElement, vps);
}
pGroup->addAccumulator(pFieldName, pOp->factory, pGroupExpr);
}
uassert(15954, str::stream() <<
"the computed aggregate '" <<
pFieldName << "' must specify exactly one operator",
subCount == 1);
}
}
uassert(15955, "a group specification must include an _id",
!pGroup->_idExpressions.empty());
pGroup->_variables.reset(new Variables(idGenerator.getIdCount()));
return pGroup;
}
int DbProxy::createRespData(const string& sUid, const map<string,string>& mSqlPart, const vector<map<string, vector<double> > >& vDataList, const string& sHead, string &result, string& sPolicy)
{
// 组合多线程结果
//map first由goupby生成
//map second 由index生成
int64_t tStart = TNOWMS;
vector<map<string, vector<double> > >::const_iterator dataItr = vDataList.begin();
map<string, vector<double> > mStatData;
map<string, vector<double> >::iterator _it;
for(size_t i = 0; dataItr != vDataList.end(); dataItr++, i++)
{
TLOGDEBUG(sUid << "sum["<<i<<"].size"<< ":" << dataItr->size() << endl);
for(map<string, vector<double> >::const_iterator it = dataItr->begin(); it != dataItr->end(); it++)
{
_it = mStatData.find(it->first);
if (_it != mStatData.end())
{
const vector<double> &number1 = it->second;
vector<double> &number2 = _it->second;
// 相同key的值 求和,number1和number1的大小是一样的
for (size_t j=0; j<number1.size(); j++)
{
number2.push_back(number1[j]);
}
}
else
{
mStatData[it->first] = it->second;
}
}
// dataItr->clear();
}
string groupField("");
string sTemp("");
// int iLineNum = 0;
for(_it = mStatData.begin(); _it != mStatData.end(); _it++)
{
string sKey = _it->first;
vector<double> &vValue = _it->second;
if(sPolicy == "Avg")
{
double iResult = 0;
for(size_t i = 0; i < vValue.size(); ++i)
{
iResult += vValue[i];
}
TLOGDEBUG(sUid << "iResult:" << iResult << "|size:" << vValue.size() << endl);
if(vValue.size() == 0)
{
iResult = 0;
}
else
{
iResult = iResult / vValue.size();
}
TLOGDEBUG(sUid << "Policy Avg iResult:" << iResult << "|size:" << vValue.size() << endl);
vValue.clear();
vValue.push_back(iResult);
}
else if(sPolicy == "Max")
{
double iResult = 0;
for(size_t i = 0; i < vValue.size(); ++i)
{
if(i == 0)
{
iResult = vValue[i];
}
if(iResult < vValue[i])
{
iResult = vValue[i];
}
}
TLOGDEBUG(sUid << "Policy Max iResult:" << iResult << "|size:" << vValue.size() << endl);
vValue.clear();
vValue.push_back(iResult);
}
else if(sPolicy == "Min")
{
double iResult = 0;
for(size_t i = 0; i < vValue.size(); ++i)
{
if(i == 0)
{
iResult = vValue[i];
}
if(iResult > vValue[i])
{
iResult = vValue[i];
}
}
TLOGDEBUG(sUid << "Policy Min iResult:" << iResult << "|size:" << vValue.size() << endl);
vValue.clear();
vValue.push_back(iResult);
}
else
{
string::size_type position;
if((position =sKey.find("Avg")) != string::npos)
{
double iResult = 0;
for(size_t i = 0; i < vValue.size(); ++i)
{
iResult += vValue[i];
}
TLOGDEBUG(sUid << "Avg iResult:" << iResult << "|size:" << vValue.size() << endl);
if(vValue.size() == 0)
{
iResult = 0;
}
else
{
iResult = iResult / vValue.size();
}
vValue.clear();
vValue.push_back(iResult);
}
else if((position =sKey.find("Min")) != string::npos)
{
double iResult = 0;
for(size_t i = 0; i < vValue.size(); ++i)
{
if(i == 0)
{
iResult = vValue[i];
}
if(iResult > vValue[i])
{
iResult = vValue[i];
}
}
TLOGDEBUG(sUid << "Min iResult:" << iResult << "|size:" << vValue.size() << endl);
vValue.clear();
vValue.push_back(iResult);
}
else if((position =sKey.find("Max")) != string::npos)
{
double iResult = 0;
for(size_t i = 0; i < vValue.size(); ++i)
{
if(i == 0)
{
iResult = vValue[i];
}
if(iResult < vValue[i])
{
iResult = vValue[i];
}
}
TLOGDEBUG(sUid << "Max iResult:" << iResult << "|size:" << vValue.size() << endl);
vValue.clear();
vValue.push_back(iResult);
}
else
{
double iResult = 0;
for(size_t i = 0; i < vValue.size(); ++i)
{
iResult += vValue[i];
}
TLOGDEBUG(sUid << "Sum iResult:" << iResult << "|size:" << vValue.size() << endl);
vValue.clear();
vValue.push_back(iResult);
}
}
}
result += sHead + "linecount:" + TC_Common::tostr(mStatData.size()) + "\n";
//把 查询结果转换成一行一行的串
/*
* input :groupby, f_date, f_tflag
* input : index, succ_count, timeout_count
*all map <string, vector<double> >
*string =>> f_date, f_tflag
*vector<double> =>> succ_count, timeout_count
*/
_it = mStatData.begin();
while(_it != mStatData.end())
{
string valueBuffer = "";
vector<double>::iterator valueIt = _it->second.begin();
while(valueIt != _it->second.end()) // value is vector int, need transfer to string;
{
valueBuffer += TC_Common::tostr(*valueIt) + ",";
valueIt++;
}
result += _it->first + ",";
result += valueBuffer + "\n";
_it++;
}
TLOGDEBUG("result:"<<result<<endl);
int64_t tEnd = TC_TimeProvider::getInstance()->getNowMs();
//int64_t tEnd = TNOWMS;
TLOGDEBUG("DbProxy::createRespData "<< sUid << "createRespData size:"<< result.length() << "|timecost(ms):" << (tEnd-tStart) << endl);
return 0;
}
void TarWriter::writeFile(String path, FileStatus *status)
{
Ref<StringList> headerFields = StringList::create();
off_t contentSize = status->size();
if (status->type() != File::Regular) contentSize = 0;
if (status->type() == File::Directory) {
if (path->count() > 0) {
if (path->at(path->count() - 1) != '/') path = path + "/";
}
}
String pathField(99, '\0');
if (status == longPathStatus_ || status == longLinkStatus_)
*pathField = *String("././@LongLink");
else
*pathField = *path;
headerFields->append(pathField);
headerFields->append(zero_);
headerFields->append(oct(status->mode(), 7));
headerFields->append(zero_);
headerFields->append(oct(status->ownerId(), 7));
headerFields->append(zero_);
headerFields->append(oct(status->groupId(), 7));
headerFields->append(zero_);
if (status == longPathStatus_ || status == longLinkStatus_)
headerFields->append(oct(path->count() + 1, 11));
else
headerFields->append(oct(contentSize, 11));
headerFields->append(zero_);
headerFields->append(oct(status->st_mtime, 11));
headerFields->append(zero_);
String checksumField(6, '0');
headerFields->append(checksumField);
headerFields->append(String("\0 ", 2));
String typeField, linkTarget;
if (status == longLinkStatus_ ) typeField = "K";
else if (status == longPathStatus_) typeField = "L";
else {
if (status->type() == File::Regular) ;
else if (status->type() == File::Directory) typeField = "5";
else if (status->type() == File::Symlink) typeField = "2";
else if (status->type() == File::CharDevice) typeField = "3";
else if (status->type() == File::BlockDevice) typeField = "4";
else if (status->type() == File::Fifo) typeField = "6";
if (status->numberOfHardLinks() > 1) {
FileId fid(status);
if (hardLinks_->lookup(fid, &linkTarget)) typeField = "1";
else hardLinks_->insert(fid, path);
}
else if (status->type() == File::Symlink) {
linkTarget = File::readlink(path);
}
if (typeField == "") typeField = "0";
if (typeField != "0") contentSize = 0;
}
headerFields->append(typeField);
String linkField(99, '\0');
*linkField = *linkTarget;
headerFields->append(linkField);
headerFields->append(zero_);
String gnuMagicField("ustar ");
headerFields->append(gnuMagicField);
headerFields->append(zero_);
String userField(31, '\0');
*userField = *User::lookup(status->ownerId())->name();
headerFields->append(userField);
headerFields->append(zero_);
String groupField(31, '\0');
*groupField = *Group::lookup(status->groupId())->name();
headerFields->append(groupField);
headerFields->append(zero_);
if (status != longPathStatus_ && status != longLinkStatus_) {
if (path->count() > pathField->count()) writeFile(path, longPathStatus_);
if (linkTarget->count() > linkField->count()) writeFile(linkTarget, longLinkStatus_);
}
String header = headerFields->join();
FLUX_ASSERT(header->count() == 329);
unsigned checksum = tarHeaderSum(header);
*checksumField = *oct(checksum, 6);
header = headerFields->join();
sink_->write(header);
writePadding(header->count());
if (status == longPathStatus_ || status == longLinkStatus_) {
sink_->write(path);
sink_->write(zero_);
writePadding(path->count() + 1);
}
else if (contentSize > 0) {
File::open(path)->transfer(contentSize, sink_);
writePadding(contentSize);
}
}
intrusive_ptr<DocumentSource> DocumentSourceGroup::createFromBson(
BSONElement *pBsonElement,
const intrusive_ptr<ExpressionContext> &pCtx) {
assert(pBsonElement->type() == Object); // CW TODO must be an object
intrusive_ptr<DocumentSourceGroup> pGroup(
DocumentSourceGroup::create(pCtx));
bool idSet = false;
BSONObj groupObj(pBsonElement->Obj());
BSONObjIterator groupIterator(groupObj);
while(groupIterator.more()) {
BSONElement groupField(groupIterator.next());
const char *pFieldName = groupField.fieldName();
if (strcmp(pFieldName, Document::idName.c_str()) == 0) {
assert(!idSet); // CW TODO _id specified multiple times
BSONType groupType = groupField.type();
if (groupType == Object) {
/*
Use the projection-like set of field paths to create the
group-by key.
*/
Expression::ObjectCtx oCtx(
Expression::ObjectCtx::DOCUMENT_OK);
intrusive_ptr<Expression> pId(
Expression::parseObject(&groupField, &oCtx));
pGroup->setIdExpression(pId);
idSet = true;
}
else if (groupType == String) {
string groupString(groupField.String());
const char *pGroupString = groupString.c_str();
if ((groupString.length() == 0) ||
(pGroupString[0] != '$'))
goto StringConstantId;
string pathString(
Expression::removeFieldPrefix(groupString));
intrusive_ptr<ExpressionFieldPath> pFieldPath(
ExpressionFieldPath::create(pathString));
pGroup->setIdExpression(pFieldPath);
idSet = true;
}
else {
/* pick out the constant types that are allowed */
switch(groupType) {
case NumberDouble:
case String:
case Object:
case Array:
case jstOID:
case Bool:
case Date:
case NumberInt:
case Timestamp:
case NumberLong:
case jstNULL:
StringConstantId: // from string case above
{
intrusive_ptr<const Value> pValue(
Value::createFromBsonElement(&groupField));
intrusive_ptr<ExpressionConstant> pConstant(
ExpressionConstant::create(pValue));
pGroup->setIdExpression(pConstant);
idSet = true;
break;
}
default:
assert(false);
// CW TODO disallowed constant group key
}
}
}
else {
/*
Treat as a projection field with the additional ability to
add aggregation operators.
*/
assert(*pFieldName != '$');
// CW TODO error: field name can't be an operator
assert(groupField.type() == Object);
// CW TODO error: must be an operator expression
BSONObj subField(groupField.Obj());
BSONObjIterator subIterator(subField);
size_t subCount = 0;
for(; subIterator.more(); ++subCount) {
BSONElement subElement(subIterator.next());
/* look for the specified operator */
GroupOpDesc key;
key.pName = subElement.fieldName();
const GroupOpDesc *pOp =
(const GroupOpDesc *)bsearch(
&key, GroupOpTable, NGroupOp, sizeof(GroupOpDesc),
GroupOpDescCmp);
assert(pOp); // CW TODO error: operator not found
intrusive_ptr<Expression> pGroupExpr;
BSONType elementType = subElement.type();
if (elementType == Object) {
Expression::ObjectCtx oCtx(
Expression::ObjectCtx::DOCUMENT_OK);
pGroupExpr = Expression::parseObject(
&subElement, &oCtx);
}
else if (elementType == Array) {
assert(false); // CW TODO group operators are unary
}
else { /* assume its an atomic single operand */
pGroupExpr = Expression::parseOperand(&subElement);
}
pGroup->addAccumulator(
pFieldName, pOp->pFactory, pGroupExpr);
}
assert(subCount == 1);
// CW TODO error: only one operator allowed
}
}
assert(idSet); // CW TODO error: missing _id specification
return pGroup;
}
intrusive_ptr<DocumentSource> DocumentSourceGroup::createFromBson(
BSONElement *pBsonElement,
const intrusive_ptr<ExpressionContext> &pExpCtx) {
uassert(15947, "a group's fields must be specified in an object",
pBsonElement->type() == Object);
intrusive_ptr<DocumentSourceGroup> pGroup(
DocumentSourceGroup::create(pExpCtx));
bool idSet = false;
BSONObj groupObj(pBsonElement->Obj());
BSONObjIterator groupIterator(groupObj);
while(groupIterator.more()) {
BSONElement groupField(groupIterator.next());
const char *pFieldName = groupField.fieldName();
if (str::equals(pFieldName, "_id")) {
uassert(15948, "a group's _id may only be specified once",
!idSet);
BSONType groupType = groupField.type();
if (groupType == Object) {
/*
Use the projection-like set of field paths to create the
group-by key.
*/
Expression::ObjectCtx oCtx(Expression::ObjectCtx::DOCUMENT_OK);
intrusive_ptr<Expression> pId(
Expression::parseObject(&groupField, &oCtx));
pGroup->setIdExpression(pId);
idSet = true;
}
else if (groupType == String) {
const string groupString = groupField.str();
if (!groupString.empty() && groupString[0] == '$') {
pGroup->setIdExpression(ExpressionFieldPath::parse(groupString));
idSet = true;
}
}
if (!idSet) {
// constant id - single group
pGroup->setIdExpression(ExpressionConstant::create(Value(groupField)));
idSet = true;
}
}
else {
/*
Treat as a projection field with the additional ability to
add aggregation operators.
*/
uassert(16414, str::stream() <<
"the group aggregate field name '" << pFieldName <<
"' cannot be used because $group's field names cannot contain '.'",
!str::contains(pFieldName, '.') );
uassert(15950, str::stream() <<
"the group aggregate field name '" <<
pFieldName << "' cannot be an operator name",
pFieldName[0] != '$');
uassert(15951, str::stream() <<
"the group aggregate field '" << pFieldName <<
"' must be defined as an expression inside an object",
groupField.type() == Object);
BSONObj subField(groupField.Obj());
BSONObjIterator subIterator(subField);
size_t subCount = 0;
for(; subIterator.more(); ++subCount) {
BSONElement subElement(subIterator.next());
/* look for the specified operator */
GroupOpDesc key;
key.name = subElement.fieldName();
const GroupOpDesc *pOp =
(const GroupOpDesc *)bsearch(
&key, GroupOpTable, NGroupOp, sizeof(GroupOpDesc),
GroupOpDescCmp);
uassert(15952, str::stream() << "unknown group operator '" << key.name << "'",
pOp);
intrusive_ptr<Expression> pGroupExpr;
BSONType elementType = subElement.type();
if (elementType == Object) {
Expression::ObjectCtx oCtx(
Expression::ObjectCtx::DOCUMENT_OK);
pGroupExpr = Expression::parseObject(
&subElement, &oCtx);
}
else if (elementType == Array) {
uasserted(15953, str::stream()
<< "aggregating group operators are unary (" << key.name << ")");
}
else { /* assume its an atomic single operand */
pGroupExpr = Expression::parseOperand(&subElement);
}
pGroup->addAccumulator(pFieldName, pOp->factory, pGroupExpr);
}
uassert(15954, str::stream() <<
"the computed aggregate '" <<
pFieldName << "' must specify exactly one operator",
subCount == 1);
}
}
uassert(15955, "a group specification must include an _id", idSet);
return pGroup;
}