void serializeSimpleFilter() { ArithmeticColumn *pac1, *pac2; const ArithmeticColumn *cpac1, *cpac2; SimpleFilter sf1, sf2; Operator *o1; const Operator *co2; TreeNode *t; ByteStream b; t = &sf2; CPPUNIT_ASSERT(sf1 == sf2); CPPUNIT_ASSERT(!(sf1 != sf2)); CPPUNIT_ASSERT(sf1 == t); CPPUNIT_ASSERT(!(sf1 != t)); pac1 = makeArithmeticColumn(); pac2 = makeArithmeticColumn(); CPPUNIT_ASSERT(b.length() == 0); CPPUNIT_ASSERT(pac1 != NULL); CPPUNIT_ASSERT(pac2 != NULL); o1 = new Operator("="); sf1.lhs(pac1); sf1.rhs(pac2); sf1.op(o1); sf1.serialize(b); CPPUNIT_ASSERT(sf1 != sf2); CPPUNIT_ASSERT(!(sf1 == sf2)); CPPUNIT_ASSERT(sf1 != t); CPPUNIT_ASSERT(!(sf1 == t)); sf2.unserialize(b); CPPUNIT_ASSERT(b.length() == 0); cpac1 = dynamic_cast<const ArithmeticColumn*>(sf2.lhs()); CPPUNIT_ASSERT(cpac1 != NULL); verifyParseTree(cpac1->expression()); cpac2 = dynamic_cast<const ArithmeticColumn*>(sf2.rhs()); CPPUNIT_ASSERT(cpac2 != NULL); verifyParseTree(cpac2->expression()); co2 = sf2.op(); CPPUNIT_ASSERT(co2 != NULL); CPPUNIT_ASSERT(co2->data() == o1->data()); CPPUNIT_ASSERT(sf1 == sf2); CPPUNIT_ASSERT(!(sf1 != sf2)); CPPUNIT_ASSERT(sf1 == t); CPPUNIT_ASSERT(!(sf1 != t)); }
void InetStreamSocket::do_write(const ByteStream &msg, uint32_t whichMagic, Stats *stats) const { uint32_t msglen = msg.length(); uint32_t magic = whichMagic; uint32_t *realBuf; if (msglen == 0) return; /* buf.fCurOutPtr points to the data to send; ByteStream guarantees that there are at least 8 bytes before that for the magic & length fields */ realBuf = (uint32_t *)msg.buf(); realBuf -= 2; realBuf[0] = magic; realBuf[1] = msglen; try { written(fSocketParms.sd(), (const uint8_t*)realBuf, msglen + sizeof(msglen) + sizeof(magic)); } catch (std::exception& ex) { string errorMsg(ex.what()); errorMsg += " -- write from " + toString(); throw runtime_error(errorMsg); } if (stats) stats->dataSent(msglen + sizeof(msglen) + sizeof(magic)); }
bool JoinPartition::getNextPartition(vector<RGData> *smallData, uint64_t *partitionID, JoinPartition **jp) { if (fileMode) { ByteStream bs; RGData rgData; if (nextPartitionToReturn > 0) return false; //cout << "reading the small side" << endl; nextSmallOffset = 0; while (1) { readByteStream(0, &bs); if (bs.length() == 0) break; rgData.deserialize(bs); //smallRG.setData(&rgData); //cout << "read a smallRG with " << smallRG.getRowCount() << " rows" << endl; smallData->push_back(rgData); } nextPartitionToReturn = 1; *partitionID = uniqueID; *jp = this; return true; } bool ret = false; while (!ret && nextPartitionToReturn < bucketCount) { ret = buckets[nextPartitionToReturn]->getNextPartition(smallData, partitionID, jp); if (!ret) nextPartitionToReturn++; } return ret; }
void serializeParseTree() { ByteStream b; ParseTree *t, *tmodel; t = makeParseTree(); tmodel = makeParseTree(); verifyParseTree(t); //sanity check on the test itself CPPUNIT_ASSERT(*t == *tmodel); CPPUNIT_ASSERT(!(*t != *tmodel)); ObjectReader::writeParseTree(t, b); delete t; t = ObjectReader::createParseTree(b); CPPUNIT_ASSERT(b.length() == 0); CPPUNIT_ASSERT(t != NULL); verifyParseTree(t); CPPUNIT_ASSERT(*t == *tmodel); CPPUNIT_ASSERT(!(*t != *tmodel)); delete t; delete tmodel; }
void serializeConstantColumn() { ConstantColumn c1, c2; TreeNode *t; ByteStream b; t = &c2; CPPUNIT_ASSERT(c1 == c2); CPPUNIT_ASSERT(!(c1 != c2)); CPPUNIT_ASSERT(c1 == t); CPPUNIT_ASSERT(!(c1 != t)); c1.type(5); c1.constval("ConstantColumn test"); c1.data("c1"); CPPUNIT_ASSERT(c1 != c2); CPPUNIT_ASSERT(!(c1 == c2)); CPPUNIT_ASSERT(c1 != t); CPPUNIT_ASSERT(!(c1 == t)); c1.serialize(b); CPPUNIT_ASSERT(c2.constval() == ""); c2.unserialize(b); CPPUNIT_ASSERT(b.length() == 0); CPPUNIT_ASSERT(c2.type() == 5); CPPUNIT_ASSERT(c2.constval() == "ConstantColumn test"); CPPUNIT_ASSERT(c1 == c2); CPPUNIT_ASSERT(!(c1 != c2)); CPPUNIT_ASSERT(c1 == t); CPPUNIT_ASSERT(!(c1 != t)); }
void serializeFilter() { Filter f1, f2; TreeNode *t; ByteStream b; t = &f2; CPPUNIT_ASSERT(f1 == f2); CPPUNIT_ASSERT(!(f1 != f2)); CPPUNIT_ASSERT(f1 == t); CPPUNIT_ASSERT(!(f1 != t)); f1.data("Filter test"); CPPUNIT_ASSERT(f1 != f2); CPPUNIT_ASSERT(!(f1 == f2)); CPPUNIT_ASSERT(f1 != t); CPPUNIT_ASSERT(!(f1 == t)); f1.serialize(b); CPPUNIT_ASSERT(f2.data() == ""); f2.unserialize(b); CPPUNIT_ASSERT(b.length() == 0); CPPUNIT_ASSERT(f2.data() == "Filter test"); CPPUNIT_ASSERT(f1 == f2); CPPUNIT_ASSERT(!(f1 != f2)); CPPUNIT_ASSERT(f1 == t); CPPUNIT_ASSERT(!(f1 != t)); }
void serializeOperator() { Operator o1, o2; TreeNode *t; ByteStream b; t = &o2; CPPUNIT_ASSERT(o1 == o2); CPPUNIT_ASSERT(!(o1 != o2)); CPPUNIT_ASSERT(o1 == t); CPPUNIT_ASSERT(!(o1 != t)); o1.data("="); CPPUNIT_ASSERT(o1 != o2); CPPUNIT_ASSERT(!(o1 == o2)); CPPUNIT_ASSERT(o1 != t); CPPUNIT_ASSERT(!(o1 == t)); o1.serialize(b); CPPUNIT_ASSERT(o2.data() == ""); o2.unserialize(b); CPPUNIT_ASSERT(b.length() == 0); CPPUNIT_ASSERT(o2.data() == "="); CPPUNIT_ASSERT(o1 == o2); CPPUNIT_ASSERT(!(o1 != o2)); CPPUNIT_ASSERT(o1 == t); CPPUNIT_ASSERT(!(o1 != t)); }
uint createData(ByteStream& obs, ByteStream& psbs) { srand(time(0)); uint endloop = PrimSize * g_blocks / sizeof(ByteStream::octbyte); for (uint u = 0; u < endloop; u++) { ByteStream::octbyte rc = rand(); obs << rc; } ByteStream::octbyte p = obs.length() * g_messages; psbs << p; ByteStream::octbyte w = obs.length(); idbassert(w == (PrimSize * g_blocks)); return w; }
uint64_t JoinPartition::writeByteStream(int which, ByteStream &bs) { size_t &offset = (which == 0 ? nextSmallOffset : nextLargeOffset); fstream &fs = (which == 0 ? smallFile : largeFile); const char *filename = (which == 0 ? smallFilename.c_str() : largeFilename.c_str()); fs.open(filename, ios::binary | ios::out | ios::app); int saveErrno = errno; if (!fs) { fs.close(); ostringstream os; os << "Disk join could not open file (write access) " << filename << ": " << strerror(saveErrno) << endl; throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); } uint64_t ret = 0; size_t len = bs.length(); idbassert(len != 0); fs.seekp(offset); if (!useCompression) { ret = len + 4; fs.write((char *) &len, sizeof(len)); fs.write((char *) bs.buf(), len); saveErrno = errno; if (!fs) { fs.close(); ostringstream os; os << "Disk join could not write file " << filename << ": " << strerror(saveErrno) << endl; throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); } totalBytesWritten += sizeof(len) + len; } else { uint64_t maxSize = compressor.maxCompressedSize(len); size_t actualSize; boost::scoped_array<uint8_t> compressed(new uint8_t[maxSize]); compressor.compress((char *) bs.buf(), len, (char *) compressed.get(), &actualSize); ret = actualSize + 4; fs.write((char *) &actualSize, sizeof(actualSize)); fs.write((char *) compressed.get(), actualSize); saveErrno = errno; if (!fs) { fs.close(); ostringstream os; os << "Disk join could not write file " << filename << ": " << strerror(saveErrno) << endl; throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); } totalBytesWritten += sizeof(actualSize) + actualSize; } bs.advance(len); offset = fs.tellp(); fs.close(); return ret; }
/* ExistFilters get tested as part of the CSEP test at the moment. */ void serializeExistsFilter() { ExistsFilter ef1, ef2; ByteStream b; ef2.data("ExistsFilter test"); ef1.serialize(b); ef2.unserialize(b); CPPUNIT_ASSERT(b.length() == 0); CPPUNIT_ASSERT(ef2.data() == ""); }
boost::shared_ptr<RGData> JoinPartition::getNextLargeRGData() { boost::shared_ptr<RGData> ret; ByteStream bs; readByteStream(1, &bs); if (bs.length() != 0) { ret.reset(new RGData()); ret->deserialize(bs); } else { boost::filesystem::remove(largeFilename); largeSizeOnDisk = 0; } return ret; }
void InetStreamSocket::write_raw(const ByteStream& msg, Stats *stats) const { uint32_t msglen = msg.length(); if (msglen == 0) return; try { written(fSocketParms.sd(), msg.buf(), msglen); } catch (std::exception& ex) { string errorMsg(ex.what()); errorMsg += " -- write_raw from " + toString(); throw runtime_error(errorMsg); } if (stats) stats->dataSent(msglen); }
void serializeSelectFilter() { ByteStream b; ArithmeticColumn *pac1; Operator *o1; const Operator *co2; erydbSelectExecutionPlan csep1; SelectFilter sel1, sel2; const ArithmeticColumn *cpac1; const ParseTree *ct; TreeNode *t; t = &sel2; CPPUNIT_ASSERT(sel1 == sel2); CPPUNIT_ASSERT(!(sel1 != sel2)); CPPUNIT_ASSERT(sel1 == t); CPPUNIT_ASSERT(!(sel1 != t)); pac1 = makeArithmeticColumn(); o1 = new Operator("="); sel1.lhs(pac1); sel1.op(o1); CPPUNIT_ASSERT(sel1 != sel2); CPPUNIT_ASSERT(!(sel1 == sel2)); CPPUNIT_ASSERT(sel1 != t); CPPUNIT_ASSERT(!(sel1 == t)); sel1.serialize(b); sel2.unserialize(b); CPPUNIT_ASSERT(b.length() == 0); CPPUNIT_ASSERT(sel1 == sel2); CPPUNIT_ASSERT(!(sel1 != sel2)); CPPUNIT_ASSERT(sel1 == t); CPPUNIT_ASSERT(!(sel1 != t)); cpac1 = dynamic_cast<const ArithmeticColumn*>(sel2.lhs()); CPPUNIT_ASSERT(cpac1 != NULL); ct = cpac1->expression(); verifyParseTree(ct); co2 = sel2.op(); CPPUNIT_ASSERT(co2 != NULL); CPPUNIT_ASSERT(co2->data() == o1->data()); }
void WriteOnceConfig::setup() { typedef EntryMap_t::value_type VT; fEntryMap.insert(VT("PrimitiveServers.LBID_Shift", &fLBID_Shift)); fEntryMap.insert(VT("SystemConfig.DBRootCount", &fDBRootCount)); fEntryMap.insert(VT("SystemConfig.DBRMRoot", &fDBRMRoot)); fEntryMap.insert(VT("SessionManager.SharedMemoryTmpFile", &fSharedMemoryTmpFile1)); fEntryMap.insert(VT("SessionManager.TxnIDFile", &fTxnIDFile)); fEntryMap.insert(VT("SessionMonitor.SharedMemoryTmpFile", &fSharedMemoryTmpFile2)); ByteStream ibs = load(); if (ibs.length() > 0) unserialize(ibs); else initializeDefaults(); }
void serializeSimpleColumn() { SimpleColumn s1, s2; TreeNode *t; ByteStream b; t = &s2; CPPUNIT_ASSERT(s1 == s2); CPPUNIT_ASSERT(!(s1 != s2)); CPPUNIT_ASSERT(s1 == t); CPPUNIT_ASSERT(!(s1 != t)); s1.schemaName("Schema Name 1"); s1.tableName("Table Name 1"); s1.columnName("Column Name 1"); //s1.tcn(5); s1.data("sc1"); s1.serialize(b); CPPUNIT_ASSERT(s2.schemaName() == ""); CPPUNIT_ASSERT(s2.tableName() == ""); CPPUNIT_ASSERT(s2.columnName() == ""); //CPPUNIT_ASSERT(s2.tcn() == 0); CPPUNIT_ASSERT(s1 != s2); CPPUNIT_ASSERT(s1 == s1); CPPUNIT_ASSERT(s1 != t); CPPUNIT_ASSERT(!(s1 == t)); s2.unserialize(b); CPPUNIT_ASSERT(b.length() == 0); CPPUNIT_ASSERT(s2.schemaName() == "Schema Name 1"); CPPUNIT_ASSERT(s2.tableName() == "Table Name 1"); CPPUNIT_ASSERT(s2.columnName() == "Column Name 1"); //CPPUNIT_ASSERT(s2.tcn() == 5); CPPUNIT_ASSERT(s2.data() == "sc1"); CPPUNIT_ASSERT(s1 == s2); CPPUNIT_ASSERT(!(s1 != s2)); CPPUNIT_ASSERT(s1 == t); CPPUNIT_ASSERT(!(s1 != t)); }
void serializeFunctionColumn() { FunctionColumn fc1, fc2; TreeNode *t; ByteStream b; t = &fc2; CPPUNIT_ASSERT(fc1 == fc2); CPPUNIT_ASSERT(!(fc1 != fc2)); CPPUNIT_ASSERT(fc1 == t); CPPUNIT_ASSERT(!(fc1 != t)); /* FunctionColumn */ fc1.sessionID(0); fc1.functionName("FunctionColumn test"); fc1.functionParms("tpch.region.r_regionkey, tpch.nation.n_nationkey"); fc1.data("fc1"); CPPUNIT_ASSERT(fc1 != fc2); CPPUNIT_ASSERT(!(fc1 == fc2)); CPPUNIT_ASSERT(fc1 != t); CPPUNIT_ASSERT(!(fc1 == t)); FunctionColumn::FunctionParm functionParms; functionParms = fc1.functionParms(); CPPUNIT_ASSERT(functionParms.size() == 2); fc1.serialize(b); CPPUNIT_ASSERT(fc2.functionName() == ""); CPPUNIT_ASSERT(fc2.functionParms().size() == 0); fc2.unserialize(b); CPPUNIT_ASSERT(b.length() == 0); CPPUNIT_ASSERT(fc2.functionName() == "FunctionColumn test"); CPPUNIT_ASSERT(fc2.functionParms().size() == 2); functionParms = fc2.functionParms(); CPPUNIT_ASSERT(functionParms.size() == 2); CPPUNIT_ASSERT(fc1 == fc2); CPPUNIT_ASSERT(!(fc1 != fc2)); CPPUNIT_ASSERT(fc1 == t); CPPUNIT_ASSERT(!(fc1 != t)); }
void serializeAggregateColumn() { AggregateColumn a1, a2; SimpleColumn *s1; TreeNode *t; ByteStream b; t = &a2; s1 = new SimpleColumn(); CPPUNIT_ASSERT(a1 == a2); CPPUNIT_ASSERT(!(a1 != a2)); CPPUNIT_ASSERT(a1 == t); CPPUNIT_ASSERT(!(a1 != t)); a1.functionName("AggregateColumn test"); a1.data("agg1"); a1.functionParms(s1); a1.serialize(b); CPPUNIT_ASSERT(a2.functionName() == ""); CPPUNIT_ASSERT(a2.data() == ""); CPPUNIT_ASSERT(a2.functionParms() == NULL); CPPUNIT_ASSERT(a1 != a2); CPPUNIT_ASSERT(!(a1 == a2)); CPPUNIT_ASSERT(a1 != t); CPPUNIT_ASSERT(!(a1 == t)); a2.unserialize(b); CPPUNIT_ASSERT(b.length() == 0); CPPUNIT_ASSERT(a2.functionName() == "AggregateColumn test"); CPPUNIT_ASSERT(a2.data() == "agg1"); CPPUNIT_ASSERT(a2.functionParms() != NULL); CPPUNIT_ASSERT(*(a2.functionParms()) == s1); CPPUNIT_ASSERT(a1 == a2); CPPUNIT_ASSERT(!(a1 != a2)); CPPUNIT_ASSERT(a1 == t); CPPUNIT_ASSERT(!(a1 != t)); }
void MessageQueueClient::write(const ByteStream& msg, const struct timespec* timeout, Stats* stats) const { if (!fClientSock.isOpen()) { fClientSock.open(); try { fClientSock.connectionTimeout(timeout); fClientSock.connect(&fServ_addr); } catch (...) { fClientSock.close(); throw; } } try { fClientSock.write(msg, stats); } catch (runtime_error& e) { try { ostringstream oss; oss << "MessageQueueClient::write: error writing " << msg.length() << " bytes to " << fClientSock << ". Socket error was " << e.what() << endl; // cerr << oss.str() << endl; logging::Message::Args args; logging::LoggingID li(31); args.add(oss.str()); fLogger.logMessage(logging::LOG_TYPE_WARNING, logging::M0000, args, li); } catch (...) { } fClientSock.close(); throw; } }
void CompressedInetStreamSocket::write(const ByteStream& msg, Stats* stats) { size_t outLen = 0; uint32_t len = msg.length(); if (useCompression && (len > 512)) { ByteStream smsg(alg.maxCompressedSize(len)); alg.compress((char*) msg.buf(), len, (char*) smsg.getInputPtr(), &outLen); smsg.advanceInputPtr(outLen); if (outLen < len) do_write(smsg, COMPRESSED_BYTESTREAM_MAGIC, stats); else InetStreamSocket::write(msg, stats); } else InetStreamSocket::write(msg, stats); }
void serializeArithmeticColumn() { ParseTree *t; const ParseTree *ct; ArithmeticColumn ac, ac2; TreeNode *tn; ByteStream b; tn = &ac2; CPPUNIT_ASSERT(ac == ac2); CPPUNIT_ASSERT(!(ac != ac2)); CPPUNIT_ASSERT(ac == tn); CPPUNIT_ASSERT(!(ac != tn)); t = makeParseTree(); ac.expression(t); ac.alias("ArithmeticColumn"); ac.data("AD"); CPPUNIT_ASSERT(ac != ac2); CPPUNIT_ASSERT(!(ac == ac2)); CPPUNIT_ASSERT(ac != tn); CPPUNIT_ASSERT(!(ac == tn)); ac.serialize(b); ac2.unserialize(b); CPPUNIT_ASSERT(b.length() == 0); ct = ac2.expression(); verifyParseTree(ct); CPPUNIT_ASSERT(ac2.alias() == "ArithmeticColumn"); CPPUNIT_ASSERT(ac2.data() == "AD"); CPPUNIT_ASSERT(ac == ac2); CPPUNIT_ASSERT(!(ac != ac2)); CPPUNIT_ASSERT(ac == tn); CPPUNIT_ASSERT(!(ac != tn)); }
void test14() { ByteStream bs; ifstream ifs("./Calpont.xml"); ifs >> bs; string id("."); string value; { ConfigStream cs(bs, id); value = cs.getConfig("Message", "Name"); CPPUNIT_ASSERT(value == "Message"); } string bss(reinterpret_cast<const char*>(bs.buf()), bs.length()); { ConfigStream cs(bss, id); value = cs.getConfig("Message", "Name"); CPPUNIT_ASSERT(value == "Message"); } { ConfigStream cs(bss.c_str(), id); value = cs.getConfig("Message", "Name"); CPPUNIT_ASSERT(value == "Message"); } }
void serializeCSEP() { /* * erydbSelectExecutionPlan * This is a large class; it makes more sense to write == operators * for everything than to write a giant equivalance test here. * For now this is mostly a regression test. */ erydbSelectExecutionPlan csep1, csep2; erydbSelectExecutionPlan::ReturnedColumnList colList; ParseTree* filterList; erydbExecutionPlan *cep; ByteStream b; cep = &csep2; CPPUNIT_ASSERT(csep1 == csep2); CPPUNIT_ASSERT(!(csep1 != csep2)); CPPUNIT_ASSERT(csep1 == cep); CPPUNIT_ASSERT(!(csep1 != cep)); // returned columns SimpleColumn *sc = new SimpleColumn("tpch.region.r_regionkey"); colList.push_back(sc); // filters erydbSelectExecutionPlan::Parser parser; std::vector<Token> tokens; Token t; SimpleFilter *sf = new SimpleFilter(); SimpleColumn *lhs = new SimpleColumn(*sc); SimpleColumn *rhs = new SimpleColumn("tpch.nation.n_regionkey"); Operator *op = new Operator("="); sf->op(op); sf->lhs(lhs); sf->rhs(rhs); t.value = sf; tokens.push_back(t); Operator *op1 = new Operator ("and"); t.value = op1; tokens.push_back(t); SimpleFilter *sf1 = new SimpleFilter(); SimpleColumn *lhs1 = new SimpleColumn (*rhs); ConstantColumn *constCol = new ConstantColumn("3", ConstantColumn::NUM); Operator *op2 = new Operator("!="); sf1->op(op2); sf1->lhs(lhs1); sf1->rhs(constCol); t.value = sf1; tokens.push_back(t); filterList = parser.parse(tokens.begin(), tokens.end()); // draw filterList tree filterList->drawTree("selectExecutionPlan_1.dot"); // erydb execution plan csep1.returnedCols (colList); csep1.filters (filterList); CPPUNIT_ASSERT(csep1 != csep2); CPPUNIT_ASSERT(!(csep1 == csep2)); CPPUNIT_ASSERT(csep1 != cep); CPPUNIT_ASSERT(!(csep1 == cep)); csep1.serialize(b); csep2.unserialize(b); CPPUNIT_ASSERT(b.length() == 0); CPPUNIT_ASSERT(csep1 == csep2); CPPUNIT_ASSERT(!(csep1 != csep2)); CPPUNIT_ASSERT(csep1 == cep); CPPUNIT_ASSERT(!(csep1 != cep)); erydbSelectExecutionPlan csep3, csep4; // subselect erydbSelectExecutionPlan *subselect = new erydbSelectExecutionPlan; subselect->location(erydbSelectExecutionPlan::WHERE); subselect->dependent (false); CPPUNIT_ASSERT (subselect->location() == erydbSelectExecutionPlan::WHERE); CPPUNIT_ASSERT (subselect->dependent() == false); erydbSelectExecutionPlan::SelectList selectList; selectList.push_back(subselect); csep3.subSelects(selectList); // exist filter erydbSelectExecutionPlan* cep1 = new erydbSelectExecutionPlan(); ExistsFilter *filter = new ExistsFilter(); delete filter; filter = new ExistsFilter(cep1); filter->exists(cep1); //erydbSelectExecutionPlan* cep2 = const_cast<erydbSelectExecutionPlan*>(filter->exists()); erydbSelectExecutionPlan::Parser parser1; std::vector<Token> tokens1; Token t1; t1.value = filter; tokens1.push_back(t1); csep3.filters(parser1.parse(tokens1.begin(), tokens1.end())); csep3.serialize(b); csep4.unserialize(b); CPPUNIT_ASSERT(csep3 == csep4); CPPUNIT_ASSERT(!(csep3 != csep4)); }
int64_t JoinPartition::convertToSplitMode() { int i, j; ByteStream bs; RGData rgData; uint32_t hash; uint64_t tmp; int64_t ret = -(int64_t)smallSizeOnDisk; // smallFile gets deleted boost::scoped_array<uint32_t> rowDist(new uint32_t[bucketCount]); uint32_t rowCount = 0; memset(rowDist.get(), 0, sizeof(uint32_t) * bucketCount); fileMode = false; htSizeEstimate = 0; smallSizeOnDisk = 0; buckets.reserve(bucketCount); for (i = 0; i < (int) bucketCount; i++) buckets.push_back(boost::shared_ptr<JoinPartition>(new JoinPartition(*this, false))); RowGroup &rg = smallRG; Row &row = smallRow; nextSmallOffset = 0; while (1) { readByteStream(0, &bs); if (bs.length() == 0) break; rgData.deserialize(bs); rg.setData(&rgData); for (j = 0; j < (int) rg.getRowCount(); j++) { rg.getRow(j, &row); if (antiWithMatchNulls && hasNullJoinColumn(row)) { if (needsAllNullRows || !gotNullRow) { for (j = 0; j < (int) bucketCount; j++) ret += buckets[j]->insertSmallSideRow(row); gotNullRow = true; } continue; } if (typelessJoin) hash = getHashOfTypelessKey(row, smallKeyCols, hashSeed) % bucketCount; else { if (UNLIKELY(row.isUnsigned(smallKeyCols[0]))) tmp = row.getUintField(smallKeyCols[0]); else tmp = row.getIntField(smallKeyCols[0]); hash = hasher((char *) &tmp, 8, hashSeed); hash = hasher.finalize(hash, 8) % bucketCount; } rowCount++; rowDist[hash]++; ret += buckets[hash]->insertSmallSideRow(row); } } boost::filesystem::remove(smallFilename); smallFilename.clear(); for (i = 0; i < (int) bucketCount; i++) if (rowDist[i] == rowCount) throw IDBExcept("All rows hashed to the same bucket", ERR_DBJ_DATA_DISTRIBUTION); rg.setData(&buffer); rg.resetRowGroup(0); rg.getRow(0, &row); return ret; }
void MasterDBRMNode::msgProcessor() { struct ThreadParams *p; ByteStream msg; vector<ByteStream *> responses; vector<ByteStream *>::iterator it; int err; uint8_t cmd; StopWatch timer; #ifdef BRM_VERBOSE cerr << "DBRM Controller: msgProcessor()" << endl; #endif mutex2.lock(); p = params; mutex2.unlock(); #ifndef __FreeBSD__ mutex.unlock(); #endif while (!die) { try { msg = p->sock->read(&MSG_TIMEOUT); } catch (...) { THREAD_EXIT throw; } if (die) // || msg.length() == 0) break; else if (msg.length() == 0) continue; /* Check for an command for the master */ msg.peek(cmd); #ifdef BRM_VERBOSE cerr << "DBRM Controller: recv'd message " << (int)cmd << " length " << msg.length() << endl; #endif switch (cmd) { case HALT: doHalt(p->sock); continue; case RESUME: doResume(p->sock); continue; case RELOAD: try { doReload(p->sock); } catch (exception &e) { cerr << e.what() << endl; } continue; case SETREADONLY: doSetReadOnly(p->sock, true); continue; case SETREADWRITE: doSetReadOnly(p->sock, false); continue; case GETREADONLY: doGetReadOnly(p->sock); continue; } /* Process SessionManager calls */ switch (cmd) { case VER_ID: doVerID(msg, p); continue; case SYSCAT_VER_ID: doSysCatVerID(msg, p); continue; case NEW_TXN_ID: doNewTxnID(msg, p); continue; case COMMITTED: doCommitted(msg, p); continue; case ROLLED_BACK: doRolledBack(msg, p); continue; case GET_TXN_ID: doGetTxnID(msg, p); continue; case SID_TID_MAP: doSIDTIDMap(msg, p); continue; case GET_UNIQUE_UINT32: doGetUniqueUint32(msg, p); continue; case GET_UNIQUE_UINT64: doGetUniqueUint64(msg, p); continue; case GET_SYSTEM_STATE: doGetSystemState(msg, p); continue; case SET_SYSTEM_STATE: doSetSystemState(msg, p); continue; case CLEAR_SYSTEM_STATE: doClearSystemState(msg, p); continue; case SM_RESET: doSessionManagerReset(msg, p); continue; } /* Process TableLock calls */ switch (cmd) { case GET_TABLE_LOCK: doGetTableLock(msg, p); continue; case RELEASE_TABLE_LOCK: doReleaseTableLock(msg, p); continue; case CHANGE_TABLE_LOCK_STATE: doChangeTableLockState(msg, p); continue; case CHANGE_TABLE_LOCK_OWNER: doChangeTableLockOwner(msg, p); continue; case GET_ALL_TABLE_LOCKS: doGetAllTableLocks(msg, p); continue; case RELEASE_ALL_TABLE_LOCKS: doReleaseAllTableLocks(msg, p); continue; case GET_TABLE_LOCK_INFO: doGetTableLockInfo(msg, p); continue; case OWNER_CHECK: doOwnerCheck(msg, p); continue; } /* Process OIDManager calls */ switch (cmd) { case ALLOC_OIDS: doAllocOIDs(msg, p); continue; case RETURN_OIDS: doReturnOIDs(msg, p); continue; case OIDM_SIZE: doOidmSize(msg, p); continue; case ALLOC_VBOID: doAllocVBOID(msg, p); continue; case GETDBROOTOFVBOID: doGetDBRootOfVBOID(msg, p); continue; case GETVBOIDTODBROOTMAP: doGetVBOIDToDBRootMap(msg, p); continue; } /* Process Autoincrement calls */ switch (cmd) { case START_AI_SEQUENCE: doStartAISequence(msg, p); continue; case GET_AI_RANGE: doGetAIRange(msg, p); continue; case RESET_AI_SEQUENCE: doResetAISequence(msg, p); continue; case GET_AI_LOCK: doGetAILock(msg, p); continue; case RELEASE_AI_LOCK: doReleaseAILock(msg, p); continue; case DELETE_AI_SEQUENCE: doDeleteAISequence(msg, p); continue; } retrycmd: uint32_t haltloops = 0; while (halting && ++haltloops < static_cast<uint32_t>(FIVE_MIN_TIMEOUT.tv_sec)) sleep(1); slaveLock.lock(); if (haltloops == FIVE_MIN_TIMEOUT.tv_sec) { ostringstream os; os << "A node is unresponsive for cmd = " << (uint32_t)cmd << ", no reconfigure in at least " << FIVE_MIN_TIMEOUT.tv_sec << " seconds. Setting read-only mode."; log(os.str()); readOnly = true; halting = false; } if (readOnly) { SEND_ALARM slaveLock.unlock(); sendError(p->sock, ERR_READONLY); goto out; } /* TODO: Separate these out-of-band items into separate functions */ /* Need to get the dbroot, convert to vbOID */ if (cmd == BEGIN_VB_COPY) { try { boost::mutex::scoped_lock lk(oidsMutex); uint8_t *buf = msg.buf(); // dbroot is currently after the cmd and transid uint16_t *dbRoot = (uint16_t *) &buf[1+4]; // If that dbroot has no vboid, create one int16_t err; err = oids.getVBOIDOfDBRoot(*dbRoot); //cout << "dbRoot " << *dbRoot << " -> vbOID " << err << endl; if (err < 0) { err = oids.allocVBOID(*dbRoot); // cout << " - allocated oid " << err << endl; } *dbRoot = err; } catch (exception& ex) { ostringstream os; os << "DBRM Controller: Begin VBCopy failure. " << ex.what(); log(os.str()); sendError(p->sock, -1); goto out; } } /* Check for deadlock on beginVBCopy */ /* if (cmd == BEGIN_VB_COPY) { ByteStream params(msg); VER_t txn; uint8_t tmp8; uint32_t tmp32; LBIDRange_v ranges; LBIDRange_v::iterator it; LBID_t end; params >> tmp8; //throw away the cmd params >> tmp32; txn = tmp32; deserializeVector(params, ranges); for (it = ranges.begin(); it != ranges.end(); it++) { end = (*it).start + (*it).size - 1; err = rg->reserveRange((*it).start, end, txn, slaveLock); if (err != ERR_OK) { pthread_mutex_unlock(&slaveLock); sendError(p->sock, err); goto out; } } } */ /* Release all blocks of lbids on vbRollback or vbCommit */ if (cmd == VB_ROLLBACK1 || cmd == VB_ROLLBACK2 || cmd == VB_COMMIT) { ByteStream params(msg); VER_t txn; uint8_t tmp8; uint32_t tmp32; params >> tmp8; params >> tmp32; txn = tmp32; rg->releaseResources(txn); } /* XXXPAT: If beginVBCopy, vbRollback, or vbCommit fail for some reason, the resource graph will be out of sync with the copylocks and/or vss. There are 2 reasons they might fail: 1) logical inconsistency between the resource graph and the copylocks & vss 2) "out of band" failures, like a network problem */ /* Need to atomically do the safety check and the clear. */ if (cmd == BRM_CLEAR) { uint32_t txnCount = sm.getTxnCount(); // do nothing if there's an active transaction if (txnCount != 0) { ByteStream *reply = new ByteStream(); *reply << (uint8_t) ERR_FAILURE; responses.push_back(reply); goto no_confirm; } } try { distribute(&msg); } catch (...) { if (!halting) { SEND_ALARM undo(); readOnly = true; slaveLock.unlock(); ostringstream ostr; ostr << "DBRM Controller: Caught network error. " "Sending command " << (uint32_t)cmd << ", length " << msg.length() << ". Setting read-only mode."; log(ostr.str()); sendError(p->sock, ERR_NETWORK); goto out; } } #ifdef BRM_VERBOSE cerr << "DBRM Controller: distributed msg" << endl; #endif bool readErrFlag; // ignore this flag in this case err = gatherResponses(cmd, msg.length(), &responses, readErrFlag); #ifdef BRM_VERBOSE cerr << "DBRM Controller: got responses" << endl; #endif CHECK_ERROR1(err) err = compareResponses(cmd, msg.length(), responses); #ifdef BRM_VERBOSE cerr << "DBRM Controller: compared responses" << endl; #endif #ifdef BRM_DEBUG if ((cmd == BEGIN_VB_COPY || cmd == VB_ROLLBACK1 || cmd == VB_ROLLBACK2 || cmd == VB_COMMIT) && err == -1) cerr << "DBRM Controller: inconsistency detected between the resource graph and the VSS or CopyLocks logic." << endl; #endif // these command will have error message carried in the response if (!responses.empty() && (cmd == DELETE_PARTITION || cmd == MARK_PARTITION_FOR_DELETION || cmd == RESTORE_PARTITION) && err) { if (err != ERR_PARTITION_DISABLED && err != ERR_PARTITION_ENABLED && err != ERR_INVALID_OP_LAST_PARTITION && err != ERR_NOT_EXIST_PARTITION && err != ERR_NO_PARTITION_PERFORMED) undo(); //goto no_confirm; } else { CHECK_ERROR1(err) } // these cmds don't need the 2-phase commit if (cmd == FLUSH_INODE_CACHES || cmd == BRM_CLEAR || cmd == TAKE_SNAPSHOT) goto no_confirm; #ifdef BRM_VERBOSE cerr << "DBRM Controller: sending confirmation" << endl; #endif try { confirm(); } catch (...) { if (!halting) { SEND_ALARM ostringstream ostr; ostr << "DBRM Controller: Caught network error. " "Confirming command " << (uint32_t)cmd << ", length " << msg.length() << ". Setting read-only mode."; log(ostr.str()); readOnly = true; } } no_confirm: slaveLock.unlock(); try { p->sock->write(*(responses.front())); } catch (...) { p->sock->close(); log("DBRM Controller: Warning: could not send the reply to a command", logging::LOG_TYPE_WARNING); } out: for (it = responses.begin(); it != responses.end(); it++) delete *it; responses.clear(); }
/****************************************************************************************** * @brief sendMsgProcMon * * purpose: Sends a Msg to ProcMon * ******************************************************************************************/ int sendMsgProcMon( std::string module, ByteStream msg, int requestID, int timeout ) { string msgPort = module + "_ProcessMonitor"; int returnStatus = API_FAILURE; Oam oam; // do a ping test to determine a quick failure Config* sysConfig = Config::makeConfig(); string IPAddr = sysConfig->getConfig(msgPort, "IPAddr"); if ( IPAddr == oam::UnassignedIpAddr ) { return returnStatus; } string cmdLine = "ping "; string cmdOption = " -w 1 >> /dev/null"; string cmd = cmdLine + IPAddr + cmdOption; if ( system(cmd.c_str()) != 0) { //ping failure return returnStatus; } try { MessageQueueClient mqRequest(msgPort); mqRequest.write(msg); if ( timeout > 0 ) { // wait for response ByteStream::byte returnACK; ByteStream::byte returnRequestID; ByteStream::byte requestStatus; ByteStream receivedMSG; struct timespec ts = { timeout, 0 }; // get current time in seconds time_t startTimeSec; time (&startTimeSec); while(true) { try { receivedMSG = mqRequest.read(&ts); } catch (...) { return returnStatus; } if (receivedMSG.length() > 0) { receivedMSG >> returnACK; receivedMSG >> returnRequestID; receivedMSG >> requestStatus; if ( returnACK == oam::ACK && returnRequestID == requestID) { // ACK for this request returnStatus = requestStatus; break; } } else { //api timeout occurred, check if retry should be done // get current time in seconds time_t endTimeSec; time (&endTimeSec); if ( timeout <= (endTimeSec - startTimeSec) ) { break; } } }
int main(int argc, char** argv) { vflg = false; uint32_t tlvl = 0; bool dflg = false; int c; int32_t sid = -1; bool Bflg = false; opterr = 0; while ((c = getopt(argc, argv, "vt:ds:Bh")) != EOF) switch (c) { case 't': tlvl = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break; case 'v': vflg = true; break; case 'd': dflg = true; break; case 's': sid = static_cast<int32_t>(strtol(optarg, 0, 0)); break; case 'B': Bflg = true; break; case 'h': case '?': default: usage(); return (c == 'h' ? 0 : 1); break; } if (dflg) vflg = true; if ((argc - optind) < 1) { usage(); return 1; } ifstream inputf; ByteStream bs; ByteStream dbs; ByteStream eoq; ByteStream tbs; ByteStream statsStream; ByteStream::quadbyte q = 0; eoq << q; uint32_t sessionid; time_t t; SJLP jl; DeliveredTableMap tm; DeliveredTableMap::iterator iter; DeliveredTableMap::iterator end; CalpontSelectExecutionPlan csep; struct timeval start_time; struct timeval end_time; MessageQueueClient* mqc = 0; if (!dflg) mqc = new MessageQueueClient("ExeMgr1"); if (sid == -1) { time(&t); sessionid = static_cast<uint32_t>(t); } else { sessionid = static_cast<uint32_t>(sid); } sessionid &= 0x7fffffff; logging::ErrorCodes errorCodes; for ( ; optind < argc; optind++) { inputf.open(argv[optind]); if (!inputf.good()) { cerr << "error opening plan stream " << argv[optind] << endl; return 1; } bs.reset(); inputf >> bs; inputf.close(); csep.unserialize(bs); csep.sessionID(sessionid); SessionManager sm; csep.verID(sm.verID()); csep.traceFlags(0); ResourceManager rm; jl = JobListFactory::makeJobList(&csep, rm); csep.traceFlags(tlvl); if (vflg) { if (dflg) cout << endl << "Query:" << endl; else { cout << endl << "Session: " << sessionid << ", Sending Query"; if (Bflg) cout << " (" << argv[optind] << ')'; cout << ':' << endl; } if (!Bflg) cout << csep.data() << endl << endl; } if (dflg) continue; try { dbs.reset(); csep.serialize(dbs); gettimeofday(&start_time, 0); //try tuples first, but expect the worst... bool expectTuples = false; ByteStream tbs; ByteStream::quadbyte tqb = 4; tbs << tqb; mqc->write(tbs); //send the CSEP mqc->write(dbs); //read the response to the tuple request tbs = mqc->read(); idbassert(tbs.length() == 4); tbs >> tqb; if (tqb == 4) expectTuples = true; if (!expectTuples) cout << "Using TableBand I/F" << endl; else cout << "Using tuple I/F" << endl; tm = jl->deliveredTables(); iter = tm.begin(); end = tm.end(); OID toid; uint64_t rowTot; bool reported = false; bool needRGCtor = true; while (iter != end) { toid = iter->first; q = static_cast<ByteStream::quadbyte>(toid); tbs.reset(); tbs << q; mqc->write(tbs); ByteStream tbbs; TableBand tb; RowGroup rg; rowTot = 0; uint16_t status = 0; TableBand::VBA::size_type rc; ofstream out; for (;;) { tbbs = mqc->read(); #if 0 cout << tbbs.length() << endl; out.open("bs1.dat"); idbassert(out.good()); out << tbbs; out.close(); tbbs = mqc->read(); cout << tbbs.length() << endl; out.open("bs2.dat"); idbassert(out.good()); out << tbbs; out.close(); tbbs = mqc->read(); cout << tbbs.length() << endl; out.open("bs3.dat"); idbassert(out.good()); out << tbbs; out.close(); #endif if(tbbs.length()) { if (!expectTuples) tb.unserialize(tbbs); else { if (needRGCtor) { rg.deserialize(tbbs); needRGCtor = false; tbbs = mqc->read(); } rg.setData((uint8_t*)tbbs.buf()); } } else { //@bug 1346 if (!status) status = logging::makeJobListErr; break; } if (!expectTuples) { rc = tb.getRowCount(); status = tb.getStatus(); } else { rc = rg.getRowCount(); status = rg.getStatus(); if (rc == 0) status = 0; } if (rc == 0) break; rowTot += rc; } BatchPrimitive* step = dynamic_cast<BatchPrimitive*>( iter->second.get() ); if (vflg && step) { cout << "For table " << step->tableName(); if (!Bflg) cout << " " << toid; cout << ": read " << rowTot << " rows" << endl; } if (status && !reported) { cout << "### Query failed: " << errorCodes.errorString(status) << " Check crit.log\n"; reported = true; } if (!step && !reported) { cout << "### Query failed: Did not return project BatchPrimitive. Check crit.log\n"; reported = true; } ++iter; } if (vflg) { gettimeofday(&end_time, 0); cout << "Query time: " << fixed << setprecision(1) << tm_diff(&start_time, &end_time) << " secs" << endl; //...Ask for query stats through special table id of 3 const OID TABLE_ID_TO_GET_QUERY_STATS = 3; if (!Bflg) cout << "Retrieving stats..." << endl; toid = TABLE_ID_TO_GET_QUERY_STATS; q = static_cast<ByteStream::quadbyte>(toid); statsStream.reset(); statsStream << q; mqc->write(statsStream); ByteStream bs_statsString; bs_statsString = mqc->read(); string statsString; bs_statsString >> statsString; string printStatsString; struct timeval startRunTime; parseStatsString (statsString, printStatsString, startRunTime); cout << printStatsString << "; QuerySetupTime-" << tm_diff(&start_time, &startRunTime) << "secs" << endl; } //...Close this query/session mqc->write(eoq); jl.reset(); } catch(const exception& ex) { cout << "### SendPlan caught an exception: " << ex.what() << endl; } } // jl.reset(); CalpontSystemCatalog::removeCalpontSystemCatalog( sessionid ); config::Config::deleteInstanceMap(); delete mqc; return 0; }
void procmonMonitor() { ServerMonitor serverMonitor; Oam oam; //wait before monitoring is started sleep(60); // get current server name string moduleName; oamModuleInfo_t st; try { st = oam.getModuleInfo(); moduleName = boost::get<0>(st); } catch (...) { // Critical error, Log this event and exit LoggingID lid(SERVER_MONITOR_LOG_ID); MessageLog ml(lid); Message msg; Message::Args args; args.add("Failed to read local module Info"); msg.format(args); ml.logCriticalMessage(msg); exit(-1); } string msgPort = moduleName + "_ProcessMonitor"; int heartbeatCount = 0; // loop forever monitoring Local Process Monitor while(true) { ByteStream msg; ByteStream::byte requestID = LOCALHEARTBEAT; msg << requestID; try { MessageQueueClient mqRequest(msgPort); mqRequest.write(msg); // wait 10 seconds for response ByteStream::byte returnACK; ByteStream::byte returnRequestID; ByteStream::byte requestStatus; ByteStream receivedMSG; struct timespec ts = { 10, 0 }; try { receivedMSG = mqRequest.read(&ts); if (receivedMSG.length() > 0) { receivedMSG >> returnACK; receivedMSG >> returnRequestID; receivedMSG >> requestStatus; if ( returnACK == oam::ACK && returnRequestID == requestID) { // ACK for this request heartbeatCount = 0; } } else { LoggingID lid(SERVER_MONITOR_LOG_ID); MessageLog ml(lid); Message msg; Message::Args args; args.add("procmonMonitor: ProcMon Msg timeout!!!"); msg.format(args); ml.logWarningMessage(msg); heartbeatCount++; if ( heartbeatCount > 2 ) { //Process Monitor not responding, restart it system("pkill ProcMon"); LoggingID lid(SERVER_MONITOR_LOG_ID); MessageLog ml(lid); Message msg; Message::Args args; args.add("procmonMonitor: Restarting ProcMon"); msg.format(args); ml.logWarningMessage(msg); sleep(60); heartbeatCount = 0; } } mqRequest.shutdown(); } catch (SocketClosed &ex) { string error = ex.what(); LoggingID lid(SERVER_MONITOR_LOG_ID); MessageLog ml(lid); Message msg; Message::Args args; args.add("procmonMonitor: EXCEPTION ERROR on mqRequest.read: " + error); msg.format(args); ml.logErrorMessage(msg); } catch (...) { LoggingID lid(SERVER_MONITOR_LOG_ID); MessageLog ml(lid); Message msg; Message::Args args; args.add("procmonMonitor: EXCEPTION ERROR on mqRequest.read: Caught unknown exception"); msg.format(args); ml.logErrorMessage(msg); } } catch (exception& ex)
int main (int argc, char *argv[] ) { g_sndbuf = 126976; g_sqns = 1000; /* parse program arguments */ const char* binary_name = strrchr (argv[0], '/'); int c; while ((c = getopt (argc, argv, "b:m:t:s:n:p:r:f:k:g:w:q:lh")) != -1) { switch (c) { case 'b': g_blocks = atoi(optarg); break; case 'm': g_messages = atoi(optarg); break; case 't': g_max_rte = atoi(optarg) * 400; break; case 'n': g_network = optarg; break; case 's': g_port = atoi (optarg); break; case 'p': g_udp_encap_port = atoi (optarg); break; case 'r': g_max_rte = atoi (optarg); break; case 'f': g_fec = TRUE; break; case 'k': g_k = atoi (optarg); break; case 'g': g_n = atoi (optarg); break; case 'w': g_sndbuf = atoi (optarg); break; case 'q': g_sqns = atoi (optarg); break; case 'l': g_multicast_loop = TRUE; break; case 'h': case '?': usage (binary_name); } } if (g_fec && ( !g_k || !g_n )) { puts ("Invalid Reed-Solomon parameters."); usage (binary_name); } log_init (); pgm_init (); /* setup signal handlers */ signal (SIGSEGV, on_sigsegv); signal (SIGHUP, SIG_IGN); ByteStream obs; ByteStream psbs; uint w = createData(obs, psbs); ByteStream::octbyte bytes_written = 0; time_t start_time = 0; if (!create_transport ()) { start_time = time(0); // Send total length for error check and to start timer on receive side gssize e = pgm_transport_send (g_transport, psbs.buf(), psbs.length(), 0); if (e < 0) g_warning ("pgm_transport_send datasize failed."); for (uint i = 0; i < g_messages; ++i) { gsize e = pgm_transport_send (g_transport, obs.buf(), obs.length(), 0); if (e < 0) g_warning ("pgm_transport_send data failed."); bytes_written +=w; } } obs.reset(); //Send 0 length to indicate transmission finished. gssize e = pgm_transport_send (g_transport, &obs, 0, 0); if (e < 0) g_warning ("pgm_transport_send failed."); double elapsed_time = time(0) - start_time; /* cleanup */ if (g_transport) { pgm_transport_destroy (g_transport, TRUE); g_transport = NULL; } std::cout << "pgmsend wrote " << bytes_written << " bytes with " << g_messages << " messages of " << w << " bytes in " << elapsed_time << " seconds "; if (elapsed_time) std::cout << bytes_written/elapsed_time/1024 << " Kbytes/second\n"; else std::cout << std::endl; return 0; }