int main(int argc, char* argv[])
{
// get and set locale language
systemLang = funcexp::utf8::idb_setlocale();
if ( systemLang != "en_US.UTF-8" &&
systemLang.find("UTF") != string::npos )
utf8 = true;
Config* cf = Config::makeConfig();
setupSignalHandlers();
setupCwd(cf);
mlp = new primitiveprocessor::Logger();
int rc;
rc = setupResources();
if (rc) {
Message::Args args;
args.add(rc);
//mlp->logMessage(logging::M0016, args);
}
int serverThreads = 1;
int serverQueueSize = 10;
int processorWeight = 8*1024;
int processorQueueSize = 10*1024;
int BRPBlocksPct = 70;
uint32_t BRPBlocks = 1887437;
int BRPThreads = 16;
int cacheCount = 1;
int maxBlocksPerRead = 256; // 1MB
bool rotatingDestination = false;
uint32_t deleteBlocks = 128;
bool PTTrace = false;
int temp;
string strTemp;
int priority = -1;
const string primitiveServers("PrimitiveServers");
const string jobListStr("JobList");
const string dbbc("DBBC");
const string ExtentMapStr("ExtentMap");
uint64_t extentRows = 8*1024*1024;
uint64_t MaxExtentSize = 0;
double prefetchThreshold;
uint64_t PMSmallSide = 67108864;
BPPCount = 16;
int numCores = -1;
int configNumCores = -1;
uint32_t highPriorityPercentage, medPriorityPercentage, lowPriorityPercentage;
utils::CGroupConfigurator cg;
gDebugLevel = primitiveprocessor::NONE;
temp = toInt(cf->getConfig(primitiveServers, "ServerThreads"));
if (temp > 0)
serverThreads = temp;
temp = toInt(cf->getConfig(primitiveServers, "ServerQueueSize"));
if (temp > 0)
serverQueueSize = temp;
temp = toInt(cf->getConfig(primitiveServers, "ProcessorThreshold"));
if (temp > 0)
processorWeight = temp;
temp = toInt(cf->getConfig(primitiveServers, "ProcessorQueueSize"));
if (temp > 0)
processorQueueSize = temp;
temp = toInt(cf->getConfig(primitiveServers, "DebugLevel"));
if (temp > 0)
gDebugLevel = (DebugLevel)temp;
highPriorityPercentage = 0;
temp = toInt(cf->getConfig(primitiveServers, "HighPriorityPercentage"));
if (temp >= 0)
highPriorityPercentage = temp;
medPriorityPercentage = 0;
temp = toInt(cf->getConfig(primitiveServers, "MediumPriorityPercentage"));
if (temp >= 0)
medPriorityPercentage = temp;
lowPriorityPercentage = 0;
temp = toInt(cf->getConfig(primitiveServers, "LowPriorityPercentage"));
if (temp >= 0)
lowPriorityPercentage = temp;
temp = toInt(cf->getConfig(ExtentMapStr, "ExtentRows"));
if (temp > 0)
extentRows = temp;
temp = toInt(cf->getConfig(primitiveServers, "ConnectionsPerPrimProc"));
if (temp > 0)
connectionsPerUM = temp;
else
connectionsPerUM = 1;
// set to smallest extent size
// do not allow to read beyond the end of an extent
const int MaxReadAheadSz = (extentRows)/BLOCK_SIZE;
//defaultBufferSize = 512 * 1024; // @bug 2627 - changed default dict buffer from 256K to 512K, allows for cols w/ length of 61.
defaultBufferSize = 100*1024; // 1/17/12 - made the dict buffer dynamic, max size for a numeric col is 80k + ovrhd
// This parm controls whether we rotate through the output sockets
// when deciding where to send response messages, or whether to simply
// send the response to the socket of origin. Should normally be set
// to 'y', for install types 1 and 3.
string strVal = cf->getConfig(primitiveServers, "RotatingDestination");
//XXX: Permanently disable for now...
strVal = "N";
if ((strVal == "y") || (strVal == "Y")) {
rotatingDestination = true;
// Disable destination rotation if UM and PM are running on same
// server, because we could accidentally end up sending DMLProc
// responses to ExeMgr and vice versa, if we rotated socket dest.
temp = toInt(cf->getConfig("Installation", "ServerTypeInstall"));
if ((temp == oam::INSTALL_COMBINE_DM_UM_PM) ||
(temp == oam::INSTALL_COMBINE_PM_UM))
rotatingDestination = false;
}
string strBlockPct = cf->getConfig(dbbc, "NumBlocksPct");
temp = atoi(strBlockPct.c_str());
if (temp > 0)
BRPBlocksPct = temp;
#ifdef _MSC_VER
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
if (GlobalMemoryStatusEx(&memStat) == 0)
//FIXME: Assume 2GB?
BRPBlocks = 2621 * BRPBlocksPct;
else
{
#ifndef _WIN64
memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys);
#endif
//We now have the total phys mem in bytes
BRPBlocks = memStat.ullTotalPhys / (8 * 1024) / 100 * BRPBlocksPct;
}
#else
// _SC_PHYS_PAGES is in 4KB units. Dividing by 200 converts to 8KB and gets ready to work in pct
// _SC_PHYS_PAGES should always be >> 200 so we shouldn't see a total loss of precision
//BRPBlocks = sysconf(_SC_PHYS_PAGES) / 200 * BRPBlocksPct;
BRPBlocks = ((BRPBlocksPct/100.0) * (double) cg.getTotalMemory()) / 8192;
#endif
#if 0
temp = toInt(cf->getConfig(dbbc, "NumThreads"));
if (temp > 0)
BRPThreads = temp;
#endif
temp = toInt(cf->getConfig(dbbc, "NumCaches"));
if (temp > 0)
cacheCount = temp;
temp = toInt(cf->getConfig(dbbc, "NumDeleteBlocks"));
if (temp > 0)
deleteBlocks = temp;
if ((uint32_t)(.01 * BRPBlocks) < deleteBlocks)
deleteBlocks = (uint32_t)(.01 * BRPBlocks);
temp = toInt(cf->getConfig(primitiveServers, "ColScanBufferSizeBlocks"));
if (temp > (int) MaxReadAheadSz || temp < 1)
maxBlocksPerRead = MaxReadAheadSz;
else if (temp > 0)
maxBlocksPerRead = temp;
temp = toInt(cf->getConfig(primitiveServers, "ColScanReadAheadBlocks"));
if (temp > (int) MaxReadAheadSz || temp < 0)
blocksReadAhead = MaxReadAheadSz;
else if (temp > 0)
{
//make sure we've got an integral factor of extent size
for (; (MaxExtentSize%temp)!=0; ++temp);
blocksReadAhead=temp;
}
temp = toInt(cf->getConfig(primitiveServers, "PTTrace"));
if (temp > 0)
PTTrace = true;
temp = toInt(cf->getConfig(primitiveServers, "PrefetchThreshold"));
if (temp < 0 || temp > 100)
prefetchThreshold = 0;
else
prefetchThreshold = temp/100.0;
int maxPct = 0; //disable by default
temp = toInt(cf->getConfig(primitiveServers, "MaxPct"));
if (temp >= 0)
maxPct = temp;
// @bug4507, configurable pm aggregation AggregationMemoryCheck
// We could use this same mechanism for other growing buffers.
int aggPct = 95;
temp = toInt(cf->getConfig("SystemConfig", "MemoryCheckPercent"));
if (temp >= 0)
aggPct = temp;
//...Start the thread to monitor our memory usage
new boost::thread(utils::MonitorProcMem(maxPct, aggPct, 28));
// config file priority is 40..1 (highest..lowest)
string sPriority = cf->getConfig(primitiveServers, "Priority");
if (sPriority.length() > 0)
temp = toInt(sPriority);
else
temp = 21;
// convert config file value to setpriority(2) value (-20..19, -1 is the default)
if (temp > 0)
priority = 20 - temp;
else if (temp < 0)
priority = 19;
if (priority < -20) priority = -20;
#ifdef _MSC_VER
//FIXME:
#else
setpriority(PRIO_PROCESS, 0, priority);
#endif
//..Instantiate UmSocketSelector singleton. Disable rotating destination
//..selection if no UM IP addresses are in the Calpo67108864LLnt.xml file.
UmSocketSelector* pUmSocketSelector = UmSocketSelector::instance();
if (rotatingDestination)
{
if (pUmSocketSelector->ipAddressCount() < 1)
rotatingDestination = false;
}
//See if we want to override the calculated #cores
temp = toInt(cf->getConfig(primitiveServers, "NumCores"));
if (temp > 0)
configNumCores = temp;
if (configNumCores <= 0)
{
//count the actual #cores
numCores = cg.getNumCores();
if (numCores == 0)
numCores = 8;
}
else
numCores = configNumCores;
//based on the #cores, calculate some thread parms
if (numCores > 0)
{
BRPThreads = 2 * numCores;
//there doesn't seem much benefit to having more than this, and sometimes it causes problems.
//DBBC.NumThreads can override this cap
BRPThreads = std::min(BRPThreads, 32);
}
// the default is ~10% low, 30% medium, 60% high, (where 2*cores = 100%)
if (highPriorityPercentage == 0 && medPriorityPercentage == 0 &&
lowPriorityPercentage == 0) {
lowPriorityThreads = max(1, (2*numCores)/10);
medPriorityThreads = max(1, (2*numCores)/3);
highPriorityThreads = (2 * numCores) - lowPriorityThreads - medPriorityThreads;
}
else {
uint32_t totalThreads = (uint32_t) ((lowPriorityPercentage + medPriorityPercentage +
highPriorityPercentage) / 100.0 * (2*numCores));
if (totalThreads == 0)
totalThreads = 1;
lowPriorityThreads = (uint32_t) (lowPriorityPercentage/100.0 * (2*numCores));
medPriorityThreads = (uint32_t) (medPriorityPercentage/100.0 * (2*numCores));
highPriorityThreads = totalThreads - lowPriorityThreads - medPriorityThreads;
}
BPPCount = highPriorityThreads + medPriorityThreads + lowPriorityThreads;
// let the user override if they want
temp = toInt(cf->getConfig(primitiveServers, "BPPCount"));
if (temp > 0 && temp < (int) BPPCount)
BPPCount = temp;
temp = toInt(cf->getConfig(dbbc, "NumThreads"));
if (temp > 0)
BRPThreads = temp;
#ifndef _MSC_VER
// @bug4598, switch for O_DIRECT to support gluster fs.
// directIOFlag == O_DIRECT, by default
strVal = cf->getConfig(primitiveServers, "DirectIO");
if ((strVal == "n") || (strVal == "N"))
directIOFlag = 0;
#endif
IDBPolicy::configIDBPolicy();
// no versionbuffer if using HDFS for performance reason
if (IDBPolicy::useHdfs())
noVB = 1;
cout << "Starting PrimitiveServer: st = " << serverThreads << ", sq = " << serverQueueSize <<
", pw = " << processorWeight << ", pq = " << processorQueueSize <<
", nb = " << BRPBlocks << ", nt = " << BRPThreads << ", nc = " << cacheCount <<
", ra = " << blocksReadAhead << ", db = " << deleteBlocks << ", mb = " << maxBlocksPerRead <<
", rd = " << rotatingDestination << ", tr = " << PTTrace <<
", ss = " << PMSmallSide << ", bp = " << BPPCount << endl;
PrimitiveServer server(serverThreads, serverQueueSize, processorWeight, processorQueueSize,
rotatingDestination, BRPBlocks, BRPThreads, cacheCount, maxBlocksPerRead, blocksReadAhead,
deleteBlocks, PTTrace, prefetchThreshold, PMSmallSide);
#ifdef QSIZE_DEBUG
thread* qszMonThd;
if (gDebugLevel >= STATS)
{
#ifdef _MSC_VER
ofstream* qszLog = new ofstream("C:/Calpont/log/trace/ppqsz.dat");
#else
ofstream* qszLog = new ofstream("/var/log/Calpont/trace/ppqsz.dat");
#endif
if (!qszLog->good())
{
qszLog->close();
delete qszLog;
qszLog = 0;
}
qszMonThd = new thread(QszMonThd(&server, qszLog));
}
#endif
#ifdef DUMP_CACHE_CONTENTS
{
//Need to use pthreads API here...
pthread_t thd1;
pthread_attr_t attr1;
pthread_attr_init(&attr1);
pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED);
pthread_create(&thd1, &attr1, waitForSIGUSR1, reinterpret_cast<void*>(cacheCount));
}
#endif
server.start();
cerr << "server.start() exited!" << endl;
return 1;
}
int main()
{
// erydb.xml file should be configured as follows:
// um1: 10.100.4.85 and 10.100.5.85
// um2: 10.101.4.85 and 10.101.5.85
sockaddr_in sa = { 1, 0, {0} , {' '} };
char* ips[] = {"10.100.4.85", "10.100.5.85", "10.101.4.85", "10.101.5.85"};
// These are the IP addresses we use to test runtime connections
// "not" in the erydb.xml file.
sockaddr_in saUnknown = { 1, 0, {0} , {' '} };
char* ipsUnknown[]={"10.102.1.1", "10.102.2.1", "10.102.3.1", "10.102.4.1"};
//--------------------------------------------------------------------------
// Test initialization
//--------------------------------------------------------------------------
UmSocketSelector* sockSel = UmSocketSelector::instance();
std::cout << "IPAddressCount: " << sockSel->ipAddressCount() << std::endl;
std::cout << std::endl << "----Dump1 after initialization..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump1 End...................." << std::endl << std::endl;
IOSocket sock[4][4];
for (int i=0; i<4; i++)
{
inet_aton(ips[i], &sa.sin_addr);
for (int j=0; j<4; j++)
{
sock[i][j].setSocketImpl(new InetStreamSocket());
sa.sin_port = htons((i*4)+j);
sock[i][j].sa( sa );
sockSel->addConnection( SP_UM_IOSOCK(new IOSocket(sock[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
std::cout << std::endl << "----Dump2 after adding 16 connections..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump2 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test socket/port selection
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k=0; k<17; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
#if 1
if (sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
#else
if (!sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
#endif
std::cout << "no nextIP found for " << sock[0][0] << std::endl;
}
for (unsigned k=0; k<7; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
if (sockSel->nextIOSocket( sock[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sock[2][0] << std::endl;
}
std::cout << std::endl;
std::cout << "----Dump3 after selecting 17 connections from IP " <<
ips[0] << "; and 7 connections from IP " << ips[2] << " ..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump3 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test connection deletions
//--------------------------------------------------------------------------
for (unsigned k=0; k<4; k++)
{
sockSel->delConnection( sock[k][0] );
}
std::cout << "----Dump4 after deleting first connection for each IP..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump4 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test addition of unknown connections
//--------------------------------------------------------------------------
IOSocket sockUnknown[4][4];
for (int i=0; i<4; i++)
{
inet_aton(ipsUnknown[i], &saUnknown.sin_addr);
for (int j=0; j<4; j++)
{
sockUnknown[i][j].setSocketImpl(new InetStreamSocket());
saUnknown.sin_port = htons((i*4)+j);
sockUnknown[i][j].sa( saUnknown );
sockSel->addConnection(
SP_UM_IOSOCK(new IOSocket(sockUnknown[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
std::cout << "----Dump5 after adding connections for unknown IP's..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump5 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test resetting of "next" indexes after deleting all sockets from a
// specific IP address for which the "next" index is pointing.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[1][1] );
sockSel->delConnection( sock[1][2] );
sockSel->delConnection( sock[1][3] );
std::cout << "----Dump6 after deleting all connections for IP " << ips[1] <<
" ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump6 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test socket/port selection for an unknown module
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k=0; k<11; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
if (sockSel->nextIOSocket( sockUnknown[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sockUnknown[2][0]<<std::endl;
}
std::cout << std::endl;
std::cout << "----Dump7 after selecting 11 connections for IP " <<
ipsUnknown[2] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump7 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test deletion of last socket/port connection and resetting if the
// "next" index that is pointing to it.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[3][3] );
std::cout << "----Dump8 after deleting last connection for IP " <<
ips[3] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump8 End...................." << std::endl << std::endl;
return 0;
}