int main(int argc, char* argv[]) { // get and set locale language systemLang = funcexp::utf8::idb_setlocale(); if ( systemLang != "en_US.UTF-8" && systemLang.find("UTF") != string::npos ) utf8 = true; Config* cf = Config::makeConfig(); setupSignalHandlers(); setupCwd(cf); mlp = new primitiveprocessor::Logger(); int rc; rc = setupResources(); if (rc) { Message::Args args; args.add(rc); //mlp->logMessage(logging::M0016, args); } int serverThreads = 1; int serverQueueSize = 10; int processorWeight = 8*1024; int processorQueueSize = 10*1024; int BRPBlocksPct = 70; uint32_t BRPBlocks = 1887437; int BRPThreads = 16; int cacheCount = 1; int maxBlocksPerRead = 256; // 1MB bool rotatingDestination = false; uint32_t deleteBlocks = 128; bool PTTrace = false; int temp; string strTemp; int priority = -1; const string primitiveServers("PrimitiveServers"); const string jobListStr("JobList"); const string dbbc("DBBC"); const string ExtentMapStr("ExtentMap"); uint64_t extentRows = 8*1024*1024; uint64_t MaxExtentSize = 0; double prefetchThreshold; uint64_t PMSmallSide = 67108864; BPPCount = 16; int numCores = -1; int configNumCores = -1; uint32_t highPriorityPercentage, medPriorityPercentage, lowPriorityPercentage; utils::CGroupConfigurator cg; gDebugLevel = primitiveprocessor::NONE; temp = toInt(cf->getConfig(primitiveServers, "ServerThreads")); if (temp > 0) serverThreads = temp; temp = toInt(cf->getConfig(primitiveServers, "ServerQueueSize")); if (temp > 0) serverQueueSize = temp; temp = toInt(cf->getConfig(primitiveServers, "ProcessorThreshold")); if (temp > 0) processorWeight = temp; temp = toInt(cf->getConfig(primitiveServers, "ProcessorQueueSize")); if (temp > 0) processorQueueSize = temp; temp = toInt(cf->getConfig(primitiveServers, "DebugLevel")); if (temp > 0) gDebugLevel = (DebugLevel)temp; highPriorityPercentage = 0; temp = toInt(cf->getConfig(primitiveServers, "HighPriorityPercentage")); if (temp >= 0) highPriorityPercentage = temp; medPriorityPercentage = 0; temp = toInt(cf->getConfig(primitiveServers, "MediumPriorityPercentage")); if (temp >= 0) medPriorityPercentage = temp; lowPriorityPercentage = 0; temp = toInt(cf->getConfig(primitiveServers, "LowPriorityPercentage")); if (temp >= 0) lowPriorityPercentage = temp; temp = toInt(cf->getConfig(ExtentMapStr, "ExtentRows")); if (temp > 0) extentRows = temp; temp = toInt(cf->getConfig(primitiveServers, "ConnectionsPerPrimProc")); if (temp > 0) connectionsPerUM = temp; else connectionsPerUM = 1; // set to smallest extent size // do not allow to read beyond the end of an extent const int MaxReadAheadSz = (extentRows)/BLOCK_SIZE; //defaultBufferSize = 512 * 1024; // @bug 2627 - changed default dict buffer from 256K to 512K, allows for cols w/ length of 61. defaultBufferSize = 100*1024; // 1/17/12 - made the dict buffer dynamic, max size for a numeric col is 80k + ovrhd // This parm controls whether we rotate through the output sockets // when deciding where to send response messages, or whether to simply // send the response to the socket of origin. Should normally be set // to 'y', for install types 1 and 3. string strVal = cf->getConfig(primitiveServers, "RotatingDestination"); //XXX: Permanently disable for now... strVal = "N"; if ((strVal == "y") || (strVal == "Y")) { rotatingDestination = true; // Disable destination rotation if UM and PM are running on same // server, because we could accidentally end up sending DMLProc // responses to ExeMgr and vice versa, if we rotated socket dest. temp = toInt(cf->getConfig("Installation", "ServerTypeInstall")); if ((temp == oam::INSTALL_COMBINE_DM_UM_PM) || (temp == oam::INSTALL_COMBINE_PM_UM)) rotatingDestination = false; } string strBlockPct = cf->getConfig(dbbc, "NumBlocksPct"); temp = atoi(strBlockPct.c_str()); if (temp > 0) BRPBlocksPct = temp; #ifdef _MSC_VER MEMORYSTATUSEX memStat; memStat.dwLength = sizeof(memStat); if (GlobalMemoryStatusEx(&memStat) == 0) //FIXME: Assume 2GB? BRPBlocks = 2621 * BRPBlocksPct; else { #ifndef _WIN64 memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys); #endif //We now have the total phys mem in bytes BRPBlocks = memStat.ullTotalPhys / (8 * 1024) / 100 * BRPBlocksPct; } #else // _SC_PHYS_PAGES is in 4KB units. Dividing by 200 converts to 8KB and gets ready to work in pct // _SC_PHYS_PAGES should always be >> 200 so we shouldn't see a total loss of precision //BRPBlocks = sysconf(_SC_PHYS_PAGES) / 200 * BRPBlocksPct; BRPBlocks = ((BRPBlocksPct/100.0) * (double) cg.getTotalMemory()) / 8192; #endif #if 0 temp = toInt(cf->getConfig(dbbc, "NumThreads")); if (temp > 0) BRPThreads = temp; #endif temp = toInt(cf->getConfig(dbbc, "NumCaches")); if (temp > 0) cacheCount = temp; temp = toInt(cf->getConfig(dbbc, "NumDeleteBlocks")); if (temp > 0) deleteBlocks = temp; if ((uint32_t)(.01 * BRPBlocks) < deleteBlocks) deleteBlocks = (uint32_t)(.01 * BRPBlocks); temp = toInt(cf->getConfig(primitiveServers, "ColScanBufferSizeBlocks")); if (temp > (int) MaxReadAheadSz || temp < 1) maxBlocksPerRead = MaxReadAheadSz; else if (temp > 0) maxBlocksPerRead = temp; temp = toInt(cf->getConfig(primitiveServers, "ColScanReadAheadBlocks")); if (temp > (int) MaxReadAheadSz || temp < 0) blocksReadAhead = MaxReadAheadSz; else if (temp > 0) { //make sure we've got an integral factor of extent size for (; (MaxExtentSize%temp)!=0; ++temp); blocksReadAhead=temp; } temp = toInt(cf->getConfig(primitiveServers, "PTTrace")); if (temp > 0) PTTrace = true; temp = toInt(cf->getConfig(primitiveServers, "PrefetchThreshold")); if (temp < 0 || temp > 100) prefetchThreshold = 0; else prefetchThreshold = temp/100.0; int maxPct = 0; //disable by default temp = toInt(cf->getConfig(primitiveServers, "MaxPct")); if (temp >= 0) maxPct = temp; // @bug4507, configurable pm aggregation AggregationMemoryCheck // We could use this same mechanism for other growing buffers. int aggPct = 95; temp = toInt(cf->getConfig("SystemConfig", "MemoryCheckPercent")); if (temp >= 0) aggPct = temp; //...Start the thread to monitor our memory usage new boost::thread(utils::MonitorProcMem(maxPct, aggPct, 28)); // config file priority is 40..1 (highest..lowest) string sPriority = cf->getConfig(primitiveServers, "Priority"); if (sPriority.length() > 0) temp = toInt(sPriority); else temp = 21; // convert config file value to setpriority(2) value (-20..19, -1 is the default) if (temp > 0) priority = 20 - temp; else if (temp < 0) priority = 19; if (priority < -20) priority = -20; #ifdef _MSC_VER //FIXME: #else setpriority(PRIO_PROCESS, 0, priority); #endif //..Instantiate UmSocketSelector singleton. Disable rotating destination //..selection if no UM IP addresses are in the Calpo67108864LLnt.xml file. UmSocketSelector* pUmSocketSelector = UmSocketSelector::instance(); if (rotatingDestination) { if (pUmSocketSelector->ipAddressCount() < 1) rotatingDestination = false; } //See if we want to override the calculated #cores temp = toInt(cf->getConfig(primitiveServers, "NumCores")); if (temp > 0) configNumCores = temp; if (configNumCores <= 0) { //count the actual #cores numCores = cg.getNumCores(); if (numCores == 0) numCores = 8; } else numCores = configNumCores; //based on the #cores, calculate some thread parms if (numCores > 0) { BRPThreads = 2 * numCores; //there doesn't seem much benefit to having more than this, and sometimes it causes problems. //DBBC.NumThreads can override this cap BRPThreads = std::min(BRPThreads, 32); } // the default is ~10% low, 30% medium, 60% high, (where 2*cores = 100%) if (highPriorityPercentage == 0 && medPriorityPercentage == 0 && lowPriorityPercentage == 0) { lowPriorityThreads = max(1, (2*numCores)/10); medPriorityThreads = max(1, (2*numCores)/3); highPriorityThreads = (2 * numCores) - lowPriorityThreads - medPriorityThreads; } else { uint32_t totalThreads = (uint32_t) ((lowPriorityPercentage + medPriorityPercentage + highPriorityPercentage) / 100.0 * (2*numCores)); if (totalThreads == 0) totalThreads = 1; lowPriorityThreads = (uint32_t) (lowPriorityPercentage/100.0 * (2*numCores)); medPriorityThreads = (uint32_t) (medPriorityPercentage/100.0 * (2*numCores)); highPriorityThreads = totalThreads - lowPriorityThreads - medPriorityThreads; } BPPCount = highPriorityThreads + medPriorityThreads + lowPriorityThreads; // let the user override if they want temp = toInt(cf->getConfig(primitiveServers, "BPPCount")); if (temp > 0 && temp < (int) BPPCount) BPPCount = temp; temp = toInt(cf->getConfig(dbbc, "NumThreads")); if (temp > 0) BRPThreads = temp; #ifndef _MSC_VER // @bug4598, switch for O_DIRECT to support gluster fs. // directIOFlag == O_DIRECT, by default strVal = cf->getConfig(primitiveServers, "DirectIO"); if ((strVal == "n") || (strVal == "N")) directIOFlag = 0; #endif IDBPolicy::configIDBPolicy(); // no versionbuffer if using HDFS for performance reason if (IDBPolicy::useHdfs()) noVB = 1; cout << "Starting PrimitiveServer: st = " << serverThreads << ", sq = " << serverQueueSize << ", pw = " << processorWeight << ", pq = " << processorQueueSize << ", nb = " << BRPBlocks << ", nt = " << BRPThreads << ", nc = " << cacheCount << ", ra = " << blocksReadAhead << ", db = " << deleteBlocks << ", mb = " << maxBlocksPerRead << ", rd = " << rotatingDestination << ", tr = " << PTTrace << ", ss = " << PMSmallSide << ", bp = " << BPPCount << endl; PrimitiveServer server(serverThreads, serverQueueSize, processorWeight, processorQueueSize, rotatingDestination, BRPBlocks, BRPThreads, cacheCount, maxBlocksPerRead, blocksReadAhead, deleteBlocks, PTTrace, prefetchThreshold, PMSmallSide); #ifdef QSIZE_DEBUG thread* qszMonThd; if (gDebugLevel >= STATS) { #ifdef _MSC_VER ofstream* qszLog = new ofstream("C:/Calpont/log/trace/ppqsz.dat"); #else ofstream* qszLog = new ofstream("/var/log/Calpont/trace/ppqsz.dat"); #endif if (!qszLog->good()) { qszLog->close(); delete qszLog; qszLog = 0; } qszMonThd = new thread(QszMonThd(&server, qszLog)); } #endif #ifdef DUMP_CACHE_CONTENTS { //Need to use pthreads API here... pthread_t thd1; pthread_attr_t attr1; pthread_attr_init(&attr1); pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); pthread_create(&thd1, &attr1, waitForSIGUSR1, reinterpret_cast<void*>(cacheCount)); } #endif server.start(); cerr << "server.start() exited!" << endl; return 1; }
int main() { // erydb.xml file should be configured as follows: // um1: 10.100.4.85 and 10.100.5.85 // um2: 10.101.4.85 and 10.101.5.85 sockaddr_in sa = { 1, 0, {0} , {' '} }; char* ips[] = {"10.100.4.85", "10.100.5.85", "10.101.4.85", "10.101.5.85"}; // These are the IP addresses we use to test runtime connections // "not" in the erydb.xml file. sockaddr_in saUnknown = { 1, 0, {0} , {' '} }; char* ipsUnknown[]={"10.102.1.1", "10.102.2.1", "10.102.3.1", "10.102.4.1"}; //-------------------------------------------------------------------------- // Test initialization //-------------------------------------------------------------------------- UmSocketSelector* sockSel = UmSocketSelector::instance(); std::cout << "IPAddressCount: " << sockSel->ipAddressCount() << std::endl; std::cout << std::endl << "----Dump1 after initialization..." << std::endl; std::cout << sockSel->toString(); std::cout << "----Dump1 End...................." << std::endl << std::endl; IOSocket sock[4][4]; for (int i=0; i<4; i++) { inet_aton(ips[i], &sa.sin_addr); for (int j=0; j<4; j++) { sock[i][j].setSocketImpl(new InetStreamSocket()); sa.sin_port = htons((i*4)+j); sock[i][j].sa( sa ); sockSel->addConnection( SP_UM_IOSOCK(new IOSocket(sock[i][j])), SP_UM_MUTEX( new boost::mutex()) ); } } std::cout << std::endl << "----Dump2 after adding 16 connections..." << std::endl; std::cout << sockSel->toString(); std::cout << "----Dump2 End...................." << std::endl << std::endl; //-------------------------------------------------------------------------- // Test socket/port selection //-------------------------------------------------------------------------- std::cout << "Test socket/port selection logic..." << std::endl; for (unsigned k=0; k<17; k++) { SP_UM_IOSOCK outIos; SP_UM_MUTEX writeLock; #if 1 if (sockSel->nextIOSocket( sock[0][0], outIos, writeLock )) std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) << "; port: " << ntohs(outIos->sa().sin_port) << std::endl; else #else if (!sockSel->nextIOSocket( sock[0][0], outIos, writeLock )) #endif std::cout << "no nextIP found for " << sock[0][0] << std::endl; } for (unsigned k=0; k<7; k++) { SP_UM_IOSOCK outIos; SP_UM_MUTEX writeLock; if (sockSel->nextIOSocket( sock[2][0], outIos, writeLock )) std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) << "; port: " << ntohs(outIos->sa().sin_port) << std::endl; else std::cout << "no nextIP found for " << sock[2][0] << std::endl; } std::cout << std::endl; std::cout << "----Dump3 after selecting 17 connections from IP " << ips[0] << "; and 7 connections from IP " << ips[2] << " ..." << std::endl; std::cout << sockSel->toString(); std::cout << "----Dump3 End...................." << std::endl << std::endl; //-------------------------------------------------------------------------- // Test connection deletions //-------------------------------------------------------------------------- for (unsigned k=0; k<4; k++) { sockSel->delConnection( sock[k][0] ); } std::cout << "----Dump4 after deleting first connection for each IP..." << std::endl; std::cout << sockSel->toString(); std::cout << "----Dump4 End...................." << std::endl << std::endl; //-------------------------------------------------------------------------- // Test addition of unknown connections //-------------------------------------------------------------------------- IOSocket sockUnknown[4][4]; for (int i=0; i<4; i++) { inet_aton(ipsUnknown[i], &saUnknown.sin_addr); for (int j=0; j<4; j++) { sockUnknown[i][j].setSocketImpl(new InetStreamSocket()); saUnknown.sin_port = htons((i*4)+j); sockUnknown[i][j].sa( saUnknown ); sockSel->addConnection( SP_UM_IOSOCK(new IOSocket(sockUnknown[i][j])), SP_UM_MUTEX( new boost::mutex()) ); } } std::cout << "----Dump5 after adding connections for unknown IP's..." << std::endl; std::cout << sockSel->toString(); std::cout << "----Dump5 End...................." << std::endl << std::endl; //-------------------------------------------------------------------------- // Test resetting of "next" indexes after deleting all sockets from a // specific IP address for which the "next" index is pointing. //-------------------------------------------------------------------------- sockSel->delConnection( sock[1][1] ); sockSel->delConnection( sock[1][2] ); sockSel->delConnection( sock[1][3] ); std::cout << "----Dump6 after deleting all connections for IP " << ips[1] << " ..." << std::endl; std::cout << sockSel->toString(); std::cout << "----Dump6 End...................." << std::endl << std::endl; //-------------------------------------------------------------------------- // Test socket/port selection for an unknown module //-------------------------------------------------------------------------- std::cout << "Test socket/port selection logic..." << std::endl; for (unsigned k=0; k<11; k++) { SP_UM_IOSOCK outIos; SP_UM_MUTEX writeLock; if (sockSel->nextIOSocket( sockUnknown[2][0], outIos, writeLock )) std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) << "; port: " << ntohs(outIos->sa().sin_port) << std::endl; else std::cout << "no nextIP found for " << sockUnknown[2][0]<<std::endl; } std::cout << std::endl; std::cout << "----Dump7 after selecting 11 connections for IP " << ipsUnknown[2] << " ..." << std::endl; std::cout << sockSel->toString(); std::cout << "----Dump7 End...................." << std::endl << std::endl; //-------------------------------------------------------------------------- // Test deletion of last socket/port connection and resetting if the // "next" index that is pointing to it. //-------------------------------------------------------------------------- sockSel->delConnection( sock[3][3] ); std::cout << "----Dump8 after deleting last connection for IP " << ips[3] << " ..." << std::endl; std::cout << sockSel->toString(); std::cout << "----Dump8 End...................." << std::endl << std::endl; return 0; }