Int32 runESP(Int32 argc, char** argv, GuaReceiveFastStart *guaReceiveFastStart)
{
// initialize ESP global data
StatsGlobals * statsGlobals;
XCONTROLMESSAGESYSTEM(XCTLMSGSYS_SETRECVLIMIT, XMAX_SETTABLE_RECVLIMIT_H);
CliGlobals *cliGlobals = NULL;
cliGlobals = CliGlobals::createCliGlobals(TRUE); // TRUE indicates a non-master process (WAIT on LREC)
if (cliGlobals == NULL) // Sanity check
NAExit(1); // Abend
Int32 shmid;
statsGlobals = shareStatsSegment(shmid);
cliGlobals->setSharedMemId(shmid);
//Lng32 numCliCalls = cliGlobals->incrNumOfCliCalls();
cliGlobals->setIsESPProcess(TRUE);
NAHeap *espExecutorHeap = cliGlobals->getExecutorMemory();
// must create default context after set IpcEnvironment in CliGlobals first
// because context's ExSqlComp object needs IpcEnvironment
cliGlobals->initiateDefaultContext();
NAHeap *espIpcHeap = cliGlobals->getIpcHeap();
IpcEnvironment *ipcEnvPtr = cliGlobals->getEnvironment();
//
// Start the memory monitor for dynamic memory management
Lng32 memMonitorWindowSize = 10;
Lng32 memMonitorSampleInterval = 10;
MemoryMonitor memMonitor(memMonitorWindowSize,
memMonitorSampleInterval,
espExecutorHeap);
cliGlobals->setMemoryMonitor(&memMonitor);
// After CLI globals are initialized but before we begin ESP message
// processing, have the CLI context set its user identity based on
// the OS user identity.
ContextCli *context = cliGlobals->currContext();
ex_assert(context, "Invalid context pointer");
context->initializeUserInfoFromOS();
ExEspFragInstanceDir espFragInstanceDir(cliGlobals,
espExecutorHeap,
espExecutorHeap,
(StatsGlobals *)statsGlobals);
ExEspControlMessage espIpcControlMessage(&espFragInstanceDir,
ipcEnvPtr,
espIpcHeap);
// handle startup (command line args, control connection)
DoEspStartup(argc,argv,*ipcEnvPtr,espFragInstanceDir,guaReceiveFastStart);
// the control message stream talks through the control connection
espIpcControlMessage.addRecipient(
ipcEnvPtr->getControlConnection()->getConnection());
// start the first receive operation
espIpcControlMessage.receive(FALSE);
NABoolean timeout;
Int64 prevWaitTime = 0;
// while there are requesters
while (espFragInstanceDir.getNumMasters() > 0)
{
// -----------------------------------------------------------------
// The ESPs most important line of code: DO THE WORK
// -----------------------------------------------------------------
espFragInstanceDir.work(prevWaitTime);
// -----------------------------------------------------------------
// After we have done work, it's necessary to wait for some I/O
// (the frag instance dir work procedure works until it is blocked).
// -----------------------------------------------------------------
ipcEnvPtr->getAllConnections()->
waitOnAll(IpcInfiniteTimeout, TRUE, &timeout, &prevWaitTime); // TRUE means: Called by ESP main
}
// nobody wants us anymore, right now that means that we stop
return 0;
}
int main()
{
// TestCase Group1:
MemPool objMemPool;
MemPool *pMemPool = &objMemPool;
int32_t nCount = 10;
char *pszArray = NEW_VEC(pMemPool, char, nCount);
int32_t *pnArray = NEW_VEC(pMemPool, int32_t, nCount);
float *pfArray = NEW_VEC(pMemPool, float, nCount);
for (int i = 0; i < nCount; i++) {
pszArray[i] = 'a' + i;
pnArray[i] = i;
pfArray[i] = 1.0 * i;
}
for (int i = 0; i < nCount; i++) {
fprintf(stdout, "pszArray[%d]:\t%c\n", i, pszArray[i]);
fprintf(stdout, "pnArray[%d]:\t%d\n", i, pnArray[i]);
fprintf(stdout, "pfArray[%d]:\t%f\n", i, pfArray[i]);
}
pMemPool->reset();
// TestCase Group2:
MemPool objMemPool2;
MemPool *pMemPool2 = &objMemPool2;
int32_t nCount2 = 10;
MyClass *pMyClass = NEW(pMemPool2, MyClass);
fprintf(stdout, "pMyClass:\t%d\n", pMyClass->getCount());
MyClass *pMyClass2 = NEW(pMemPool2, MyClass)(100);
fprintf(stdout, "pMyClass2:\t%d\n", pMyClass2->getCount());
MyClass *pMyClassArray = NEW_ARRAY(pMemPool2, MyClass, nCount2);
for (int i = 0; i < nCount2; i++) {
fprintf(stdout, "pMyClassArray[%d]:\t%d\n", i, pMyClassArray[i].getCount());
}
pMemPool2->reset();
// TestCase Group3:
MemPool objMemPool3;
MemPool *pMemPool3 = &objMemPool3;
MemMonitor memMonitor(pMemPool3, 1);
pMemPool3->setMonitor(&memMonitor);
memMonitor.enableException();
int32_t nCount3 = 1;
char *pszArray3 = NEW_VEC(pMemPool3, char, nCount3);
if (!pszArray3) {
printf("pszArray3 NEW_VEC Err!\n");
}
nCount3 = 1024;
int32_t *pnArray3 = NEW_VEC(pMemPool3, int32_t, nCount3);
if (!pnArray3) {
printf("pnArray3 NEW_VEC Err!\n");
}
float *pfArray3 = NEW_VEC(pMemPool3, float, nCount3);
if (!pfArray3) {
printf("pfArray3 NEW_VEC Err!\n");
}
for (int i = 0; i < nCount3; i++) {
pszArray3[i] = 'a' + i;
pnArray3[i] = i;
pfArray3[i] = 1.0 * i;
}
for (int i = 0; i < nCount3; i++) {
fprintf(stdout, "pszArray3[%d]:\t%c\n", i, pszArray3[i]);
fprintf(stdout, "pnArray3[%d]:\t%d\n", i, pnArray3[i]);
fprintf(stdout, "pfArray3[%d]:\t%f\n", i, pfArray3[i]);
}
pMemPool3->reset();
// TestCase Group4:
MemPool objMemPool4;
MemPool *pMemPool4 = &objMemPool4;
MemMonitor memMonitor4(pMemPool4, 1);
pMemPool4->setMonitor(&memMonitor4);
memMonitor4.enableException();
int32_t nCountsz4 = 1024;
char *pszArraysz4 = NEW_VEC(pMemPool4, char, nCountsz4);
if (!pszArraysz4) {
printf("pszArraysz4 is NULL\n");
}
MyClass *pMyClass4 = NEW(pMemPool4, MyClass);
if (!pMyClass4) {
printf("pMyClass4 is NULL\n");
}
fprintf(stdout, "pMyClass4:\t%d\n", pMyClass4->getCount());
int32_t nCountsz4_1 = 1024;
char *pszArraysz4_1 = NEW_VEC(pMemPool4, char, nCountsz4_1);
if (!pszArraysz4_1) {
printf("pszArraysz4_1 is NULL\n");
}
MyClass *pMyClass4_1 = NEW(pMemPool4, MyClass)(100);
if (!pMyClass4_1) {
printf("pMyClass4_1 is NULL\n");
}
fprintf(stdout, "pMyClass4_1:\t%d\n", pMyClass4_1->getCount());
int32_t nCountsz4_2 = 1024;
char *pszArraysz4_2 = NEW_VEC(pMemPool4, char, nCountsz4_2);
if (!pszArraysz4_2) {
printf("pszArraysz4_2 is NULL\n");
}
int32_t nCount4_2 = 1;
MyClass *pMyClassArray4_2 = NEW_ARRAY(pMemPool4, MyClass, nCount4_2);
if (!pMyClassArray4_2) {
printf("pMyClassArray4_2 is NULL\n");
}
for (int i = 0; i < nCount4_2; i++) {
fprintf(stdout, "pMyClassArray4_2[%d]:\t%d\n", i, pMyClassArray4_2[i].getCount());
}
pMemPool4->reset();
return 0;
}
int32_t SearcherWorker::run() {
const char *pureString = NULL;
uint32_t pureSize = 0;
char *resData = NULL;
uint32_t resSize = 0;
char *resDetailData = NULL;
int32_t resDetailSize = 0;
int32_t ret = 0;
MemPool *memPool = NULL;
int64_t *pNid = NULL;
commdef::ClusterResult *pClusterResult = NULL;
queryparser::QueryRewriterResult *pQRWResult = NULL;
queryparser::QPResult *pQueryResult = NULL;
SearchResult *pSearchResult = NULL;
statistic::StatisticResult *pStatisticResult = NULL;
sort_framework::SortResult *pSortResult = NULL;
ResultSerializer resultSerial;
FRAMEWORK::Context context;
FILE *pOutput = NULL;
//check session status
FRAMEWORK::session_status_t status = _session.getStatus();
if (status == FRAMEWORK::ss_timeout) {
handleTimeout();
return KS_SUCCESS;
}
//get query infomation
FRAMEWORK::Query &query = _session.getQuery();
pureSize = query.getPureQuerySize();
pureString = query.getPureQueryData();
if (!pureString || pureSize == 0) {
_session.setStatus(FRAMEWORK::ss_error);
_session.response();
return KS_EFAILED;
}
//set LogInfo level
_session._logInfo._eRole = FRAMEWORK::sr_simple;
//get MemPool from factory
memPool = _memFactory.make((uint64_t)(getOwner()));
if (memPool == NULL) {
TWARN("Make mem pool failed!");
return KS_EFAILED;
}
//create memory pool monitor
MemMonitor memMonitor(memPool, _memLimit);
memPool->setMonitor(&memMonitor);
memMonitor.enableException();
//initialize context class
context.setMemPool(memPool);
//initialize format processor
_formatProcessor.init(memPool);
//Deal with search proccess
do{
if(_session.isHttp()){
pQRWResult = NEW(memPool, queryparser::QueryRewriterResult)();
if (pQRWResult == NULL) {
TWARN("SEARCHER: new Result no mem");
_session.setStatus(FRAMEWORK::ss_error);
break;
}
}
pQueryResult = NEW(memPool, queryparser::QPResult)(memPool);
pSearchResult = NEW(memPool, SearchResult);
pStatisticResult = NEW(memPool, statistic::StatisticResult);
pSortResult = NEW(memPool, sort_framework::SortResult)(memPool);
if(unlikely(!pQueryResult || !pSearchResult || !pStatisticResult
|| !pSortResult)) {
TWARN("SEARCHER: new Result no mem");
_session.setStatus(FRAMEWORK::ss_error);
break;
}
//add queryrewrite process
if(_session.isHttp()){
ret = _qrewriter.doRewrite(&context, pureString, pureSize, pQRWResult);
if (timeoutCheck && (_timeout > 0) && (_session.getLatencyTime() > _timeout)) {
_session.setStatus(FRAMEWORK::ss_timeout);
TWARN("SEARCHER: qrewriter.doRewrite function over time. query is %s", pureString);
break;
}
if (unlikely(ret != KS_SUCCESS)) {
_session.setStatus(FRAMEWORK::ss_error);
TWARN("qrewriter.doRewrite function error. query is %s", pureString);
break;
}
pureString = pQRWResult->getRewriteQuery();
}
//end add
ret = _qp.doParse(&context, pQueryResult, pureString);
if (timeoutCheck && (_timeout > 0) &&
(_session.getLatencyTime() > _timeout))
{
_session.setStatus(FRAMEWORK::ss_timeout);
TWARN("SEARCHER: qp.doParse function over time. query is %s", pureString);
break;
}
if (unlikely(ret != KS_SUCCESS)){
TWARN("SEARCHER: queryparser doParse function error. query is %s", pureString);
_session.setStatus(FRAMEWORK::ss_error);
break;
}
// cache命中情况下, mempool reset时调用CacheResult析构函数,释放命中的节点，
// 否则sortResult对节点内存的引用失效，变为野指针(bug#118631)
{
ret = _is.doQuery(&context, pQueryResult, &_sort, pSearchResult);
if (timeoutCheck && (_timeout > 0) &&
(_session.getLatencyTime() > _timeout))
{
_session.setStatus(FRAMEWORK::ss_timeout);
TWARN("SEARCHER: is.doQuery function over time. query is %s", pureString);
break;
}
if (unlikely(ret != KS_SUCCESS)){
TWARN("SEARCHER: search doQuery function error. query is %s", pureString);
_session.setStatus(FRAMEWORK::ss_error);
break;
}
ret = _stat.doStatisticOnSearcher(&context,
*(pQueryResult->getParam()),
pSearchResult, pStatisticResult);
if (timeoutCheck && (_timeout > 0) &&
(_session.getLatencyTime() > _timeout))
{
_session.setStatus(FRAMEWORK::ss_timeout);
TWARN("SEARCHER: doStatisticOnSearcher function over time. query is %s", pureString);
break;
}
if (unlikely(ret != KS_SUCCESS)){
TWARN("SEARCHER: statistic doStatisticOnSearcher function error. query is %s", pureString);
_session.setStatus(FRAMEWORK::ss_error);
break;
}
ret = _sort.doProcess(context, *(pQueryResult->getParam()),
*pSearchResult, *pSortResult);
if (timeoutCheck && (_timeout > 0) &&
(_session.getLatencyTime() > _timeout))
{
_session.setStatus(FRAMEWORK::ss_timeout);
TWARN("SEARCHER: sort.doProcess function over time. query is %s", pureString);
break;
}
if (unlikely(ret)) {
TWARN("SEARCHER: sort doProcess function error. query is %s", pureString);
_session.setStatus(FRAMEWORK::ss_error);
break;
}
}
pNid = NULL;
//get docs return number
int32_t num = pSortResult->getNidList(pNid);
pClusterResult = NEW(memPool, commdef::ClusterResult)();
if ( NULL == pClusterResult ){
TWARN("SEARCHER: malloc from memPool failed ");
_session.setStatus(FRAMEWORK::ss_error);
break;
}
memset(pClusterResult, 0x0, sizeof(commdef::ClusterResult));
pClusterResult->_nDocsFound = pSortResult->getDocsFound();//pSearchResult->nDocFound;
pClusterResult->_nEstimateDocsFound = pSortResult->getEstimateDocsFound();//pSearchResult->nEstimateDocFound;
pClusterResult->_nDocsSearch = pSearchResult->nDocSearch;
pClusterResult->_nDocsReturn = num;
pClusterResult->_nDocsRestrict = pSearchResult->nEstimateDocFound;//pSearchResult->nDocFound;
pClusterResult->_pStatisticResult = pStatisticResult;
pClusterResult->_pSortResult = pSortResult;
pClusterResult->_pQPResult = pQueryResult;
pClusterResult->_ppDocuments = NULL;
if(_session.isHttp() && _detail){
if(!pClusterResult->_nDocsReturn){
break;
}
char* pid = NULL;
uint32_t pid_size = 0;
ret = get_nid_str(pNid, num, memPool, &pid, &pid_size);
if(KS_SUCCESS != ret){
_session.setStatus(FRAMEWORK::ss_error);
break;
}
ret = getDetailInfo(pid, pid_size, pureString, &resDetailData, &resDetailSize);
if(KS_SUCCESS != ret){
_session.setStatus(FRAMEWORK::ss_error);
break;
}
if(timeoutCheck && _timeout>0 && _session.getLatencyTime()>_timeout){
_session.setStatus(FRAMEWORK::ss_timeout);
break;
}
if (!translateV3(resDetailData, pClusterResult->_ppDocuments, (uint32_t &)pClusterResult->_nDocsReturn, memPool)) {
_session.setStatus(FRAMEWORK::ss_error);
break;
}
}
} while (0);
if(resDetailData)std::free(resDetailData);
if(_session.isHttp() && _detail){
//fromat result
get_outfmt_type(pureString, pureSize);
result_container_t container;
container.pClusterResult = pClusterResult;
container.cost = _session.getLatencyTime();
if(_formatProcessor.frmt(_ofmt_type, container, &resData , &resSize) < 0){
TERR("FORMATRESULT: format_processor process error!");
_session.setStatus(FRAMEWORK::ss_error);
}
}
else{
//serialize search result
ret = resultSerial.serialClusterResult(pClusterResult,
resData, resSize, "Z");
if (ret != KS_SUCCESS) {
_session.setStatus(FRAMEWORK::ss_error);
}
}
//recycle mem pool
context.getMemPool()->reset();
_session.setResponseData(resData, resSize);
_session.response();
return KS_SUCCESS;
}
