//---------------------------------------------------------------------------
// @function:
// CICGTest::EresUnittest_NegativeIndexApplyTests
//
// @doc:
// Negative IndexApply tests;
// optimizer should not be able to generate a plan
//
//---------------------------------------------------------------------------
GPOS_RESULT
CICGTest::EresUnittest_NegativeIndexApplyTests()
{
// enable (Redistribute, Broadcast) hash join plans
CAutoTraceFlag atf(EopttraceEnableRedistributeBroadcastHashJoin, true /*fVal*/);
// disable physical scans and NLJ to force using index-apply
CAutoTraceFlag atfDTS(EopttraceDisableXformBase + CXform::ExfDynamicGet2DynamicTableScan, true);
CAutoTraceFlag atfTS(EopttraceDisableXformBase + CXform::ExfGet2TableScan, true);
CAutoTraceFlag atfNLJ(EopttraceDisableXformBase + CXform::ExfInnerJoin2NLJoin, true);
CAutoMemoryPool amp(CAutoMemoryPool::ElcNone);
IMemoryPool *pmp = amp.Pmp();
GPOS_RESULT eres = GPOS_OK;
const ULONG ulTests = GPOS_ARRAY_SIZE(rgszNegativeIndexApplyFileNames);
for (ULONG ul = m_ulNegativeIndexApplyTestCounter; ul < ulTests; ul++)
{
GPOS_TRY
{
ICostModel *pcm = CTestUtils::Pcm(pmp);
COptimizerConfig *poconf = GPOS_NEW(pmp) COptimizerConfig
(
CEnumeratorConfig::Pec(pmp, 0 /*ullPlanId*/),
CStatisticsConfig::PstatsconfDefault(pmp),
CCTEConfig::PcteconfDefault(pmp),
pcm,
CHint::PhintDefault(pmp)
);
CDXLNode *pdxlnPlan = CMinidumperUtils::PdxlnExecuteMinidump
(
pmp,
rgszNegativeIndexApplyFileNames[ul],
GPOPT_TEST_SEGMENTS /*ulSegments*/,
1 /*ulSessionId*/,
1, /*ulCmdId*/
poconf,
NULL /*pceeval*/
);
GPOS_CHECK_ABORT;
poconf->Release();
pdxlnPlan->Release();
pcm->Release();
// test should have thrown
eres = GPOS_FAILED;
break;
}
GPOS_CATCH_EX(ex)
{
if (GPOS_MATCH_EX(ex, gpopt::ExmaGPOPT, gpopt::ExmiNoPlanFound))
{
GPOS_RESET_EX;
}
else
{
GPOS_RETHROW(ex);
}
}
GPOS_CATCH_END;
m_ulNegativeIndexApplyTestCounter++;
}
if (GPOS_OK == eres)
{
m_ulNegativeIndexApplyTestCounter = 0;
}
return eres;
}
//---------------------------------------------------------------------------
// @function:
// CICGTest::EresUnittest_RunUnsupportedMinidumpTests
//
// @doc:
// Run all unsupported Minidump-based tests
//
//---------------------------------------------------------------------------
GPOS_RESULT
CICGTest::EresUnittest_RunUnsupportedMinidumpTests()
{
// enable (Redistribute, Broadcast) hash join plans
CAutoTraceFlag atf1(EopttraceEnableRedistributeBroadcastHashJoin, true /*fVal*/);
CAutoTraceFlag atf2(EopttraceDisableXformBase + CXform::ExfDynamicGet2DynamicTableScan, true);
CAutoMemoryPool amp(CAutoMemoryPool::ElcNone);
IMemoryPool *pmp = amp.Pmp();
GPOS_RESULT eres = GPOS_OK;
const ULONG ulTests = GPOS_ARRAY_SIZE(unSupportedTestCases);
for (ULONG ul = m_ulUnsupportedTestCounter; ul < ulTests; ul++)
{
const CHAR *szFilename = unSupportedTestCases[ul].szFilename;
CDXLMinidump *pdxlmd = CMinidumperUtils::PdxlmdLoad(pmp, szFilename);
bool unmatchedException = false;
ULONG unmatchedExceptionMajor = 0;
ULONG unmatchedExceptionMinor = 0;
GPOS_TRY
{
ICostModel *pcm = CTestUtils::Pcm(pmp);
COptimizerConfig *poconf = pdxlmd->Poconf();
CDXLNode *pdxlnPlan = CMinidumperUtils::PdxlnExecuteMinidump
(
pmp,
szFilename,
poconf->Pcm()->UlHosts() /*ulSegments*/,
1 /*ulSessionId*/,
1, /*ulCmdId*/
poconf,
NULL /*pceeval*/
);
GPOS_CHECK_ABORT;
pdxlnPlan->Release();
pcm->Release();
// test should have thrown
eres = GPOS_FAILED;
break;
}
GPOS_CATCH_EX(ex)
{
unmatchedExceptionMajor = ex.UlMajor();
unmatchedExceptionMinor = ex.UlMinor();
// verify expected exception
if (unSupportedTestCases[ul].ulMajor == unmatchedExceptionMajor
&& unSupportedTestCases[ul].ulMinor == unmatchedExceptionMinor)
{
eres = GPOS_OK;
}
else
{
unmatchedException = true;
eres = GPOS_FAILED;
}
GPOS_RESET_EX;
}
GPOS_CATCH_END;
GPOS_DELETE(pdxlmd);
m_ulUnsupportedTestCounter++;
if (GPOS_FAILED == eres && unmatchedException)
{
CAutoTrace at(pmp);
at.Os() << "Test failed due to unmatched exceptions." << std::endl;
at.Os() << " Expected result: " << unSupportedTestCases[ul].ulMajor << "." << unSupportedTestCases[ul].ulMinor << std::endl;
at.Os() << " Actual result: " << unmatchedExceptionMajor << "." << unmatchedExceptionMinor << std::endl;
}
}
if (GPOS_OK == eres)
{
m_ulUnsupportedTestCounter = 0;
}
return eres;
}
//---------------------------------------------------------------------------
// @function:
// CCostContext::ComputeCost
//
// @doc:
// Compute cost of current context,
//
// the function extracts cardinality and row width of owner operator
// and child operators, and then adjusts row estimate obtained from
// statistics based on data distribution obtained from plan properties,
//
// statistics row estimate is computed on logical expressions by
// estimating the size of the whole relation regardless data
// distribution, on the other hand, optimizer's cost model computes
// the cost of a plan instance on some segment,
//
// when a plan produces tuples distributed to multiple segments, we
// need to divide statistics row estimate by the number segments to
// provide a per-segment row estimate for cost computation,
//
// Note that this scaling of row estimate cannot happen during
// statistics derivation since plans are not created yet at this point
//
// this function also extracts number of rebinds of owner operator child
// operators, if statistics are computed using predicates with external
// parameters (outer references), number of rebinds is the total number
// of external parameters' values
//
//---------------------------------------------------------------------------
CCost
CCostContext::CostCompute
(
IMemoryPool *pmp,
DrgPcost *pdrgpcostChildren
)
{
// derive context stats
DeriveStats();
ULONG ulArity = 0;
if (NULL != m_pdrgpoc)
{
ulArity = Pdrgpoc()->UlLength();
}
m_pstats->AddRef();
ICostModel::SCostingInfo ci(pmp, ulArity, GPOS_NEW(pmp) ICostModel::CCostingStats(m_pstats));
ICostModel *pcm = COptCtxt::PoctxtFromTLS()->Pcm();
CExpressionHandle exprhdl(pmp);
exprhdl.Attach(this);
// extract local costing info
DOUBLE dRows = m_pstats->DRows().DVal();
if (CDistributionSpec::EdptPartitioned == Pdpplan()->Pds()->Edpt())
{
// scale statistics row estimate by number of segments
dRows = DRowsPerHost().DVal();
}
ci.SetRows(dRows);
DOUBLE dWidth = m_pstats->DWidth(pmp, m_poc->Prpp()->PcrsRequired()).DVal();
ci.SetWidth(dWidth);
DOUBLE dRebinds = m_pstats->DRebinds().DVal();
ci.SetRebinds(dRebinds);
GPOS_ASSERT_IMP(!exprhdl.FHasOuterRefs(), GPOPT_DEFAULT_REBINDS == (ULONG) (dRebinds) && "invalid number of rebinds when there are no outer references");
// extract children costing info
for (ULONG ul = 0; ul < ulArity; ul++)
{
COptimizationContext *pocChild = (*m_pdrgpoc)[ul];
CCostContext *pccChild = pocChild->PccBest();
GPOS_ASSERT(NULL != pccChild);
IStatistics *pstatsChild = pccChild->Pstats();
DOUBLE dRowsChild = pstatsChild->DRows().DVal();
if (CDistributionSpec::EdptPartitioned == pccChild->Pdpplan()->Pds()->Edpt())
{
// scale statistics row estimate by number of segments
dRowsChild = pccChild->DRowsPerHost().DVal();
}
ci.SetChildRows(ul, dRowsChild);
DOUBLE dWidthChild = pstatsChild->DWidth(pmp, pocChild->Prpp()->PcrsRequired()).DVal();
ci.SetChildWidth(ul, dWidthChild);
DOUBLE dRebindsChild = pstatsChild->DRebinds().DVal();
ci.SetChildRebinds(ul, dRebindsChild);
GPOS_ASSERT_IMP(!exprhdl.FHasOuterRefs(ul), GPOPT_DEFAULT_REBINDS == (ULONG) (dRebindsChild) && "invalid number of rebinds when there are no outer references");
DOUBLE dCostChild = (*pdrgpcostChildren)[ul]->DVal();
ci.SetChildCost(ul, dCostChild);
}
// compute cost using the underlying cost model
return pcm->Cost(exprhdl, &ci);
}
//---------------------------------------------------------------------------
// @function:
// CCostTest::EresUnittest_SetParams
//
// @doc:
// Test of setting cost model params
//
//---------------------------------------------------------------------------
GPOS_RESULT
CCostTest::EresUnittest_SetParams()
{
CAutoMemoryPool amp;
IMemoryPool *mp = amp.Pmp();
// setup a file-based provider
CMDProviderMemory *pmdp = CTestUtils::m_pmdpf;
pmdp->AddRef();
CMDAccessor mda(mp, CMDCache::Pcache(), CTestUtils::m_sysidDefault, pmdp);
ICostModel *pcm = GPOS_NEW(mp) CCostModelGPDB(mp, GPOPT_TEST_SEGMENTS);
// install opt context in TLS
CAutoOptCtxt aoc(mp, &mda, NULL, /* pceeval */ pcm);
// generate in-equality join expression
CExpression *pexprOuter = CTestUtils::PexprLogicalGet(mp);
const CColRef *pcrOuter = CDrvdPropRelational::GetRelationalProperties(pexprOuter->PdpDerive())->PcrsOutput()->PcrAny();
CExpression *pexprInner = CTestUtils::PexprLogicalGet(mp);
const CColRef *pcrInner = CDrvdPropRelational::GetRelationalProperties(pexprInner->PdpDerive())->PcrsOutput()->PcrAny();
CExpression *pexprPred = CUtils::PexprScalarCmp(mp, pcrOuter, pcrInner, IMDType::EcmptNEq);
CExpression *pexpr = CUtils::PexprLogicalJoin<CLogicalInnerJoin>(mp, pexprOuter, pexprInner, pexprPred);
// optimize in-equality join based on default cost model params
CExpression *pexprPlan1 = NULL;
{
CEngine eng(mp);
// generate query context
CQueryContext *pqc = CTestUtils::PqcGenerate(mp, pexpr);
// Initialize engine
eng.Init(pqc, NULL /*search_stage_array*/);
// optimize query
eng.Optimize();
// extract plan
pexprPlan1 = eng.PexprExtractPlan();
GPOS_ASSERT(NULL != pexprPlan1);
GPOS_DELETE(pqc);
}
// change NLJ cost factor
ICostModelParams::SCostParam *pcp = pcm->GetCostModelParams()->PcpLookup(CCostModelParamsGPDB::EcpNLJFactor);
CDouble dNLJFactor = CDouble(2.0);
CDouble dVal = pcp->Get() * dNLJFactor;
pcm->GetCostModelParams()->SetParam(pcp->Id(), dVal, dVal - 0.5, dVal + 0.5);
// optimize again after updating NLJ cost factor
CExpression *pexprPlan2 = NULL;
{
CEngine eng(mp);
// generate query context
CQueryContext *pqc = CTestUtils::PqcGenerate(mp, pexpr);
// Initialize engine
eng.Init(pqc, NULL /*search_stage_array*/);
// optimize query
eng.Optimize();
// extract plan
pexprPlan2 = eng.PexprExtractPlan();
GPOS_ASSERT(NULL != pexprPlan2);
GPOS_DELETE(pqc);
}
{
CAutoTrace at(mp);
at.Os() << "\nPLAN1: \n" << *pexprPlan1;
at.Os() << "\nNLJ Cost1: " << (*pexprPlan1)[0]->Cost();
at.Os() << "\n\nPLAN2: \n" << *pexprPlan2;
at.Os() << "\nNLJ Cost2: " << (*pexprPlan2)[0]->Cost();
}
GPOS_ASSERT((*pexprPlan2)[0]->Cost() >= (*pexprPlan1)[0]->Cost() * dNLJFactor &&
"expected NLJ cost in PLAN2 to be larger than NLJ cost in PLAN1");
// clean up
pexpr->Release();
pexprPlan1->Release();
pexprPlan2->Release();
return GPOS_OK;
}
//---------------------------------------------------------------------------
// @function:
// CCostTest::EresUnittest_CalibratedCostModel
//
// @doc:
// GPDB's calibrated cost model test
//
//---------------------------------------------------------------------------
GPOS_RESULT
CCostTest::EresUnittest_CalibratedCostModel()
{
CAutoTraceFlag atf1(EtraceSimulateOOM, false);
CAutoTraceFlag atf2(EtraceSimulateAbort, false);
CAutoTraceFlag atf3(EtraceSimulateIOError, false);
CAutoTraceFlag atf4(EtraceSimulateNetError, false);
CAutoMemoryPool amp;
IMemoryPool *pmp = amp.Pmp();
// setup a file-based provider
CMDProviderMemory *pmdp = CTestUtils::m_pmdpf;
pmdp->AddRef();
CMDAccessor mda(pmp, CMDCache::Pcache(), CTestUtils::m_sysidDefault, pmdp);
ICostModel *pcm = GPOS_NEW(pmp) CCostModelGPDB(pmp, GPOPT_TEST_SEGMENTS);
pcm->AddRef();
{
// install opt context in TLS
CAutoOptCtxt aoc
(
pmp,
&mda,
NULL, /* pceeval */
pcm
);
TestParams(pmp, true /*fCalibrated*/);
}
// minidump files
const CHAR *rgszFileNamesCalibratedCostModel[] =
{
"../data/dxl/minidump/PartTbl-MultiWayJoinWithDPE.mdp",
"../data/dxl/tpch/q2.mdp",
"../data/dxl/minidump/CTE-4.mdp",
"../data/dxl/minidump/Lead-Lag-WinFuncs.mdp",
};
COptimizerConfig* poconf = COptimizerConfig::PoconfDefault(pmp, pcm);
for (ULONG ul = 0; ul < GPOS_ARRAY_SIZE(rgszFileNamesCalibratedCostModel); ul++)
{
CDXLNode *pdxlnPlan = CMinidumperUtils::PdxlnExecuteMinidump
(
pmp,
rgszFileNamesCalibratedCostModel[ul],
GPOPT_TEST_SEGMENTS,
1 /*ulSessionId*/,
1 /*ulCmdId*/,
poconf,
NULL /*pceeval*/
);
pdxlnPlan->Release();
}
poconf->Release();
return GPOS_OK;
}
//---------------------------------------------------------------------------
// @function:
// CParseHandlerOptimizerConfig::EndElement
//
// @doc:
// Invoked by Xerces to process a closing tag
//
//---------------------------------------------------------------------------
void
CParseHandlerOptimizerConfig::EndElement
(
const XMLCh* const, // element_uri,
const XMLCh* const element_local_name,
const XMLCh* const // element_qname
)
{
if (0 != XMLString::compareString(CDXLTokens::XmlstrToken(EdxltokenOptimizerConfig), element_local_name))
{
CWStringDynamic *str = CDXLUtils::CreateDynamicStringFromXMLChArray(m_parse_handler_mgr->GetDXLMemoryManager(), element_local_name);
GPOS_RAISE( gpdxl::ExmaDXL, gpdxl::ExmiDXLUnexpectedTag, str->GetBuffer());
}
GPOS_ASSERT(NULL == m_optimizer_config);
GPOS_ASSERT(7 >= this->Length());
CParseHandlerEnumeratorConfig *pphEnumeratorConfig = dynamic_cast<CParseHandlerEnumeratorConfig *>((*this)[0]);
CEnumeratorConfig *pec = pphEnumeratorConfig->GetEnumeratorCfg();
pec->AddRef();
CParseHandlerStatisticsConfig *pphStatisticsConfig = dynamic_cast<CParseHandlerStatisticsConfig *>((*this)[1]);
CStatisticsConfig *stats_config = pphStatisticsConfig->GetStatsConf();
stats_config->AddRef();
CParseHandlerCTEConfig *pphCTEConfig = dynamic_cast<CParseHandlerCTEConfig *>((*this)[2]);
CCTEConfig *pcteconfig = pphCTEConfig->GetCteConf();
pcteconfig->AddRef();
CParseHandlerWindowOids *pphDefoidsGPDB = dynamic_cast<CParseHandlerWindowOids *>((*this)[3]);
CWindowOids *pwindowoidsGPDB = pphDefoidsGPDB->GetWindowOids();
GPOS_ASSERT(NULL != pwindowoidsGPDB);
pwindowoidsGPDB->AddRef();
ICostModel *pcm = NULL;
CHint *phint = NULL;
if (5 == this->Length())
{
// no cost model: use default one
pcm = ICostModel::PcmDefault(m_mp);
phint = CHint::PhintDefault(m_mp);
}
else
{
CParseHandlerCostModel *pphCostModelConfig = dynamic_cast<CParseHandlerCostModel *>((*this)[4]);
pcm = pphCostModelConfig->GetCostModel();
GPOS_ASSERT(NULL != pcm);
pcm->AddRef();
if (6 == this->Length())
{
phint = CHint::PhintDefault(m_mp);
}
else
{
CParseHandlerHint *pphHint = dynamic_cast<CParseHandlerHint *>((*this)[5]);
phint = pphHint->GetHint();
GPOS_ASSERT(NULL != phint);
phint->AddRef();
}
}
m_optimizer_config = GPOS_NEW(m_mp) COptimizerConfig(pec, stats_config, pcteconfig, pcm, phint, pwindowoidsGPDB);
CParseHandlerTraceFlags *pphTraceFlags = dynamic_cast<CParseHandlerTraceFlags *>((*this)[this->Length() - 1]);
pphTraceFlags->GetTraceFlagBitSet()->AddRef();
m_pbs = pphTraceFlags->GetTraceFlagBitSet();
// deactivate handler
m_parse_handler_mgr->DeactivateHandler();
}