virtual void vRun(void* threadData) override
{
MGPExecutor::ThreadData* data = (MGPExecutor::ThreadData*)threadData;
auto function = data->get();
GPContents* oldContent = mCollector->lock();
GPContents* input = NULL;
GPContents* mergeInput = NULL;
{
MGPAutoMutex __m(mInputMutex);
auto tempMergeInput = new GPContents;
input = _loadContent(mInputNum, mInput, mInputKeys);
for (int i=0; i<mVariableKey.size(); ++i)
{
auto v = mVariableKey[i];
MGPASSERT(v.first<=1);
if (v.first == 1)
{
tempMergeInput->pushContent(oldContent->getContent(v.second));
}
else
{
tempMergeInput->pushContent(input->getContent(v.second));
}
}
mergeInput = tempMergeInput->copyAsNoOwner();
tempMergeInput->decRef();
}
GPContents* tempOutput = function->vRun(mergeInput);
{
MGPAutoMutex __m(mInputMutex);
mergeInput->decRef();
input->decRef();
}
{
oldContent->setContent(0, tempOutput->getContent(0));
tempOutput->decRef();
mCollector->unlock(oldContent);
}
}
virtual void vRun(void* threadData) override
{
//GPCLOCK;
MGPExecutor::ThreadData* data = (MGPExecutor::ThreadData*)threadData;
auto function = data->get();
GPContents* orderedInput = NULL;
GPContents* orderedOriginInput = NULL;
{
//GPCLOCK;
MGPAutoMutex __m(mInputMutex);
GPContents* totalInput = _loadContent(mInputNum, mInput, mInputKeys);
orderedOriginInput = new GPContents;
for (int i=0; i<mVariableKey.size(); ++i)
{
orderedOriginInput->pushContent(totalInput->getContent(mVariableKey[i].second));
}
totalInput->decRef();
orderedInput = orderedOriginInput->copyAsNoOwner();
}
GPContents* output = NULL;
{
//GPCLOCK;
output = function->vRun(orderedInput);
}
{
//GPCLOCK;
MGPAutoMutex __m(mInputMutex);
orderedInput->decRef();
orderedOriginInput->decRef();
}
{
//GPCLOCK;
MGPAutoMutex __m(mOutputMutex);
mOutput->vSave(mOutputKeys, mOutput->nKeyNumber, output);
output->decRef();
}
}
Buffer *
packetRepo_Lookup(PacketRepo *repo, Buffer *name, Buffer *hash)
{
return _loadContent(repo, name, hash);
}
bool MGPExecutor::_reduceRun(GPPieces* output, GPPieces** inputs, int inputNumber) const
{
/*Collect Keys*/
GPPtr<IGPKeyIterator> iterator = mFactory->create(inputs, inputNumber, output);
MGPKeyMatcher matcher(iterator.get());
auto keymaps = matcher.get();
size_t threadNumber = mPool->getThreadNumber();
std::map<MGPKeyMatcher::Key*, std::vector<Collector*>> collectorMaps;
std::vector<GPPtr<MGPSema>> waitSemas;
MGPMutex inputMutex;
MGPMutex outputMutex;
for (auto& k : keymaps)
{
auto& outputKey = k.first;
auto& inputKeys = k.second;
/*Generate Collectors*/
collectorMaps.insert(std::make_pair(outputKey, std::vector<Collector*>()));
std::vector<Collector*>& collectors = collectorMaps.find(outputKey)->second;
auto eachCollectSize = threadNumber;
if (inputKeys.size() < threadNumber)
{
eachCollectSize = 1;
}
for (int i=0; i<eachCollectSize; ++i)
{
GPContents* target = _loadContent(inputNumber, inputs, inputKeys[i]->getKey());
collectors.push_back(new Collector(target));
}
std::vector<std::vector<MGPThreadPool::Runnable*>> runnableList;
for (int i=0; i<collectors.size(); ++i)
{
std::vector<MGPThreadPool::Runnable*> rr;
runnableList.push_back(rr);
}
/*For Remain outputKey, Parralelly to reduce*/
for (size_t i=eachCollectSize; i<inputKeys.size(); ++i)
{
size_t index = i % collectors.size();
runnableList[index].push_back(new ReduceRunnable(collectors[index], inputs, inputNumber, inputKeys[i]->getKey(), mVariableKey, inputMutex));
}
for (int i=0; i<collectors.size(); ++i)
{
MGPThreadPool::Runnable* merge = MGPThreadPool::mergeRunnables(runnableList[i]);
waitSemas.push_back(mPool->pushTask(merge));
}
}
for (auto s : waitSemas)
{
s->wait();
}
waitSemas.clear();
/*Collect All Collector*/
for (auto& kv : collectorMaps)
{
auto& collectors = kv.second;
if (collectors.size()>1)
{
waitSemas.push_back(mPool->pushTask(new CollectRunnable(collectors, mVariableKey)));
}
}
for (auto s : waitSemas)
{
s->wait();
}
waitSemas.clear();
/*Write to output*/
for (auto& kv : collectorMaps)
{
GPContents* result = kv.second[0]->lock();
output->vSave(kv.first->getKey(), kv.first->getKeyNumber(), result);
kv.second[0]->unlock(NULL);
for (auto c : kv.second)
{
delete c;
}
}
return true;
}
