virtual void vRun(void* threadData) override { MGPExecutor::ThreadData* data = (MGPExecutor::ThreadData*)threadData; auto function = data->get(); GPContents* oldContent = mCollector->lock(); GPContents* input = NULL; GPContents* mergeInput = NULL; { MGPAutoMutex __m(mInputMutex); auto tempMergeInput = new GPContents; input = _loadContent(mInputNum, mInput, mInputKeys); for (int i=0; i<mVariableKey.size(); ++i) { auto v = mVariableKey[i]; MGPASSERT(v.first<=1); if (v.first == 1) { tempMergeInput->pushContent(oldContent->getContent(v.second)); } else { tempMergeInput->pushContent(input->getContent(v.second)); } } mergeInput = tempMergeInput->copyAsNoOwner(); tempMergeInput->decRef(); } GPContents* tempOutput = function->vRun(mergeInput); { MGPAutoMutex __m(mInputMutex); mergeInput->decRef(); input->decRef(); } { oldContent->setContent(0, tempOutput->getContent(0)); tempOutput->decRef(); mCollector->unlock(oldContent); } }
virtual void vRun(void* threadData) override { //GPCLOCK; MGPExecutor::ThreadData* data = (MGPExecutor::ThreadData*)threadData; auto function = data->get(); GPContents* orderedInput = NULL; GPContents* orderedOriginInput = NULL; { //GPCLOCK; MGPAutoMutex __m(mInputMutex); GPContents* totalInput = _loadContent(mInputNum, mInput, mInputKeys); orderedOriginInput = new GPContents; for (int i=0; i<mVariableKey.size(); ++i) { orderedOriginInput->pushContent(totalInput->getContent(mVariableKey[i].second)); } totalInput->decRef(); orderedInput = orderedOriginInput->copyAsNoOwner(); } GPContents* output = NULL; { //GPCLOCK; output = function->vRun(orderedInput); } { //GPCLOCK; MGPAutoMutex __m(mInputMutex); orderedInput->decRef(); orderedOriginInput->decRef(); } { //GPCLOCK; MGPAutoMutex __m(mOutputMutex); mOutput->vSave(mOutputKeys, mOutput->nKeyNumber, output); output->decRef(); } }
Buffer * packetRepo_Lookup(PacketRepo *repo, Buffer *name, Buffer *hash) { return _loadContent(repo, name, hash); }
bool MGPExecutor::_reduceRun(GPPieces* output, GPPieces** inputs, int inputNumber) const { /*Collect Keys*/ GPPtr<IGPKeyIterator> iterator = mFactory->create(inputs, inputNumber, output); MGPKeyMatcher matcher(iterator.get()); auto keymaps = matcher.get(); size_t threadNumber = mPool->getThreadNumber(); std::map<MGPKeyMatcher::Key*, std::vector<Collector*>> collectorMaps; std::vector<GPPtr<MGPSema>> waitSemas; MGPMutex inputMutex; MGPMutex outputMutex; for (auto& k : keymaps) { auto& outputKey = k.first; auto& inputKeys = k.second; /*Generate Collectors*/ collectorMaps.insert(std::make_pair(outputKey, std::vector<Collector*>())); std::vector<Collector*>& collectors = collectorMaps.find(outputKey)->second; auto eachCollectSize = threadNumber; if (inputKeys.size() < threadNumber) { eachCollectSize = 1; } for (int i=0; i<eachCollectSize; ++i) { GPContents* target = _loadContent(inputNumber, inputs, inputKeys[i]->getKey()); collectors.push_back(new Collector(target)); } std::vector<std::vector<MGPThreadPool::Runnable*>> runnableList; for (int i=0; i<collectors.size(); ++i) { std::vector<MGPThreadPool::Runnable*> rr; runnableList.push_back(rr); } /*For Remain outputKey, Parralelly to reduce*/ for (size_t i=eachCollectSize; i<inputKeys.size(); ++i) { size_t index = i % collectors.size(); runnableList[index].push_back(new ReduceRunnable(collectors[index], inputs, inputNumber, inputKeys[i]->getKey(), mVariableKey, inputMutex)); } for (int i=0; i<collectors.size(); ++i) { MGPThreadPool::Runnable* merge = MGPThreadPool::mergeRunnables(runnableList[i]); waitSemas.push_back(mPool->pushTask(merge)); } } for (auto s : waitSemas) { s->wait(); } waitSemas.clear(); /*Collect All Collector*/ for (auto& kv : collectorMaps) { auto& collectors = kv.second; if (collectors.size()>1) { waitSemas.push_back(mPool->pushTask(new CollectRunnable(collectors, mVariableKey))); } } for (auto s : waitSemas) { s->wait(); } waitSemas.clear(); /*Write to output*/ for (auto& kv : collectorMaps) { GPContents* result = kv.second[0]->lock(); output->vSave(kv.first->getKey(), kv.first->getKeyNumber(), result); kv.second[0]->unlock(NULL); for (auto c : kv.second) { delete c; } } return true; }