void CSSHScheduler::makeScheduleImpl(const CTopology& _topology,
const CAgentChannel::weakConnectionPtrVector_t& _channels)
{
m_schedule.clear();
size_t nofChannels = _channels.size();
// Fill host name to channel index map in order to reduce number of regex matches and speed up scheduling.
map<string, vector<size_t>> hostToChannelMap;
for (size_t iChannel = 0; iChannel < nofChannels; ++iChannel)
{
const auto& v = _channels[iChannel];
if (v.expired())
continue;
auto ptr = v.lock();
const SHostInfoCmd& hostInfo = ptr->getRemoteHostInfo();
hostToChannelMap[hostInfo.m_host].push_back(iChannel);
}
// TODO: before scheduling the collections we have to sort them by a number of tasks in the collection.
// TODO: for the moment we are not able to schedule collection without requirements.
// Collect all tasks that belong to collections
set<uint64_t> tasksInCollections;
CollectionMap_t collectionMap;
CTopology::TaskCollectionIteratorPair_t collections = _topology.getTaskCollectionIterator();
for (auto it = collections.first; it != collections.second; it++)
{
const vector<uint64_t>& taskHashes = _topology.getTaskHashesByTaskCollectionHash(it->first);
tasksInCollections.insert(taskHashes.begin(), taskHashes.end());
collectionMap[it->second->getNofTasks()].push_back(it->first);
}
set<uint64_t> scheduledTasks;
scheduleCollections(_topology, _channels, hostToChannelMap, scheduledTasks, collectionMap, true);
scheduleTasks(_topology, _channels, hostToChannelMap, scheduledTasks, tasksInCollections, true);
scheduleCollections(_topology, _channels, hostToChannelMap, scheduledTasks, collectionMap, false);
scheduleTasks(_topology, _channels, hostToChannelMap, scheduledTasks, tasksInCollections, false);
size_t totalNofTasks = _topology.getMainGroup()->getTotalNofTasks();
if (totalNofTasks != m_schedule.size())
{
printSchedule();
stringstream ss;
ss << "Unable to make a schedule for tasks. Number of requested tasks: " << totalNofTasks
<< ". Number of scheduled tasks: " << m_schedule.size();
throw runtime_error(ss.str());
}
printSchedule();
}
ALERROR CSystemMap::ExecuteCreator (STopologyCreateCtx &Ctx, CTopology &Topology, CXMLElement *pCreator)
// ExecuteCreator
//
// Runs a specific creator
{
ALERROR error;
int i;
// If this is a root node tag then we add it and all its connections.
if (strEquals(pCreator->GetTag(), ROOT_NODE_TAG))
{
if (error = Topology.AddTopologyNode(Ctx, pCreator->GetAttribute(ID_ATTRIB)))
return error;
}
// Otherwise we process the creator element
else
{
for (i = 0; i < pCreator->GetContentElementCount(); i++)
{
CXMLElement *pDirective = pCreator->GetContentElement(i);
if (strEquals(pDirective->GetTag(), NODE_TAG))
{
if (error = Topology.AddTopologyNode(Ctx, pDirective->GetAttribute(ID_ATTRIB)))
return error;
}
else if (strEquals(pDirective->GetTag(), STARGATE_TAG) || strEquals(pDirective->GetTag(), STARGATES_TAG))
{
if (error = Topology.AddStargateFromXML(Ctx, pDirective))
return error;
}
else
{
Ctx.sError = strPatternSubst(CONSTLIT("Unknown TopologyCreator directive: %s."), pDirective->GetTag());
return ERR_FAIL;
}
}
}
return NOERROR;
}
void check_topology_maps(const string& _topoName)
{
CTopology topology;
string topoFile(_topoName + ".xml");
topology.init(topoFile, true);
output_test_stream output1(_topoName + "_maps_1.txt", true);
for (const auto& v : topology.getTopoIndexToTopoElementMap())
{
output1 << v.first.getPath() << " " << v.second->getPath() << "\n";
// std::cout << v.first.getPath() << " " << v.second->getPath() << "\n";
}
BOOST_CHECK(output1.match_pattern());
output_test_stream output2(_topoName + "_maps_2.txt", true);
check_topology_map_task(topology.getHashToTaskInfoMap(), output2);
output_test_stream output3(_topoName + "_maps_3.txt", true);
check_topology_map(topology.getHashToTaskCollectionMap(), output3);
output_test_stream output4(_topoName + "_maps_4.txt", true);
check_topology_map(topology.getHashPathToTaskMap(), output4);
output_test_stream output5(_topoName + "_maps_5.txt", true);
check_topology_map(topology.getHashPathToTaskCollectionMap(), output5);
}
void ITopologyProcessor::RestoreMarks (CTopology &Topology, TArray<bool> &Saved)
// RestoreMarks
//
// Restore the mark for nodes
{
Topology.GetTopologyNodeList().RestoreMarks(Saved);
}
void ITopologyProcessor::SaveAndMarkNodes (CTopology &Topology, CTopologyNodeList &NodeList, TArray<bool> *retSaved)
// SaveAndMarkNodes
//
// Saves the mark for all topology nodes. Then it marks all nodes in the NodeList
// and clears the mark on all others.
{
int i;
Topology.GetTopologyNodeList().SaveAndSetMarks(false, retSaved);
for (i = 0; i < NodeList.GetCount(); i++)
NodeList[i]->SetMarked(true);
}
void CSSHScheduler::scheduleCollections(const CTopology& _topology,
const CAgentChannel::weakConnectionPtrVector_t& _channels,
map<string, vector<size_t>>& _hostToChannelMap,
set<uint64_t>& _scheduledTasks,
const CollectionMap_t& _collectionMap,
bool useRequirement)
{
for (const auto& it_col : _collectionMap)
{
for (auto id : it_col.second)
{
TaskCollectionPtr_t collection = _topology.getTaskCollectionByHash(id);
// First path only for collections with requirements;
// Second path for collections without requirements.
if ((useRequirement && collection->getRequirement() == nullptr) ||
(!useRequirement && collection->getRequirement() != nullptr))
continue;
bool collectionAssigned = false;
for (auto& v : _hostToChannelMap)
{
if (v.second.size() >= collection->getNofTasks() &&
(!useRequirement || (useRequirement && collection->getRequirement()->hostPatterMatches(v.first))))
{
const vector<uint64_t>& taskHashes = _topology.getTaskHashesByTaskCollectionHash(id);
for (auto hash : taskHashes)
{
const STaskInfo& info = _topology.getTaskInfoByHash(hash);
size_t channelIndex = v.second.back();
const auto& channel = _channels[channelIndex];
SSchedule schedule;
schedule.m_channel = channel;
schedule.m_task = info.m_task;
schedule.m_taskID = hash;
schedule.m_taskIndex = info.m_taskIndex;
schedule.m_collectionIndex = info.m_collectionIndex;
m_schedule.push_back(schedule);
v.second.pop_back();
_scheduledTasks.insert(hash);
}
collectionAssigned = true;
break;
}
}
if (!collectionAssigned)
{
printSchedule();
stringstream ss;
ss << "Unable to schedule collection <" << id << "> with path " << collection->getPath();
throw runtime_error(ss.str());
}
}
}
}
int main(int argc, char* argv[])
{
try
{
chrono::high_resolution_clock::time_point t1 = chrono::high_resolution_clock::now();
CUserDefaults::instance(); // Initialize user defaults
Logger::instance().init(); // Initialize log
size_t nInstances(0);
size_t nMaxValue(g_maxValue);
size_t type(0);
size_t timeout(g_timeout);
bool testErrors(true);
// Generic options
bpo::options_description options("task-test_key_value options");
options.add_options()("help,h", "Produce help message");
options.add_options()(
"instances,i", bpo::value<size_t>(&nInstances)->default_value(0), "A number of instances");
options.add_options()(
"max-value", bpo::value<size_t>(&nMaxValue)->default_value(g_maxValue), "A max value of the property");
options.add_options()("type,t", bpo::value<size_t>(&type)->default_value(0), "Type of task. Must be 0 or 1.");
options.add_options()("test-errors",
"Indicates that taks will also put incorrect data and test the error messages.");
options.add_options()("timeout",
bpo::value<size_t>(&timeout)->default_value(g_timeout),
"A max timeout for a task to get all properties.");
// Parsing command-line
bpo::variables_map vm;
bpo::store(bpo::command_line_parser(argc, argv).options(options).run(), vm);
bpo::notify(vm);
if (vm.count("help") || vm.empty())
{
cout << options;
return 0;
}
testErrors = vm.count("test-errors");
LOG(info) << "Start task with type " << type;
// Get list of READ and WRITE properties from topology
CTopology topo;
topo.init();
uint64_t taskId = env_prop<EEnvProp::task_id>();
const STopoRuntimeTask& runtimeTask = topo.getRuntimeTaskById(taskId);
const CTopoProperty::PtrMap_t& properties = runtimeTask.m_task->getProperties();
vector<string> readPropertyNames;
vector<string> writePropertyNames;
for (auto property : properties)
{
CTopoProperty::EAccessType accessType = property.second->getAccessType();
string propertyName(property.first);
if (accessType == CTopoProperty::EAccessType::READ)
{
readPropertyNames.push_back(propertyName);
}
else
{
writePropertyNames.push_back(propertyName);
}
}
// DDS Intercom API
CIntercomService service;
CKeyValue keyValue(service);
mutex keyMutex;
condition_variable keyCondition;
// on task done condition
condition_variable onTaskDoneCondition;
map<string, string> keyValueCache;
size_t numWaits = 0;
size_t numUpdateKeyValueCalls = 0;
size_t currentIteration = (type == 0) ? 1 : 0;
size_t numTaskDone = 0;
// Subscribe on error events
service.subscribeOnError([/*&keyCondition*/](EErrorCode _errorCode, const string& _msg) {
LOG(error) << "Key-value error code: " << _errorCode << ", message: " << _msg;
});
// Subscribe on task done notifications
service.subscribeOnTaskDone(
[&numTaskDone, nInstances, &onTaskDoneCondition](uint64_t _taskID, uint32_t _exitCode) {
++numTaskDone;
LOG(info) << "Task Done notification received for task " << _taskID << " with exit code " << _exitCode;
// TODO: In order to properly account finished tasks, use taskID to get task's name
if (numTaskDone >= nInstances)
onTaskDoneCondition.notify_all();
});
// Subscribe on key update events
// DDS garantees that this callback function will not be called in parallel from multiple threads.
// It is safe to update global data without locks inside the callback.
keyValue.subscribe([&keyCondition, ¤tIteration, &keyValueCache, &nInstances, &numUpdateKeyValueCalls](
const string& _propertyName, const string& _value, uint64_t _senderTaskID) {
numUpdateKeyValueCalls++;
string key = _propertyName + "." + to_string(_senderTaskID);
keyValueCache[key] = _value;
// Check that all values in the key-value cache have a correct value
size_t counter = 0;
string currentValue = to_string(currentIteration);
for (const auto& v : keyValueCache)
{
if (v.second == currentValue)
counter++;
}
if (counter == nInstances * 5)
{
currentIteration += 2;
keyCondition.notify_all();
}
});
// Start listening to events we have subscribed on
service.start();
for (size_t i = 0; i < nMaxValue; ++i)
{
LOG(info) << "Start iteration " << i << ". Current value: " << currentIteration;
// For tasks with type 0 we start with writing the properties.
if ((i % 2 == 0 && type == 0) || (i % 2 == 1 && type == 1))
{
LOG(info) << "Iteration " << i << " start sending values.";
string writePropValue = to_string(i);
for (const auto& prop : writePropertyNames)
{
keyValue.putValue(prop, writePropValue);
}
LOG(info) << "Iteration " << i << " all values have been sent.";
// Writing non existing and readonly properties to test the errors
if (testErrors)
{
LOG(info) << "Iteration " << i << " sending wrong properties.";
keyValue.putValue("non_existing_property", "non_existing_property_name");
for (const auto& prop : readPropertyNames)
{
keyValue.putValue(prop, writePropValue);
}
}
}
// For tasks with type 1 we start with subscribtion to properties.
else if ((i % 2 == 0 && type == 1) || (i % 2 == 1 && type == 0))
{
LOG(info) << "Iteration " << i << " subscribe on property updates. Current value: " << currentIteration;
unique_lock<mutex> lock(keyMutex);
bool waitStatus = keyCondition.wait_for(
lock, chrono::seconds(timeout), [¤tIteration, &i] { return currentIteration > i; });
// Timeout waiting for property updates
if (waitStatus == false)
{
LOG(error) << "Iteration " << i << " timed out waiting for property updates.";
LOG(error) << "Number of key-value update calls: " << numUpdateKeyValueCalls
<< "; currentIteration: " << currentIteration << "; numWaits: " << numWaits;
LOG(error) << "Key value cache.\n" << map_to_string(keyValueCache);
if (currentIteration == nMaxValue - 1)
{
LOG(warning) << "Some properties of the LAST iteration are missing.";
}
else
{
LOG(fatal) << "Task failed: timeout wait for property updates.";
return EXIT_FAILURE;
}
}
LOG(info) << "Iteration " << i << " got all properties. Current value: " << currentIteration;
}
}
if ((nMaxValue % 2 == 0 && type == 0) || (nMaxValue % 2 == 1 && type == 1))
{
unique_lock<mutex> lock(keyMutex);
bool waitStatus = onTaskDoneCondition.wait_for(
lock, chrono::seconds(timeout), [&numTaskDone, &nInstances] { return numTaskDone >= nInstances; });
if (waitStatus == false)
{
LOG(error) << "Task failed: Timed out on waiting Task Done.";
LOG(error) << "Finished tasks: " << numTaskDone << " expected: " << nInstances;
return EXIT_FAILURE;
}
}
LOG(info) << "Key value cache.\n" << map_to_string(keyValueCache);
LOG(info) << "Task successfully done";
chrono::high_resolution_clock::time_point t2 = chrono::high_resolution_clock::now();
auto duration = chrono::duration_cast<chrono::seconds>(t2 - t1).count();
LOG(info) << "Calculation time: " << duration << " seconds";
}
catch (exception& _e)
{
LOG(fatal) << "USER TASK Error: " << _e.what() << endl;
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
ALERROR CSystemMap::AddFixedTopology (CTopology &Topology, CString *retsError)
// AddFixedTopology
//
// Adds all the nodes in its fixed topology
{
ALERROR error;
int i;
// If we already added this map, then we're done
if (m_bAdded)
return NOERROR;
// Mark this map as added so we don't recurse back here when we
// process all the Uses statments.
m_bAdded = true;
// Load all the maps that this map requires
for (i = 0; i < m_Uses.GetCount(); i++)
{
if (error = m_Uses[i]->AddFixedTopology(Topology, retsError))
return error;
}
// Iterate over all creators and execute them
CTopologyNodeList NodesAdded;
STopologyCreateCtx Ctx;
Ctx.pMap = GetDisplayMap();
Ctx.pNodesAdded = &NodesAdded;
// We need to include any maps that we use.
Ctx.Tables.Insert(&m_FixedTopology);
for (i = 0; i < m_Uses.GetCount(); i++)
Ctx.Tables.Insert(&m_Uses[i]->m_FixedTopology);
for (i = 0; i < m_Creators.GetCount(); i++)
{
if (error = ExecuteCreator(Ctx, Topology, m_Creators[i]))
{
*retsError = strPatternSubst(CONSTLIT("SystemMap (%x): %s"), GetUNID(), Ctx.sError);
return error;
}
}
// Add any additional nodes marked as "root" (this is here only for backwards compatibility)
// NOTE: This call only worries about the first table (Ctx.Tables[0])
if (error = Topology.AddTopology(Ctx))
{
*retsError = strPatternSubst(CONSTLIT("SystemMap (%x): %s"), GetUNID(), Ctx.sError);
return error;
}
// Apply any topology processors (in order) on all the newly added nodes
for (i = 0; i < m_Processors.GetCount(); i++)
{
// Make a copy of the node list because each call will destroy it
CTopologyNodeList NodeList = NodesAdded;
// Process
if (error = m_Processors[i]->Process(this, Topology, NodeList, retsError))
{
*retsError = strPatternSubst(CONSTLIT("SystemMap (%x): %s"), GetUNID(), *retsError);
return error;
}
}
// Make sure every node added has a system UNID
for (i = 0; i < NodesAdded.GetCount(); i++)
if (NodesAdded[i]->GetSystemDescUNID() == 0)
{
*retsError = strPatternSubst(CONSTLIT("SystemMap (%x): NodeID %s: No system specified"), GetUNID(), NodesAdded[i]->GetID());
return ERR_FAIL;
}
return NOERROR;
}