// GetSourceNodes
//------------------------------------------------------------------------------
bool FunctionCopy::GetSourceNodes( const BFFIterator & iter, Node * node, Array< Node * > & nodes ) const
{
if ( node->GetType() == Node::ALIAS_NODE )
{
// resolve aliases to real nodes
AliasNode * aliasNode = node->CastTo< AliasNode >();
const Dependencies & aliasedNodes = aliasNode->GetAliasedNodes();
const Dependency * const end = aliasedNodes.End();
for ( const Dependency * it = aliasedNodes.Begin(); it != end; ++it )
{
if ( !GetSourceNodes( iter, it->GetNode(), nodes ) )
{
return false;
}
}
return true;
}
else if ( node->IsAFile() )
{
// anything that results in a file is ok
nodes.Append( node );
return true;
}
// something we don't know how to handle
Error::Error_1005_UnsupportedNodeType( iter, this, ".Source", node->GetName(), node->GetType() );
return false;
}
void SchedulerCPU::LaunchTasks(const FrameContextEx& context, std::function<jobs::Future<std::any>(const FrameContextEx&, const Pipeline&, lemon::ListDigraph::Node, std::any)> onNode) {
struct DependencyCounter {
public:
explicit DependencyCounter(size_t total = 0) {
this->total = total;
}
void Reset() {
*counter = 0;
}
bool operator++() {
return (counter->fetch_add(1) + 1) == total;
}
private:
std::shared_ptr<std::atomic_size_t> counter = std::make_shared<std::atomic_size_t>(0);
std::size_t total = 0;
};
lemon::ListDigraph::NodeMap<DependencyCounter> depCounter(m_pipeline->GetTaskGraph());
for (lemon::ListDigraph::NodeIt nodeIt(m_pipeline->GetTaskGraph()); nodeIt != lemon::INVALID; ++nodeIt) {
depCounter[nodeIt] = DependencyCounter{ (size_t)lemon::countInArcs(m_pipeline->GetTaskGraph(), nodeIt) };
}
auto stopFlag = std::make_shared<std::atomic_bool>(false);
auto Traverse = [this, &context, stopFlag, &onNode, &depCounter](lemon::ListDigraph::Node node, std::any passThrough, auto self) -> jobs::Future<void> {
if (stopFlag->load()) {
co_return;
}
try {
std::any passToNext = co_await onNode(context, *m_pipeline, node, passThrough);
std::vector<jobs::Future<void>> childJobs;
for (lemon::ListDigraph::OutArcIt outArc(m_pipeline->GetTaskGraph(), node); outArc != lemon::INVALID; ++outArc) {
auto nextNode = m_pipeline->GetTaskGraph().target(outArc);
if (++depCounter[nextNode]) {
childJobs.push_back(m_scheduler->Enqueue(self, nextNode, passToNext, self));
}
}
for (auto& childFuture : childJobs) {
co_await childFuture;
}
}
catch (...) {
stopFlag->store(true);
throw;
}
};
auto sourceNodes = GetSourceNodes(m_pipeline->GetTaskGraph());
std::vector<jobs::Future<void>> childJobs;
for (auto sourceNode : sourceNodes) {
childJobs.push_back(m_scheduler->Enqueue(Traverse, sourceNode, std::any{}, Traverse));
}
std::vector<std::exception_ptr> exceptions;
for (auto& job : childJobs) {
try {
job.get();
}
catch (...) {
exceptions.push_back(std::current_exception());
}
}
if (!exceptions.empty()) {
// Rest of the exceptions is ignored.
// TODO: throw some aggregate exception?
std::rethrow_exception(exceptions[0]);
}
}
// Commit
//------------------------------------------------------------------------------
/*virtual*/ bool FunctionCopy::Commit( NodeGraph & nodeGraph, const BFFIterator & funcStartIter ) const
{
// make sure all required variables are defined
Array< AString > sources( 16, true );
const BFFVariable * dstFileV;
if ( !GetStrings( funcStartIter, sources, ".Source", true ) ||
!GetString( funcStartIter, dstFileV, ".Dest", true ) )
{
return false; // GetString will have emitted errors
}
// Optional
AStackString<> sourceBasePath;
if ( !GetString( funcStartIter, sourceBasePath, ".SourceBasePath", false ) )
{
return false; // GetString will have emitted errors
}
// Canonicalize the SourceBasePath
if ( !sourceBasePath.IsEmpty() )
{
AStackString<> cleanValue;
NodeGraph::CleanPath( sourceBasePath, cleanValue );
PathUtils::EnsureTrailingSlash( cleanValue );
sourceBasePath = cleanValue;
}
// check sources are not paths
{
const AString * const end = sources.End();
for ( const AString * it = sources.Begin(); it != end; ++it )
{
const AString & srcFile( *it );
// source must be a file, not a path
if ( PathUtils::IsFolderPath( srcFile ) )
{
Error::Error_1105_PathNotAllowed( funcStartIter, this, ".Source", srcFile );
return false;
}
}
}
// Pre-build dependencies
Dependencies preBuildDependencies;
if ( !GetNodeList( nodeGraph, funcStartIter, ".PreBuildDependencies", preBuildDependencies, false ) )
{
return false; // GetNodeList will have emitted an error
}
Array< AString > preBuildDependencyNames( preBuildDependencies.GetSize(), false );
for ( const auto & dep : preBuildDependencies )
{
preBuildDependencyNames.Append( dep.GetNode()->GetName() );
}
// get source node
Array< Node * > srcNodes;
{
const AString * const end = sources.End();
for ( const AString * it = sources.Begin(); it != end; ++it )
{
Node * srcNode = nodeGraph.FindNode( *it );
if ( srcNode )
{
if ( GetSourceNodes( funcStartIter, srcNode, srcNodes ) == false )
{
return false;
}
}
else
{
// source file not defined by use - assume an external file
srcNodes.Append( nodeGraph.CreateFileNode( *it ) );
}
}
}
AStackString<> dstFile;
NodeGraph::CleanPath( dstFileV->GetString(), dstFile );
const bool dstIsFolderPath = PathUtils::IsFolderPath( dstFile );
// make all the nodes for copies
Dependencies copyNodes( srcNodes.GetSize(), false );
for ( const Node * srcNode : srcNodes )
{
AStackString<> dst( dstFile );
// dest can be a file OR a path. If it's a path, use the source filename part
if ( dstIsFolderPath )
{
// find filename part of source
const AString & srcName = srcNode->GetName();
// If the sourceBasePath is specified (and valid) use the name relative to that
if ( !sourceBasePath.IsEmpty() && PathUtils::PathBeginsWith( srcName, sourceBasePath ) )
{
// Use everything relative to the SourceBasePath
dst += srcName.Get() + sourceBasePath.GetLength();
}
else
{
// Use just the file name
const char * lastSlash = srcName.FindLast( NATIVE_SLASH );
dst += lastSlash ? ( lastSlash + 1 ) // append filename part if found
: srcName.Get(); // otherwise append whole thing
}
}
// check node doesn't already exist
if ( nodeGraph.FindNode( dst ) )
{
// TODO:C could have a specific error for multiple sources with only 1 output
// to differentiate from two rules creating the same dst target
Error::Error_1100_AlreadyDefined( funcStartIter, this, dst );
return false;
}
// create our node
CopyFileNode * copyFileNode = nodeGraph.CreateCopyFileNode( dst );
copyFileNode->m_Source = srcNode->GetName();
copyFileNode->m_PreBuildDependencyNames = preBuildDependencyNames;
if ( !copyFileNode->Initialize( nodeGraph, funcStartIter, this ) )
{
return false; // Initialize will have emitted an error
}
copyNodes.Append( Dependency( copyFileNode ) );
}
// handle alias creation
return ProcessAlias( nodeGraph, funcStartIter, copyNodes );
}
