static list asList( FrameListPtr l )
{
std::vector<FrameList::Frame> frames;
l->asList( frames );
list result;
for ( std::vector<FrameList::Frame>::const_iterator it = frames.begin(); it != frames.end(); ++it )
{
result.append( *it );
}
return result;
}
void IECore::findSequences( const std::vector< std::string > &names, std::vector< FileSequencePtr > &sequences, size_t minSequenceSize )
{
sequences.clear();
/// this matches names of the form $prefix$frameNumber$suffix
/// placing each of those in a group of the resulting match.
/// both $prefix and $suffix may be the empty string and $frameNumber
/// may be preceded by a minus sign.
/// It also matches file extensions with 3 or 4 characters that contain numbers (for example: CR2, MP3 )
boost::regex matchExpression( std::string( "^([^#]*?)(-?[0-9]+)([^0-9#]*|[^0-9#]*\\.[a-zA-Z]{2,3}[0-9])$" ) );
/// build a mapping from ($prefix, $suffix) to a list of $frameNumbers
typedef std::vector< std::string > Frames;
typedef std::map< std::pair< std::string, std::string >, Frames > SequenceMap;
SequenceMap sequenceMap;
for ( std::vector< std::string >::const_iterator it = names.begin(); it != names.end(); ++it )
{
boost::smatch matches;
if ( boost::regex_match( *it, matches, matchExpression ) )
{
sequenceMap[
SequenceMap::key_type(
std::string( matches[1].first, matches[1].second ),
std::string( matches[3].first, matches[3].second )
)
].push_back( std::string( matches[2].first, matches[2].second ) );
}
}
for ( SequenceMap::const_iterator it = sequenceMap.begin(); it != sequenceMap.end(); ++it )
{
const SequenceMap::key_type &fixes = it->first;
const Frames &frames = it->second;
// todo: could be more efficient by writing a custom comparison function that uses indexes
// into the const Frames vector rather than duplicating the strings and sorting them directly
Frames sortedFrames = frames;
std::sort( sortedFrames.begin(), sortedFrames.end() );
/// in diabolical cases the elements of frames may not all have the same padding
/// so we'll sort them out into padded and unpadded frame sequences here, by creating
/// a map of padding->list of frames. unpadded things will be considered to have a padding
/// of 1.
typedef std::vector< FrameList::Frame > NumericFrames;
typedef std::map< unsigned int, NumericFrames > PaddingToFramesMap;
PaddingToFramesMap paddingToFrames;
for ( Frames::const_iterator fIt = sortedFrames.begin(); fIt != sortedFrames.end(); ++fIt )
{
std::string frame = *fIt;
int sign = 1;
assert( frame.size() );
if ( *frame.begin() == '-' )
{
frame = frame.substr( 1, frame.size() - 1 );
sign = -1;
}
if ( *frame.begin() == '0' || paddingToFrames.find( frame.size() ) != paddingToFrames.end() )
{
paddingToFrames[ frame.size() ].push_back( sign * boost::lexical_cast<FrameList::Frame>( frame ) );
}
else
{
paddingToFrames[ 1 ].push_back( sign * boost::lexical_cast<FrameList::Frame>( frame ) );
}
}
for ( PaddingToFramesMap::iterator pIt = paddingToFrames.begin(); pIt != paddingToFrames.end(); ++pIt )
{
const PaddingToFramesMap::key_type &padding = pIt->first;
NumericFrames &numericFrames = pIt->second;
std::sort( numericFrames.begin(), numericFrames.end() );
FrameListPtr frameList = frameListFromList( numericFrames );
std::vector< FrameList::Frame > expandedFrameList;
frameList->asList( expandedFrameList );
/// remove any sequences with less than the given minimum.
if ( expandedFrameList.size() >= minSequenceSize )
{
std::string frameTemplate;
for ( PaddingToFramesMap::key_type i = 0; i < padding; i++ )
{
frameTemplate += "#";
}
sequences.push_back(
new FileSequence(
fixes.first + frameTemplate + fixes.second,
frameList
)
);
}
}
}
}
void Dispatcher::dispatch( const std::vector<NodePtr> &nodes ) const
{
// clear job directory, so that if our node validation fails,
// jobDirectory() won't return the result from the previous dispatch.
m_jobDirectory = "";
// validate the nodes we've been given
if ( nodes.empty() )
{
throw IECore::Exception( getName().string() + ": Must specify at least one node to dispatch." );
}
std::vector<ExecutableNodePtr> executables;
const ScriptNode *script = (*nodes.begin())->scriptNode();
for ( std::vector<NodePtr>::const_iterator nIt = nodes.begin(); nIt != nodes.end(); ++nIt )
{
const ScriptNode *currentScript = (*nIt)->scriptNode();
if ( !currentScript || currentScript != script )
{
throw IECore::Exception( getName().string() + ": Dispatched nodes must all belong to the same ScriptNode." );
}
if ( ExecutableNode *executable = runTimeCast<ExecutableNode>( nIt->get() ) )
{
executables.push_back( executable );
}
else if ( const Box *box = runTimeCast<const Box>( nIt->get() ) )
{
for ( RecursiveOutputPlugIterator plugIt( box ); plugIt != plugIt.end(); ++plugIt )
{
Node *sourceNode = plugIt->get()->source<Plug>()->node();
if ( ExecutableNode *executable = runTimeCast<ExecutableNode>( sourceNode ) )
{
executables.push_back( executable );
}
}
}
else
{
throw IECore::Exception( getName().string() + ": Dispatched nodes must be ExecutableNodes or Boxes containing ExecutableNodes." );
}
}
// create the job directory now, so it's available in preDispatchSignal().
const Context *context = Context::current();
m_jobDirectory = createJobDirectory( context );
// this object calls this->preDispatchSignal() in its constructor and this->postDispatchSignal()
// in its destructor, thereby guaranteeing that we always call this->postDispatchSignal().
DispatcherSignalGuard signalGuard( this, executables );
if ( signalGuard.cancelledByPreDispatch() )
{
return;
}
std::vector<FrameList::Frame> frames;
FrameListPtr frameList = frameRange( script, context );
frameList->asList( frames );
size_t i = 0;
ExecutableNode::Tasks tasks;
tasks.reserve( executables.size() * frames.size() );
for ( std::vector<FrameList::Frame>::const_iterator fIt = frames.begin(); fIt != frames.end(); ++fIt )
{
for ( std::vector<ExecutableNodePtr>::const_iterator nIt = executables.begin(); nIt != executables.end(); ++nIt, ++i )
{
ContextPtr frameContext = new Context( *context, Context::Borrowed );
frameContext->setFrame( *fIt );
tasks.push_back( ExecutableNode::Task( *nIt, frameContext ) );
}
}
TaskBatchPtr rootBatch = batchTasks( tasks );
if ( !rootBatch->requirements().empty() )
{
doDispatch( rootBatch.get() );
}
// inform the guard that the process has been completed, so it can pass this info to
// postDispatchSignal():
signalGuard.success();
}
void Dispatcher::dispatch( const std::vector<NodePtr> &nodes ) const
{
// clear job directory, so that if our node validation fails,
// jobDirectory() won't return the result from the previous dispatch.
m_jobDirectory = "";
// validate the nodes we've been given
if ( nodes.empty() )
{
throw IECore::Exception( getName().string() + ": Must specify at least one node to dispatch." );
}
std::vector<TaskNodePtr> taskNodes;
const ScriptNode *script = (*nodes.begin())->scriptNode();
for ( std::vector<NodePtr>::const_iterator nIt = nodes.begin(); nIt != nodes.end(); ++nIt )
{
const ScriptNode *currentScript = (*nIt)->scriptNode();
if ( !currentScript || currentScript != script )
{
throw IECore::Exception( getName().string() + ": Dispatched nodes must all belong to the same ScriptNode." );
}
if ( TaskNode *taskNode = runTimeCast<TaskNode>( nIt->get() ) )
{
taskNodes.push_back( taskNode );
}
else if ( const SubGraph *subGraph = runTimeCast<const SubGraph>( nIt->get() ) )
{
for ( RecursiveOutputPlugIterator plugIt( subGraph ); !plugIt.done(); ++plugIt )
{
Node *sourceNode = plugIt->get()->source()->node();
if ( TaskNode *taskNode = runTimeCast<TaskNode>( sourceNode ) )
{
taskNodes.push_back( taskNode );
}
}
}
else
{
throw IECore::Exception( getName().string() + ": Dispatched nodes must be TaskNodes or SubGraphs containing TaskNodes." );
}
}
Context::EditableScope jobScope( Context::current() );
// create the job directory now, so it's available in preDispatchSignal().
/// \todo: move directory creation between preDispatchSignal() and dispatchSignal()
m_jobDirectory = createJobDirectory( Context::current() );
jobScope.set( g_jobDirectoryContextEntry, m_jobDirectory );
// this object calls this->preDispatchSignal() in its constructor and this->postDispatchSignal()
// in its destructor, thereby guaranteeing that we always call this->postDispatchSignal().
DispatcherSignalGuard signalGuard( this, taskNodes );
if ( signalGuard.cancelledByPreDispatch() )
{
return;
}
dispatchSignal()( this, taskNodes );
std::vector<FrameList::Frame> frames;
FrameListPtr frameList = frameRange( script, Context::current() );
frameList->asList( frames );
Batcher batcher;
for( std::vector<FrameList::Frame>::const_iterator fIt = frames.begin(); fIt != frames.end(); ++fIt )
{
for( std::vector<TaskNodePtr>::const_iterator nIt = taskNodes.begin(); nIt != taskNodes.end(); ++nIt )
{
jobScope.setFrame( *fIt );
batcher.addTask( TaskNode::Task( *nIt, Context::current() ) );
}
}
executeAndPruneImmediateBatches( batcher.rootBatch() );
if( !batcher.rootBatch()->preTasks().empty() )
{
doDispatch( batcher.rootBatch() );
}
// inform the guard that the process has been completed, so it can pass this info to
// postDispatchSignal():
signalGuard.success();
}
void Dispatcher::dispatch( const std::vector<NodePtr> &nodes ) const
{
if ( nodes.empty() )
{
throw IECore::Exception( getName().string() + ": Must specify at least one node to dispatch." );
}
std::vector<ExecutableNodePtr> executables;
const ScriptNode *script = (*nodes.begin())->scriptNode();
for ( std::vector<NodePtr>::const_iterator nIt = nodes.begin(); nIt != nodes.end(); ++nIt )
{
const ScriptNode *currentScript = (*nIt)->scriptNode();
if ( !currentScript || currentScript != script )
{
throw IECore::Exception( getName().string() + ": Dispatched nodes must all belong to the same ScriptNode." );
}
if ( ExecutableNode *executable = runTimeCast<ExecutableNode>( nIt->get() ) )
{
executables.push_back( executable );
}
else if ( const Box *box = runTimeCast<const Box>( nIt->get() ) )
{
for ( RecursiveOutputPlugIterator plugIt( box ); plugIt != plugIt.end(); ++plugIt )
{
Node *sourceNode = plugIt->get()->source<Plug>()->node();
if ( ExecutableNode *executable = runTimeCast<ExecutableNode>( sourceNode ) )
{
executables.push_back( executable );
}
}
}
else
{
throw IECore::Exception( getName().string() + ": Dispatched nodes must be ExecutableNodes or Boxes containing ExecutableNodes." );
}
}
if ( preDispatchSignal()( this, executables ) )
{
/// \todo: communicate the cancellation to the user
return;
}
const Context *context = Context::current();
std::vector<FrameList::Frame> frames;
FrameListPtr frameList = frameRange( script, context );
frameList->asList( frames );
size_t i = 0;
ExecutableNode::Tasks tasks;
tasks.reserve( executables.size() * frames.size() );
for ( std::vector<FrameList::Frame>::const_iterator fIt = frames.begin(); fIt != frames.end(); ++fIt )
{
for ( std::vector<ExecutableNodePtr>::const_iterator nIt = executables.begin(); nIt != executables.end(); ++nIt, ++i )
{
ContextPtr frameContext = new Context( *context, Context::Borrowed );
frameContext->setFrame( *fIt );
tasks.push_back( ExecutableNode::Task( *nIt, frameContext ) );
}
}
TaskBatchPtr rootBatch = batchTasks( tasks );
if ( !rootBatch->requirements().empty() )
{
doDispatch( rootBatch.get() );
}
postDispatchSignal()( this, executables );
}