// change the node associated with nodeName to newNode; used in the KL-reg based adaptation to reduce feature copy
// need to update all the mappings as well childrens
void ComputationNetwork::ChangeNode(wstring nodeName, ComputationNodeBasePtr newNode)
{
InvalidateCompiledNetwork();
ComputationNodeBasePtr oldNode = GetNodeFromName(nodeName);
if (oldNode->OperationName() != newNode->OperationName())
InvalidArgument("newNode must have the same type as the old node.");
// change children
for (auto nodeIter = m_nameToNodeMap.begin(); nodeIter != m_nameToNodeMap.end(); nodeIter++)
{
ComputationNodeBasePtr node = nodeIter->second;
for (int i = 0; i < node->GetNumInputs(); i++)
if (node->GetInputs()[i] == oldNode)
node->SetInput(i, newNode);
}
// change name map
m_nameToNodeMap[nodeName] = newNode;
for (int i = 0; i < oldNode->GetNumInputs(); i++)
newNode->SetInput(i, oldNode->GetInputs()[i]);
// change other maps
for (auto groupIter : GetAllNodeGroups())
{
auto& group = *groupIter;
for (int i = 0; i < group.size(); i++)
if (group[i] == oldNode)
group[i] = newNode;
}
}
// We only remove the node, not delete it.
void ComputationNetwork::RemoveFeatureNode(ComputationNodeBasePtr featureNode)
{
InvalidateCompiledNetwork();
wstring nodeName = featureNode->NodeName();
if (!NodeNameExists(nodeName))
RuntimeError("RemoveFeatureNode: feature node does not exist.");
// Removes links.
for (auto nodeIter = m_nameToNodeMap.begin(); nodeIter != m_nameToNodeMap.end(); ++nodeIter)
{
ComputationNodeBasePtr node = nodeIter->second;
for (size_t i = 0; i < node->GetNumInputs(); ++i)
{
ComputationNodeBasePtr child = node->GetInputs()[i];
if (child == featureNode)
{
node->SetInput(i, NULL);
break;
}
}
}
// Removes from feature list.
auto search = std::find(m_features.begin(), m_features.end(), featureNode);
if (search != m_features.end())
m_features.erase(search);
m_nameToNodeMap.erase(nodeName);
}
void ComputationNetwork::ReplaceFinalCriterionNode(wstring oldNodeName, ComputationNodeBasePtr newNode)
{
InvalidateCompiledNetwork();
// Checks if the node is a criterion node.
int index = -1;
for (int i = 0; i < m_finalCriteria.size(); ++i)
{
if (m_finalCriteria[i]->NodeName() == oldNodeName)
{
index = i;
break;
}
}
if (index == -1)
RuntimeError("ReplaceFinalCriterionNode: the node to be replaced is not a criterion node.");
// Replaces children.
for (int i = 0; i < newNode->GetNumInputs(); ++i)
{
if (m_nameToNodeMap.find(newNode->GetInputs()[i]->NodeName()) == m_nameToNodeMap.end())
RuntimeError("Child node does not exist.");
newNode->SetInput(i, m_nameToNodeMap[newNode->GetInputs()[i]->NodeName()]);
}
// Addes it to criterion node list.
m_finalCriteria[index] = newNode;
m_nameToNodeMap[newNode->NodeName()] = newNode;
}
// add a new criterion node and at the same time orphan the previous one (it won't be removed)
// The newNode can have the same name and come with pre-connected inputs, which will be used to connect to existing nodes of the same name.
// BUGBUG: Can this operate on both new and existing nodes?
void ComputationNetwork::ReplaceFinalCriterionNode(wstring oldNodeName, ComputationNodeBasePtr newNode)
{
InvalidateCompiledNetwork();
// remove old criterion node
// BUGBUG: The old node is not removed from the network. Seems strangely inconsistent.
bool wasThere = RemoveFromNodeGroup(L"criterion", GetNodeFromName(oldNodeName));
if (!wasThere)
RuntimeError("ReplaceFinalCriterionNode: The node to be replaced is not a criterion node.");
// replace children
// This looks for nodes in the network that have the same name as its current inputs, and then relinks its inputs to those.
// I.e. this allows to move a node from network to another and reconnect by the names if its inputs.
for (int i = 0; i < newNode->GetNumInputs(); ++i)
{
if (m_nameToNodeMap.find(newNode->GetInputs()[i]->NodeName()) == m_nameToNodeMap.end())
RuntimeError("Child node %ls is not part of the network.", newNode->GetInputs()[i]->NodeName().c_str());
newNode->SetInput(i, m_nameToNodeMap[newNode->GetInputs()[i]->NodeName()]);
}
// add it to the network
AddNodeToNetIfNotYet(newNode);
// add new node to criterion node group
AddToNodeGroup(L"criterion", newNode);
}
// replace a named node by newNode of the same type under the same name, including moving over all network links
// This is used in
// 1. Update nodes to quantized versions.
// 2. The KL-reg based adaptation to reduce feature copy (deprecated)
// need to update all the mappings as well childrens.
void ComputationNetwork::ReplaceNode(wstring nodeName, ComputationNodeBasePtr newNode)
{
ComputationNodeBasePtr oldNode = GetNodeFromName(nodeName);
if (newNode->NodeName() != nodeName) // TODO: This was not tested for earlier; I hope no code depends on this.
InvalidArgument("ChangeNode: newNode must have the same name as the old node.");
InvalidateCompiledNetwork();
// change all nodes that have old node as input to point to the new node instead
ChangeNodeInputs(oldNode, newNode);
// change all inputs of this new node to share the old one's inputs
for (int i = 0; i < oldNode->GetNumInputs(); i++)
{
newNode->SetInput(i, oldNode->GetInputs()[i]); // TODO: use AttachInput()?
//oldNode->SetInput(i, nullptr); // BUGBUG: old node should no longer point into the network
}
// replace the node in the network
RemoveNodeFromNet(oldNode);
AddNodeToNet(newNode);
// also update node groups
for (auto groupIter : GetAllNodeGroups())
{
auto& group = *groupIter;
for (int i = 0; i < group.size(); i++)
if (group[i] == oldNode)
group[i] = newNode;
}
}
// change all nodes that have fromNode as input to have toNode as input instead
void ComputationNetwork::ChangeNodeInputs(ComputationNodeBasePtr fromNode, ComputationNodeBasePtr toNode)
{
for (auto nodeIter = m_nameToNodeMap.begin(); nodeIter != m_nameToNodeMap.end(); nodeIter++)
{
ComputationNodeBasePtr node = nodeIter->second;
for (int i = 0; i < node->GetNumInputs(); i++)
if (node->GetInputs()[i] == fromNode)
node->SetInput(i, toNode);
}
}
// traverse sub-graph feeding this node (which is a top-level node at start, e.g. training criterion) and list
// - all nodes that participate in a loop -> recurrentResult[loopId][]
// - all nodes that don't -> noRecurrentResult[]
// in order of traversal (depth-first).
// This is part of the FormRecurrentLoops() process, and only called from there from one place.
void ComputationNetwork::GatherLoopNodesR(const ComputationNodeBasePtr& node, unordered_set<ComputationNodeBasePtr>& visited,
map<int, list<ComputationNodeBasePtr>>& recurrentResult,
list<ComputationNodeBasePtr>& noRecurrentResult)
{
if (visited.find(node) != visited.end())
return; // do each node only once
visited.insert(node);
for (int i = 0; i < node->GetNumInputs(); i++)
GatherLoopNodesR(node->Input(i), visited, recurrentResult, noRecurrentResult);
if (node->m_loopId >= 0)
recurrentResult[node->m_loopId].push_back(node);
else
noRecurrentResult.push_back(node);
}
// replace the old node with the current node, assuming the old node is a leaf node
// need to update those nodes who use oldNode as their child
// TODO: Can this be called with a node that's already part of the network? This is currently allowed, but should it?
// BUGBUG: Seems ReplaceNode() also updates node groups. Why doesn't this function?
// BUGBUG: What if newNode is the one referenced by oldNodeName?
// BUGBUG: Or what if an unrelated node of the same name exists?
void ComputationNetwork::ReplaceLeafNode(wstring oldNodeName, ComputationNodeBasePtr newNode)
{
InvalidateCompiledNetwork();
ComputationNodeBasePtr oldNode = GetNodeFromName(oldNodeName);
// relink the input of those nodes whose child is oldNode to point to the new one instead
for (auto nodeIter = m_nameToNodeMap.begin(); nodeIter != m_nameToNodeMap.end(); nodeIter++)
{
ComputationNodeBasePtr node = nodeIter->second;
for (int i = 0; i < node->GetNumInputs(); i++)
if (node->GetInputs()[i] == oldNode)
node->SetInput(i, newNode);
}
// add the new, remove the old
AddNodeToNetIfNotYet(newNode);
DeleteNode(oldNodeName); // TODO: can this just be RemoveNodeFromNet()?
}
// replace the old node with the current node, assuming the old node is a leaf node
// need to update those nodes who use oldNode as their child
void ComputationNetwork::ReplaceLeafNode(wstring oldNodeName, ComputationNodeBasePtr newNode)
{
InvalidateCompiledNetwork();
ComputationNodeBasePtr oldNode = GetNodeFromName(oldNodeName);
// change the input of those nodes whose child is oldNode
for (auto nodeIter = m_nameToNodeMap.begin(); nodeIter != m_nameToNodeMap.end(); nodeIter++)
{
ComputationNodeBasePtr node = nodeIter->second;
for (int i = 0; i < node->GetNumInputs(); i++)
if (node->GetInputs()[i] == oldNode)
node->SetInput(i, newNode);
}
m_nameToNodeMap[newNode->GetName()] = newNode;
// now the old node becomes a orphan node , remove it
DeleteNode(oldNodeName);
// RemoveOrphanNode(oldNode);
}
void ComputationNetwork::DeleteNode(const std::wstring& nodeName)
{
InvalidateCompiledNetwork();
ComputationNodeBasePtr nodeToDelete = GetNodeFromName(nodeName);
// first delete links, if this node is involved, the whole connection will be removed
for (auto nodeIter = m_nameToNodeMap.begin(); nodeIter != m_nameToNodeMap.end(); nodeIter++)
{
ComputationNodeBasePtr node = nodeIter->second;
for (size_t i = 0; i < node->GetNumInputs(); i++)
{
ComputationNodeBasePtr child = node->GetInputs()[i];
// nodeToDelete is a child
if (child == nodeToDelete)
{
// this used to call DetatchInputs(), but it's better for MEL to retain other inputs
node->SetInput(i, nullptr);
break;
}
}
}
// nodeToDelete is a parent
nodeToDelete->DetachInputs(); // deref all its inputs; if we don't do that, we might end up with a mem leak due to a circular reference
// unlink from all node-group sets
for (auto groupIter : GetAllNodeGroups())
{
auto search = std::find(groupIter->begin(), groupIter->end(), nodeToDelete);
if (search != groupIter->end())
groupIter->erase(search);
}
// Note: the necessary update of m_allSEQNodes is hanlded by the InvalidateCompiledNetwork() call above
// delete the node itself
m_nameToNodeMap.erase(nodeName); // this will deref the node and possibly deallocate it
}
// recovers the processing order within a recurrent loop
// TODO: Once we only use the nested network for recurrent traversal, this will be no longer necessary.
void ComputationNetwork::DetermineLoopForwardOrder(unordered_set<ComputationNodeBasePtr>& visited,
unordered_set<ComputationNodeBasePtr>& recStack,
list<ComputationNodeBasePtr>& nodesStack,
ComputationNodeBasePtr cur)
{
if (visited.find(cur) == visited.end())
{
visited.insert(cur);
recStack.insert(cur);
if (GetRecurrenceSteppingDirection(cur) == 0) // recurrence stops at delay nodes
{
for (size_t i = 0; i < cur->GetNumInputs(); i++)
if (cur->Input(i)->m_loopId == cur->m_loopId)
DetermineLoopForwardOrder(visited, recStack, nodesStack, cur->Input(i));
}
recStack.erase(cur);
nodesStack.push_back(cur);
}
else if (recStack.find(cur) != recStack.end())
LogicError("%ls %ls operation is part of an infinite loop that cannot be unrolled.", cur->NodeName().c_str(), cur->OperationName().c_str());
}
// TODO: how does the file distinguish float vs double nodes?
void ComputationNetwork::SaveToFileImpl(const wstring& fileName, const FileOptions fileFormat) const
{
File fstream(fileName, fileFormat | FileOptions::fileOptionsWrite);
fstream.PutMarker(FileMarker::fileMarkerBeginSection, L"BCN");
// model version
fstream.PutMarker(FileMarker::fileMarkerBeginSection, L"BVersion");
fstream << (size_t) CURRENT_CNTK_MODEL_VERSION;
fstream.PutMarker(FileMarker::fileMarkerEndSection, L"EVersion");
fstream << (size_t) m_nameToNodeMap.size();
// put all node info first
fstream.PutMarker(FileMarker::fileMarkerBeginSection, L"BNodeList");
for (auto nodeIter = m_nameToNodeMap.begin(); nodeIter != m_nameToNodeMap.end(); nodeIter++)
{
ComputationNodeBasePtr nodePtr = nodeIter->second;
nodePtr->Save(fstream);
}
fstream.PutMarker(FileMarker::fileMarkerEndSection, L"ENodeList");
// put relationship
fstream.PutMarker(FileMarker::fileMarkerBeginSection, L"BRelation");
for (auto nodeIter = m_nameToNodeMap.begin(); nodeIter != m_nameToNodeMap.end(); nodeIter++)
{
ComputationNodeBasePtr nodePtr = nodeIter->second;
fstream << nodePtr->NodeName() << nodePtr->GetNumInputs();
for (size_t i = 0; i < nodePtr->GetNumInputs(); i++)
{
if (!nodePtr->Input(i))
fprintf(stderr, "Warning: node %ls 's child is null, please check your ndl/mel file.\n", nodePtr->NodeName().c_str());
else
fstream << nodePtr->Input(i)->NodeName();
}
}
fstream.PutMarker(FileMarker::fileMarkerEndSection, L"ERelation");
fstream.PutMarker(FileMarker::fileMarkerBeginSection, L"BRootNodes");
fstream.PutMarker(FileMarker::fileMarkerBeginSection, L"BFeatureNodes");
fstream << m_features.size();
for (size_t i = 0; i < m_features.size(); i++)
fstream << m_features[i]->NodeName();
fstream.PutMarker(FileMarker::fileMarkerEndSection, L"EFeatureNodes");
fstream.PutMarker(FileMarker::fileMarkerBeginSection, L"BLabelNodes");
fstream << m_labels.size();
for (size_t i = 0; i < m_labels.size(); i++)
fstream << m_labels[i]->NodeName();
fstream.PutMarker(FileMarker::fileMarkerEndSection, L"ELabelNodes");
fstream.PutMarker(FileMarker::fileMarkerBeginSection, L"BCriterionNodes");
fstream << m_finalCriteria.size();
for (size_t i = 0; i < m_finalCriteria.size(); i++)
fstream << m_finalCriteria[i]->NodeName();
fstream.PutMarker(FileMarker::fileMarkerEndSection, L"ECriterionNodes");
fstream.PutMarker(FileMarker::fileMarkerBeginSection, L"BEvalNodes");
fstream << m_evalNodes.size();
for (size_t i = 0; i < m_evalNodes.size(); i++)
fstream << m_evalNodes[i]->NodeName();
fstream.PutMarker(FileMarker::fileMarkerEndSection, L"EEvalNodes");
fstream.PutMarker(FileMarker::fileMarkerBeginSection, L"BOutputNodes");
fstream << m_outputNodes.size();
for (size_t i = 0; i < m_outputNodes.size(); i++)
{
fstream << m_outputNodes[i]->NodeName();
}
fstream.PutMarker(FileMarker::fileMarkerEndSection, L"EOutputNodes");
fstream.PutMarker(FileMarker::fileMarkerEndSection, L"ERootNodes");
fstream.PutMarker(FileMarker::fileMarkerEndSection, L"ECN");
fstream.Flush();
}
// (recursive part of DetermineSCCs())
void ComputationNetwork::DetermineSCCsR(ComputationNodeBasePtr cur,
list<ComputationNodeBasePtr>& sccStack,
size_t& index, size_t& loopId)
{
assert(!cur->m_visited);
// set the index (in order of visitation)
cur->m_index = index; // TODO: can this be used as m_visitedOrder?
cur->m_minIndex = index; // also set m_minIndex
index++;
cur->m_visited = true;
sccStack.push_back(cur);
cur->m_inStack = true;
// set m_minIndex to min over m_lowLinks of children
for (int i = 0; i < cur->GetNumInputs(); i++)
{
if (!cur->Input(i)->m_visited)
{
DetermineSCCsR(cur->Input(i), sccStack, index, loopId);
cur->m_minIndex = min(cur->m_minIndex, cur->Input(i)->m_minIndex);
}
else if (cur->Input(i)->m_inStack)
{
cur->m_minIndex = min(cur->m_minIndex, cur->Input(i)->m_minIndex);
}
}
// if we closed a loop then create an entry in m_allSEQNodes
if (cur->m_minIndex == cur->m_index) // m_minIndex is still equal to m_index, as we set it at the start of this function: we closed a loop
{
// gather the list of all nodes in this loop
vector<ComputationNodeBasePtr> nestedNodes;
// TODO: build array first in a local array. Only if succeeds, then construct the node off it.
SEQTraversalFlowControlNode rInfo(m_allSEQNodes.size() /*loopId*/, cur);
for (;;)
{
ComputationNodeBasePtr w = sccStack.back();
sccStack.pop_back();
w->m_inStack = false;
nestedNodes.push_back(w);
if (w == cur) // hit our starting point: done
break;
}
// insert loop into m_allSEQNodes
if (nestedNodes.size() > 1) // non-looped nodes are detected here as loops of size 1 --skip those
{
// only add to the array if the loop is not already there
// We end up producing the same loop multiple times because:
// - FormRecurrentLoops() is called multiple times from different roots
// - depth-first traversal might have led us to enter a loop multiple times?
// TODO: Check whether this edge case of idempotence is done correctly:
// - a recurrent loop with two delay nodes
// - two root nodes
// - the first root takes the first delay node's value, the second root that of the second delay node
// I.e. the depth-first tree traversals enter the loop at two different places (m_sourceNode).
// -> Are these two loops detected as identical? (determined by m_minIndex, but m_index depends on traversal from each root, so maybe not)
bool bFound = false; // find a dup --TODO: check whether there is an STL algorithm for this
for (const auto& iter2 : m_allSEQNodes)
{
if (iter2->m_sourceNode == cur)
{
bFound = true;
break;
}
}
if (!bFound)
{
#if 1
if (loopId != m_allSEQNodes.size())
LogicError("DetermineSCCsR(): inconsistent loopId (%d) vs. m_allSEQNodes.size() (%d)", (int) loopId, (int) m_allSEQNodes.size());
SEQTraversalFlowControlNode rInfo(m_allSEQNodes.size(), cur);
#else
assert(loopId == m_allSEQNodes.size()); // BUGBUG: Only true if all loops are shared among roots. Fix: use m_allSEQNodes.size() instead
SEQTraversalFlowControlNode rInfo(loopId, cur);
#endif
// TODO: can we prove that 'cur' == nestedNodes.front()? If so, we won't need to store it separately.
rInfo.m_nestedNodes = move(nestedNodes); // TODO: make these two part of the constructor
rInfo.m_steppingDirection = DetermineLoopDirection(rInfo.m_nestedNodes);
m_allSEQNodes.push_back(make_shared<SEQTraversalFlowControlNode>(move(rInfo)));
loopId++; // and count it TODO: may be removed
}
}
}
}
