static NeuralNetwork extractTileFromNetwork(const NeuralNetwork& network,
unsigned int outputNeuron)
{
// Get the connected subgraph
auto newNetwork = network.getSubgraphConnectedToThisOutput(outputNeuron);
util::log("NeuronVisualizer")
<< "sliced out tile with shape: " << newNetwork.shapeString() << ".\n";
// Remove all other connections from the final layer
#if 1
size_t block = (outputNeuron % newNetwork.getOutputNeurons()) / newNetwork.getOutputBlockingFactor();
size_t offset = (outputNeuron % newNetwork.getOutputNeurons()) % newNetwork.getOutputBlockingFactor();
auto& outputLayer = newNetwork.back();
assert(block < outputLayer.blocks());
Matrix weights = outputLayer[block].slice(0, offset, outputLayer.getInputBlockingFactor(), 1);
Matrix bias = outputLayer.at_bias(block).slice(0, offset, 1, 1);
outputLayer.resize(1, outputLayer.getInputBlockingFactor(), 1);
outputLayer[0] = weights;
outputLayer.at_bias(0) = bias;
util::log("NeuronVisualizer")
<< " trimmed to: " << newNetwork.shapeString() << ".\n";
#endif
return newNetwork;
}
void SoftmaxLayer::runForwardImplementation(Bundle& bundle)
{
auto& inputActivationsVector = bundle[ "inputActivations"].get<MatrixVector>();
auto& outputActivationsVector = bundle["outputActivations"].get<MatrixVector>();
assert(inputActivationsVector.size() == 1);
auto inputActivations = foldTime(inputActivationsVector.back());
util::log("SoftmaxLayer") << " Running forward propagation of matrix "
<< inputActivations.shapeString() << "\n";
if(util::isLogEnabled("SoftmaxLayer::Detail"))
{
util::log("SoftmaxLayer::Detail") << " input: "
<< inputActivations.debugString();
}
auto outputActivations = softmax(inputActivations);
if(util::isLogEnabled("SoftmaxLayer::Detail"))
{
util::log("SoftmaxLayer::Detail") << " outputs: "
<< outputActivations.debugString();
}
saveMatrix("outputActivations", outputActivations);
outputActivationsVector.push_back(unfoldTime(outputActivations,
inputActivationsVector.front().size()));
}
Layer::BlockSparseMatrix Layer::runReverse(const BlockSparseMatrix& m) const
{
if(util::isLogEnabled("Layer"))
{
util::log("Layer") << " Running reverse propagation on matrix (" << m.rows()
<< " rows, " << m.columns() << " columns) through layer with dimensions ("
<< blocks() << " blocks, "
<< getInputCount() << " inputs, " << getOutputCount()
<< " outputs, " << blockStep() << " block step).\n";
util::log("Layer") << " layer: " << m_sparseMatrix.shapeString() << "\n";
}
auto result = m.reverseConvolutionalMultiply(m_sparseMatrix.transpose());
if(util::isLogEnabled("Layer"))
{
util::log("Layer") << " output: " << result.shapeString() << "\n";
}
if(util::isLogEnabled("Layer::Detail"))
{
util::log("Layer::Detail") << " output: " << result.debugString() << "\n";
}
return result;
}
Layer::BlockSparseMatrix Layer::runInputs(const BlockSparseMatrix& m) const
{
if(util::isLogEnabled("Layer"))
{
util::log("Layer") << " Running forward propagation on matrix (" << m.rows()
<< " rows, " << m.columns() << " columns) through layer with dimensions ("
<< blocks() << " blocks, "
<< getInputCount() << " inputs, " << getOutputCount()
<< " outputs, " << blockStep() << " block step).\n";
util::log("Layer") << " layer: " << m_sparseMatrix.shapeString() << "\n";
}
if(util::isLogEnabled("Layer::Detail"))
{
util::log("Layer::Detail") << " input: " << m.debugString() << "\n";
util::log("Layer::Detail") << " layer: " << m_sparseMatrix.debugString() << "\n";
util::log("Layer::Detail") << " bias: " << m_bias.debugString() << "\n";
}
auto unbiasedOutput = m.convolutionalMultiply(m_sparseMatrix, blockStep());
auto output = unbiasedOutput.convolutionalAddBroadcastRow(m_bias);
output.sigmoidSelf();
if(util::isLogEnabled("Layer"))
{
util::log("Layer") << " output: " << output.shapeString() << "\n";
}
if(util::isLogEnabled("Layer::Detail"))
{
util::log("Layer::Detail") << " output: " << output.debugString() << "\n";
}
return output;
}
std::string Matrix::debugString() const
{
return shapeString() + "\n" + matrix::toString(*this, 16) + "\n";
}
void CTCDecoderLayer::runReverseImplementation(Bundle& bundle)
{
auto& inputDeltaVector = bundle["inputDeltas"].get<matrix::MatrixVector>();
auto& outputActivationWeightDeltas =
bundle["outputActivationWeightDeltas"].get<matrix::Matrix>();
auto outputActivationWeights = loadMatrix("outputActivationWeights");
auto inputPaths = loadMatrix("inputPaths");
auto inputActivations = loadMatrix("inputActivations");
Matrix inputDeltas = zeros(inputActivations.size(), inputActivations.precision());
matrix::ctcBeamSearchInputGradients(inputDeltas, outputActivationWeights,
inputPaths, outputActivationWeightDeltas, inputActivations);
if(util::isLogEnabled("CTCDecoderLayer::Detail"))
{
util::log("CTCDecoderLayer::Detail") << " beam search input deltas: "
<< inputDeltas.debugString();
}
else
{
util::log("CTCDecoderLayer") << " beam search input deltas size: "
<< inputDeltas.shapeString() << "\n";
}
if(_implementation->hasCostFunction())
{
auto inputTimesteps = _implementation->getInputTimesteps();
auto inputLabels = _implementation->getInputLabels();
auto ctcInputDeltas = computeCtcInputDeltas(_implementation->getCostFunctionName(),
_implementation->getCostFunctionWeight(),
inputActivations, inputLabels, inputTimesteps);
// TODO: see if it is possible to remove this line, it is only necessary because
// by CTC can't tell the difference between a beam and a minibatch
size_t miniBatchSize = inputActivations.size()[1];
apply(ctcInputDeltas, ctcInputDeltas, matrix::Multiply(miniBatchSize));
if(util::isLogEnabled("CTCDecoderLayer::Detail"))
{
util::log("CTCDecoderLayer::Detail") << " ctc input deltas: "
<< ctcInputDeltas.debugString();
}
else
{
util::log("CTCDecoderLayer") << " ctc input deltas size: "
<< ctcInputDeltas.shapeString() << "\n";
}
apply(inputDeltas, inputDeltas, ctcInputDeltas, matrix::Add());
if(util::isLogEnabled("CTCDecoderLayer::Detail"))
{
util::log("CTCDecoderLayer::Detail") << " combined input deltas: "
<< inputDeltas.debugString();
}
else
{
util::log("CTCDecoderLayer") << " combined input deltas size: "
<< inputDeltas.shapeString() << "\n";
}
}
inputDeltaVector.push_back(inputDeltas);
}
