void PackedValue::Unpack() const
{
if (m_packedDataLayout && (m_packedDataLayout->GetNumTimeSteps() != 1) && (m_packedDataLayout->GetNumSequences() != 1) && Internal::IsAutomaticUnpackingOfPackedValuesDisabled())
LogicError("PackedValue::Unpack: Automatic unpacking of PackedValue objects is disabled");
if (m_isPacked)
{
ValuePtr valueObject;
auto dataType = m_packedData->GetDataType();
switch (dataType)
{
case DataType::Float:
valueObject = CompositeFunction::GetValueObjectFromCNTKImplMatrixAndMBLayout(m_sampleShape, *(m_packedData->GetMatrix<float>()), m_packedDataLayout, m_isReadOnly);
break;
case DataType::Double:
valueObject = CompositeFunction::GetValueObjectFromCNTKImplMatrixAndMBLayout(m_sampleShape, *(m_packedData->GetMatrix<double>()), m_packedDataLayout, m_isReadOnly);
break;
default:
LogicError("Unsupported DataType %s", DataTypeName(dataType));
}
m_data = valueObject->Data();
m_mask = valueObject->Mask();
m_packedData = nullptr;
m_packedDataLayout = nullptr;
m_isPacked = false;
if (m_unpackedShape != m_data->Shape())
LogicError("The computed unpacked shape of the PackedValue object does not match the actual Data NDArrayView's shape after unpacking");
}
}
void RunEvaluationOneHidden(FunctionPtr evalFunc, const DeviceDescriptor& device)
{
const std::wstring inputNodeName = L"features";
const std::wstring outputNodeName = L"out.z_output";
Variable inputVar;
if (!GetInputVariableByName(evalFunc, inputNodeName, inputVar))
{
fprintf(stderr, "Input variable %S is not available.\n", inputNodeName.c_str());
throw("Input variable not found error.");
}
Variable outputVar;
if (!GetOutputVaraiableByName(evalFunc, outputNodeName, outputVar))
{
fprintf(stderr, "Output variable %S is not available.\n", outputNodeName.c_str());
throw("Output variable not found error.");
}
// Evaluate the network in several runs
size_t iterationCount = 4;
size_t numSamples = 3;
for (size_t t = 0; t < iterationCount; ++t)
{
std::vector<float> inputData(inputVar.Shape().TotalSize() * numSamples);
for (size_t i = 0; i < inputData.size(); ++i)
{
inputData[i] = static_cast<float>(i % 255);
}
NDShape inputShape = inputVar.Shape().AppendShape({1, numSamples});
ValuePtr inputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(inputShape, inputData, true));
ValuePtr outputValue;
std::unordered_map<Variable, ValuePtr> outputs = {{outputVar, outputValue}};
evalFunc->Forward({{inputVar, inputValue}}, outputs, device);
outputValue = outputs[outputVar];
NDShape outputShape = outputVar.Shape().AppendShape({1, numSamples});
std::vector<float> outputData(outputShape.TotalSize());
NDArrayViewPtr cpuArrayOutput = MakeSharedObject<NDArrayView>(outputShape, outputData, false);
cpuArrayOutput->CopyFrom(*outputValue->Data());
assert(outputData.size() == outputVar.Shape()[0] * numSamples);
fprintf(stderr, "Evaluation result:\n");
size_t dataIndex = 0;
auto outputDim = outputVar.Shape()[0];
for (size_t i = 0; i < numSamples; i++)
{
fprintf(stderr, "Iteration:%lu, Sample %lu:\n", t, i);
fprintf(stderr, "Ouput:");
for (size_t j = 0; j < outputDim; j++)
{
fprintf(stderr, "%f ", outputData[dataIndex++]);
}
fprintf(stderr, "\n");
}
}
}
void CheckValue(const ValuePtr testValue, const size_t dimension, const vector<vector<size_t>>& expectedData, const vector<size_t>& seqLenList, const vector<bool>& seqStartFlags = {})
{
// Check parameters
BOOST_TEST(expectedData.size() == seqLenList.size(), "Parameter error: the sequence number in the exepected data and sequence list does not match.");
for (size_t i = 0; i < expectedData.size(); i++)
{
if (expectedData[i].size() != seqLenList[i])
{
ReportFailure("Parameter erroe: the number of data for sequence %" PRIu64 " in the expected data does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".",
i, seqLenList[i], expectedData[i].size());
}
}
// Check shape
NDShape shape = testValue->Shape();
size_t valueRank = shape.Rank();
if (valueRank < 2 || valueRank > 3 || shape[0] != dimension)
{
ReportFailure("The shape of the value does not match\n");
}
size_t numOfSequences = valueRank == 2 ? 1 : shape[2];
if (numOfSequences != expectedData.size())
{
ReportFailure("The sequence number in the Value does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", expectedData.size(), numOfSequences);
}
CheckMask(testValue, seqLenList, seqStartFlags);
// Get data from Value
vector<ElementType> outputData(shape.TotalSize());
NDArrayViewPtr arrayOutput = MakeSharedObject<NDArrayView>(shape, outputData, false);
arrayOutput->CopyFrom(*testValue->Data());
size_t maxSeqLen = *max_element(seqLenList.begin(), seqLenList.end());
size_t oIndex = 0;
for (size_t seq = 0; seq < seqLenList.size(); seq++)
{
size_t seqLen = seqLenList[seq];
for (size_t sample = 0; sample < seqLen; sample++)
{
for (size_t c = 0; c < dimension; c++, oIndex++)
{
if (outputData[oIndex] != 0)
{
if (outputData[oIndex] != 1)
{
ReportFailure("OneHot vector contains value other than 0 and 1 at seqNo=%" PRIu64 " sampleNo=%" PRIu64 " position=%" PRIu64 "\n", seq, sample, c);
}
if (c != expectedData[seq][sample])
{
ReportFailure("OneHot Index does match at seqNo=%" PRIu64 ", sampleNo=%" PRIu64 ", expected: %" PRIu64 ", actual: %" PRIu64 "\n", seq, sample, expectedData[seq][sample], c);
}
}
}
}
// Skip mask data
oIndex += (maxSeqLen - seqLen) * dimension;
}
}
void RunEvaluationClassifier(FunctionPtr evalFunc, const DeviceDescriptor& device)
{
const std::wstring inputNodeName = L"features";
Variable inputVar;
if (!GetInputVariableByName(evalFunc, inputNodeName, inputVar))
{
fprintf(stderr, "Input variable %S is not available.\n", inputNodeName.c_str());
throw("Input variable not found error.");
}
// Evaluate the network in several runs
size_t iterationCount = 4;
unsigned int randSeed = 2;
srand(randSeed);
size_t numSamples = 3;
std::vector<float> inputData(inputVar.Shape().TotalSize() * numSamples);
for (size_t t = 0; t < iterationCount; ++t)
{
for (size_t i = 0; i < inputData.size(); ++i)
{
inputData[i] = ((float)rand()) / RAND_MAX;
}
// Create input data shape. Adding sequence length and numSamples as axes.
// Todo: remove sequence length when only numSamples is supported.
// Todo: add convenience APIs to simplify data preparation here.
NDShape inputShape = inputVar.Shape().AppendShape({1, numSamples});
ValuePtr inputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(inputShape, inputData, true));
// Define output.
ValuePtr outputValue;
auto outputVar = evalFunc->Output();
std::unordered_map<Variable, ValuePtr> outputs = {{outputVar, outputValue}};
// Evaluate the model
evalFunc->Forward({{inputVar, inputValue}}, outputs, device);
// Get output value
outputValue = outputs[outputVar];
// Todo: remove sequence length when only numSamples is supported.
// Todo: add convenience APIs to simplify retrieval of output results.
NDShape outputShape = outputVar.Shape().AppendShape({1, numSamples});
std::vector<float> outputData(outputShape.TotalSize());
NDArrayViewPtr cpuArrayOutput = MakeSharedObject<NDArrayView>(outputShape, outputData, false);
cpuArrayOutput->CopyFrom(*outputValue->Data());
assert(outputData.size() == outputVar.Shape()[0] * numSamples);
fprintf(stderr, "Evaluation result:\n");
size_t dataIndex = 0;
auto outputDim = outputVar.Shape()[0];
for (size_t i = 0; i < numSamples; i++)
{
fprintf(stderr, "Iteration:%lu, Sample %lu:\n", t, i);
fprintf(stderr, " ");
dataIndex = i * outputDim;
for (size_t j = 0; j < std::min((size_t)10, outputDim); j++)
{
fprintf(stderr, "%f ", outputData[dataIndex++]);
}
if (outputDim > 10)
{
fprintf(stderr, "...");
}
fprintf(stderr, "\n");
}
}
}
void TestTimesAndPlus(size_t inputDim,
size_t outputDim,
size_t numSamples,
const DeviceDescriptor& device,
size_t numIterations,
bool usePreAllocatedOutputs,
bool outputOnSpecifiedDevice,
bool testSaveAndReLoad,
unsigned int seed = 1)
{
Parameter timesParam(MakeSharedObject<NDArrayView>((ElementType)0.5, NDShape({ outputDim, inputDim }), device), L"timesParameters");
Parameter plusParam(MakeSharedObject<NDArrayView>((ElementType)1.2, std::initializer_list<size_t>({ outputDim }), device), L"plusParameters");
Variable inputVar({ inputDim }, AsDataType<ElementType>(), L"input");
auto timesAndPlusFunc = Plus(plusParam, Times(timesParam, inputVar));
if (testSaveAndReLoad)
SaveAndReloadModel<ElementType>(timesAndPlusFunc, { &inputVar, ×Param, &plusParam }, device);
srand(seed);
for (size_t iterIdx = 0; iterIdx < numIterations; ++iterIdx)
{
std::vector<ElementType> inputData(inputDim * numSamples);
for (size_t i = 0; i < inputData.size(); ++i)
inputData[i] = ((ElementType)rand()) / RAND_MAX;
NDShape inputShape = inputVar.Shape().AppendShape({ 1, numSamples });
ValuePtr inputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(inputShape, inputData.data(), inputData.size(), DeviceDescriptor::CPUDevice(), true));
NDShape outputShape = timesAndPlusFunc->Output().Shape().AppendShape({ 1, numSamples });
std::vector<ElementType> outputData(outputShape.TotalSize());
ValuePtr outputValue;
if (usePreAllocatedOutputs)
{
auto outputAllocationDevice = outputOnSpecifiedDevice ? device : DeviceDescriptor::CPUDevice();
if (outputAllocationDevice.Type() == DeviceKind::CPU)
outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputShape, outputData.data(), outputData.size(), outputAllocationDevice, false));
else
outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), outputShape, outputAllocationDevice));
}
std::unordered_map<Variable, ValuePtr> outputs = { { timesAndPlusFunc->Output(), outputValue } };
auto backpropState = timesAndPlusFunc->Forward({ { inputVar, inputValue } }, outputs, device, { timesAndPlusFunc->Output() });
if (!usePreAllocatedOutputs)
outputValue = outputs[timesAndPlusFunc->Output()];
// Perform backprop
std::vector<ElementType> rootGradientsData(outputShape.TotalSize(), 1);
ValuePtr rootGradientValue;
if (device.Type() == DeviceKind::CPU)
rootGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputShape, rootGradientsData.data(), rootGradientsData.size(), device, true));
else
{
NDArrayViewPtr cpuArrayView = MakeSharedObject<NDArrayView>(outputShape, rootGradientsData.data(), rootGradientsData.size(), DeviceDescriptor::CPUDevice(), true);
NDArrayViewPtr gpuArrayView = MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), outputShape, device);
gpuArrayView->CopyFrom(*cpuArrayView);
rootGradientValue = MakeSharedObject<Value>(gpuArrayView);
}
std::vector<ElementType> plusParameterGradientData(plusParam.Shape().TotalSize());
std::vector<ElementType> timesParameterGradientData(timesParam.Shape().TotalSize());
ValuePtr plusParameterGradientValue, timesParameterGradientValue;
if (usePreAllocatedOutputs)
{
auto outputAllocationDevice = outputOnSpecifiedDevice ? device : DeviceDescriptor::CPUDevice();
if (outputAllocationDevice.Type() == DeviceKind::CPU)
{
plusParameterGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(plusParam.Shape(), plusParameterGradientData.data(), plusParameterGradientData.size(), outputAllocationDevice, false));
timesParameterGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(timesParam.Shape(), timesParameterGradientData.data(), timesParameterGradientData.size(), outputAllocationDevice, false));
}
else
{
plusParameterGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), plusParam.Shape(), outputAllocationDevice));
timesParameterGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), timesParam.Shape(), outputAllocationDevice));
}
}
std::unordered_map<Variable, ValuePtr> paramGradients = { { plusParam, plusParameterGradientValue }, { timesParam, timesParameterGradientValue } };
timesAndPlusFunc->Backward(backpropState, { { timesAndPlusFunc->Output(), rootGradientValue } }, paramGradients);
if (!usePreAllocatedOutputs)
{
plusParameterGradientValue = paramGradients[plusParam];
timesParameterGradientValue = paramGradients[timesParam];
}
// Verify forward prop results
if (!usePreAllocatedOutputs || (outputOnSpecifiedDevice && (device.Type() != DeviceKind::CPU)))
{
NDArrayViewPtr cpuArrayView = MakeSharedObject<NDArrayView>(outputShape, outputData.data(), outputData.size(), DeviceDescriptor::CPUDevice(), false);
cpuArrayView->CopyFrom(*outputValue->Data());
}
std::vector<ElementType> expectedOutputValues(outputShape.TotalSize());
for (size_t i = 0; i < numSamples; ++i)
{
ElementType expectedVal = (ElementType)1.2;
for (size_t j = 0; j < inputDim; ++j)
expectedVal += (ElementType)(inputData[i * inputDim + j] * 0.5);
for (size_t j = 0; j < outputDim; ++j)
expectedOutputValues[i * outputDim + j] = expectedVal;
}
FloatingPointVectorCompare(outputData, expectedOutputValues, "TestTimesAndPlus: Forward prop results do not match expected results");
// Verify backward prop results
if (device.Type() != DeviceKind::CPU)
{
NDArrayViewPtr cpuArrayView = MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), plusParam.Shape(), DeviceDescriptor::CPUDevice());
cpuArrayView->CopyFrom(*plusParameterGradientValue->Data());
const ElementType* cpuArrayViewBuffer = cpuArrayView->DataBuffer<ElementType>();
memcpy(plusParameterGradientData.data(), cpuArrayViewBuffer, plusParam.Shape().TotalSize() * sizeof(ElementType));
cpuArrayView = MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), timesParam.Shape(), DeviceDescriptor::CPUDevice());
cpuArrayView->CopyFrom(*timesParameterGradientValue->Data());
cpuArrayViewBuffer = cpuArrayView->DataBuffer<ElementType>();
memcpy(timesParameterGradientData.data(), cpuArrayViewBuffer, timesParam.Shape().TotalSize() * sizeof(ElementType));
}
for (size_t i = 0; i < outputDim; ++i)
if (plusParameterGradientData[i] != numSamples)
throw std::runtime_error("TestTimesAndPlus: Backprop prop results do not match expected results for Plus params gradients");
std::vector<ElementType> expectedTimesParamsGradientValues(timesParam.Shape().TotalSize());
for (size_t i = 0; i < inputDim; ++i)
{
ElementType expectedVal = 0;
for (size_t j = 0; j < numSamples; ++j)
expectedVal += inputData[j * inputDim + i];
for (size_t j = 0; j < outputDim; ++j)
expectedTimesParamsGradientValues[i * outputDim + j] = expectedVal;
}
FloatingPointVectorCompare(timesParameterGradientData, expectedTimesParamsGradientValues, "TestTimesAndPlus: Backprop prop results do not match expected results for Times params gradients");
}
}
void CheckValue(const ValuePtr testValue, const NDShape& sampleShape, const vector<vector<ElementType>>& expectedData, const vector<size_t>& seqLenList, const vector<bool>& seqStartFlags = {})
{
size_t sampleSize = sampleShape.TotalSize();
// Check parameters
BOOST_TEST(expectedData.size() == seqLenList.size(), "Parameter error: the sequence number in the exepected data and sequence list does not match.");
for (size_t i = 0; i < expectedData.size(); i++)
{
if (expectedData[i].size() != seqLenList[i] * sampleSize)
{
ReportFailure("Parameter erroe: the number of data for sequence %" PRIu64 " in the expected data does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".",
i, seqLenList[i] * sampleSize, expectedData[i].size());
}
}
// Check shape
auto valueRank = testValue->Shape().Rank();
auto sampleRank = sampleShape.Rank();
auto shapeIsCorrect = !((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != testValue->Shape().SubShape(0, sampleRank)));
BOOST_TEST(shapeIsCorrect, "The Value does not have the expected shape.");
size_t numOfSequences;
if (valueRank == sampleShape.Rank() + 1)
{
// no batch axis, only sequence axis
numOfSequences = 1;
}
else
{
assert(valueRank == sampleShape.Rank() + 2);
numOfSequences = testValue->Shape()[valueRank - 1];
}
if (numOfSequences != expectedData.size())
{
ReportFailure("The sequence number in the Value does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", expectedData.size(), numOfSequences);
}
CheckMask(testValue, seqLenList, seqStartFlags);
// Get data from Value
vector<ElementType> outputData(testValue->Shape().TotalSize());
NDArrayViewPtr arrayOutput = MakeSharedObject<NDArrayView>(testValue->Shape(), outputData, false);
arrayOutput->CopyFrom(*testValue->Data());
size_t maxSeqLen = *max_element(seqLenList.begin(), seqLenList.end());
size_t oIndex = 0;
for (size_t seq = 0; seq < seqLenList.size(); seq++)
{
size_t seqLen = seqLenList[seq];
for (size_t sIndex = 0; sIndex < seqLen * sampleSize; sIndex++, oIndex++)
{
if (expectedData[seq][sIndex] != outputData[oIndex])
{
ReportFailure("Data does match at position %" PRIu64 ", expected: %f, actual: %f\n", oIndex, expectedData[seq][sIndex], outputData[oIndex]);
}
}
// Skip mask data
oIndex += (maxSeqLen - seqLen) * sampleSize;
}
}
bool Trainer::TrainMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, const DeviceDescriptor& computeDevice /*= DeviceDescriptor::UseDefaultDevice()*/)
{
std::unordered_map<Variable, ValuePtr> outputs = { { m_aggregatedLossFunction, nullptr }, { m_trainingSampleCountVar, nullptr } };
if (m_aggregatedEvaluationFunction)
outputs.insert({ m_aggregatedEvaluationFunction, nullptr });
outputs.insert(outputsToFetch.begin(), outputsToFetch.end());
if (m_distributedTrainer)
m_distributedTrainer->PreMinibatchCallback(*this);
auto backPropSate = m_combinedTrainingFunction->Forward(arguments, outputs, computeDevice, { m_aggregatedLossFunction });
m_prevMinibatchAggregateTrainingLossValue = outputs[m_aggregatedLossFunction];
if (m_aggregatedEvaluationFunction)
m_prevMinibatchAggregateEvalCriterionValue = outputs[m_aggregatedEvaluationFunction];
for (auto outputToFetch : outputsToFetch)
{
if (outputToFetch.second == nullptr)
outputsToFetch[outputToFetch.first] = outputs[outputToFetch.first];
}
ValuePtr rootGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(m_aggregatedLossFunction->Output().GetDataType(), m_prevMinibatchAggregateTrainingLossValue->Shape(), computeDevice), outputs.at(m_aggregatedLossFunction)->Mask());
if (m_aggregatedLossFunction->Output().GetDataType() == DataType::Float)
rootGradientValue->Data()->SetValue(1.0f);
else
rootGradientValue->Data()->SetValue(1.0);
auto modelParameters = m_combinedTrainingFunction->Parameters();
std::unordered_map<Variable, ValuePtr> parameterGradients;
for (const auto& parameter : modelParameters)
{
parameterGradients[parameter] = nullptr;
}
m_combinedTrainingFunction->Backward(backPropSate, { { m_aggregatedLossFunction, rootGradientValue } }, parameterGradients);
m_prevMinibatchNumSamples = GetSampleCount(m_trainingSampleCountVar, outputs[m_trainingSampleCountVar]);
bool endOfData = m_prevMinibatchNumSamples == 0;
if (m_distributedTrainer)
{
// Aggregation should happen in the same order, the order of parmaters is guaranteed to be the same.
std::vector<std::pair<Parameter, NDArrayViewPtr>> gradients;
gradients.reserve(modelParameters.size());
for (const auto& parameter : modelParameters)
gradients.push_back(std::make_pair(parameter, parameterGradients[parameter]->Data()));
MinibatchInfo info
{
arguments.empty(),
m_prevMinibatchNumSamples,
m_prevMinibatchAggregateTrainingLossValue->Data(),
m_prevMinibatchAggregateEvalCriterionValue->Data()
};
endOfData = m_distributedTrainer->PreParameterUpdateCallback(*this, gradients, info);
m_prevMinibatchNumSamples = info.numberOfSamples;
}
bool anyUpdatesPerformed = false;
for (auto learner : m_parameterLearners)
{
std::unordered_map<Parameter, NDArrayViewPtr> learnerParameterGradients;
const auto& learnerParameters = learner->Parameters();
for (const auto& parameter : learnerParameters)
{
learnerParameterGradients[parameter] = parameterGradients[parameter]->Data();
if (parameterGradients[parameter]->Mask())
LogicError("The gradient value for a Parameter cannot have an associated mask!");
}
anyUpdatesPerformed |= learner->Update(learnerParameterGradients, m_prevMinibatchNumSamples);
}
return anyUpdatesPerformed && !endOfData;
}