void CheckValue(const ValuePtr testValue, const size_t dimension, const vector<vector<size_t>>& expectedData, const vector<size_t>& seqLenList, const vector<bool>& seqStartFlags = {})
{
// Check parameters
BOOST_TEST(expectedData.size() == seqLenList.size(), "Parameter error: the sequence number in the exepected data and sequence list does not match.");
for (size_t i = 0; i < expectedData.size(); i++)
{
if (expectedData[i].size() != seqLenList[i])
{
ReportFailure("Parameter erroe: the number of data for sequence %" PRIu64 " in the expected data does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".",
i, seqLenList[i], expectedData[i].size());
}
}
// Check shape
NDShape shape = testValue->Shape();
size_t valueRank = shape.Rank();
if (valueRank < 2 || valueRank > 3 || shape[0] != dimension)
{
ReportFailure("The shape of the value does not match\n");
}
size_t numOfSequences = valueRank == 2 ? 1 : shape[2];
if (numOfSequences != expectedData.size())
{
ReportFailure("The sequence number in the Value does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", expectedData.size(), numOfSequences);
}
CheckMask(testValue, seqLenList, seqStartFlags);
// Get data from Value
vector<ElementType> outputData(shape.TotalSize());
NDArrayViewPtr arrayOutput = MakeSharedObject<NDArrayView>(shape, outputData, false);
arrayOutput->CopyFrom(*testValue->Data());
size_t maxSeqLen = *max_element(seqLenList.begin(), seqLenList.end());
size_t oIndex = 0;
for (size_t seq = 0; seq < seqLenList.size(); seq++)
{
size_t seqLen = seqLenList[seq];
for (size_t sample = 0; sample < seqLen; sample++)
{
for (size_t c = 0; c < dimension; c++, oIndex++)
{
if (outputData[oIndex] != 0)
{
if (outputData[oIndex] != 1)
{
ReportFailure("OneHot vector contains value other than 0 and 1 at seqNo=%" PRIu64 " sampleNo=%" PRIu64 " position=%" PRIu64 "\n", seq, sample, c);
}
if (c != expectedData[seq][sample])
{
ReportFailure("OneHot Index does match at seqNo=%" PRIu64 ", sampleNo=%" PRIu64 ", expected: %" PRIu64 ", actual: %" PRIu64 "\n", seq, sample, expectedData[seq][sample], c);
}
}
}
}
// Skip mask data
oIndex += (maxSeqLen - seqLen) * dimension;
}
}
size_t Evaluator::GetSampleCount(const Variable& var, const ValuePtr& value)
{
auto valueDataShape = value->Shape();
size_t numMaskedSamples = value->MaskedCount();
size_t numSamplesInDataArrayView = valueDataShape.SubShape(var.Shape().Rank()).TotalSize();
if (numMaskedSamples > numSamplesInDataArrayView)
LogicError("Number (%d) of masked values cannot exceed the number (%d) of samples that the Value object's Data NDArrayView can hold.",
(int)numMaskedSamples, (int)numSamplesInDataArrayView);
return (numSamplesInDataArrayView - numMaskedSamples);
}
void TestReduceSum(size_t sampleRank, const DeviceDescriptor& device)
{
size_t numSequences = 7;
size_t maxAllowedSequenceLength = 11;
size_t maxDimSize = 23;
NDShape inputShape(sampleRank);
for (size_t i = 0; i < sampleRank; ++i)
inputShape[i] = (rand() % maxDimSize) + 1;
auto sequenceLengths = GenerateSequenceLengths(numSequences, maxAllowedSequenceLength);
auto sequences = GenerateSequences<float>(sequenceLengths, inputShape);
ValuePtr sequencesValue = Value::Create(inputShape, sequences, device, true);
// Test ReduceSum along a static axis
{
auto testReduceSum = [&sequences, &sequenceLengths, inputShape, sequencesValue, device, sampleRank](int reductionAxis, bool useNegativeAxisIndex)
{
size_t maxActualSequenceLength = sequencesValue->Shape()[inputShape.Rank()];
size_t numSequences = sequencesValue->Shape()[inputShape.Rank() + 1];
auto inputVar = InputVariable(inputShape, DataType::Float, L"input");
FunctionPtr reduceSumFunc;
bool reduceAll = (reductionAxis < 0);
if (reduceAll)
reduceSumFunc = ReduceSum(inputVar);
else
reduceSumFunc = ReduceSum(inputVar, Axis(useNegativeAxisIndex ? (reductionAxis - (int)sampleRank) : reductionAxis));
NDShape outputShape = reduceSumFunc->Output().Shape();
NDShape outputDataShape = outputShape;
if (!reduceAll)
outputDataShape = outputDataShape.AppendShape({ maxActualSequenceLength, numSequences });
std::vector<float> outputData(outputDataShape.TotalSize());
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputDataShape, outputData, false), reduceAll ? nullptr : sequencesValue->Mask()->DeepClone());
std::unordered_map<Variable, ValuePtr> outputs = { { reduceSumFunc->Output(), outputValue } };
reduceSumFunc->Forward({ { inputVar, sequencesValue } }, outputs, device);
std::vector<size_t> inputShapeStrides = GetStrides(inputShape);
std::vector<size_t> outputShapeStrides = GetStrides(outputShape);
std::vector<float> expectedPerFrameTotals(outputShape.TotalSize() * maxActualSequenceLength * numSequences, 0.0f);
float expectedTotal = 0.0f;
for (size_t i = 0; i < numSequences; ++i)
{
size_t currentSequenceLength = sequenceLengths[i];
for (size_t j = 0; j < currentSequenceLength; ++j)
{
for (size_t k = 0; k < inputShape.TotalSize(); ++k)
{
auto inputIdx = UnflattenedShape(k, inputShapeStrides);
auto outputIdx = inputIdx;
if (!reduceAll)
outputIdx[reductionAxis] = 0;
else
outputIdx = {};
auto flatOutputIdx = FlattenedIndex(outputIdx, outputShapeStrides);
float value = sequences[i][(j * inputShape.TotalSize()) + k];
expectedPerFrameTotals[(((i * maxActualSequenceLength) + j) * outputShape.TotalSize()) + flatOutputIdx] += value;
expectedTotal += value;
}
}
}
if (reduceAll)
FloatingPointVectorCompare(outputData, std::vector<float>({ expectedTotal }), "testReduceSum: Forward prop results do not match expected results");
else
FloatingPointVectorCompare(outputData, expectedPerFrameTotals, "testReduceSum: Forward prop results do not match expected results");
};
// Reduce over all axes
testReduceSum(-1, false);
int reductionAxis = 0;
testReduceSum(reductionAxis, true);
if (reductionAxis < (inputShape.Rank() - 1))
reductionAxis++;
testReduceSum(reductionAxis, false);
if (reductionAxis < (inputShape.Rank() - 1))
reductionAxis++;
testReduceSum(reductionAxis, true);
}
// Test ReduceSum along a dynamic axis
{
auto testReduceSum = [&sequences, &sequenceLengths, inputShape, sequencesValue, device](const Axis& axis)
{
if (!axis.IsDynamicAxis())
RuntimeError("Called the dynamic axis ReduceSum test with a static axis");
size_t maxActualSequenceLength = sequencesValue->Shape()[inputShape.Rank()];
size_t numSequences = sequencesValue->Shape()[inputShape.Rank() + 1];
auto inputVar = InputVariable({ inputShape }, DataType::Float, L"input");
FunctionPtr reduceSumFunc = ReduceSum(inputVar, axis);
NDShape maskShape = { ((axis == Axis::DefaultBatchAxis()) ? maxActualSequenceLength : 1), ((axis == Axis::DefaultBatchAxis()) ? 1 : numSequences) };
NDShape outputShape = reduceSumFunc->Output().Shape();
auto outputDataShape = outputShape.AppendShape(maskShape);
std::vector<float> outputData(outputDataShape.TotalSize());
auto maskPtr = MakeSharedObject<NDMask>(maskShape, device);
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputDataShape, outputData, false), maskPtr);
std::unordered_map<Variable, ValuePtr> outputs = { { reduceSumFunc->Output(), outputValue } };
reduceSumFunc->Forward({ { inputVar, sequencesValue } }, outputs, device);
std::vector<float> expectedTotals(outputDataShape.TotalSize(), 0.0f);
for (size_t i = 0; i < numSequences; ++i)
{
size_t currentSequenceLength = sequenceLengths[i];
for (size_t j = 0; j < currentSequenceLength; ++j)
{
for (size_t k = 0; k < inputShape.TotalSize(); ++k)
{
float value = sequences[i][(j * inputShape.TotalSize()) + k];
if (axis == Axis::DefaultBatchAxis())
expectedTotals[(j * inputShape.TotalSize()) + k] += value;
else
expectedTotals[(i * inputShape.TotalSize()) + k] += value;
}
}
}
FloatingPointVectorCompare(outputData, expectedTotals, "testReduceSum: Forward prop results do not match expected results");
};
testReduceSum(Axis::DefaultDynamicAxis());
}
}
void TestSlice(size_t sampleRank, const DeviceDescriptor& device)
{
size_t numSequences = 7;
size_t maxAllowedSequenceLength = 11;
size_t maxDimSize = 23;
size_t minDimSize = 5;
NDShape inputShape(sampleRank);
for (size_t i = 0; i < sampleRank; ++i)
inputShape[i] = (rand() % maxDimSize) + minDimSize;
auto sequenceLengths = GenerateSequenceLengths(numSequences, maxAllowedSequenceLength);
auto sequences = GenerateSequences<float>(sequenceLengths, inputShape);
ValuePtr sequencesValue = Value::Create(inputShape, sequences, device, true);
// Test slice along a static axis
{
auto testStaticAxisSlice = [&sequences, &sequenceLengths, inputShape, sequencesValue, device, sampleRank](int sliceAxis, int beginOffset, int endOffset, bool useNegativeAxisIndex)
{
size_t maxActualSequenceLength = sequencesValue->Shape()[inputShape.Rank()];
size_t numSequences = sequencesValue->Shape()[inputShape.Rank() + 1];
auto inputVar = InputVariable(inputShape, DataType::Float, L"input");
auto sliceFunc = Slice(inputVar, Axis(useNegativeAxisIndex ? (sliceAxis - (int)sampleRank) : sliceAxis), beginOffset, endOffset);
NDShape outputShape = sliceFunc->Output().Shape();
auto outputDataShape = outputShape.AppendShape({ maxActualSequenceLength, numSequences });
std::vector<float> outputData(outputDataShape.TotalSize());
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputDataShape, outputData, false), sequencesValue->Mask()->DeepClone());
std::unordered_map<Variable, ValuePtr> outputs = { { sliceFunc->Output(), outputValue } };
sliceFunc->Forward({ { inputVar, sequencesValue } }, outputs, device);
std::vector<size_t> inputShapeStrides = GetStrides(inputShape);
std::vector<size_t> outputShapeStrides = GetStrides(outputShape);
size_t sliceStartOffset = (beginOffset >= 0) ? beginOffset : (inputShape[sliceAxis] + beginOffset);
std::vector<float> expectedOutputValues(outputShape.TotalSize() * maxActualSequenceLength * numSequences);
for (size_t i = 0; i < numSequences; ++i)
{
size_t currentSequenceLength = sequenceLengths[i];
for (size_t j = 0; j < currentSequenceLength; ++j)
{
for (size_t k = 0; k < outputShape.TotalSize(); ++k)
{
auto outputIdx = UnflattenedShape(k, outputShapeStrides);
auto inputIdx = outputIdx;
inputIdx[sliceAxis] += sliceStartOffset;
auto flatInputIdx = FlattenedIndex(inputIdx, inputShapeStrides);
expectedOutputValues[(((i * maxActualSequenceLength) + j) * outputShape.TotalSize()) + k] = sequences[i][(j * inputShape.TotalSize()) + flatInputIdx];
}
}
}
FloatingPointVectorCompare(outputData, expectedOutputValues, "testStaticAxisSlice: Forward prop results do not match expected results");
};
int sliceAxis = 0;
testStaticAxisSlice(sliceAxis, 3, 5, true);
if (sliceAxis < (inputShape.Rank() - 1))
sliceAxis++;
testStaticAxisSlice(sliceAxis, -1, 0, false);
if (sliceAxis < (inputShape.Rank() - 1))
sliceAxis++;
testStaticAxisSlice(sliceAxis, -3, -1, true);
}
// Test slice along a dynamic axis
{
auto testDynamicAxisSlice = [&sequences, &sequenceLengths, inputShape, sequencesValue, device](const Axis& axis, int beginOffset, int endOffset)
{
if (!axis.IsDynamicAxis())
RuntimeError("Called the dynamic axis slice test with a static axis");
size_t maxActualSequenceLength = sequencesValue->Shape()[inputShape.Rank()];
size_t numSequences = sequencesValue->Shape()[inputShape.Rank() + 1];
int endAndBeginOffsetDiff = endOffset - beginOffset;
size_t maxSliceLength = (endAndBeginOffsetDiff > 0) ? endAndBeginOffsetDiff : maxActualSequenceLength + endAndBeginOffsetDiff;
auto inputVar = InputVariable(inputShape, DataType::Float, L"input");
auto sliceFunc = Slice(inputVar, axis, beginOffset, endOffset);
sliceFunc = sliceFunc + sliceFunc;
size_t outputSequenceAxisLength = (axis == Axis::DefaultDynamicAxis()) ? maxSliceLength : maxActualSequenceLength;
size_t outputBatchAxisLength = (axis == Axis::DefaultBatchAxis()) ? maxSliceLength : numSequences;
NDShape outputShape = sliceFunc->Output().Shape().AppendShape({ outputSequenceAxisLength, outputBatchAxisLength });
std::vector<float> outputData(outputShape.TotalSize(), 0);
NDMaskPtr mask;
if (endAndBeginOffsetDiff < 0)
{
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputShape, outputData, false));
mask = MakeSharedObject<NDMask>(std::initializer_list<size_t>({ outputSequenceAxisLength, outputBatchAxisLength }), device);
}
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputShape, outputData, false), mask);
std::unordered_map<Variable, ValuePtr> outputs = { { sliceFunc->Output(), outputValue } };
sliceFunc->Forward({ { inputVar, sequencesValue } }, outputs, device);
size_t startSequenceIdx = (axis == Axis::DefaultBatchAxis()) ? ((beginOffset >= 0) ? beginOffset : (numSequences + beginOffset)) : 0;
size_t endSequenceIdx = (axis == Axis::DefaultBatchAxis()) ? ((endOffset > 0) ? endOffset : (numSequences + endOffset)) : numSequences;
std::vector<float> expectedOutputValues(inputShape.TotalSize() * outputSequenceAxisLength * outputBatchAxisLength);
for (size_t i = startSequenceIdx; i < endSequenceIdx; ++i)
{
size_t currentSequenceLength = sequenceLengths[i];
size_t startFrameIdx = (axis == Axis::DefaultDynamicAxis()) ? ((beginOffset >= 0) ? beginOffset : (currentSequenceLength + beginOffset)) : 0;
size_t endFrameIdx = (axis == Axis::DefaultDynamicAxis()) ? ((endOffset > 0) ? endOffset : (currentSequenceLength + endOffset)) : currentSequenceLength;
size_t j = startFrameIdx;
for (; j < endFrameIdx; ++j)
{
for (size_t k = 0; k < inputShape.TotalSize(); ++k)
expectedOutputValues[((((i - startSequenceIdx) * outputSequenceAxisLength) + (j - startFrameIdx)) * inputShape.TotalSize()) + k] = 2 * sequences[i][(j * inputShape.TotalSize()) + k];
}
// Zero out the invalid portions of the actual output
for (; j < (outputSequenceAxisLength + startFrameIdx); ++j)
for (size_t k = 0; k < inputShape.TotalSize(); ++k)
outputData[((((i - startSequenceIdx) * outputSequenceAxisLength) + (j - startFrameIdx)) * inputShape.TotalSize()) + k] = 0;
}
FloatingPointVectorCompare(outputData, expectedOutputValues, "testDynamicAxisSlice: Forward prop results do not match expected results");
};
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), 0, 1);
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), 0, 2);
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), -1, 0);
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), -2, 0);
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), 0, -1);
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), 1, 0);
}
}
void CheckValue(const ValuePtr testValue, const NDShape& sampleShape, const vector<vector<ElementType>>& expectedData, const vector<size_t>& seqLenList, const vector<bool>& seqStartFlags = {})
{
size_t sampleSize = sampleShape.TotalSize();
// Check parameters
BOOST_TEST(expectedData.size() == seqLenList.size(), "Parameter error: the sequence number in the exepected data and sequence list does not match.");
for (size_t i = 0; i < expectedData.size(); i++)
{
if (expectedData[i].size() != seqLenList[i] * sampleSize)
{
ReportFailure("Parameter erroe: the number of data for sequence %" PRIu64 " in the expected data does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".",
i, seqLenList[i] * sampleSize, expectedData[i].size());
}
}
// Check shape
auto valueRank = testValue->Shape().Rank();
auto sampleRank = sampleShape.Rank();
auto shapeIsCorrect = !((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != testValue->Shape().SubShape(0, sampleRank)));
BOOST_TEST(shapeIsCorrect, "The Value does not have the expected shape.");
size_t numOfSequences;
if (valueRank == sampleShape.Rank() + 1)
{
// no batch axis, only sequence axis
numOfSequences = 1;
}
else
{
assert(valueRank == sampleShape.Rank() + 2);
numOfSequences = testValue->Shape()[valueRank - 1];
}
if (numOfSequences != expectedData.size())
{
ReportFailure("The sequence number in the Value does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", expectedData.size(), numOfSequences);
}
CheckMask(testValue, seqLenList, seqStartFlags);
// Get data from Value
vector<ElementType> outputData(testValue->Shape().TotalSize());
NDArrayViewPtr arrayOutput = MakeSharedObject<NDArrayView>(testValue->Shape(), outputData, false);
arrayOutput->CopyFrom(*testValue->Data());
size_t maxSeqLen = *max_element(seqLenList.begin(), seqLenList.end());
size_t oIndex = 0;
for (size_t seq = 0; seq < seqLenList.size(); seq++)
{
size_t seqLen = seqLenList[seq];
for (size_t sIndex = 0; sIndex < seqLen * sampleSize; sIndex++, oIndex++)
{
if (expectedData[seq][sIndex] != outputData[oIndex])
{
ReportFailure("Data does match at position %" PRIu64 ", expected: %f, actual: %f\n", oIndex, expectedData[seq][sIndex], outputData[oIndex]);
}
}
// Skip mask data
oIndex += (maxSeqLen - seqLen) * sampleSize;
}
}