void TestTimesNodeShapeInference()
{
auto timesNodeShapeInferenceTest = [](size_t inputRank, size_t outputRank, int inputRankToMap) {
auto device = DeviceDescriptor::CPUDevice();
size_t maxDimSize = 15;
NDShape outputShape(outputRank);
for (size_t i = 0; i < outputRank; ++i)
outputShape[i] = (rand() % maxDimSize) + 1;
NDShape paramShape = outputShape;
if (inputRankToMap > 0)
paramShape = paramShape.AppendShape({ NDShape::InferredDimension });
else
paramShape = paramShape.AppendShape(NDShape(inputRank));
auto timesParam = Parameter(paramShape, DataType::Float, ConstantInitializer(), device);
auto placeholderInput = PlaceholderVariable();
auto timesFunction = Times(timesParam, placeholderInput, outputRank, inputRankToMap);
NDShape inputShape(inputRank);
for (size_t i = 0; i < inputRank; ++i)
inputShape[i] = (rand() % maxDimSize) + 1;
auto actualInput = InputVariable(inputShape, DataType::Float);
timesFunction->ReplacePlaceholders({ { placeholderInput, actualInput } });
// Verify that the inferred shape of the param, input and output matches expectation
auto expectedInputShape = inputShape;
auto expectedParamShape = outputShape;
if (inputRankToMap > 0)
expectedParamShape = expectedParamShape.AppendShape(inputShape.SubShape(0, inputRank - inputRankToMap));
else
expectedParamShape = expectedParamShape.AppendShape(inputShape);
auto expectedOutputShape = outputShape;
if (inputRankToMap > 0)
expectedOutputShape = expectedOutputShape.AppendShape(inputShape.SubShape(inputRank - inputRankToMap));
auto actualInputShape = timesFunction->Arguments()[0].Shape();
auto actualParamShape = timesFunction->Parameters()[0].Shape();
auto actualOutputShape = timesFunction->Output().Shape();
if (actualInputShape != expectedInputShape)
ReportFailure("Times nodes actual input shape (%S) does not match expectation (%S)", actualInputShape.AsString().c_str(), expectedInputShape.AsString().c_str());
if (actualParamShape != expectedParamShape)
ReportFailure("Times nodes actual parameter shape (%S) does not match expectation (%S)", actualParamShape.AsString().c_str(), expectedParamShape.AsString().c_str());
if (actualOutputShape != expectedOutputShape)
ReportFailure("Times nodes actual output shape (%S) does not match expectation (%S)", actualOutputShape.AsString().c_str(), expectedOutputShape.AsString().c_str());
};
timesNodeShapeInferenceTest(2, 2, -1);
timesNodeShapeInferenceTest(2, 1, 1);
timesNodeShapeInferenceTest(1, 2, 0);
timesNodeShapeInferenceTest(3, 2, 2);
}
void NDMask::MaskSection(const std::vector<size_t>& sectionOffset, const NDShape& sectionShape)
{
// TODO: Implement batching of masking operation for masks residing on GPUs to avoid making
// GPU invocations for each MaskSection call.
if (sectionOffset.size() > m_maskShape.NumAxes())
LogicError("NDMask::MaskSection: The sectionOffset cannot have dimensionality higher than the number of axes of 'this' mask");
if (sectionShape.NumAxes() > m_maskShape.NumAxes())
LogicError("NDMask::MaskSection: The section shape cannot have an axes count higher than the number of axes of 'this' mask");
std::vector<size_t> offset(m_maskShape.NumAxes(), 0);
for (size_t i = 0; i < sectionOffset.size(); ++i)
offset[i] = sectionOffset[i];
NDShape shape = sectionShape.AppendShape(NDShape(m_maskShape.NumAxes() - sectionShape.NumAxes(), NDShape::InferredDimension));
auto maskMatrix = GetMatrix();
size_t rowOffset = offset[0];
size_t colOffset = offset[1];
size_t sliceRowLength = (shape[0] != NDShape::InferredDimension) ? shape[0] : (maskMatrix->GetNumRows() - rowOffset);
size_t sliceColLength = (shape[1] != NDShape::InferredDimension) ? shape[1] : (maskMatrix->GetNumCols() - colOffset);
if ((rowOffset == 0) && (sliceRowLength == maskMatrix->GetNumRows()))
maskMatrix->ColumnSlice(colOffset, sliceColLength).SetValue(0);
else
{
// Since Matrix does not support strides in the row dimension, we will need to create separate slices for each column
for (size_t i = colOffset; i < (colOffset + sliceColLength); ++i)
{
auto column = maskMatrix->ColumnSlice(i, 1);
column.Reshape(1, maskMatrix->GetNumRows());
column.ColumnSlice(rowOffset, sliceRowLength).SetValue(0);
}
}
}
/*static*/ ValuePtr Value::Create(const NDShape& sampleShape, const std::vector<std::vector<ElementType>>& sequences, const DeviceDescriptor& device, bool readOnly/* = false*/)
{
size_t sampleSize = sampleShape.TotalSize();
NDMaskPtr deviceValueMask = CreateMask(sampleSize, sequences, device);
size_t maxSequenceLength = (deviceValueMask == nullptr) ? sequences[0].size() : deviceValueMask->Shape()[0];
size_t numSequences = sequences.size();
NDShape valueDataShape = sampleShape.AppendShape({ maxSequenceLength, numSequences });
NDArrayViewPtr valueData(new NDArrayView(AsDataType<ElementType>(), valueDataShape, DeviceDescriptor::CPUDevice()), [](ReferenceCount* ptr) { delete ptr; });
ElementType* dataBuffer = valueData->WritableDataBuffer<ElementType>();
for (size_t i = 0; i < numSequences; ++i)
std::copy(sequences[i].data(), sequences[i].data() + sequences[i].size(), dataBuffer + (maxSequenceLength * i * sampleSize));
NDArrayViewPtr deviceValueData;
if (device == DeviceDescriptor::CPUDevice())
{
if (readOnly)
deviceValueData = valueData->Alias(true);
else
deviceValueData = valueData;
}
else
{
deviceValueData = NDArrayViewPtr(new NDArrayView(AsDataType<ElementType>(), valueDataShape, device), [](ReferenceCount* ptr) { delete ptr; });
deviceValueData->CopyFrom(*valueData);
if (readOnly)
deviceValueData = deviceValueData->Alias(true);
}
return ValuePtr(new Value(deviceValueData, deviceValueMask), [](ReferenceCount* ptr) { delete ptr; });
}
/*static*/ ValuePtr Value::Create(size_t vocabularySize, const std::vector<std::vector<size_t>>& oneHotSequences, const DeviceDescriptor& device, bool readOnly/* = false*/)
{
NDMaskPtr deviceValueMask = CreateMask(1, oneHotSequences, device);
size_t maxSequenceLength = (deviceValueMask == nullptr) ? oneHotSequences[0].size() : deviceValueMask->Shape()[0];
size_t numSequences = oneHotSequences.size();
NDShape sampleShape = { vocabularySize };
NDShape valueDataShape = sampleShape.AppendShape({ maxSequenceLength, numSequences });
size_t numCSCCols = valueDataShape.SubShape(1).TotalSize() + 1;
std::vector<SparseIndexType> colStarts(numCSCCols);
std::vector<ElementType> nonZeroValues;
std::vector<SparseIndexType> rowIndices;
for (size_t i = 0; i < numSequences; ++i)
{
size_t currentSequenceLength = oneHotSequences[i].size();
size_t j = 0;
for (; j < currentSequenceLength; ++j)
{
colStarts[(i * maxSequenceLength) + j] = (SparseIndexType)nonZeroValues.size();
nonZeroValues.push_back(1);
rowIndices.push_back((SparseIndexType)(oneHotSequences[i][j]));
}
for (; j < maxSequenceLength; ++j)
colStarts[(i * maxSequenceLength) + j] = (SparseIndexType)(nonZeroValues.size());
}
colStarts[numSequences * maxSequenceLength] = (SparseIndexType)(nonZeroValues.size());
NDArrayViewPtr deviceValueData(new NDArrayView(valueDataShape, colStarts.data(), rowIndices.data(), nonZeroValues.data(), nonZeroValues.size(), device, readOnly), [](ReferenceCount* ptr) { delete ptr; });
return ValuePtr(new Value(deviceValueData, deviceValueMask), [](ReferenceCount* ptr) { delete ptr; });
}
void TestTranspose(size_t numAxes, int axis1, int axis2, const DeviceDescriptor& device)
{
srand(1);
size_t maxDimSize = 15;
NDShape inputShape(numAxes);
for (size_t i = 0; i < numAxes; ++i)
inputShape[i] = (rand() % maxDimSize) + 1;
auto inputVar = InputVariable(inputShape, DataType::Float, false, L"leftInput");
auto transposeFunc = TransposeAxes(inputVar, Axis(axis1), Axis(axis2));
std::vector<float> inputData(inputShape.TotalSize());
for (size_t i = 0; i < inputData.size(); ++i)
inputData[i] = ((float)rand()) / RAND_MAX;
auto inputValueShape = inputShape.AppendShape({ 1, 1 });
ValuePtr inputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(inputValueShape, inputData, true));
NDShape outputShape = transposeFunc->Output().Shape();
NDShape outputValueShape = outputShape.AppendShape({ 1, 1 });
std::vector<float> outputData(outputValueShape.TotalSize());
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputValueShape, outputData, false));
std::unordered_map<Variable, ValuePtr> outputs = { { transposeFunc->Output(), outputValue } };
transposeFunc->Forward({ { inputVar, inputValue } }, outputs, device);
std::vector<size_t> inputShapeStrides = GetStrides(inputShape);
std::vector<size_t> outputShapeStrides = GetStrides(outputShape);
// Verify forward prop results
std::vector<float> expectedOutputValues(outputShape.TotalSize());
for (size_t i = 0; i < expectedOutputValues.size(); ++i)
{
auto unflattenedShape = UnflattenedShape(i, outputShapeStrides);
std::swap(unflattenedShape[axis1], unflattenedShape[axis2]);
size_t flattenedIndex = FlattenedIndex(unflattenedShape, inputShapeStrides);
expectedOutputValues[i] = inputData[flattenedIndex];
}
FloatingPointVectorCompare(outputData, expectedOutputValues, "TestTimesAndPlus: Forward prop results do not match expected results");
}
ValuePtr CreateBatchWithVariableSequence(const NDShape& sampleShape, size_t batchSize, const std::vector<size_t>& sequenceSize, const std::vector<ElementType>& batchData, const DeviceDescriptor& device, bool readOnly = false)
{
auto shapeSize = sampleShape.TotalSize();
if (batchData.size() % shapeSize != 0)
InvalidArgument("The number of elements (%zu) in the vector containing batch data must be a multiple of the size (%zu) of the sample shape '%S'.",
batchData.size(), shapeSize, sampleShape.AsString().c_str());
if (sequenceSize.size() != batchSize)
InvalidArgument("The number of sequences (%zu) in the vector containing sequence size must match batch size (%zu)", sequenceSize.size(), batchSize);
std::vector<NDArrayViewPtr> sequencesView(batchSize);
size_t curBatchDataIdx = 0;
for (size_t i = 0; i < batchSize; i++)
{
auto sequenceDataShape = sampleShape.AppendShape({sequenceSize[i]});
sequencesView[i] = MakeSharedObject<NDArrayView>(sequenceDataShape, batchData.data() + curBatchDataIdx, shapeSize * sequenceSize[i], DeviceDescriptor::CPUDevice());
curBatchDataIdx += shapeSize * sequenceSize[i];
}
return Value::Create(sampleShape, sequencesView, {}, device, readOnly, true);
}
void TestReduceSum(size_t sampleRank, const DeviceDescriptor& device)
{
size_t numSequences = 7;
size_t maxAllowedSequenceLength = 11;
size_t maxDimSize = 23;
NDShape inputShape(sampleRank);
for (size_t i = 0; i < sampleRank; ++i)
inputShape[i] = (rand() % maxDimSize) + 1;
auto sequenceLengths = GenerateSequenceLengths(numSequences, maxAllowedSequenceLength);
auto sequences = GenerateSequences<float>(sequenceLengths, inputShape);
ValuePtr sequencesValue = Value::Create(inputShape, sequences, device, true);
// Test ReduceSum along a static axis
{
auto testReduceSum = [&sequences, &sequenceLengths, inputShape, sequencesValue, device, sampleRank](int reductionAxis, bool useNegativeAxisIndex)
{
size_t maxActualSequenceLength = sequencesValue->Shape()[inputShape.Rank()];
size_t numSequences = sequencesValue->Shape()[inputShape.Rank() + 1];
auto inputVar = InputVariable(inputShape, DataType::Float, L"input");
FunctionPtr reduceSumFunc;
bool reduceAll = (reductionAxis < 0);
if (reduceAll)
reduceSumFunc = ReduceSum(inputVar);
else
reduceSumFunc = ReduceSum(inputVar, Axis(useNegativeAxisIndex ? (reductionAxis - (int)sampleRank) : reductionAxis));
NDShape outputShape = reduceSumFunc->Output().Shape();
NDShape outputDataShape = outputShape;
if (!reduceAll)
outputDataShape = outputDataShape.AppendShape({ maxActualSequenceLength, numSequences });
std::vector<float> outputData(outputDataShape.TotalSize());
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputDataShape, outputData, false), reduceAll ? nullptr : sequencesValue->Mask()->DeepClone());
std::unordered_map<Variable, ValuePtr> outputs = { { reduceSumFunc->Output(), outputValue } };
reduceSumFunc->Forward({ { inputVar, sequencesValue } }, outputs, device);
std::vector<size_t> inputShapeStrides = GetStrides(inputShape);
std::vector<size_t> outputShapeStrides = GetStrides(outputShape);
std::vector<float> expectedPerFrameTotals(outputShape.TotalSize() * maxActualSequenceLength * numSequences, 0.0f);
float expectedTotal = 0.0f;
for (size_t i = 0; i < numSequences; ++i)
{
size_t currentSequenceLength = sequenceLengths[i];
for (size_t j = 0; j < currentSequenceLength; ++j)
{
for (size_t k = 0; k < inputShape.TotalSize(); ++k)
{
auto inputIdx = UnflattenedShape(k, inputShapeStrides);
auto outputIdx = inputIdx;
if (!reduceAll)
outputIdx[reductionAxis] = 0;
else
outputIdx = {};
auto flatOutputIdx = FlattenedIndex(outputIdx, outputShapeStrides);
float value = sequences[i][(j * inputShape.TotalSize()) + k];
expectedPerFrameTotals[(((i * maxActualSequenceLength) + j) * outputShape.TotalSize()) + flatOutputIdx] += value;
expectedTotal += value;
}
}
}
if (reduceAll)
FloatingPointVectorCompare(outputData, std::vector<float>({ expectedTotal }), "testReduceSum: Forward prop results do not match expected results");
else
FloatingPointVectorCompare(outputData, expectedPerFrameTotals, "testReduceSum: Forward prop results do not match expected results");
};
// Reduce over all axes
testReduceSum(-1, false);
int reductionAxis = 0;
testReduceSum(reductionAxis, true);
if (reductionAxis < (inputShape.Rank() - 1))
reductionAxis++;
testReduceSum(reductionAxis, false);
if (reductionAxis < (inputShape.Rank() - 1))
reductionAxis++;
testReduceSum(reductionAxis, true);
}
// Test ReduceSum along a dynamic axis
{
auto testReduceSum = [&sequences, &sequenceLengths, inputShape, sequencesValue, device](const Axis& axis)
{
if (!axis.IsDynamicAxis())
RuntimeError("Called the dynamic axis ReduceSum test with a static axis");
size_t maxActualSequenceLength = sequencesValue->Shape()[inputShape.Rank()];
size_t numSequences = sequencesValue->Shape()[inputShape.Rank() + 1];
auto inputVar = InputVariable({ inputShape }, DataType::Float, L"input");
FunctionPtr reduceSumFunc = ReduceSum(inputVar, axis);
NDShape maskShape = { ((axis == Axis::DefaultBatchAxis()) ? maxActualSequenceLength : 1), ((axis == Axis::DefaultBatchAxis()) ? 1 : numSequences) };
NDShape outputShape = reduceSumFunc->Output().Shape();
auto outputDataShape = outputShape.AppendShape(maskShape);
std::vector<float> outputData(outputDataShape.TotalSize());
auto maskPtr = MakeSharedObject<NDMask>(maskShape, device);
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputDataShape, outputData, false), maskPtr);
std::unordered_map<Variable, ValuePtr> outputs = { { reduceSumFunc->Output(), outputValue } };
reduceSumFunc->Forward({ { inputVar, sequencesValue } }, outputs, device);
std::vector<float> expectedTotals(outputDataShape.TotalSize(), 0.0f);
for (size_t i = 0; i < numSequences; ++i)
{
size_t currentSequenceLength = sequenceLengths[i];
for (size_t j = 0; j < currentSequenceLength; ++j)
{
for (size_t k = 0; k < inputShape.TotalSize(); ++k)
{
float value = sequences[i][(j * inputShape.TotalSize()) + k];
if (axis == Axis::DefaultBatchAxis())
expectedTotals[(j * inputShape.TotalSize()) + k] += value;
else
expectedTotals[(i * inputShape.TotalSize()) + k] += value;
}
}
}
FloatingPointVectorCompare(outputData, expectedTotals, "testReduceSum: Forward prop results do not match expected results");
};
testReduceSum(Axis::DefaultDynamicAxis());
}
}
void TestSlice(size_t sampleRank, const DeviceDescriptor& device)
{
size_t numSequences = 7;
size_t maxAllowedSequenceLength = 11;
size_t maxDimSize = 23;
size_t minDimSize = 5;
NDShape inputShape(sampleRank);
for (size_t i = 0; i < sampleRank; ++i)
inputShape[i] = (rand() % maxDimSize) + minDimSize;
auto sequenceLengths = GenerateSequenceLengths(numSequences, maxAllowedSequenceLength);
auto sequences = GenerateSequences<float>(sequenceLengths, inputShape);
ValuePtr sequencesValue = Value::Create(inputShape, sequences, device, true);
// Test slice along a static axis
{
auto testStaticAxisSlice = [&sequences, &sequenceLengths, inputShape, sequencesValue, device, sampleRank](int sliceAxis, int beginOffset, int endOffset, bool useNegativeAxisIndex)
{
size_t maxActualSequenceLength = sequencesValue->Shape()[inputShape.Rank()];
size_t numSequences = sequencesValue->Shape()[inputShape.Rank() + 1];
auto inputVar = InputVariable(inputShape, DataType::Float, L"input");
auto sliceFunc = Slice(inputVar, Axis(useNegativeAxisIndex ? (sliceAxis - (int)sampleRank) : sliceAxis), beginOffset, endOffset);
NDShape outputShape = sliceFunc->Output().Shape();
auto outputDataShape = outputShape.AppendShape({ maxActualSequenceLength, numSequences });
std::vector<float> outputData(outputDataShape.TotalSize());
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputDataShape, outputData, false), sequencesValue->Mask()->DeepClone());
std::unordered_map<Variable, ValuePtr> outputs = { { sliceFunc->Output(), outputValue } };
sliceFunc->Forward({ { inputVar, sequencesValue } }, outputs, device);
std::vector<size_t> inputShapeStrides = GetStrides(inputShape);
std::vector<size_t> outputShapeStrides = GetStrides(outputShape);
size_t sliceStartOffset = (beginOffset >= 0) ? beginOffset : (inputShape[sliceAxis] + beginOffset);
std::vector<float> expectedOutputValues(outputShape.TotalSize() * maxActualSequenceLength * numSequences);
for (size_t i = 0; i < numSequences; ++i)
{
size_t currentSequenceLength = sequenceLengths[i];
for (size_t j = 0; j < currentSequenceLength; ++j)
{
for (size_t k = 0; k < outputShape.TotalSize(); ++k)
{
auto outputIdx = UnflattenedShape(k, outputShapeStrides);
auto inputIdx = outputIdx;
inputIdx[sliceAxis] += sliceStartOffset;
auto flatInputIdx = FlattenedIndex(inputIdx, inputShapeStrides);
expectedOutputValues[(((i * maxActualSequenceLength) + j) * outputShape.TotalSize()) + k] = sequences[i][(j * inputShape.TotalSize()) + flatInputIdx];
}
}
}
FloatingPointVectorCompare(outputData, expectedOutputValues, "testStaticAxisSlice: Forward prop results do not match expected results");
};
int sliceAxis = 0;
testStaticAxisSlice(sliceAxis, 3, 5, true);
if (sliceAxis < (inputShape.Rank() - 1))
sliceAxis++;
testStaticAxisSlice(sliceAxis, -1, 0, false);
if (sliceAxis < (inputShape.Rank() - 1))
sliceAxis++;
testStaticAxisSlice(sliceAxis, -3, -1, true);
}
// Test slice along a dynamic axis
{
auto testDynamicAxisSlice = [&sequences, &sequenceLengths, inputShape, sequencesValue, device](const Axis& axis, int beginOffset, int endOffset)
{
if (!axis.IsDynamicAxis())
RuntimeError("Called the dynamic axis slice test with a static axis");
size_t maxActualSequenceLength = sequencesValue->Shape()[inputShape.Rank()];
size_t numSequences = sequencesValue->Shape()[inputShape.Rank() + 1];
int endAndBeginOffsetDiff = endOffset - beginOffset;
size_t maxSliceLength = (endAndBeginOffsetDiff > 0) ? endAndBeginOffsetDiff : maxActualSequenceLength + endAndBeginOffsetDiff;
auto inputVar = InputVariable(inputShape, DataType::Float, L"input");
auto sliceFunc = Slice(inputVar, axis, beginOffset, endOffset);
sliceFunc = sliceFunc + sliceFunc;
size_t outputSequenceAxisLength = (axis == Axis::DefaultDynamicAxis()) ? maxSliceLength : maxActualSequenceLength;
size_t outputBatchAxisLength = (axis == Axis::DefaultBatchAxis()) ? maxSliceLength : numSequences;
NDShape outputShape = sliceFunc->Output().Shape().AppendShape({ outputSequenceAxisLength, outputBatchAxisLength });
std::vector<float> outputData(outputShape.TotalSize(), 0);
NDMaskPtr mask;
if (endAndBeginOffsetDiff < 0)
{
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputShape, outputData, false));
mask = MakeSharedObject<NDMask>(std::initializer_list<size_t>({ outputSequenceAxisLength, outputBatchAxisLength }), device);
}
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputShape, outputData, false), mask);
std::unordered_map<Variable, ValuePtr> outputs = { { sliceFunc->Output(), outputValue } };
sliceFunc->Forward({ { inputVar, sequencesValue } }, outputs, device);
size_t startSequenceIdx = (axis == Axis::DefaultBatchAxis()) ? ((beginOffset >= 0) ? beginOffset : (numSequences + beginOffset)) : 0;
size_t endSequenceIdx = (axis == Axis::DefaultBatchAxis()) ? ((endOffset > 0) ? endOffset : (numSequences + endOffset)) : numSequences;
std::vector<float> expectedOutputValues(inputShape.TotalSize() * outputSequenceAxisLength * outputBatchAxisLength);
for (size_t i = startSequenceIdx; i < endSequenceIdx; ++i)
{
size_t currentSequenceLength = sequenceLengths[i];
size_t startFrameIdx = (axis == Axis::DefaultDynamicAxis()) ? ((beginOffset >= 0) ? beginOffset : (currentSequenceLength + beginOffset)) : 0;
size_t endFrameIdx = (axis == Axis::DefaultDynamicAxis()) ? ((endOffset > 0) ? endOffset : (currentSequenceLength + endOffset)) : currentSequenceLength;
size_t j = startFrameIdx;
for (; j < endFrameIdx; ++j)
{
for (size_t k = 0; k < inputShape.TotalSize(); ++k)
expectedOutputValues[((((i - startSequenceIdx) * outputSequenceAxisLength) + (j - startFrameIdx)) * inputShape.TotalSize()) + k] = 2 * sequences[i][(j * inputShape.TotalSize()) + k];
}
// Zero out the invalid portions of the actual output
for (; j < (outputSequenceAxisLength + startFrameIdx); ++j)
for (size_t k = 0; k < inputShape.TotalSize(); ++k)
outputData[((((i - startSequenceIdx) * outputSequenceAxisLength) + (j - startFrameIdx)) * inputShape.TotalSize()) + k] = 0;
}
FloatingPointVectorCompare(outputData, expectedOutputValues, "testDynamicAxisSlice: Forward prop results do not match expected results");
};
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), 0, 1);
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), 0, 2);
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), -1, 0);
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), -2, 0);
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), 0, -1);
testDynamicAxisSlice(Axis::DefaultDynamicAxis(), 1, 0);
}
}
void TestTensorPlus(size_t numAxesLeftOperand, size_t numAxesRightOperand, const DeviceDescriptor& device, bool useConstantInputsOnly)
{
srand(1);
size_t maxDimSize = 15;
NDShape leftInputShape(numAxesLeftOperand);
for (size_t i = 0; i < numAxesLeftOperand; ++i)
leftInputShape[i] = (rand() % maxDimSize) + 1;
NDShape rightInputShape(numAxesRightOperand);
for (size_t i = 0; i < std::min(numAxesLeftOperand, numAxesRightOperand); ++i)
rightInputShape[i] = leftInputShape[i];
for (size_t i = std::min(numAxesLeftOperand, numAxesRightOperand); i < numAxesRightOperand; ++i)
rightInputShape[i] = (rand() % maxDimSize) + 1;
std::vector<ElementType> leftInputData(leftInputShape.TotalSize());
for (size_t i = 0; i < leftInputData.size(); ++i)
leftInputData[i] = ((ElementType)rand()) / RAND_MAX;
auto leftInputValueShape = leftInputShape.AppendShape({ 1, 1 });
auto leftInputValue = MakeSharedObject<NDArrayView>(leftInputValueShape, leftInputData, true);
std::vector<ElementType> rightInputData(rightInputShape.TotalSize());
for (size_t i = 0; i < rightInputData.size(); ++i)
rightInputData[i] = ((ElementType)rand()) / RAND_MAX;
auto rightInputValueShape = rightInputShape.AppendShape({ 1, 1 });
auto rightInputValue = MakeSharedObject<NDArrayView>(rightInputValueShape, rightInputData, true);
Variable leftInputVar, rightInputVar;
if (useConstantInputsOnly)
{
leftInputValue = leftInputValue->DeepClone(device, false);
rightInputValue = rightInputValue->DeepClone(device, false);
leftInputVar = Parameter(leftInputValue, L"leftInput");
rightInputVar = Parameter(rightInputValue, L"rightInput");
}
else
{
leftInputVar = InputVariable(leftInputShape, AsDataType<ElementType>(), true, L"leftInput");
rightInputVar = InputVariable(rightInputShape, AsDataType<ElementType>(), true, L"rightInput");
}
auto plusFunc = Plus(leftInputVar, rightInputVar);
NDShape outputShape = plusFunc->Output().Shape();
if (!useConstantInputsOnly)
outputShape = outputShape.AppendShape({ 1, 1 });
std::vector<ElementType> outputData(outputShape.TotalSize());
ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputShape, outputData, false));
std::unordered_map<Variable, ValuePtr> outputs = { { plusFunc->Output(), outputValue } };
BackPropStatePtr backPropState;
if (useConstantInputsOnly)
backPropState = plusFunc->Forward(std::unordered_map<Variable, ValuePtr>({}), outputs, device, { plusFunc->Output() });
else
backPropState = plusFunc->Forward({ { leftInputVar, MakeSharedObject<Value>(leftInputValue) }, { rightInputVar, MakeSharedObject<Value>(rightInputValue) } }, outputs, device, { plusFunc->Output() });
// Perform backprop
std::vector<ElementType> rootGradientsData(outputShape.TotalSize(), 1);
ValuePtr rootGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputShape, rootGradientsData, true));
std::vector<ElementType> leftInputGradientsData(leftInputValueShape.TotalSize());
ValuePtr leftInputGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(leftInputValueShape, leftInputGradientsData, false));
std::vector<ElementType> rightInputGradientsData(rightInputValueShape.TotalSize());
ValuePtr rightInputGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(rightInputValueShape, rightInputGradientsData, false));
std::unordered_map<Variable, ValuePtr> gradients = { { leftInputVar, leftInputGradientValue }, { rightInputVar, rightInputGradientValue } };
plusFunc->Backward(backPropState, { { plusFunc->Output(), rootGradientValue } }, gradients);
// Verify forward prop results
auto& smallerInput = (numAxesLeftOperand < numAxesRightOperand) ? leftInputData : rightInputData;
auto& largerInput = (numAxesLeftOperand < numAxesRightOperand) ? rightInputData : leftInputData;
std::vector<ElementType> expectedOutputValues = largerInput;
for (size_t i = 0; i < (expectedOutputValues.size() / smallerInput.size()); ++i)
{
for (size_t j = 0; j < smallerInput.size(); ++j)
expectedOutputValues[(i * smallerInput.size()) + j] += smallerInput[j];
}
FloatingPointVectorCompare(outputData, expectedOutputValues, "Forward prop results do not match expected results");
auto& smallerInputGradients = (numAxesLeftOperand < numAxesRightOperand) ? leftInputGradientsData : rightInputGradientsData;
auto& largerInputGradients = (numAxesLeftOperand < numAxesRightOperand) ? rightInputGradientsData : leftInputGradientsData;
std::vector<ElementType> expectedLargerInputGradientValues(largerInputGradients.size(), (ElementType)1);
std::vector<ElementType> expectedSmallerInputGradientValues(smallerInputGradients.size(), (ElementType)(largerInputGradients.size() / smallerInputGradients.size()));
FloatingPointVectorCompare(smallerInputGradients, expectedSmallerInputGradientValues, "TestTimesAndPlus: Backward prop results do not match expected results");
FloatingPointVectorCompare(largerInputGradients, expectedLargerInputGradientValues, "TestTimesAndPlus: Backward prop results do not match expected results");
}