/*static*/ ValuePtr Value::Create(const NDShape& sampleShape, const std::vector<std::vector<ElementType>>& sequences, const DeviceDescriptor& device, bool readOnly/* = false*/)
{
size_t sampleSize = sampleShape.TotalSize();
NDMaskPtr deviceValueMask = CreateMask(sampleSize, sequences, device);
size_t maxSequenceLength = (deviceValueMask == nullptr) ? sequences[0].size() : deviceValueMask->Shape()[0];
size_t numSequences = sequences.size();
NDShape valueDataShape = sampleShape.AppendShape({ maxSequenceLength, numSequences });
NDArrayViewPtr valueData(new NDArrayView(AsDataType<ElementType>(), valueDataShape, DeviceDescriptor::CPUDevice()), [](ReferenceCount* ptr) { delete ptr; });
ElementType* dataBuffer = valueData->WritableDataBuffer<ElementType>();
for (size_t i = 0; i < numSequences; ++i)
std::copy(sequences[i].data(), sequences[i].data() + sequences[i].size(), dataBuffer + (maxSequenceLength * i * sampleSize));
NDArrayViewPtr deviceValueData;
if (device == DeviceDescriptor::CPUDevice())
{
if (readOnly)
deviceValueData = valueData->Alias(true);
else
deviceValueData = valueData;
}
else
{
deviceValueData = NDArrayViewPtr(new NDArrayView(AsDataType<ElementType>(), valueDataShape, device), [](ReferenceCount* ptr) { delete ptr; });
deviceValueData->CopyFrom(*valueData);
if (readOnly)
deviceValueData = deviceValueData->Alias(true);
}
return ValuePtr(new Value(deviceValueData, deviceValueMask), [](ReferenceCount* ptr) { delete ptr; });
}
void DistributedLearnerBase::ConvertToOrdered(const std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, std::vector<std::pair<Parameter, NDArrayViewPtr>>& result, std::unordered_map<Parameter, NDArrayViewPtr>* convertedGradientValues)
{
result.reserve(gradientValues.size());
result.clear();
if (convertedGradientValues)
convertedGradientValues->clear();
for (auto g : gradientValues)
{
NDArrayViewPtr p = g.second;
// convert sparse gradient to dense for accumulation
if (m_convertSparseToDense && p->GetStorageFormat() != StorageFormat::Dense)
{
NDArrayViewPtr pDense = MakeSharedObject<NDArrayView>(0, p->GetDataType(), p->Shape(), p->Device());
pDense->CopyFrom(*p);
p = pDense;
}
auto pair = std::make_pair(g.first, p);
result.push_back(pair);
if (convertedGradientValues)
convertedGradientValues->insert(pair);
}
std::sort(result.begin(), result.end(),
[](const std::pair<Parameter, NDArrayViewPtr>& a, const std::pair<Parameter, NDArrayViewPtr>& b) { return a.first.Uid() < b.first.Uid(); });
}
void RunEvaluationOneHidden(FunctionPtr evalFunc, const DeviceDescriptor& device)
{
const std::wstring inputNodeName = L"features";
const std::wstring outputNodeName = L"out.z_output";
Variable inputVar;
if (!GetInputVariableByName(evalFunc, inputNodeName, inputVar))
{
fprintf(stderr, "Input variable %S is not available.\n", inputNodeName.c_str());
throw("Input variable not found error.");
}
Variable outputVar;
if (!GetOutputVaraiableByName(evalFunc, outputNodeName, outputVar))
{
fprintf(stderr, "Output variable %S is not available.\n", outputNodeName.c_str());
throw("Output variable not found error.");
}
// Evaluate the network in several runs
size_t iterationCount = 4;
size_t numSamples = 3;
for (size_t t = 0; t < iterationCount; ++t)
{
std::vector<float> inputData(inputVar.Shape().TotalSize() * numSamples);
for (size_t i = 0; i < inputData.size(); ++i)
{
inputData[i] = static_cast<float>(i % 255);
}
NDShape inputShape = inputVar.Shape().AppendShape({1, numSamples});
ValuePtr inputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(inputShape, inputData, true));
ValuePtr outputValue;
std::unordered_map<Variable, ValuePtr> outputs = {{outputVar, outputValue}};
evalFunc->Forward({{inputVar, inputValue}}, outputs, device);
outputValue = outputs[outputVar];
NDShape outputShape = outputVar.Shape().AppendShape({1, numSamples});
std::vector<float> outputData(outputShape.TotalSize());
NDArrayViewPtr cpuArrayOutput = MakeSharedObject<NDArrayView>(outputShape, outputData, false);
cpuArrayOutput->CopyFrom(*outputValue->Data());
assert(outputData.size() == outputVar.Shape()[0] * numSamples);
fprintf(stderr, "Evaluation result:\n");
size_t dataIndex = 0;
auto outputDim = outputVar.Shape()[0];
for (size_t i = 0; i < numSamples; i++)
{
fprintf(stderr, "Iteration:%lu, Sample %lu:\n", t, i);
fprintf(stderr, "Ouput:");
for (size_t j = 0; j < outputDim; j++)
{
fprintf(stderr, "%f ", outputData[dataIndex++]);
}
fprintf(stderr, "\n");
}
}
}
void CheckValue(const ValuePtr testValue, const size_t dimension, const vector<vector<size_t>>& expectedData, const vector<size_t>& seqLenList, const vector<bool>& seqStartFlags = {})
{
// Check parameters
BOOST_TEST(expectedData.size() == seqLenList.size(), "Parameter error: the sequence number in the exepected data and sequence list does not match.");
for (size_t i = 0; i < expectedData.size(); i++)
{
if (expectedData[i].size() != seqLenList[i])
{
ReportFailure("Parameter erroe: the number of data for sequence %" PRIu64 " in the expected data does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".",
i, seqLenList[i], expectedData[i].size());
}
}
// Check shape
NDShape shape = testValue->Shape();
size_t valueRank = shape.Rank();
if (valueRank < 2 || valueRank > 3 || shape[0] != dimension)
{
ReportFailure("The shape of the value does not match\n");
}
size_t numOfSequences = valueRank == 2 ? 1 : shape[2];
if (numOfSequences != expectedData.size())
{
ReportFailure("The sequence number in the Value does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", expectedData.size(), numOfSequences);
}
CheckMask(testValue, seqLenList, seqStartFlags);
// Get data from Value
vector<ElementType> outputData(shape.TotalSize());
NDArrayViewPtr arrayOutput = MakeSharedObject<NDArrayView>(shape, outputData, false);
arrayOutput->CopyFrom(*testValue->Data());
size_t maxSeqLen = *max_element(seqLenList.begin(), seqLenList.end());
size_t oIndex = 0;
for (size_t seq = 0; seq < seqLenList.size(); seq++)
{
size_t seqLen = seqLenList[seq];
for (size_t sample = 0; sample < seqLen; sample++)
{
for (size_t c = 0; c < dimension; c++, oIndex++)
{
if (outputData[oIndex] != 0)
{
if (outputData[oIndex] != 1)
{
ReportFailure("OneHot vector contains value other than 0 and 1 at seqNo=%" PRIu64 " sampleNo=%" PRIu64 " position=%" PRIu64 "\n", seq, sample, c);
}
if (c != expectedData[seq][sample])
{
ReportFailure("OneHot Index does match at seqNo=%" PRIu64 ", sampleNo=%" PRIu64 ", expected: %" PRIu64 ", actual: %" PRIu64 "\n", seq, sample, expectedData[seq][sample], c);
}
}
}
}
// Skip mask data
oIndex += (maxSeqLen - seqLen) * dimension;
}
}
void RunEvaluationClassifier(FunctionPtr evalFunc, const DeviceDescriptor& device)
{
const std::wstring inputNodeName = L"features";
Variable inputVar;
if (!GetInputVariableByName(evalFunc, inputNodeName, inputVar))
{
fprintf(stderr, "Input variable %S is not available.\n", inputNodeName.c_str());
throw("Input variable not found error.");
}
// Evaluate the network in several runs
size_t iterationCount = 4;
unsigned int randSeed = 2;
srand(randSeed);
size_t numSamples = 3;
std::vector<float> inputData(inputVar.Shape().TotalSize() * numSamples);
for (size_t t = 0; t < iterationCount; ++t)
{
for (size_t i = 0; i < inputData.size(); ++i)
{
inputData[i] = ((float)rand()) / RAND_MAX;
}
// Create input data shape. Adding sequence length and numSamples as axes.
// Todo: remove sequence length when only numSamples is supported.
// Todo: add convenience APIs to simplify data preparation here.
NDShape inputShape = inputVar.Shape().AppendShape({1, numSamples});
ValuePtr inputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(inputShape, inputData, true));
// Define output.
ValuePtr outputValue;
auto outputVar = evalFunc->Output();
std::unordered_map<Variable, ValuePtr> outputs = {{outputVar, outputValue}};
// Evaluate the model
evalFunc->Forward({{inputVar, inputValue}}, outputs, device);
// Get output value
outputValue = outputs[outputVar];
// Todo: remove sequence length when only numSamples is supported.
// Todo: add convenience APIs to simplify retrieval of output results.
NDShape outputShape = outputVar.Shape().AppendShape({1, numSamples});
std::vector<float> outputData(outputShape.TotalSize());
NDArrayViewPtr cpuArrayOutput = MakeSharedObject<NDArrayView>(outputShape, outputData, false);
cpuArrayOutput->CopyFrom(*outputValue->Data());
assert(outputData.size() == outputVar.Shape()[0] * numSamples);
fprintf(stderr, "Evaluation result:\n");
size_t dataIndex = 0;
auto outputDim = outputVar.Shape()[0];
for (size_t i = 0; i < numSamples; i++)
{
fprintf(stderr, "Iteration:%lu, Sample %lu:\n", t, i);
fprintf(stderr, " ");
dataIndex = i * outputDim;
for (size_t j = 0; j < std::min((size_t)10, outputDim); j++)
{
fprintf(stderr, "%f ", outputData[dataIndex++]);
}
if (outputDim > 10)
{
fprintf(stderr, "...");
}
fprintf(stderr, "\n");
}
}
}
void TestTimesAndPlus(size_t inputDim,
size_t outputDim,
size_t numSamples,
const DeviceDescriptor& device,
size_t numIterations,
bool usePreAllocatedOutputs,
bool outputOnSpecifiedDevice,
bool testSaveAndReLoad,
unsigned int seed = 1)
{
Parameter timesParam(MakeSharedObject<NDArrayView>((ElementType)0.5, NDShape({ outputDim, inputDim }), device), L"timesParameters");
Parameter plusParam(MakeSharedObject<NDArrayView>((ElementType)1.2, std::initializer_list<size_t>({ outputDim }), device), L"plusParameters");
Variable inputVar({ inputDim }, AsDataType<ElementType>(), L"input");
auto timesAndPlusFunc = Plus(plusParam, Times(timesParam, inputVar));
if (testSaveAndReLoad)
SaveAndReloadModel<ElementType>(timesAndPlusFunc, { &inputVar, ×Param, &plusParam }, device);
srand(seed);
for (size_t iterIdx = 0; iterIdx < numIterations; ++iterIdx)
{
std::vector<ElementType> inputData(inputDim * numSamples);
for (size_t i = 0; i < inputData.size(); ++i)
inputData[i] = ((ElementType)rand()) / RAND_MAX;
NDShape inputShape = inputVar.Shape().AppendShape({ 1, numSamples });
ValuePtr inputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(inputShape, inputData.data(), inputData.size(), DeviceDescriptor::CPUDevice(), true));
NDShape outputShape = timesAndPlusFunc->Output().Shape().AppendShape({ 1, numSamples });
std::vector<ElementType> outputData(outputShape.TotalSize());
ValuePtr outputValue;
if (usePreAllocatedOutputs)
{
auto outputAllocationDevice = outputOnSpecifiedDevice ? device : DeviceDescriptor::CPUDevice();
if (outputAllocationDevice.Type() == DeviceKind::CPU)
outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputShape, outputData.data(), outputData.size(), outputAllocationDevice, false));
else
outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), outputShape, outputAllocationDevice));
}
std::unordered_map<Variable, ValuePtr> outputs = { { timesAndPlusFunc->Output(), outputValue } };
auto backpropState = timesAndPlusFunc->Forward({ { inputVar, inputValue } }, outputs, device, { timesAndPlusFunc->Output() });
if (!usePreAllocatedOutputs)
outputValue = outputs[timesAndPlusFunc->Output()];
// Perform backprop
std::vector<ElementType> rootGradientsData(outputShape.TotalSize(), 1);
ValuePtr rootGradientValue;
if (device.Type() == DeviceKind::CPU)
rootGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(outputShape, rootGradientsData.data(), rootGradientsData.size(), device, true));
else
{
NDArrayViewPtr cpuArrayView = MakeSharedObject<NDArrayView>(outputShape, rootGradientsData.data(), rootGradientsData.size(), DeviceDescriptor::CPUDevice(), true);
NDArrayViewPtr gpuArrayView = MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), outputShape, device);
gpuArrayView->CopyFrom(*cpuArrayView);
rootGradientValue = MakeSharedObject<Value>(gpuArrayView);
}
std::vector<ElementType> plusParameterGradientData(plusParam.Shape().TotalSize());
std::vector<ElementType> timesParameterGradientData(timesParam.Shape().TotalSize());
ValuePtr plusParameterGradientValue, timesParameterGradientValue;
if (usePreAllocatedOutputs)
{
auto outputAllocationDevice = outputOnSpecifiedDevice ? device : DeviceDescriptor::CPUDevice();
if (outputAllocationDevice.Type() == DeviceKind::CPU)
{
plusParameterGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(plusParam.Shape(), plusParameterGradientData.data(), plusParameterGradientData.size(), outputAllocationDevice, false));
timesParameterGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(timesParam.Shape(), timesParameterGradientData.data(), timesParameterGradientData.size(), outputAllocationDevice, false));
}
else
{
plusParameterGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), plusParam.Shape(), outputAllocationDevice));
timesParameterGradientValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), timesParam.Shape(), outputAllocationDevice));
}
}
std::unordered_map<Variable, ValuePtr> paramGradients = { { plusParam, plusParameterGradientValue }, { timesParam, timesParameterGradientValue } };
timesAndPlusFunc->Backward(backpropState, { { timesAndPlusFunc->Output(), rootGradientValue } }, paramGradients);
if (!usePreAllocatedOutputs)
{
plusParameterGradientValue = paramGradients[plusParam];
timesParameterGradientValue = paramGradients[timesParam];
}
// Verify forward prop results
if (!usePreAllocatedOutputs || (outputOnSpecifiedDevice && (device.Type() != DeviceKind::CPU)))
{
NDArrayViewPtr cpuArrayView = MakeSharedObject<NDArrayView>(outputShape, outputData.data(), outputData.size(), DeviceDescriptor::CPUDevice(), false);
cpuArrayView->CopyFrom(*outputValue->Data());
}
std::vector<ElementType> expectedOutputValues(outputShape.TotalSize());
for (size_t i = 0; i < numSamples; ++i)
{
ElementType expectedVal = (ElementType)1.2;
for (size_t j = 0; j < inputDim; ++j)
expectedVal += (ElementType)(inputData[i * inputDim + j] * 0.5);
for (size_t j = 0; j < outputDim; ++j)
expectedOutputValues[i * outputDim + j] = expectedVal;
}
FloatingPointVectorCompare(outputData, expectedOutputValues, "TestTimesAndPlus: Forward prop results do not match expected results");
// Verify backward prop results
if (device.Type() != DeviceKind::CPU)
{
NDArrayViewPtr cpuArrayView = MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), plusParam.Shape(), DeviceDescriptor::CPUDevice());
cpuArrayView->CopyFrom(*plusParameterGradientValue->Data());
const ElementType* cpuArrayViewBuffer = cpuArrayView->DataBuffer<ElementType>();
memcpy(plusParameterGradientData.data(), cpuArrayViewBuffer, plusParam.Shape().TotalSize() * sizeof(ElementType));
cpuArrayView = MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), timesParam.Shape(), DeviceDescriptor::CPUDevice());
cpuArrayView->CopyFrom(*timesParameterGradientValue->Data());
cpuArrayViewBuffer = cpuArrayView->DataBuffer<ElementType>();
memcpy(timesParameterGradientData.data(), cpuArrayViewBuffer, timesParam.Shape().TotalSize() * sizeof(ElementType));
}
for (size_t i = 0; i < outputDim; ++i)
if (plusParameterGradientData[i] != numSamples)
throw std::runtime_error("TestTimesAndPlus: Backprop prop results do not match expected results for Plus params gradients");
std::vector<ElementType> expectedTimesParamsGradientValues(timesParam.Shape().TotalSize());
for (size_t i = 0; i < inputDim; ++i)
{
ElementType expectedVal = 0;
for (size_t j = 0; j < numSamples; ++j)
expectedVal += inputData[j * inputDim + i];
for (size_t j = 0; j < outputDim; ++j)
expectedTimesParamsGradientValues[i * outputDim + j] = expectedVal;
}
FloatingPointVectorCompare(timesParameterGradientData, expectedTimesParamsGradientValues, "TestTimesAndPlus: Backprop prop results do not match expected results for Times params gradients");
}
}
void TestNDArrayView(size_t numAxes, const DeviceDescriptor& device)
{
srand(1);
size_t maxDimSize = 15;
NDShape viewShape(numAxes);
for (size_t i = 0; i < numAxes; ++i)
viewShape[i] = (rand() % maxDimSize) + 1;
// Create a NDArrayView over a std::array
std::array<ElementType, 1> arrayData = { 3 };
auto arrayDataView = MakeSharedObject<NDArrayView>(NDShape({}), arrayData);
if (arrayDataView->template DataBuffer<ElementType>() != arrayData.data())
throw std::runtime_error("The DataBuffer of the NDArrayView does not match the original buffer it was created over");
std::vector<ElementType> data(viewShape.TotalSize());
ElementType scale = 19.0;
ElementType offset = -4.0;
for (size_t i = 0; i < viewShape.TotalSize(); ++i)
data[i] = offset + ((((ElementType)rand()) / RAND_MAX) * scale);
auto cpuDataView = MakeSharedObject<NDArrayView>(viewShape, data);
if (cpuDataView->template DataBuffer<ElementType>() != data.data())
throw std::runtime_error("The DataBuffer of the NDArrayView does not match the original buffer it was created over");
NDArrayViewPtr dataView;
if ((device.Type() == DeviceKind::CPU))
dataView = cpuDataView;
else
{
dataView = MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), viewShape, device);
dataView->CopyFrom(*cpuDataView);
}
if (dataView->Device() != device)
throw std::runtime_error("Device of NDArrayView does not match 'device' it was created on");
// Test clone
auto clonedView = dataView->DeepClone(false);
ElementType* first = nullptr;
const ElementType* second = cpuDataView->template DataBuffer<ElementType>();
NDArrayViewPtr temp1CpuDataView, temp2CpuDataView;
if ((device.Type() == DeviceKind::CPU))
{
if (dataView->DataBuffer<ElementType>() != data.data())
throw std::runtime_error("The DataBuffer of the NDArrayView does not match the original buffer it was created over");
first = clonedView->WritableDataBuffer<ElementType>();
}
else
{
temp1CpuDataView = MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), viewShape, DeviceDescriptor::CPUDevice());
temp1CpuDataView->CopyFrom(*clonedView);
first = temp1CpuDataView->WritableDataBuffer<ElementType>();
}
for (size_t i = 0; i < viewShape.TotalSize(); ++i)
{
if (first[i] != second[i])
throw std::runtime_error("The contents of the clone do not match expected");
}
first[0] += 1;
if ((device.Type() != DeviceKind::CPU))
clonedView->CopyFrom(*temp1CpuDataView);
if ((device.Type() == DeviceKind::CPU))
{
first = clonedView->WritableDataBuffer<ElementType>();
second = dataView->DataBuffer<ElementType>();
}
else
{
temp1CpuDataView = MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), viewShape, DeviceDescriptor::CPUDevice());
temp1CpuDataView->CopyFrom(*clonedView);
first = temp1CpuDataView->WritableDataBuffer<ElementType>();
temp2CpuDataView = MakeSharedObject<NDArrayView>(AsDataType<ElementType>(), viewShape, DeviceDescriptor::CPUDevice());
temp2CpuDataView->CopyFrom(*dataView);
second = temp2CpuDataView->DataBuffer<ElementType>();
}
if (first[0] != (second[0] + 1))
throw std::runtime_error("The clonedView's contents do not match expected");
// Test alias
auto aliasView = clonedView->Alias(true);
const ElementType* aliasViewBuffer = aliasView->DataBuffer<ElementType>();
const ElementType* clonedDataBuffer = clonedView->DataBuffer<ElementType>();
if (aliasViewBuffer != clonedDataBuffer)
throw std::runtime_error("The buffers underlying the alias view and the view it is an alias of are different!");
clonedView->CopyFrom(*dataView);
if (aliasViewBuffer != clonedDataBuffer)
throw std::runtime_error("The buffers underlying the alias view and the view it is an alias of are different!");
// Test readonliness
auto errorMsg = "Was incorrectly able to get a writable buffer pointer from a readonly view";
// Should not be able to get the WritableDataBuffer for a read-only view
VerifyException([&aliasView]() {
ElementType* aliasViewBuffer = aliasView->WritableDataBuffer<ElementType>();
aliasViewBuffer;
}, errorMsg);
// Should not be able to copy into a read-only view
VerifyException([&aliasView, &dataView]() {
aliasView->CopyFrom(*dataView);
}, errorMsg);
}
void CheckValue(const ValuePtr testValue, const NDShape& sampleShape, const vector<vector<ElementType>>& expectedData, const vector<size_t>& seqLenList, const vector<bool>& seqStartFlags = {})
{
size_t sampleSize = sampleShape.TotalSize();
// Check parameters
BOOST_TEST(expectedData.size() == seqLenList.size(), "Parameter error: the sequence number in the exepected data and sequence list does not match.");
for (size_t i = 0; i < expectedData.size(); i++)
{
if (expectedData[i].size() != seqLenList[i] * sampleSize)
{
ReportFailure("Parameter erroe: the number of data for sequence %" PRIu64 " in the expected data does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".",
i, seqLenList[i] * sampleSize, expectedData[i].size());
}
}
// Check shape
auto valueRank = testValue->Shape().Rank();
auto sampleRank = sampleShape.Rank();
auto shapeIsCorrect = !((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != testValue->Shape().SubShape(0, sampleRank)));
BOOST_TEST(shapeIsCorrect, "The Value does not have the expected shape.");
size_t numOfSequences;
if (valueRank == sampleShape.Rank() + 1)
{
// no batch axis, only sequence axis
numOfSequences = 1;
}
else
{
assert(valueRank == sampleShape.Rank() + 2);
numOfSequences = testValue->Shape()[valueRank - 1];
}
if (numOfSequences != expectedData.size())
{
ReportFailure("The sequence number in the Value does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", expectedData.size(), numOfSequences);
}
CheckMask(testValue, seqLenList, seqStartFlags);
// Get data from Value
vector<ElementType> outputData(testValue->Shape().TotalSize());
NDArrayViewPtr arrayOutput = MakeSharedObject<NDArrayView>(testValue->Shape(), outputData, false);
arrayOutput->CopyFrom(*testValue->Data());
size_t maxSeqLen = *max_element(seqLenList.begin(), seqLenList.end());
size_t oIndex = 0;
for (size_t seq = 0; seq < seqLenList.size(); seq++)
{
size_t seqLen = seqLenList[seq];
for (size_t sIndex = 0; sIndex < seqLen * sampleSize; sIndex++, oIndex++)
{
if (expectedData[seq][sIndex] != outputData[oIndex])
{
ReportFailure("Data does match at position %" PRIu64 ", expected: %f, actual: %f\n", oIndex, expectedData[seq][sIndex], outputData[oIndex]);
}
}
// Skip mask data
oIndex += (maxSeqLen - seqLen) * sampleSize;
}
}