void DoCommand(const ConfigParameters& configRoot) { ConfigArray command = configRoot("command", "train"); ConfigParameters config = configRoot(command[0]); ConfigParameters readerConfig(config("reader")); readerConfig.Insert("traceLevel", config("traceLevel", "0")); ConfigArray minibatchSize = config("minibatchSize", "256"); intargvector mbSizeArr = minibatchSize; size_t mbSize = 20000; // mbSizeArr[0]; size_t epochSize = config("epochSize", "0"); if (epochSize == 0) { epochSize = requestDataSize; } ConfigParameters configFeatures = readerConfig(L"features"); size_t dimFeatures = configFeatures("dim"); ConfigParameters configLabels = readerConfig(L"labels"); size_t dimLabels = configLabels("labelDim"); ConfigParameters configSgd = config("SGD"); std::wstring modelPath = configSgd("modelPath"); StreamMinibatchInputs inputMatrices; StreamMinibatchInputs outputMatrices; std::wstring inputName = L"features"; std::wstring outputName = L"CE.BFF.FF.P"; int deviceId = 0; auto matrix = make_shared<Matrix<ElemType>>(dimFeatures, mbSize, deviceId); MBLayoutPtr pMBLayout = make_shared<MBLayout>(); inputMatrices.AddInput(inputName, matrix, pMBLayout, TensorShape(dimFeatures)); outputMatrices.AddInput(outputName, make_shared<Matrix<ElemType>>(dimLabels, mbSize, deviceId), pMBLayout, TensorShape(dimLabels)); std::map<std::wstring, std::vector<ElemType>*> input; std::map<std::wstring, std::vector<ElemType>*> output; std::vector<ElemType>* arr = input[inputName] = new std::vector<ElemType>(dimFeatures * mbSize); output[outputName] = new std::vector<ElemType>(dimLabels * mbSize); Eval<ElemType> eval(config); auto dataReader = make_shared<DataReader>(readerConfig); eval.CreateNetwork(Microsoft::MSR::CNTK::ToLegacyString(Microsoft::MSR::CNTK::ToUTF8(modelPath))); dataReader->StartMinibatchLoop(mbSize, 0, inputMatrices.GetStreamDescriptions(), epochSize); eval.StartEvaluateMinibatchLoop(outputName); while (dataReader->GetMinibatch(inputMatrices)) { void* data = (void*) arr->data(); size_t dataSize = arr->size() * sizeof(ElemType); void* mat = &(*matrix)(0, 0); size_t matSize = matrix->GetNumElements() * sizeof(ElemType); memcpy_s(data, dataSize, mat, matSize); eval.Evaluate(input, output); } }
void TestReader(const ConfigParameters& configBase) { // int nonexistant = configBase("nonexistant"); // use to test global exception handler ConfigParameters config(configBase("mnistTest")); ConfigParameters readerConfig(config("reader")); readerConfig.Insert("traceLevel", config("traceLevel", "0")); size_t mbSize = config("minibatchSize"); size_t epochSize = config("epochSize", "0"); if (epochSize == 0) { epochSize = requestDataSize; } DataReader dataReader(readerConfig); // get names of features and labels std::vector<std::wstring> featureNames; std::vector<std::wstring> labelNames; GetFileConfigNames(readerConfig, featureNames, labelNames); // setup minibatch matrices int deviceId = 0; auto featuresMatrix = make_shared<Matrix<ElemType>>(deviceId); auto labelsMatrix = make_shared<Matrix<ElemType>>(deviceId); MBLayoutPtr pMBLayout = make_shared<MBLayout>(); StreamMinibatchInputs matrices; matrices.AddInput(featureNames[0], featuresMatrix, pMBLayout, TensorShape()); matrices.AddInput(labelNames[0], labelsMatrix , pMBLayout, TensorShape()); auto start = std::chrono::system_clock::now(); int epochs = config("maxEpochs"); epochs *= 2; for (int epoch = 0; epoch < epochs; epoch++) { dataReader.StartMinibatchLoop(mbSize, epoch, epochSize); int i = 0; while (dataReader.GetMinibatch(matrices)) { Matrix<ElemType>& features = matrices.GetInputMatrix<ElemType>(featureNames[0]); Matrix<ElemType>& labels = matrices.GetInputMatrix<ElemType>(labelNames[0]); if (labels.GetNumRows() == 0) { fprintf(stderr, "%4d: features dim: %lu x %lu - [%.8g, %.8g, ...]\n", i++, features.GetNumRows(), features.GetNumCols(), features(0, 0), features(0, 1)); } else { fprintf(stderr, "%4d: features dim: %lu x %lu - [%.8g, %.8g, ...] label dim: %lu x %lu - [%d, %d, ...]\n", i++, features.GetNumRows(), features.GetNumCols(), features(0, 0), features(0, 1), labels.GetNumRows(), labels.GetNumCols(), (int) labels(0, 0), (int) labels(0, 1)); } } } auto end = std::chrono::system_clock::now(); auto elapsed = end - start; fprintf(stderr, "%f seconds elapsed", (float) (std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count()) / 1000); }