STATIC Adagrad *Adagrad::instance(EasyCL *cl, float learningRate) { Adagrad *sgd = new Adagrad(cl); sgd->setLearningRate(learningRate); return sgd; }
void go(Config config) { Timer timer; int Ntrain; int Ntest; int numPlanes; int imageSize; float *trainData = 0; float *testData = 0; int *trainLabels = 0; int *testLabels = 0; int trainAllocateN = 0; int testAllocateN = 0; // int totalLinearSize; GenericLoaderv2 trainLoader( config.dataDir + "/" + config.trainFile ); Ntrain = trainLoader.getN(); numPlanes = trainLoader.getPlanes(); imageSize = trainLoader.getImageSize(); // GenericLoader::getDimensions( , &Ntrain, &numPlanes, &imageSize ); Ntrain = config.numTrain == -1 ? Ntrain : config.numTrain; // long allocateSize = (long)Ntrain * numPlanes * imageSize * imageSize; cout << "Ntrain " << Ntrain << " numPlanes " << numPlanes << " imageSize " << imageSize << endl; if( config.loadOnDemand ) { trainAllocateN = config.batchSize; // can improve this later } else { trainAllocateN = Ntrain; } trainData = new float[ (long)trainAllocateN * numPlanes * imageSize * imageSize ]; trainLabels = new int[trainAllocateN]; if( !config.loadOnDemand && Ntrain > 0 ) { trainLoader.load( trainData, trainLabels, 0, Ntrain ); } GenericLoaderv2 testLoader( config.dataDir + "/" + config.validateFile ); Ntest = testLoader.getN(); numPlanes = testLoader.getPlanes(); imageSize = testLoader.getImageSize(); Ntest = config.numTest == -1 ? Ntest : config.numTest; if( config.loadOnDemand ) { testAllocateN = config.batchSize; // can improve this later } else { testAllocateN = Ntest; } testData = new float[ (long)testAllocateN * numPlanes * imageSize * imageSize ]; testLabels = new int[testAllocateN]; if( !config.loadOnDemand && Ntest > 0 ) { testLoader.load( testData, testLabels, 0, Ntest ); } cout << "Ntest " << Ntest << " Ntest" << endl; timer.timeCheck("after load images"); const int inputCubeSize = numPlanes * imageSize * imageSize; float translate; float scale; int normalizationExamples = config.normalizationExamples > Ntrain ? Ntrain : config.normalizationExamples; if( !config.loadOnDemand ) { if( config.normalization == "stddev" ) { float mean, stdDev; NormalizationHelper::getMeanAndStdDev( trainData, normalizationExamples * inputCubeSize, &mean, &stdDev ); cout << " image stats mean " << mean << " stdDev " << stdDev << endl; translate = - mean; scale = 1.0f / stdDev / config.normalizationNumStds; } else if( config.normalization == "maxmin" ) { float mean, stdDev; NormalizationHelper::getMinMax( trainData, normalizationExamples * inputCubeSize, &mean, &stdDev ); translate = - mean; scale = 1.0f / stdDev; } else { cout << "Error: Unknown normalization: " << config.normalization << endl; return; } } else { if( config.normalization == "stddev" ) { float mean, stdDev; NormalizeGetStdDev normalizeGetStdDev( trainData, trainLabels ); BatchProcessv2::run( &trainLoader, 0, config.batchSize, normalizationExamples, inputCubeSize, &normalizeGetStdDev ); normalizeGetStdDev.calcMeanStdDev( &mean, &stdDev ); cout << " image stats mean " << mean << " stdDev " << stdDev << endl; translate = - mean; scale = 1.0f / stdDev / config.normalizationNumStds; } else if( config.normalization == "maxmin" ) { NormalizeGetMinMax normalizeGetMinMax( trainData, trainLabels ); BatchProcessv2::run( &trainLoader, 0, config.batchSize, normalizationExamples, inputCubeSize, &normalizeGetMinMax ); normalizeGetMinMax.calcMinMaxTransform( &translate, &scale ); } else { cout << "Error: Unknown normalization: " << config.normalization << endl; return; } } cout << " image norm translate " << translate << " scale " << scale << endl; timer.timeCheck("after getting stats"); // const int numToTrain = Ntrain; // const int batchSize = config.batchSize; EasyCL *cl = 0; if( config.gpuIndex >= 0 ) { cl = EasyCL::createForIndexedGpu( config.gpuIndex ); } else { cl = EasyCL::createForFirstGpuOtherwiseCpu(); } NeuralNet *net; net = new NeuralNet(cl); WeightsInitializer *weightsInitializer = 0; if( toLower( config.weightsInitializer ) == "original" ) { weightsInitializer = new OriginalInitializer(); } else if( toLower( config.weightsInitializer ) == "uniform" ) { weightsInitializer = new UniformInitializer( config.initialWeights ); } else { cout << "Unknown weights initializer " << config.weightsInitializer << endl; return; } // net->inputMaker<unsigned char>()->numPlanes(numPlanes)->imageSize(imageSize)->insert(); net->addLayer( InputLayerMaker::instance()->numPlanes(numPlanes)->imageSize(imageSize) ); net->addLayer( NormalizationLayerMaker::instance()->translate(translate)->scale(scale) ); if( !NetdefToNet::createNetFromNetdef( net, config.netDef, weightsInitializer ) ) { return; } // apply the trainer Trainer *trainer = 0; if( toLower( config.trainer ) == "sgd" ) { SGD *sgd = new SGD( cl ); sgd->setLearningRate( config.learningRate ); sgd->setMomentum( config.momentum ); sgd->setWeightDecay( config.weightDecay ); trainer = sgd; } else if( toLower( config.trainer ) == "anneal" ) { Annealer *annealer = new Annealer( cl ); annealer->setLearningRate( config.learningRate ); annealer->setAnneal( config.anneal ); trainer = annealer; } else if( toLower( config.trainer ) == "nesterov" ) { Nesterov *nesterov = new Nesterov( cl ); nesterov->setLearningRate( config.learningRate ); nesterov->setMomentum( config.momentum ); trainer = nesterov; } else if( toLower( config.trainer ) == "adagrad" ) { Adagrad *adagrad = new Adagrad( cl ); adagrad->setLearningRate( config.learningRate ); trainer = adagrad; } else if( toLower( config.trainer ) == "rmsprop" ) { Rmsprop *rmsprop = new Rmsprop( cl ); rmsprop->setLearningRate( config.learningRate ); trainer = rmsprop; } else if( toLower( config.trainer ) == "adadelta" ) { Adadelta *adadelta = new Adadelta( cl, config.rho ); trainer = adadelta; } else { cout << "trainer " << config.trainer << " unknown." << endl; return; } cout << "Using trainer " << trainer->asString() << endl; // trainer->bindTo( net ); // net->setTrainer( trainer ); net->setBatchSize( config.batchSize ); net->print(); bool afterRestart = false; int restartEpoch = 0; int restartBatch = 0; float restartAnnealedLearningRate = 0; int restartNumRight = 0; float restartLoss = 0; if( config.loadWeights && config.weightsFile != "" ) { cout << "loadingweights" << endl; afterRestart = WeightsPersister::loadWeights( config.weightsFile, config.getTrainingString(), net, &restartEpoch, &restartBatch, &restartAnnealedLearningRate, &restartNumRight, &restartLoss ); if( !afterRestart && FileHelper::exists( config.weightsFile ) ) { // try old trainingstring afterRestart = WeightsPersister::loadWeights( config.weightsFile, config.getOldTrainingString(), net, &restartEpoch, &restartBatch, &restartAnnealedLearningRate, &restartNumRight, &restartLoss ); } if( !afterRestart && FileHelper::exists( config.weightsFile ) ) { cout << "Weights file " << config.weightsFile << " exists, but doesnt match training options provided." << endl; cout << "Continue loading anyway (might crash, or weights might be completely inappropriate)? (y/n)" << endl; string response; cin >> response; if( response != "y" ) { cout << "Please either check the training options, or choose a weights file that doesnt exist yet" << endl; return; } }