TEST(testbackward, squareloss) {
// here's the plan:
// generate some input, randomly
// generate some expected output, randomly
// forward propagate
// calculate loss
// calculate gradInput
// change some of the inputs, forward prop, recalculate loss, check corresponds
// to the gradient
EasyCL *cl = EasyCL::createForFirstGpuOtherwiseCpu();
NeuralNet *net = new NeuralNet(cl, 3, 5);
net->addLayer(ForceBackpropLayerMaker::instance());
net->addLayer(SquareLossMaker::instance());
cout << net->asString() << endl;
int batchSize = 32;
net->setBatchSize(batchSize);
int inputCubeSize = net->getInputCubeSize();
int outputCubeSize = net->getOutputCubeSize();
int inputTotalSize = inputCubeSize * batchSize;
int outputTotalSize = outputCubeSize * batchSize;
cout << "inputtotalsize=" << inputTotalSize << " outputTotalSize=" << outputTotalSize << endl;
float *input = new float[inputTotalSize];
float *expectedOutput = new float[outputTotalSize];
WeightRandomizer::randomize(0, input, inputTotalSize, -2.0f, 2.0f);
WeightRandomizer::randomize(1, expectedOutput, outputTotalSize, -2.0f, 2.0f);
// now, forward prop
// net->input(input);
net->forward(input);
net->print();
// net->printOutput();
// calculate loss
float lossBefore = net->calcLoss(expectedOutput);
// calculate gradInput
net->backward(expectedOutput);
// modify input slightly
mt19937 random;
const int numSamples = 10;
for(int i = 0; i < numSamples; i++) {
int inputIndex;
WeightRandomizer::randomizeInts(i, &inputIndex, 1, 0, inputTotalSize);
// cout << "i=" << i << " index " << inputIndex << endl;
float oldValue = input[inputIndex];
// grad for this index is....
float grad = net->getLayer(2)->getGradInput()[inputIndex];
// cout << "grad=" << grad << endl;
// tweak slightly
float newValue = oldValue * 1.01f;
float inputDelta = newValue - oldValue;
float predictedLossChange = inputDelta * grad;
input[inputIndex] = newValue;
// cout << "oldvalue=" << oldValue << " newvalue=" << newValue << endl;
// forwardProp
net->forward(input);
input[inputIndex] = oldValue;
// net->printOutput();
float lossAfter = net->calcLoss(expectedOutput);
float lossChange = lossAfter - lossBefore;
cout << "idx=" << inputIndex << " predicted losschange=" << predictedLossChange << " actual=" << lossChange << endl;
}
delete[] expectedOutput;
delete[] input;
delete net;
delete cl;
}
TEST(testbackward, softmaxloss) {
// here's the plan:
// generate some input, randomly
// generate some expected output, randomly
// forward propagate
// calculate loss
// calculate gradInput
// change some of the inputs, forward prop, recalculate loss, check corresponds
// to the gradient
EasyCL *cl = EasyCL::createForFirstGpuOtherwiseCpu();
NeuralNet *net = new NeuralNet(cl, 5, 1);
net->addLayer(ForceBackpropLayerMaker::instance());
net->addLayer(SoftMaxMaker::instance());
cout << net->asString() << endl;
const int batchSize = 2;
net->setBatchSize(batchSize);
const int outputPlanes = net->getOutputPlanes();
int inputCubeSize = net->getInputCubeSize();
int outputCubeSize = net->getOutputCubeSize();
int inputTotalSize = inputCubeSize * batchSize;
int outputTotalSize = outputCubeSize * batchSize;
cout << "inputtotalsize=" << inputTotalSize << " outputTotalSize=" << outputTotalSize << endl;
float *input = new float[inputTotalSize];
float *expectedOutput = new float[outputTotalSize];
WeightRandomizer::randomize(0, input, inputTotalSize, 0.0f, 1.0f);
WeightRandomizer::randomize(1, expectedOutput, outputTotalSize, 0.0f, 1.0f);
// we should make the input and output a probability distribution I think
// so: add up the input, and divide each by that. do same for expectedoutput (?)
// normalizeAsProbabilityDistribution(input, inputTotalSize);
normalizeAsProbabilityDistribution(outputPlanes, expectedOutput, outputTotalSize);
// set all to zero, and one to 1, ie like labelled data
// for(int i = 0; i < outputTotalSize; i++) {
// expectedOutput[i] = 0;
// }
// for(int n = 0; n < batchSize; n++) {
// int chosenLabel = 0;
// WeightRandomizer::randomizeInts(n, &chosenLabel, 1, 0, net->getOutputPlanes());
// expectedOutput[ n * outputPlanes + chosenLabel ] = 1;
// }
// for(int i = 0; i < outputTotalSize; i++) {
// cout << "expected[" << i << "]=" << expectedOutput[i] << endl;
// }
//
// now, forward prop
// net->input(input);
net->forward(input);
net->print();
// net->printOutput();
// calculate loss
float lossBefore = net->calcLoss(expectedOutput);
// calculate gradInput
net->backward(expectedOutput);
// modify input slightly
mt19937 random;
const int numSamples = 10;
for(int i = 0; i < numSamples; i++) {
int inputIndex;
WeightRandomizer::randomizeInts(i, &inputIndex, 1, 0, inputTotalSize);
// cout << "i=" << i << " index " << inputIndex << endl;
float oldValue = input[inputIndex];
// grad for this index is....
float grad = net->getLayer(2)->getGradInput()[inputIndex];
// cout << "grad=" << grad << endl;
// tweak slightly
float newValue = oldValue * 1.001f;
float inputDelta = newValue - oldValue;
float predictedLossChange = inputDelta * grad;
input[inputIndex] = newValue;
// cout << "oldvalue=" << oldValue << " newvalue=" << newValue << endl;
// forwardProp
net->forward(input);
input[inputIndex] = oldValue;
// net->printOutput();
float lossAfter = net->calcLoss(expectedOutput);
float lossChange = lossAfter - lossBefore;
cout << "idx=" << inputIndex << " predicted losschange=" << predictedLossChange << " actual=" << lossChange << endl;
}
delete[] expectedOutput;
delete[] input;
delete net;
delete cl;
}
