bool SlaveTrainer::LayerActivation(double* up_ao, const double* low_ao, Matrix& w, const EActType eActType)
{
if(!up_ao || !low_ao || w.IsNull())
return false;
double e = 0.0;
for(int32_t j = 0; j < w.Cols(); j++)
{
// forward propagation
up_ao[j] = w[w.Rows()-1][j]; // bias
for(int32_t i = 0; i < w.Rows() - 1; i++)
up_ao[j] += w[i][j] * low_ao[i];
// activation
if(eActType == _ACT_SOFTMAX)
e += exp(up_ao[j]);
else
up_ao[j] = Activation::Activate(up_ao[j], eActType);
}
if(eActType == _ACT_SOFTMAX)
{ // softmax
for(int32_t j = 0; j < w.Cols(); j++)
up_ao[j] = exp(up_ao[j]) / e;
}
return true;
}
Matrix operator-(Matrix a,Matrix b){
Matrix result(a.Rows(),a.Cols());
for(int i = 0 ; i < a.Rows() ; i++)
for( int j = 0 ; j < a.Cols() ; j++)
result.Set(j,i,a.Get(j,i) - b.Get(j,i));
return result;
}
// C = A*b
Matrix operator*(const Matrix& A, const double b) {
Matrix C(A.Rows(),A.Cols());
for (size_t j=0; j<A.Cols(); j++)
for (size_t i=0; i<A.Rows(); i++)
C.data[j][i] = A.data[j][i] * b;
return C;
}
// C = a*B
Matrix operator*(const double a, const Matrix& B) {
Matrix C(B.Rows(),B.Cols());
for (size_t j=0; j<B.Cols(); j++)
for (size_t i=0; i<B.Rows(); i++)
C.data[j][i] = a * B.data[j][i];
return C;
}
Matrix operator*(const Matrix &left, const double right)
{
Matrix M(left.Rows(), left.Cols());
int i, j;
for(i=0; i<left.Rows(); i++)
for(j=0; j<left.Cols(); j++)
M[i][j] = left[i][j] * right;
return M;
}
Matrix operator*(Matrix a,double b){
Matrix result(a.Rows(),a.Cols());
for( int i = 0 ; i < a.Rows() ; i++ ){
for( int j = 0 ; j < a.Cols() ; j++ ){
result.Set(j,i,a.Get(j,i) * b);
}
}
return result;
}
Matrix TransposeMatrix(const Matrix& other)
{
Matrix M(other.Cols(), other.Rows());
int i, j;
for(i=0; i<other.Rows(); i++)
for(j=0; j<other.Cols(); j++)
M[j][i] = other[i][j];
return M;
}
// standard matrix-vector product
vector<double> MatVec(const Matrix& A, const vector<double>& v) {
vector<double> res(0.0, A.Rows());
if (A.Cols() != v.size()) {
cerr << "MatVec: incompatible matrix/vector sizes in A*v\n";
} else {
for (size_t i=0; i<A.Rows(); i++)
for (size_t j=0; j<A.Cols(); j++)
res[i] += A(i,j)*v[j];
}
return res;
}
// column/row vector dot product, something else for matrices?
double Dot(const Matrix& A, const Matrix& B) {
double sum=0.0;
if ((A.Cols() != B.Cols()) || (A.Rows() != B.Rows())) {
cerr << "Dot error, matrix objects must be the same size\n";
} else {
for (size_t j=0; j<A.Cols(); j++)
for (size_t i=0; i<A.Rows(); i++)
sum += A(i,j) * B(i,j);
}
return sum;
}
Matrix ScalarMultiply(const Matrix &left, const Matrix &right)
{
wxASSERT(left.Rows() == right.Rows());
wxASSERT(left.Cols() == right.Cols());
Matrix M(left.Rows(), left.Cols());
int i, j;
for(i=0; i<left.Rows(); i++)
for(j=0; j<left.Cols(); j++)
M[i][j] = left[i][j] * right[i][j];
return M;
}
Matrix operator+(const Matrix &left, const Matrix &right)
{
wxASSERT(left.Rows() == right.Rows());
wxASSERT(left.Cols() == right.Cols());
Matrix M(left.Rows(), left.Cols());
int i, j;
for(i=0; i<left.Rows(); i++)
for(j=0; j<left.Cols(); j++)
M[i][j] = left[i][j] + right[i][j];
return M;
}
Vector operator*(const Matrix &left, const Vector &right)
{
wxASSERT(left.Cols() == right.Len());
Vector v(left.Rows());
int i, j;
for(i=0; i<left.Rows(); i++) {
v[i] = 0.0;
for(j=0; j<left.Cols(); j++)
v[i] += left[i][j] * right[j];
}
return v;
}
Matrix operator*(const Matrix &mat1, const Matrix &mat2)
{
Matrix result;
for (int ix = 0; ix < mat1.Rows(); ++ix) {
for (int jx = 0; jx < mat1.Cols(); ++jx) {
result(ix, jx) = 0;
for (int kx = 0; kx < mat1.Cols(); ++kx)
result(ix, jx) += mat1(ix, kx) * mat2(kx, jx);
}
}
return result;
}
Matrix MatrixConcatenateCols(const Matrix& left, const Matrix& right)
{
wxASSERT(left.Rows() == right.Rows());
Matrix M(left.Rows(), left.Cols() + right.Cols());
int i, j;
for(i=0; i<left.Rows(); i++) {
for(j=0; j<left.Cols(); j++)
M[i][j] = left[i][j];
for(j=0; j<right.Cols(); j++)
M[i][j+left.Cols()] = right[i][j];
}
return M;
}
Matrix MatrixMultiply(const Matrix &left, const Matrix &right)
{
wxASSERT(left.Cols() == right.Rows());
Matrix M(left.Rows(), right.Cols());
int i, j, k;
for(i=0; i<left.Rows(); i++)
for(j=0; j<right.Cols(); j++) {
M[i][j] = 0.0;
for(k=0; k<left.Cols(); k++)
M[i][j] += left[i][k] * right[k][j];
}
return M;
}
// backward substitution on U*x = b, returning x as a new vector<double>
// U and b remain unchanged in this operation
vector<double> BackSub(const Matrix& U, const vector<double>& b) {
if (U.Rows() != b.size() || U.Rows() != U.Cols()) {
cerr << "BackSub error, illegal matrix/vector dimensions\n";
cerr << " Matrix is " << U.Rows() << " x " << U.Cols()
<< ", rhs is " << b.size() << " x 1\n";
return vector<double>(0);
}
// create output vector, call existing BackSub routine, and return
vector<double> x(0.0, U.Cols());
if (BackSub(U, x, b) != 0)
cerr << "BackSub Warning: error in BackSub call\n";
return x;
}
// forward substitution on L*x = b, returning x as a new vector<double>
// L and b remain unchanged in this operation
vector<double> FwdSub(const Matrix& L, const vector<double>& b) {
// check that matrix sizes match
if (L.Rows() != b.size() || L.Rows() != L.Cols()) {
cerr << "FwdSub error, illegal matrix/vector dimensions\n";
cerr << " Matrix is " << L.Rows() << " x " << L.Cols()
<< ", rhs is " << b.size() << " x 1\n";
return vector<double>(0);
}
// create output vector and return
vector<double> x(0.0, L.Cols());
if (FwdSub(L, x, b) != 0)
cerr << "FwdSub Warning: error in FwdSub call\n";
return x;
}
// forward substitution on the linear system L*X = B, returning X as a new Matrix
// L and B remain unchanged in this operation
Matrix FwdSub(const Matrix& L, const Matrix& B) {
// check that matrix sizes match
if (L.Rows() != B.Rows() || L.Rows() != L.Cols()) {
cerr << "FwdSub error, illegal matrix/vector dimensions\n";
cerr << " Matrix is " << L.Rows() << " x " << L.Cols()
<< ", rhs is " << B.Rows() << " x " << B.Cols() << endl;
return Matrix(0,0);
} else {
// create new Matrix for output and call existing BackSub routine
Matrix X(L.Rows(),B.Cols());
if (FwdSub(L, X, B) != 0)
cerr << "FwdSub Warning: error in FwdSub call\n";
return X;
}
}
//exponential of a matrix (element-wise)
Matrix exp(const Matrix& A){
size_t zeros = UNDEFINED;
Matrix E(A.Rows(), A.Cols());
for (size_t a = 0; a < A.Rows(); a++)
for (size_t b = 0; b < A.Cols(); b++) {
E.M[a * (A.Cols()) + b] = exp(A.M[a * (A.Cols()) + b]);
CheckZero(&zeros, E.M[a * (A.Cols()) + b]);
}
if (zeros == ZERO)
E._isZero = true;
return E;
}
// solves a linear system A*x = b, returning x as a new vector<double>
// A and b are modified in this operation; x holds the result
vector<double> Solve(Matrix& A, vector<double>& b) {
// check that matrix sizes match
if (A.Rows() != b.size() || A.Rows() != A.Cols()) {
cerr << "Solve error, illegal matrix/vector dimensions\n";
cerr << " Matrix is " << A.Rows() << " x " << A.Cols()
<< ", rhs is " << b.size() << " x 1\n";
return vector<double>(0);
}
// create output, call existing Solve routine and return
vector<double> x = b;
if (Solve(A, x, b) != 0)
cerr << "Solve: error in in-place Solve call\n";
return x;
}
// matrix Frobenius norm (column/row vector 2-norm)
double Norm(const Matrix& A) {
double sum=0.0;
for (size_t j=0; j<A.Cols(); j++)
for (size_t i=0; i<A.Rows(); i++)
sum += A(i,j)*A(i,j);
return sqrt(sum);
}
// solves a linear system A*X = B, returning X as a new Matrix
Matrix Solve(Matrix& A, Matrix& B) {
// check that matrix sizes match
if (A.Rows() != B.Rows() || A.Rows() != A.Cols()) {
cerr << "Solve error, illegal matrix/vector dimensions\n";
cerr << " Matrix is " << A.Rows() << " x " << A.Cols()
<< ", rhs is " << B.Rows() << " x " << B.Cols() << endl;
return Matrix(0,0);
} else {
// create new Mat for output, and call existing Solve routine
Matrix X(A.Rows(),B.Cols());
if (Solve(A, X, B) != 0)
cerr << "Solve: error in in-place Solve call\n";
return X;
}
}
Matrix operator*(Matrix a,Matrix b){
int i,j,k;
Matrix result(a.Rows(),b.Cols());
if(a.Cols() != b.Rows())
cerr << "WARNING : matrix inner dimen. not equal\n";
for(i=0;i<a.Rows();i++){
for(k=0;k<b.Cols();k++){
result.Set(i,k,0);
for(j=0;j<a.Cols();j++){
result.Set(i,k,(a.Get(i,j) * b.Get(j,k)) + result.Get(i,k));
}
}
}
return result;
}
bool InvertMatrix(const Matrix& input, Matrix& Minv)
{
// Very straightforward implementation of
// Gauss-Jordan elimination to invert a matrix.
// Returns true if successful
wxASSERT(input.Rows() == input.Cols());
int N = input.Rows();
int i, j, k;
Matrix M = input;
Minv = IdentityMatrix(N);
// Do the elimination one column at a time
for(i=0; i<N; i++) {
// Pivot the row with the largest absolute value in
// column i, into row i
double absmax = 0.0;
int argmax=0;
for(j=i; j<N; j++)
if (fabs(M[j][i]) > absmax) {
absmax = fabs(M[j][i]);
argmax = j;
}
// If no row has a nonzero value in that column,
// the matrix is singular and we have to give up.
if (absmax == 0)
return false;
if (i != argmax) {
M.SwapRows(i, argmax);
Minv.SwapRows(i, argmax);
}
// Divide this row by the value of M[i][i]
double factor = 1.0 / M[i][i];
M[i] = M[i] * factor;
Minv[i] = Minv[i] * factor;
// Eliminate the rest of the column
for(j=0; j<N; j++) {
if (j==i)
continue;
if (fabs(M[j][i]) > 0) {
// Subtract a multiple of row i from row j
double factor = M[j][i];
for(k=0; k<N; k++) {
M[j][k] -= (M[i][k] * factor);
Minv[j][k] -= (Minv[i][k] * factor);
}
}
}
}
return true;
}
EqSys::EqSys(const Matrix& coeff, const Vector& res)
{
if((coeff.Rows() != coeff.Cols()) || (res.Size() != coeff.Rows()))
{
std::cout << "Zle wymiary macierzy/wektora do ukl rown\n";
std::terminate();
}
A=coeff; b=res; size=A.Rows();
}
// matrix one norm (column vector one norm, row vector infinity norm)
double OneNorm(const Matrix& A) {
double mx=0.0;
for (size_t j=0; j<A.Cols(); j++) {
double sum=0.0;
for (size_t i=0; i<A.Rows(); i++)
sum += std::abs(A(i,j));
mx = std::max(mx,sum);
}
return mx;
}
static void MakeWater (Matrix& M, float waterlevel = 0.0,
float scale_x = 80, float scale_y = 120, float scale_z = 0.1)
{
for (int i = M.Rlo(); i <= M.Rhi(); i++)
for (int j = M.Clo(); j <= M.Chi(); j++)
if (M(i,j) < waterlevel) { // apply noise to values below
float x = scale_x * (i-M.Rlo())/M.Rows(), // the water level to simulate
y = scale_y * (j-M.Clo())/M.Cols(); // waves on the sea
M(i,j) = scale_z * Noise(x,y,0);
}
}
void Activation::InitTransformMatrix(Matrix& w, const EActType eActType)
{
if(eActType == _ACT_SIGMOID)
w.Init_RandNormal(0.0, sqrt(1.0 / (double)w.Rows()));
else if(eActType == _ACT_TANH)
w.Init_RandNormal(0.0, sqrt(1.0 / (double)w.Rows()));
else if(eActType == _ACT_RELU)
w.Init_RandNormal(0.0, sqrt(1.0 / (double)w.Rows()));
else if(eActType == _ACT_SOFTMAX)
w.Init_RandNormal(0.0, sqrt(1.0 / (double)w.Rows()));
else
w.Init_RandUni(-4.0 * sqrt(6.0 / (double)(w.Rows() + w.Cols())), 4.0 * sqrt(6.0 / (double)(w.Rows() + w.Cols())));
}
// forward substitution on L*x = b, filling in an existing vector<double) x
// L and b remain unchanged in this operation; x holds the result
int FwdSub(const Matrix& L, vector<double>& x, const vector<double>& b) {
// check that matrix sizes match
if (L.Rows() != b.size() || L.Rows() != L.Cols() ||
x.size() != L.Rows()) {
cerr << "FwdSub error, illegal matrix/vector dimensions\n";
cerr << " Matrix is " << L.Rows() << " x " << L.Cols()
<< ", rhs is " << b.size() << " x 1"
<< ", solution is " << x.size() << " x 1\n";
return 1;
}
// copy B into X
x = b;
// analyze matrix for magnitude
double Lmax = InfNorm(L);
// perform column-oriented Forwards Substitution algorithm
for (long int j=0; j<L.Rows(); j++) {
// check for nonzero matrix diagonal
if (fabs(L.data[j][j]) < STOL*Lmax) {
cerr << "FwdSub error: singular matrix!\n";
return 1;
}
// solve for this row of solution
x[j] /= L.data[j][j];
// update all remaining rhs
for (long int i=j+1; i<L.Rows(); i++)
x[i] -= L.data[j][i]*x[j];
}
// return success
return 0;
}
// backward substitution on U*x = b, filling in an existing vector<double> x
int BackSub(const Matrix& U, vector<double>& x, const vector<double>& b) {
// check that matrix sizes match
if (U.Rows() != b.size() || U.Rows() != U.Cols() || x.size() != U.Rows()) {
cerr << "BackSub error, incompatible matrix/vector dimensions\n";
cerr << " Matrix is " << U.Rows() << " x " << U.Cols()
<< ", rhs is " << b.size() << " x 1"
<< ", solution is " << x.size() << " x 1\n";
return 1;
}
// copy b into x
x = b;
// analyze matrix for magnitude
double Umax = InfNorm(U);
// perform column-oriented Backward Substitution algorithm
for (long int j=U.Rows()-1; j>=0; j--) {
// check for nonzero matrix diagonal
if (fabs(U.data[j][j]) < STOL*Umax) {
cerr << "BackSub error: numerically singular matrix!\n";
return 1;
}
// solve for this row of solution
x[j] /= U.data[j][j];
// update all remaining rhs
for (long int i=0; i<j; i++)
x[i] -= U.data[j][i]*x[j];
}
// return success
return 0;
}