/*
void
matrix_get_row(Matrix *m, index_t i, Matrix *v)
{
// Extracts the ith row of m to the column vector v
index_t j;
const index_t nc=numcols(m);
#ifdef _DBG_
len=numrows(v),
if (len!=nc)
error("Incompatible dimensions in matrix_get_row()");
#endif
for (j=0; j<nc; j++)
matrix_set_element(v,j,0, matrix_get_element(m,i,j));
return;
}
*/
int
matrix_is_symmetric(Matrix *xx)
{
// Checks a matrix for symmetry, aindex_t with other basic checks.
// Returns 1 if the matrix is
// symmetric, 0 if not symmetric (if something else is wrong).
int retval = 1;
index_t ii, jj, nrow_xx = numrows(xx), ncol_xx = numcols(xx);
Matrix *yy = matrix_new(nrow_xx, ncol_xx);
matrix_transpose(xx, yy);
matrix_scalar_multiply(yy, -1.0, yy);
matrix_add(xx, yy, yy);
for (ii=0; ii<nrow_xx; ii++)
for (jj=0; jj<ncol_xx; jj++)
if (matrix_get_element(yy, ii, jj) > DBL_EPSILON)
retval = 0;
matrix_free(yy);
return retval;
}
void gradient_descent(int num_threads, matrix_t* rolled_theta, unsigned int layer_sizes[], unsigned int num_layers,
unsigned int num_labels, matrix_t* X, matrix_t* y, double lamda, unsigned int iteration_number)
{
double start, end;
double cpu_time_used;
start = omp_get_wtime();
unsigned int theta_sizes[][2] = {{25, 401}, {10, 26}};
matrix_t* gradient;
unsigned int i;
for(i=0; i < iteration_number; i++)
{
NN_cost_function(num_threads, &gradient, rolled_theta, layer_sizes, num_layers, num_labels, X, y, lamda);
matrix_t* tmp;
tmp = matrix_scalar_multiply(gradient, ALPHA);
free_matrix(gradient);
gradient = tmp;
tmp = matrix_subtract(rolled_theta, gradient);
free_matrix(rolled_theta);
rolled_theta = tmp;
free_matrix(gradient);
if((i+1) % 100 == 0)
{
end = omp_get_wtime();
cpu_time_used = end - start;
matrix_list_t* theta = unroll_matrix_list(rolled_theta, num_layers-1, theta_sizes);
printf("iteration #%d, accuracy: %f, time used: %f\n", i+1, accuracy(theta, X, y), cpu_time_used);
free_matrix_list(theta);
}
}
free_matrix(rolled_theta);
}
double NN_cost_function(int num_threads, matrix_t** gradient, matrix_t* rolled_theta, unsigned int layer_sizes[], unsigned int num_layers,
unsigned int num_labels, matrix_t* X, matrix_t* y, double lamda)
{
unsigned int theta_sizes[][2] = {{25, 401}, {10, 26}};
matrix_list_t* theta = unroll_matrix_list(rolled_theta, num_layers-1, theta_sizes);
unsigned int m = X->rows;
//unsigned int n = X->cols;
matrix_list_t* theta_gradient_total = matrix_list_constructor(theta->num);
unsigned int i, j;
for(i=0; i<theta_gradient_total->num; i++)
{
theta_gradient_total->matrix_list[i] = matrix_constructor(theta->matrix_list[i]->rows, theta->matrix_list[i]->cols);
}
omp_set_num_threads(num_threads);
int nthreads, tid;
#pragma omp parallel private(nthreads, tid)
{
int indexes[2];
tid = omp_get_thread_num();
nthreads = omp_get_num_threads();
get_indexes(m, nthreads, tid, indexes);
unsigned int i, j;
matrix_t* temp;
matrix_t* temp2;
matrix_t* temp3;
matrix_list_t* theta_gradient = matrix_list_constructor(theta->num);
for(i=0; i<theta_gradient->num; i++)
{
theta_gradient->matrix_list[i] = matrix_constructor(theta->matrix_list[i]->rows, theta->matrix_list[i]->cols);
}
for(i=indexes[0]; i<indexes[1]; i++)
{
matrix_list_t* A = matrix_list_constructor(num_layers);
matrix_list_t* Z = matrix_list_constructor(num_layers-1);
matrix_list_t* delta = matrix_list_constructor(num_layers-1);
A->matrix_list[0] = row_to_vector(X, i);
temp = matrix_prepend_col(A->matrix_list[0], 1.0);
free_matrix(A->matrix_list[0]);
A->matrix_list[0] = matrix_transpose(temp);
free_matrix(temp);
for(j=0; j<num_layers-1; j++)
{
Z->matrix_list[j] = matrix_multiply(theta->matrix_list[j], A->matrix_list[j]);
temp = matrix_sigmoid(Z->matrix_list[j]);
A->matrix_list[j+1] = matrix_prepend_row(temp, 1.0);
free_matrix(temp);
}
temp = matrix_remove_row(A->matrix_list[num_layers-1]);
free_matrix(A->matrix_list[num_layers-1]);
A->matrix_list[num_layers-1] = temp;
matrix_t* result_matrix = matrix_constructor(1, num_labels);
for(j = 0; j < num_labels; j++)
{
if(vector_get(y, i) == j)
{
vector_set(result_matrix, j, 1.0);
}
}
temp = matrix_transpose(result_matrix);
free_matrix(result_matrix);
result_matrix= temp;
delta->matrix_list[1] = matrix_subtract(A->matrix_list[num_layers-1], result_matrix);
free_matrix(result_matrix);
matrix_t* theta_transpose = matrix_transpose(theta->matrix_list[1]);
temp = matrix_multiply(theta_transpose, delta->matrix_list[1]);
matrix_t* sig_gradient = matrix_sigmoid_gradient(Z->matrix_list[0]);
temp2 = matrix_prepend_row(sig_gradient, 1.0);
temp3 = matrix_cell_multiply(temp, temp2);
delta->matrix_list[0] = matrix_remove_row(temp3);
free_matrix(temp);
free_matrix(temp2);
free_matrix(temp3);
free_matrix(sig_gradient);
free_matrix(theta_transpose);
for(j=0; j<num_layers-1; j++)
{
matrix_t* A_transpose = matrix_transpose(A->matrix_list[j]);
temp = matrix_multiply(delta->matrix_list[j], A_transpose);
temp2 = matrix_add(theta_gradient->matrix_list[j], temp);
free_matrix(theta_gradient->matrix_list[j]);
theta_gradient->matrix_list[j] = temp2;
free_matrix(A_transpose);
free_matrix(temp);
}
free_matrix_list(A);
free_matrix_list(Z);
free_matrix_list(delta);
}
#pragma omp critical
{
matrix_list_t* temp_list;
temp_list = matrix_list_add(theta_gradient_total, theta_gradient);
free_matrix_list(theta_gradient_total);
free_matrix_list(theta_gradient);
theta_gradient_total = temp_list;
}
}
for(i=0; i<num_layers-1; i++)
{
matrix_t* temp;
matrix_t* temp2;
matrix_t* temp3;
temp = matrix_scalar_multiply(theta_gradient_total->matrix_list[i], 1.0/m);
temp2 = copy_matrix(theta->matrix_list[i]);
for(j=0; j<theta->matrix_list[i]->rows; j++)
{
matrix_set(temp2, j, 0, 0.0);
}
free_matrix(theta_gradient_total->matrix_list[i]);
temp3 = matrix_scalar_multiply(temp2, lamda/m);
theta_gradient_total->matrix_list[i] = matrix_add(temp, temp3);
free_matrix(temp);
free_matrix(temp2);
free_matrix(temp3);
}
*gradient = roll_matrix_list(theta_gradient_total);
free_matrix_list(theta);
free_matrix_list(theta_gradient_total);
return 0.0;
}