JNIEXPORT jint JNICALL Java_utils_SVD_svd___3DII_3D_3D_3D
(JNIEnv *env, jclass obj, jdoubleArray _A, jint _N, jint _M,
jdoubleArray _S, jdoubleArray _Ut, jdoubleArray _Vt)
{
//double svd_end[2] = { -1e-10, 1e-10 };
SVDVerbosity = 0;
double* At = env->GetDoubleArrayElements(_A, 0);
DMat dAt = svdNewDMat(_N, _M);
for (int i = 0; i < _N; ++i)
for (int j = 0; j < _M; ++j)
dAt->value[i][j] = At[i * _M + j];
SMat sAt = svdConvertDtoS(dAt);
svdFreeDMat(dAt);
env->ReleaseDoubleArrayElements(_A, At, 0);
jsize maxRank = env->GetArrayLength(_S);
//SVDRec Arec = svdLAS2(sAt, 0, SVD_MAX_ITER, svd_end, SVD_KAPPA);
SVDRec Arec = svdLAS2A(sAt, maxRank);
svdFreeSMat(sAt);
if (Arec == NULL) {
svdFreeSVDRec(Arec);
printf("WARNING: SVDLIBC las2a returns NULL \n\n");
return 0;
}
int rank = Arec->d;
double* S = env->GetDoubleArrayElements(_S, 0);
double* Ut = env->GetDoubleArrayElements(_Ut, 0);
double* Vt = env->GetDoubleArrayElements(_Vt, 0);
for (int i = 0; i < rank; ++i) S[i] = Arec->S[i];
for (int i = 0; i < rank; ++i)
for (int j = 0; j < _N; ++j) {
int p = i * _N + j;
Ut[p] = Arec->Ut->value[i][j];
}
for (int i = 0; i < rank; ++i)
for (int j = 0; j < _M; ++j) {
int p = i * _M + j;
Vt[p] += Arec->Vt->value[i][j];
}
env->ReleaseDoubleArrayElements(_S, S, 0);
env->ReleaseDoubleArrayElements(_Ut, Ut, 0);
env->ReleaseDoubleArrayElements(_Vt, Vt, 0);
svdFreeSVDRec(Arec);
return rank;
}
Dataset::Dataset(std::string dataset_file)
:
ratings(read_mtx(std::ifstream(dataset_file), 1).csr()),
n_users(ratings.dim1()),
n_items(ratings.dim2()),
ica(generateICA(n_items)), // n_items * M
blooms(mult(ratings, ica, 1)), // n_users * M
sim(compute_sim()),
sim_dense(to_dense(sim)),
max_profile_size(0),
uniform_priors(n_items, 1),
empirical_priors(compute_item_priors()),
bloom_bits_priors(compute_bits_priors()),
ratings_dense(to_dense(ratings)),
rank(std::min(n_users, n_items)), // full rank
colMajorU(ratings.dim1() , rank),
diagS(rank),
colMajorV(ratings.dim2() , rank)
{
std::cerr << dataset_file << std::endl;
std::cerr << n_users << " users, and " << n_items << " items" << std::endl;
// if(graphs[dataset_no] != "null")//(2 == dataset || 5 == dataset || 7 == dataset)
// {
// std::cerr << graphs[dataset_no] << std::endl;
// CompressedFormat graph = read_mtx(std::ifstream(base + graphs[dataset_no])).csr();
// }
stack_assert(n_items == ica.dim1());
this->max_profile_size = 0;
for(index_t user = 0; user < n_users; user++)
this->max_profile_size = std::max(this->max_profile_size, nnz_row(user , ratings.starts()));
// SVD
smat mat = {this->ratings.dim2()/*!!*/, this->ratings.dim1()/*!!*/,
this->ratings.nnz(),
const_cast<index_t*>(this->ratings.starts()),
const_cast<index_t*>(this->ratings.index()),
const_cast<double*>(this->ratings.values())};
SVDVerbosity = 0;
SVDRec svd = svdLAS2A(&mat, this->rank);
std::cerr << "SVD computed!" << std::endl;
// i would take the pointer from SVDRec, but i'd have to remember calling free instead of delete[].
std::copy(svd->S, svd->S + this->rank, this->diagS.get_data());
// svd->Ut is row-major, so it is U col-major, save for V
std::copy(svd->Vt/*!!*/->value[0], svd->Vt/*!!*/->value[0] + this->ratings.dim1() * this->rank, this->colMajorU.get_data());
std::copy(svd->Ut/*!!*/->value[0], svd->Ut/*!!*/->value[0] + this->ratings.dim2() * this->rank, this->colMajorV.get_data());
svdFreeSVDRec(svd);
}
//Computes a "D" eigensystem of a semidefinite sparse matrix S.
ESys sparseSDEig(SMat S, long d) {
unsigned dim = S->cols;
SVDRec svd = svdLAS2A(S,d);
ESys E = newESys(dim, d);
for (unsigned i = 0 ; i < d ; i++) {
for(unsigned j = 0 ; j < dim ; j++) {
E -> vecs[i][j] = svd->Vt->value[i][j];
}
E ->vals[i] = (svd->S[i]);
}
return E;
}
//#define SVD_KAPPA 1e-4
//#define SVD_MAX_ITER 100
//
JNIEXPORT jint JNICALL Java_utils_SVD_svd__III_3I_3I_3D_3D_3D_3D
(JNIEnv *env, jclass obj, jint _N, jint _M, jint _R,
jintArray _x, jintArray _y, jdoubleArray _z,
jdoubleArray _S, jdoubleArray _Ut, jdoubleArray _Vt)
{
SVDVerbosity = 0;
int* x = env->GetIntArrayElements(_x, 0);
int* y = env->GetIntArrayElements(_y, 0);
double* z = env->GetDoubleArrayElements(_z, 0);
jsize K = env->GetArrayLength(_x);
int rank = 0;
SMat smat = svdNewSMat(_N, _M, K);
if (smat) {
for (int i = 0, p = 0; i < _M; ++i) {
smat->pointr[i] = p;
while (p < K && y[p] == i) {
smat->rowind[p] = x[p];
smat->value[p] = z[p];
++p;
}
}
smat->pointr[_M] = K;
SVDRec rec = svdLAS2A(smat, _R);
if (rec) {
double* S = env->GetDoubleArrayElements(_S, 0);
double* Ut = env->GetDoubleArrayElements(_Ut, 0);
double* Vt = env->GetDoubleArrayElements(_Vt, 0);
rank = rec->d;
for (int i = 0; i < rank; ++i) S[i] = rec->S[i];
for (int i = 0; i < rank; ++i)
for (int j = 0; j < _N; ++j) Ut[i*_N+j] = rec->Ut->value[i][j];
for (int i = 0; i < rank; ++i)
for (int j = 0; j < _M; ++j) Vt[i*_M+j] = rec->Vt->value[i][j];
env->ReleaseDoubleArrayElements(_S, S, 0);
env->ReleaseDoubleArrayElements(_Ut, Ut, 0);
env->ReleaseDoubleArrayElements(_Vt, Vt, 0);
}
svdFreeSMat(smat);
svdFreeSVDRec(rec);
}
env->ReleaseIntArrayElements(_x, x, 0);
env->ReleaseIntArrayElements(_y, y, 0);
env->ReleaseDoubleArrayElements(_z, z, 0);
return rank;
}
void getSVDscore(unsigned int *split_taxa, int sze)
{
int j, jj;
int num_unique_rows, num_unique_cols;
long int* firstnz;
// initialize score
double score=0;
SMat S = NULL;
SVDRec R = NULL;
/* initialize split_taxa vector */
for (j=0; j<nints; j++) {
split_taxaR[j] = 0;
split_taxaC[j] = 0;
}
/* for counting number of SVs returned */
int numberFound=0;
/* setting a tolerance for checking when the sum of the squares
of the requested number of singular values equals the sum of
the squares of all of the singular values (the Frobenius norm).
If these two values are equal to within tol, then the score
is set to zero. */
double tol=10e-12;
int flag=1;
// transfer split
for (j=0; j<nints; j++) split_taxaR[j] = split_taxa[j];
// complementary mask
for (j=0; j<nints; j++) split_taxaC[j] =~ split_taxaR[j];
/* sort on rows, to get row indices,
then on columns, to get sparse encoding of flattening */
/* sort for rows */
qsort(*ppBase_u_bin, num_unique_bin, (nints+2)*sizeof(unsigned int), bin_compR);
/* Enter row numbers into array */
jj=0; // initialize row number
ppBase_u_bin[0][nints+1]=0; // this is always in first row
for (j=1; j<num_unique_bin; j++)
{
if (bin_compR(ppBase_u_bin[j],ppBase_u_bin[j-1])==1) jj++;
ppBase_u_bin[j][nints+1]=jj; // record row number
}
num_unique_rows=jj+1;
/* sort for columns */
qsort(*ppBase_u_bin, num_unique_bin, (nints+2)*sizeof(unsigned int), bin_compC);
/* allocate space for sparse matrix encoding */
long int* rowindex = (long int*) calloc(num_unique_bin,sizeof(long int));
double* values = (double*) calloc(num_unique_bin,sizeof(double));
/* create vector encoding column numbers */
jj=0; // first column number
// copy value of first non-zero for SVD
values[0] = (double)ppBase_u_bin[0][nints]/(double)num_no_gaps;
// copy row index of first entry for SVD
rowindex[0]=ppBase_u_bin[0][nints+1];
ppBase_u_bin[0][nints+1]=0; // first entry is always in first column
/* Note: we're reusing this space to store column info as we remove row info */
for (j=1; j<num_unique_bin; j++)
{
// copy non-zeros for SVD
values[j] = (double)(ppBase_u_bin[j][nints])/(double)num_no_gaps;
rowindex[j]=ppBase_u_bin[j][nints+1]; // copy row indices for SVD
if (bin_compC(ppBase_u_bin[j],ppBase_u_bin[j-1])==1) // if new column...
{
jj++;
ppBase_u_bin[jj][nints+1]=j; // save column coding here temporarily
}
}
num_unique_cols=jj+1;
/* allocate final space for sparse matrix encoding */
firstnz = (long int*) calloc( num_unique_cols+1 , sizeof(long int));
for (j=0; j<num_unique_cols; j++)
{
firstnz[j]=ppBase_u_bin[j][nints+1];
}
/* copy over column coding for SVD */
firstnz[j]=num_unique_bin;/* sparse format requires this too */
/* Create Smat object to use in SVD routines */
S = svdNewSMat(num_unique_rows,num_unique_cols,num_unique_bin);
R = svdNewSVDRec();
S->pointr = firstnz;
S->rowind = rowindex;
S->value = values;
/* Compute SVD */
extern long SVDVerbosity;
SVDVerbosity=0; // print nothing from SVD routine
R = svdLAS2A(S,rank);
/** Scores computations **/
// count number of singular values requested are found
numberFound = R->d;
if (numberFound < rank)
{
score = NAN;
flag = 0;
}
// only compute score if the requested number of singular values was returned
if (flag)
{
for (j=0; j<rank; j++) score = score+pow(R->S[j],2);
/* sometimes sum of squares of r singular values > fnorm2 due to numerical error,
i.e. in the 10e-16 decimal place. Test how close these values are and
reset score to 0 if difference less than tolerance. */
if (fabs(score-fnorm2)>tol)
{
score = sqrt(1-score/fnorm2);
}
else
{
score=0;
}
}
fprintf(scoresfile, "%1.15f\n",score);
fflush(0);
free(firstnz);
free(rowindex);
free(values);
S->pointr = NULL;
S->rowind = NULL;
S->value = NULL;
svdFreeSMat(S);
svdFreeSVDRec(R);
num_unique_rows=0;
num_unique_cols=0;
}
SEXP svdLAS2_(SEXP dim, SEXP i, SEXP p, SEXP x, SEXP dimensions, SEXP exclude, SEXP kappa) {
struct smat M;
SVDRec svd;
SEXP res, res_d, res_u, res_v, res_names;
double *u_dbl, *v_dbl, *mark, *point;
int *i_int = INTEGER(i);
int *p_int = INTEGER(p);
int nR = INTEGER(dim)[0];
int nC = INTEGER(dim)[1];
int n_cells = length(x);
int rank, k, j, n_row, n_col;
/* copy M to SMat structure (column-compressed format) */
M.rows = nR;
M.cols = nC;
M.vals = n_cells;
M.value = REAL(x);
/* need to make copy of i and p because of different data type (long vs. int) */
M.pointr = (long *) R_alloc(nC + 1, sizeof(long));
for (k = 0; k <= nC; k++)
M.pointr[k] = p_int[k];
M.rowind = (long *) R_alloc(n_cells, sizeof(long));
for (k = 0; k < n_cells; k++)
M.rowind[k] = i_int[k];
/* execute sparse SVD */
SVDVerbosity = 0;
svd = svdLAS2A(&M, INTEGER(dimensions)[0]);
rank = svd->d;
/* check matrix dimensions */
n_row = svd->Ut->cols; /* Ut is the transposed matrix, hence swap row/col counts */
n_col = svd->Ut->rows;
if ((n_col < rank) || (n_row != nR)) {
svdFreeSVDRec(svd);
error("internal error (U is %d x %d matrix, expected %d x %d)", n_row, n_col, nR, rank);
}
n_row = svd->Vt->cols; /* same for Vt */
n_col = svd->Vt->rows;
if ((n_col < rank) || n_row != nC) {
svdFreeSVDRec(svd);
error("internal error (V is %d x %d matrix, expected %d x %d)", n_row, n_col, nC, rank);
}
/* note that Ut and Vt may contain more eigenvectors than there are significant eigenvalues;
this is expected if some singular values are culled because of the kappa criterion */
/* extract singular values and matrices of singluar vectors into R objects */
res_d = PROTECT(allocVector(REALSXP, rank));
for (k = 0; k < rank; k++)
REAL(res_d)[k] = svd->S[k];
res_u = PROTECT(allocMatrix(REALSXP, nR, rank));
u_dbl = REAL(res_u);
for (k = 0; k < rank; k++) {
mark = svd->Ut->value[k];
point = u_dbl + k * nR;
for (j = 0; j < nR; j++)
*point++ = *mark++;
}
res_v = PROTECT(allocMatrix(REALSXP, nC, rank));
v_dbl = REAL(res_v);
for (k = 0; k < rank; k++) {
mark = svd->Vt->value[k];
point = v_dbl + k * nC;
for (j = 0; j < nC; j++)
*point++ = *mark++;
}
/* free SVDRec after copying to R objects */
svdFreeSVDRec(svd);
/* construct result list */
res = PROTECT(allocVector(VECSXP, 3));
SET_VECTOR_ELT(res, 0, res_d);
SET_VECTOR_ELT(res, 1, res_u);
SET_VECTOR_ELT(res, 2, res_v);
res_names = PROTECT(allocVector(STRSXP, 3));
SET_STRING_ELT(res_names, 0, mkChar("d"));
SET_STRING_ELT(res_names, 1, mkChar("u"));
SET_STRING_ELT(res_names, 2, mkChar("v"));
setAttrib(res, R_NamesSymbol, res_names);
UNPROTECT(5);
return res;
}
JNIEXPORT jint JNICALL Java_utils_SVD_lowRankSvd
(JNIEnv *env, jclass obj, jdoubleArray _At, jdoubleArray _Bt,
jint _N, jint _M, jint _R, jdoubleArray _S, jdoubleArray _Ut,
jdoubleArray _Vt)
{
//double svd_end[2] = { -1e-10, 1e-10 };
SVDVerbosity = 0;
double* At = env->GetDoubleArrayElements(_At, 0);
double* Bt = env->GetDoubleArrayElements(_Bt, 0);
DMat dAt = svdNewDMat(_R, _N);
for (int i = 0; i < _R; ++i)
for (int j = 0; j < _N; ++j)
dAt->value[i][j] = At[i * _N + j];
SMat sAt = svdConvertDtoS(dAt);
svdFreeDMat(dAt);
DMat dBt = svdNewDMat(_R, _M);
for (int i = 0; i < _R; ++i)
for (int j = 0; j < _M; ++j)
dBt->value[i][j] = Bt[i * _M + j];
SMat sBt = svdConvertDtoS(dBt);
svdFreeDMat(dBt);
env->ReleaseDoubleArrayElements(_At, At, 0);
env->ReleaseDoubleArrayElements(_Bt, Bt, 0);
//SVDRec Arec = svdLAS2(sAt, 0, SVD_MAX_ITER, svd_end, SVD_KAPPA);
//SVDRec Brec = svdLAS2(sBt, 0, SVD_MAX_ITER, svd_end, SVD_KAPPA);
SVDRec Arec = svdLAS2A(sAt, 0);
SVDRec Brec = svdLAS2A(sBt, 0);
svdFreeSMat(sAt);
svdFreeSMat(sBt);
if (Arec == NULL || Brec == NULL) {
svdFreeSVDRec(Arec);
svdFreeSVDRec(Brec);
printf("WARNING: SVDLIBC las2a returns NULL \n\n");
return 0;
}
int ranka = Arec->d, rankb = Brec->d;
if (ranka == 0 || rankb == 0) {
printf("WARNING: one matrix has 0 rank. %d %d\n\n", ranka, rankb);
return 0;
}
DMat dM = svdNewDMat(ranka, rankb);
for (int i = 0; i < ranka; ++i)
for (int j = 0; j < rankb; ++j) {
double va = 0;
for (int k = 0; k < _R; ++k)
va += Arec->Ut->value[i][k] * Brec->Ut->value[j][k];
dM->value[i][j] = va * Arec->S[i] * Brec->S[j];
}
SMat sM = svdConvertDtoS(dM);
svdFreeDMat(dM);
//SVDRec Mrec = svdLAS2(sM, 0, SVD_MAX_ITER, svd_end, SVD_KAPPA);
SVDRec Mrec = svdLAS2A(sM, 0);
svdFreeSMat(sM);
int rank = 0;
if (Mrec != NULL && Mrec->d > 0) {
double* S = env->GetDoubleArrayElements(_S, 0);
double* Ut = env->GetDoubleArrayElements(_Ut, 0);
double* Vt = env->GetDoubleArrayElements(_Vt, 0);
rank = Mrec->d;
for (int i = 0; i < rank; ++i) S[i] = Mrec->S[i];
for (int i = 0; i < rank; ++i)
for (int j = 0; j < _N; ++j) {
int p = i * _N + j;
for (int k = 0; k < ranka; ++k)
Ut[p] += Mrec->Ut->value[i][k] * Arec->Vt->value[k][j];
}
for (int i = 0; i < rank; ++i)
for (int j = 0; j < _M; ++j) {
int p = i * _M + j;
for (int k = 0; k < rankb; ++k)
Vt[p] += Mrec->Vt->value[i][k] * Brec->Vt->value[k][j];
}
env->ReleaseDoubleArrayElements(_S, S, 0);
env->ReleaseDoubleArrayElements(_Ut, Ut, 0);
env->ReleaseDoubleArrayElements(_Vt, Vt, 0);
} else {
printf("WARNING: matrix M has 0 rank or las2a returns NULL \n\n");
}
svdFreeSVDRec(Arec);
svdFreeSVDRec(Brec);
svdFreeSVDRec(Mrec);
return rank;
}
