void r_sum_w_x(double **x, double *w, int n, int p,
double *tmp,
double *sum)
{
/*
// given a matrix x (n x p) and a vector w of n
// weights, it computes the vector
// \sumin w_i x_i
// need space for p doubles in *tmp
*/
void scalar_vec(double *a, double b, double *c, int n);
void sum_vec(double *a, double *b, double *c, int n);
void reset_vec(double *a, int n);
register int i;
reset_vec(sum, p);
for(i=0; i<n; i++) {
scalar_vec(x[i], w[i], tmp, p);
sum_vec(sum , tmp , sum, p);
};
}
void calc_search_template_window(
variogram_search_template_t * templ,
search_template_window_t * window)
{
double max = 1e10;
double mini, maxi, minj, maxj, mink, maxk;
mini = minj = mink = max;
maxi = maxj = maxk = -max;
for (int i = 0; i < 2; ++i)
for (int j = -1; j < 3; j += 2)
for (int k = -1; k < 3; k+=2)
{
vector_t DI = {0};
vector_t DJ = {0};
vector_t DK = {0};
vector_t V = {0};
vec_by_scalar(&templ->m_ellipsoid.m_direction1, templ->m_lag_separation, &DI);
vec_by_scalar(&DI, templ->m_num_lags, &DI);
vec_by_scalar(&DI, i, &DI);
vec_by_scalar(&templ->m_ellipsoid.m_direction2, templ->m_ellipsoid.m_R2, &DJ);
vec_by_scalar(&DJ, j, &DJ);
vec_by_scalar(&templ->m_ellipsoid.m_direction3, templ->m_ellipsoid.m_R3, &DK);
vec_by_scalar(&DK, k, &DK);
sum_vec(&DI, &DJ, &V);
sum_vec(&V, &DK, &V);
set_min(&mini, V.m_data[0]);
set_max(&maxi, V.m_data[0]);
set_min(&minj, V.m_data[1]);
set_max(&maxj, V.m_data[1]);
set_min(&mink, V.m_data[2]);
set_max(&maxk, V.m_data[2]);
}
window->m_min_i = mini;
window->m_max_i = maxi;
window->m_min_j = minj;
window->m_max_j = maxj;
window->m_min_k = mink;
window->m_max_k = maxk;
}
int main(int argc, char* argv[])
{
double twoThrd = 0, sqrts = 0, Flint = 0, Cookson = 0;
v2df Harmonic, zeta, poly, alt, Gregory;
v2df zero, one, two, init, m_one, kv, av;
double k, k3, s, c;
int n; n = atoi(argv[1]);
zero = make_vec( 0.0, 0.0); one = make_vec( 1.0, 1.0);
two = make_vec( 2.0, 2.0); m_one = make_vec(-1.0, -1.0);
init = make_vec( 1.0, 2.0); av = make_vec( 1.0, -1.0);
Harmonic = zeta = poly = alt = Gregory = zero;
for (k=1; k<=n; k++)
{
twoThrd += pow(2.0/3.0, k-1);
sqrts += 1.0/sqrt(k);
k3 = k*k*k;
s = sin(k); c = cos(k);
Flint += 1.0/(k3 * s*s);
Cookson += 1.0/(k3 * c*c);
}
for (kv=init; *(double *)(&kv)<=n; kv+=two)
{
poly += one /(kv*(kv+one));
Harmonic+= one / kv;
zeta += one /(kv*kv);
alt += av / kv;
Gregory += av /(two*kv - one);
}
#define psum(name,num) printf("%.9f\t%s\n",num,name)
psum("(2/3)^k", twoThrd); psum("k^-0.5", sqrts);
psum("1/k(k+1)", sum_vec(poly)); psum("Flint Hills", Flint);
psum("Cookson Hills", Cookson); psum("Harmonic", sum_vec(Harmonic));
psum("Riemann Zeta",sum_vec(zeta)); psum("Alternating Harmonic",sum_vec(alt));
psum("Gregory", sum_vec(Gregory));
return 0;
}
inline double entropy( const dataType *vec, int dim )
{
double s = static_cast<double>( sum_vec( vec, dim ) );
double re = 0.0;
const dataType *x = vec;
for ( int i=0; i<dim; i++ ) {
double v = *(x++) / s;
if ( v >= 1e-6 ) {
re += v * log( v );
}
}
return -re;
}
vector<int> plusOne(vector<int>& digits) {
if (digits.size() == 0) return {1};
vector<int> sum_vec(digits.size()+1, 0);
int index = sum_vec.size() - 1, carry = 0;
for (int i = digits.size() - 1; i >= 0; --i) {
int sum = digits[i] + carry + (i == digits.size() - 1 ? 1 : 0);
sum_vec[index--] = sum % 10;
carry = sum / 10;
}
if (carry) {
sum_vec[index] = carry;
return sum_vec;
}
else return vector<int>(sum_vec.begin()+1, sum_vec.end());
}
void initialize( string filename )
{
LispFormParser lisp;
lisp.parse( filename );
_to_name.clear();
_to_id.clear();
int id = 0;
for ( auto& every : lisp ) {
_to_name.push_back( every );
_to_id.insert( std::make_pair( every, id ) );
wt.push_back( 1.0 / lisp[every].toInt() );
id ++;
}
classes = static_cast<int>( _to_name.size() );
inv = 1.0 / classes;
// normalize the weights
double s = 1.0 / sum_vec( &wt[0], classes );
for ( auto& w : wt ) {
w *= s;
}
}
void stocml_ridge_sdca_idx(double *Y, double * X, double * X_maxrn, double * beta, double * intcpt, int * nn, int * dd, int * ite_lamb, int * ite_in, double *runt, double *lambda, int *nnlambda, int *mmax_ite, double *pprec) {
int i, j, n, d, max_ite1, max_ite2, nlambda, ite1, ite2, c_idx;
double prec1, prec2, ilambda, dif1, dif2, dbn;
double R, kappa, mu, rho, eta, bt, lambdak, one, delta_alp;
clock_t start, stop;
n = *nn;
d = *dd;
max_ite1 = *mmax_ite;
max_ite2 = n*3;
prec1 = *pprec;
prec2 = 1e-3;
nlambda = *nnlambda;
dbn = (double)n;
one = 1;
double *beta2 = (double *) Calloc(d, double);
double *beta1 = (double *) Calloc(d, double);
double *betay = (double *) Calloc(d, double);
double *v = (double *) Calloc(d, double);
double *z = (double *) Calloc(d, double);
double *alp = (double *) Calloc(n, double);
double *y_tild = (double *) Calloc(n, double);
int *idx = (int *) Calloc(max_ite2, int);
for(i=0; i<d; i++) {
beta2[i] = 0;
beta1[i] = 0;
betay[i] = 0;
}
for(i=0; i<n; i++) {
alp[i] = 0;
}
for(i=0; i<max_ite2; i++) {
idx[i] = i%n;
}
//printf("idx: 0=%d,5=%d,203=%d,416=%d \n",idx[0],idx[5],idx[203],idx[416]);
start = clock();
for (i=0; i<nlambda; i++) {
intcpt[i] = 0;
ilambda = lambda[i];
R = max(*X_maxrn, sqrt(11*dbn*ilambda)); // parameter set-up
kappa = R*R/dbn - ilambda;
mu = ilambda/2;
rho = mu + kappa;
eta = sqrt(mu/rho);
bt = (1-eta)/(1+eta);
lambdak = ilambda+kappa;
ite1 = 0;
dif1 = 1;
while (dif1>prec1 && ite1<max_ite1) {
prod_vec_const(z, betay, kappa/lambdak, d); // z=betay*kappa/(lambda+kappa)
get_residual_dense(y_tild, Y, X, z, n, d); // y_tild = Y - X * z;
vec_mat_prod_coef(v, alp, X, 1/(lambdak*dbn), n, d); // v = X^T alp / ((lambda+kappa)*n);
shuffle(idx, max_ite2);
ite2 = 0;
dif2 = 1;
while (dif2>prec2 && ite2<max_ite2) {
c_idx = idx[ite2];
//if(c_idx<0 || c_idx>n-1) printf("c_idx=%d \n",c_idx);
delta_alp = -(alp[c_idx]+vec_matrow_inprod(X+c_idx, v, one, n, d)-y_tild[c_idx])/(1+norm2sq_matrow_coef(X+c_idx, 1/(lambdak*dbn), n, d));
//if(i==0&&ite1==0&&ite2==2)printf("y_tild=%f,Xv=%f,X2=%f,c_idx=%d,delta_alp=%f,v=%f,beta2=%f \n",y_tild[0],vec_matrow_inprod(X+c_idx, v, one, n, d),norm2sq_matrow_coef(X+c_idx, 1/(lambdak*dbn), n, d),c_idx,delta_alp,v[0],beta2[0]);
alp[c_idx] = alp[c_idx]+delta_alp;
sum_vec_matrow(v, X+c_idx, delta_alp/(lambdak*dbn), n, d); // v = v+X[c_idx,]*delta_alp/(lambdak*dbn)
sum_vec(beta2,v,z,d); //beta2 = v+z
dif2 = (residual_l2sq_dense(Y,X,beta2,n,d)+vec_sum_l2sq_dense(alp,Y,n)-vec_2normsq(Y,n))/(2*dbn)+lambdak*vec_inprod(beta2,v,d);
ite2++;
//if(ite2%100==0) printf("i=%d,ite1=%d,ite2=%d,dif2=%f \n",i,ite1,ite2,dif2);
}
ite_in[i] += ite2;
dif1 = dif_2norm_dense(beta1, beta2, d); // ||beta1-beta2||_2
//printf("dif1=%f \n",dif1);
for(j=0; j<d; j++) {
betay[j] = beta2[j]+bt*(beta2[j]-beta1[j]);
beta1[j] = beta2[j];
}
ite1++;
}
ite_lamb[i] = ite1;
for(j=0; j<d; j++) {
beta[i*d+j] = beta2[j];
}
//printf("i=%d,ite1=%d,b1=%f,b2=%f,b3=%f,b4=%f,b5=%f \n",i,ite1,beta[i*d+0],beta[i*d+1],beta[i*d+2],beta[i*d+3],beta[i*d+4]);
stop = clock();
runt[i] = (double)(stop - start)/CLOCKS_PER_SEC;
}
Free(beta2);
Free(beta1);
Free(betay);
Free(v);
Free(z);
Free(alp);
Free(y_tild);
Free(idx);
}
int main(int argc, char * argv[])
{
if (argc > 1){
parse_args(argc, argv);
}
const int size = n * n;
float *mat = malloc(size * sizeof(float));
float *vec = malloc(n * sizeof(float));
float *output = malloc(n * sizeof(float));
float *expected = malloc(n * sizeof(float));
float *mat_transposed = malloc(n * n * sizeof(float));
generate_matrix(n, mat, range);
generate_vector(n, vec, range);
timing_t timer1;
timer_start(&timer1);
transpose(n, mat, mat_transposed);
MatVecMultiply(size, n, mat_transposed, vec, output);
timer_stop(&timer1);
float sum = sum_vec(n, output);
printf("%d %f %ld %ld\n", n, sum, timer1.realtime, timer1.cputime);
if (trace == 1) {
printf("\nInput matrix\n");
for (int i=0; i<n; i++){
for (int j=0; j<n; j++){
printf("%f " , mat[i*n+j]);
}
printf("\n");
}
printf("\nInput vector \n");
for (int i=0; i<n; i++){
printf("%f " , vec[i]);
}
printf("\n\nResult\n");
for (int i=0; i<n; i++){
printf("%f " , output[i]);
}
printf("\n");
}
else if (trace == 2) {
multiply_CPU(n, mat, vec, expected);
int status = check(n, output, expected);
if (status)
printf("Test failed.\n");
else
printf("Test passed OK!\n");
return status;
}
free(mat);
free(vec);
free(output);
free(expected);
free(mat_transposed);
return 0;
}
void read_desc(char *desc, double *P)
{
char action;
double arg;
int i, jump = 0, push = 0, pop = 0;
/* Matriz para almacenar transformaciones a lo largo de iteraciones
* En un comienzo apunta hacia Y+, ya que la primera columna indica
* el Heading (hacia dónde apunta), la seguna cuál es la dirección
* hacia la izquierda (en este caso hacia X+) y cual es la dirección
* hacia arriba (en este caso Z+). Cuando fue descrita por Lindenmayer
* et al en "The Algorithmic Beauty of Plants" esta matriz es mencionada
* como [H L U] (por Heading, Left, Up). */
double T[DIM][DIM] = {{0, 1, 0}, {1, 0, 0}, {0, 0, 1}};
/* Matriz para guardar rotaciones y resultados de operaciones */
double R[DIM][DIM];
double M[DIM][DIM];
/* Vector para almacenar tamaño de segmento a dibujar */
double L[DIM] = {0, 0, 0};
/* Punto inicial */
double P0[DIM] = {P[0], P[1], P[2]};
/* Estado inicial */
assign_mat(EstadoActual.T, T);
assign_vec(EstadoActual.P, P0);
for(i = 0; i < strlen(desc); i++)
{
/* Lee caracter y verifica si existe argumento */
action = desc[i];
get_argument(desc, i, &arg, &jump);
/* Obtiene estado actual */
assign_mat(T, EstadoActual.T);
assign_vec(P0, EstadoActual.P);
/* Los casos se describen en "L-systems: from the Theory to Visual Models of Plants"
* Apartado num. 5: The turtle interpretation of L-systems */
switch(action)
{
case 'F':
/* Si no hay argumento, entonces tomar valor por defecto */
if(!jump) arg = DEFAULT_STEP;
/* Tamaño del segmento a dibujar */
L[0] = arg;
mat_by_vec(P, T, L);
printf("Dibujar segmento (%f, %f, %f), ", P0[0], P0[1], P0[2]);
sum_vec(P0, P, P0);
printf("(%f, %f, %f)\n", P0[0], P0[1], P0[2]);
break;
case '+':
if(!jump) arg = DEFAULT_ANGLE;
Ru_matrix(R, arg);
/* Aplicar la transformación R a T: T*R */
mat_by_mat(M, T, R);
assign_mat(T, M);
if(DEBUG) printf("Rotar hacia izquierda en torno a eje U. Ru(%f)\n", arg);
break;
case '-':
if(!jump) arg = DEFAULT_ANGLE;
Ru_matrix(R, arg*-1.0);
/* Aplicar la transformación R a T: T*R */
mat_by_mat(M, T, R);
assign_mat(T, M);
if(DEBUG) printf("Rotar hacia derecha en torno a eje U. Ru(-%f)\n", arg);
break;
case '&':
if(!jump) arg = DEFAULT_ANGLE;
Rl_matrix(R, arg);
/* Aplicar la transformación R a T: T*R */
mat_by_mat(M, T, R);
assign_mat(T, M);
if(DEBUG) printf("Rotar hacia izquierda en torno a eje L. Rl(%f)\n", arg);
break;
case '^':
if(!jump) arg = DEFAULT_ANGLE;
Rl_matrix(R, arg*-1.0);
/* Aplicar la transformación R a T: T*R */
mat_by_mat(M, T, R);
assign_mat(T, M);
if(DEBUG) printf("Rotar hacia derecha en torno a eje L. Rl(-%f)\n", arg);
break;
case '\\':
if(!jump) arg = DEFAULT_ANGLE;
Rh_matrix(R, arg);
/* Aplicar la transformación R a T: T*R */
mat_by_mat(M, T, R);
assign_mat(T, M);
if(DEBUG) printf("Rotar hacia izquierda en torno a eje H. Rh(%f)\n", arg);
break;
case '/':
if(!jump) arg = DEFAULT_ANGLE;
Rh_matrix(R, arg*-1.0);
/* Aplicar la transformación R a T: T*R */
mat_by_mat(M, T, R);
assign_mat(T, M);
if(DEBUG) printf("Rotar hacia derecha en torno a eje H. Rh(-%f)\n", arg);
break;
case '[':
/* Seteamos flag para hacer push del estado actual */
push = 1;
if(DEBUG) printf("Guardar el estado actual en la pila.\n");
break;
case ']':
/* Seteamos flag para hacer pop de la pila y actualizar estado */
pop = 1;
if(DEBUG) printf("Obtener estado desde la pila y actualizarlo como estado actual.\n");
break;
default:
break;
}
i += jump;
/* Guardar estado actual */
assign_mat(EstadoActual.T, T);
assign_vec(EstadoActual.P, P0);
/* Verificamos si debemos sacar el estado actual desde la pila */
if(pop && !PilaEstados.empty())
{
EstadoActual = PilaEstados.top();
PilaEstados.pop();
pop = 0;
}
/* Verificamos si debemos guardar estado actual en la pila */
else if(push)
{
PilaEstados.push(EstadoActual);
push = 0;
}
}
/* Vaciar pila */
while(!PilaEstados.empty())
{
PilaEstados.pop();
}
}
int main(int argc, char* argv[])
{
TEST_PARAS myparas = parse_test_paras(argc, argv, testfile, embeddingfile, trainfile);
printf("Predicting...\n");
if(!myparas.allow_self_transition)
printf("Do not allow self-transtion.\n");
if (!myparas.underflow_correction)
printf("Underflow correction disabled\n");
int new_test_song_exp = (myparas.train_test_hash_file[0] != '\0');
if(myparas.tagfile[0] == '\0' && new_test_song_exp)
{
printf("Have to support with a tag file if you want to test on unseen songs.\n");
exit(1);
}
int d;
int m;
int l;
int i;
int j;
int s;
int fr;
int to;
double* bias_terms = 0;
double** X = read_matrix_file(embeddingfile, &l, &d, &bias_terms);
double** realX;
PDATA pd = read_playlists_data(testfile);
//int k = pd.num_songs;
int k;
double llhood = 0.0;
double uniform_llhood = 0.0;
double realn = 0.0;
double not_realn= 0.0;
int* train_test_hash;
int k_train;
int k_test;
TDATA td;
if(!new_test_song_exp)
{
k = pd.num_songs;
if(myparas.tagfile[0] != '\0')
{
td = read_tag_data(myparas.tagfile);
m = td.num_tags;
myparas.num_points = l / (k + m);
realX = zerosarray(k * myparas.num_points, d);
calculate_realX(X, realX, td, k, m, d, myparas.num_points);
free_tag_data(td);
if(myparas.tag_ebd_filename[0] != '\0')
write_embedding_to_file(X + k * myparas.num_points, m * myparas.num_points, d, myparas.tag_ebd_filename, 0);
}
else
{
myparas.num_points = l / k;
realX = zerosarray(k * myparas.num_points, d);
Array2Dcopy(X, realX, l, d);
}
Array2Dfree(X, l, d);
}
else
{
printf("Prediction on unseen songs.\n");
td = read_tag_data(myparas.tagfile);
m = td.num_tags;
k = td.num_songs;
train_test_hash = read_hash(myparas.train_test_hash_file, &k_train);
k_test = k - k_train;
printf("Number of new songs %d.\n", k_test);
myparas.num_points = l / (k_train + m);
realX = zerosarray(k * myparas.num_points, d);
calculate_realX_with_hash(X, realX, td, k, m, d, myparas.num_points, k_train, train_test_hash);
free_tag_data(td);
Array2Dfree(X, l, d);
}
if(myparas.song_ebd_filename[0] != '\0')
write_embedding_to_file(realX, k * myparas.num_points, d, myparas.song_ebd_filename, 0);
if(myparas.bias_ebd_filename[0] != '\0')
{
FILE* fp = fopen(myparas.bias_ebd_filename, "w");
for( i = 0; i < k ;i++)
{
fprintf(fp, "%f", bias_terms[i]);
if ( i != k - 1)
fputc('\n', fp);
}
fclose(fp);
}
double** square_dist;
if(myparas.square_dist_filename[0] != '\0')
square_dist = zerosarray(k, k);
int n = 0;
for(i = 0; i < pd.num_playlists; i ++)
if(pd.playlists_length[i] > 0)
n += pd.playlists_length[i] - 1;
printf("Altogether %d transitions.\n", n);fflush(stdout);
PHASH* tcount;
PHASH* tcount_train;
double** tcount_full;
double** tcount_full_train;
if(myparas.use_hash_TTable)
tcount = create_empty_hash(2 * n);
else
tcount_full = zerosarray(k, k);
HELEM temp_elem;
TPAIR temp_pair;
int idx;
double temp_val;
for(i = 0; i < pd.num_playlists; i ++)
{
if(pd.playlists_length[i] > myparas.range)
{
for(j = 0; j < pd.playlists_length[i] - 1; j++)
{
temp_pair.fr = pd.playlists[i][j];
temp_pair.to = pd.playlists[i][j + myparas.range];
//printf("(%d, %d)\n", temp_pair.fr, temp_pair.to);
if(temp_pair.fr >= 0 && temp_pair.to >= 0)
{
if(myparas.use_hash_TTable)
{
idx = exist_in_hash(tcount, temp_pair);
if(idx < 0)
{
temp_elem.key = temp_pair;
temp_elem.val = 1.0;
add_entry(tcount, temp_elem);
}
else
update_with(tcount, idx, 1.0);
}
else
tcount_full[temp_pair.fr][temp_pair.to] += 1.0;
}
}
}
}
TRANSITIONTABLE ttable;
TRANSITIONTABLE BFStable;
//Need to use the training file
if(myparas.output_distr)
{
PDATA pd_train = read_playlists_data(trainfile);
if(myparas.use_hash_TTable)
tcount_train = create_empty_hash(2 * n);
else
tcount_full_train = zerosarray(k, k);
for(i = 0; i < pd_train.num_playlists; i ++)
{
if(pd_train.playlists_length[i] > 1)
{
for(j = 0; j < pd_train.playlists_length[i] - 1; j++)
{
temp_pair.fr = pd_train.playlists[i][j];
temp_pair.to = pd_train.playlists[i][j + 1];
if(myparas.use_hash_TTable)
{
idx = exist_in_hash(tcount_train, temp_pair);
if(idx < 0)
{
temp_elem.key = temp_pair;
temp_elem.val = 1.0;
add_entry(tcount_train, temp_elem);
}
else
update_with(tcount_train, idx, 1.0);
}
else
tcount_full_train[temp_pair.fr][temp_pair.to] += 1.0;
}
}
}
}
FILE* song_distr_file;
FILE* trans_distr_file;
double* song_sep_ll;
if(myparas.output_distr)
{
printf("Output likelihood distribution file turned on.\n");
if(myparas.output_distr)
{
song_distr_file = fopen(songdistrfile, "w");
trans_distr_file = fopen(transdistrfile, "w");
song_sep_ll = (double*)calloc(k, sizeof(double));
}
}
int* test_ids_for_new_songs;
if(new_test_song_exp)
test_ids_for_new_songs = get_test_ids(k, k_train, train_test_hash);
for(fr = 0; fr < k; fr++)
{
int collection_size;
int* collection_idx;
if(myparas.fast_collection)
{
collection_size = (BFStable.parray)[fr].length;
if (collection_size == 0)
continue;
collection_idx = (int*)malloc(collection_size * sizeof(int));
LINKEDELEM* tempp = (BFStable.parray)[fr].head;
for(i = 0; i < collection_size; i++)
{
collection_idx[i] = tempp -> idx;
tempp = tempp -> pnext;
}
}
else if(new_test_song_exp)
{
collection_size = k_test;
collection_idx = (int*)malloc(collection_size * sizeof(int));
int_list_copy(test_ids_for_new_songs, collection_idx, k_test);
}
else
collection_size = k;
double** delta = zerosarray(collection_size, d);
double* p = (double*)calloc(collection_size, sizeof(double));
double** tempkd = zerosarray(collection_size, d);
double* tempk = (double*)calloc(collection_size, sizeof(double));
double** mid_delta = 0;
double* mid_p = 0;
double** mid_tempkd = 0;
// I get a seg fault when these get freed. Don't understand.
if (myparas.num_points == 3) {
mid_delta = zerosarray(collection_size, d);
mid_p = (double*)calloc(collection_size, sizeof(double));
mid_tempkd = zerosarray(collection_size, d);
}
for(j = 0; j < collection_size; j++)
{
for(i = 0; i < d; i++)
{
if(myparas.fast_collection || new_test_song_exp)
delta[j][i] = realX[fr][i] - realX[(myparas.num_points - 1) * k + collection_idx[j]][i];
else
delta[j][i] = realX[fr][i] - realX[(myparas.num_points - 1) * k + j][i];
}
if(myparas.num_points == 3) {
if(myparas.fast_collection || new_test_song_exp)
mid_delta[j][i] =
realX[k + fr][i] - realX[k + collection_idx[j]][i];
else
mid_delta[j][i] = realX[k + fr][i] - realX[k + j][i];
}
}
mat_mult(delta, delta, tempkd, collection_size, d);
scale_mat(tempkd, collection_size, d, -1.0);
sum_along_direct(tempkd, p, collection_size, d, 1);
if(myparas.square_dist_filename[0] != '\0')
for(i = 0; i < k; i++)
square_dist[fr][i] = -p[i];
if (bias_terms != 0)
add_vec(p, bias_terms, collection_size, 1.0);
if (myparas.num_points == 3) {
// Just use the mid_deltas (midpoint differences): square them,
// then sum and add to the p vector directly, then the midpoint
// probability is incorporated
mat_mult(mid_delta, mid_delta, mid_tempkd, collection_size, d);
scale_mat(mid_tempkd, collection_size, d, -1.0);
sum_along_direct(mid_tempkd, mid_p, collection_size, d, 1);
add_vec(p, mid_p, collection_size, 1.0);
}
if (myparas.underflow_correction == 1) {
double max_val = p[0];
for(i = 0; i < collection_size; i++)
max_val = p[i] > max_val? p[i] : max_val;
vec_scalar_sum(p, -max_val, collection_size);
}
Veccopy(p, tempk, collection_size);
exp_on_vec(tempk, collection_size);
//exp_on_vec(p, collection_size);
// underflow checking:
// for (i = 0; i < collection_size; i++)
// if (p[i] < 0.000001)
// p[i] = 0.000001;
double temp_sum;
if(myparas.allow_self_transition)
temp_sum = sum_vec(tempk, collection_size);
else
{
temp_sum = 0.0;
for(i = 0; i < collection_size; i++)
if(!myparas.fast_collection || new_test_song_exp)
temp_sum += (i != fr)? tempk[i] : 0.0;
else
temp_sum += (collection_idx[i] != fr)? tempk[i] : 0.0;
}
vec_scalar_sum(p, -log(temp_sum), collection_size);
//scale_vec(p, collection_size, 1.0 / temp_sum);
//printf("done...\n");
for(to = 0; to < k; to++)
{
if(myparas.allow_self_transition || (!myparas.allow_self_transition && fr != to))
{
temp_pair.fr = fr;
temp_pair.to = to;
//printf("(%d, %d)\n", fr, to);
if(myparas.use_hash_TTable)
idx = exist_in_hash(tcount, temp_pair);
else
idx = tcount_full[fr][to] > 0.0? 1 : -1;
//printf("%d\n", idx);fflush(stdout);
int idx_train;
//printf("done...\n");fflush(stdout);
if(myparas.output_distr)
{
if(myparas.use_hash_TTable)
idx_train = exist_in_hash(tcount_train, temp_pair);
else
idx_train = tcount_full_train[fr][to] > 0.0? 1 : -1;
}
if(idx >= 0)
{
if(myparas.fast_collection || new_test_song_exp)
{
s = -1;
for(i = 0; i < collection_size; i++)
{
if(collection_idx[i] == to)
{
s = i;
break;
}
}
}
else
s = to;
//printf("%d\n", idx);fflush(stdout);
if(myparas.use_hash_TTable)
temp_val = retrieve_value_with_idx(tcount, idx);
else
temp_val = tcount_full[fr][to];
if(s < 0)
not_realn += temp_val;
else
{
//printf("s = %d\n", s);
llhood += temp_val * p[s];
if(new_test_song_exp)
uniform_llhood += temp_val * log(1.0 / (double) k_test);
realn += temp_val;
if(myparas.output_distr)
{
//double temp_val_train = idx_train >= 0? retrieve_value_with_idx(tcount_train, idx_train): 0.0;
double temp_val_train;
if(idx_train < 0)
temp_val_train = 0.0;
else
temp_val_train = myparas.use_hash_TTable ? retrieve_value_with_idx(tcount_train, idx_train) : tcount_full_train[fr][to];
song_sep_ll[fr] += temp_val * p[s];
song_sep_ll[to] += temp_val * p[s];
fprintf(trans_distr_file, "%d %d %f\n", (int)temp_val_train, (int)temp_val, temp_val * p[s]);
}
}
}
}
}
Array2Dfree(delta, collection_size, d);
free(p);
Array2Dfree(tempkd, collection_size, d);
free(tempk);
if (myparas.num_points == 3) {
Array2Dfree(mid_delta, collection_size, d);
free(mid_p);
Array2Dfree(mid_tempkd, collection_size, d);
}
if(myparas.fast_collection || new_test_song_exp)
free(collection_idx);
}
if(myparas.output_distr)
{
printf("Writing song distr.\n");
for(i = 0; i < k; i++)
fprintf(song_distr_file, "%d %f\n", (int)(pd.id_counts[i]), song_sep_ll[i]);
fclose(song_distr_file);
fclose(trans_distr_file);
free(song_sep_ll);
}
llhood /= realn;
printf("Avg log-likelihood on test: %f\n", llhood);
if(myparas.fast_collection)
printf("Ratio of transitions that do not appear in the training set: %f\n", not_realn / (realn + not_realn));
if(new_test_song_exp)
{
uniform_llhood /= realn;
printf("Avg log-likelihood for uniform baseline: %f\n", uniform_llhood);
}
if(myparas.use_hash_TTable)
free_hash(tcount);
else
Array2Dfree(tcount_full, k, k);
free_playlists_data(pd);
if(myparas.output_distr)
{
if(myparas.use_hash_TTable)
free_hash(tcount_train);
else
Array2Dfree(tcount_full_train, k, k);
}
Array2Dfree(realX, k * myparas.num_points, d);
if(new_test_song_exp)
{
free(train_test_hash);
free(test_ids_for_new_songs);
}
if(myparas.square_dist_filename[0] != '\0')
{
write_embedding_to_file(square_dist, k, k, myparas.square_dist_filename, 0);
Array2Dfree(square_dist, k, k);
}
}
double R_rlm_rand(double *X, double *y, int *N, int *P,
int *Boot_Samp, int *Nres,
int *M, int *size_boot, double *ours, double *full,
double *Beta_m, double *Beta_s, double *Scale,
int *Seed, int *calc_full,
double *C, double *Psi_c, int *max_it,
int *converged_mm,
int *groups, int *n_group, int *k_fast_s)
{
void initialize_mat(double **a, int n, int m);
void initialize_vec(double *a, int n);
void R_S_rlm(double *X, double *y, int *n, int *P,
int *nres, int *max_it,
double *SCale, double *beta_s, double *beta_m,
int *converged_mm,
int *seed_rand, double *C, double *Psi_c,
int *Groups, int *N_group, int *K_fast_s);
double Psi_reg(double,double);
double Psi_reg_prime(double,double);
double Chi_prime(double,double);
double Chi(double,double);
void sampler_i(int, int, int *);
int inverse(double **,double **, int);
void matias_vec_vec(double **, double *, double *, int);
void scalar_mat(double **, double, double **, int, int);
void scalar_vec(double *, double, double *, int);
void sum_mat(double **,double **, double **, int, int);
void sum_vec(double *, double *, double *, int);
void dif_mat(double **, double **, double **, int , int );
void dif_vec(double *, double *, double *, int);
void mat_vec(double **, double *, double *, int, int);
void mat_mat(double **, double **, double **, int, int, int);
// void disp_vec(double *, int);
// void disp_mat(double **, int, int);
// void disp_mat_i(int **, int, int);
// void disp_vec(double *, int);
/* double **xb; */
double *Xb, **xb;
int **boot_samp;
double **x, **x2, **x3, **x4, *beta_m, *beta_s,*beta_aux;
double *Fi, *res, *res_s, *w, *ww, dummyscale, scale;
double *v, *v2, *v_aux, *yb; // , timefinish, timestart;
double u,u2,s,c,Psi_constant;
// double test_chi=0, test_psi=0;
int n,p,m,seed; // ,*indices;
int nboot=*size_boot;
// int fake_p = 0;
register int i,j,k;
setbuf(stdout,NULL);
c = *C; Psi_constant = *Psi_c;
n = *N; p = *P; m = *M; seed = *Seed;
boot_samp = (int **) malloc(m * sizeof(int*) );
for(i=0;i<m;i++)
boot_samp[i] = (int*) malloc(nboot *sizeof(int));
// indices = (int *) malloc( n * sizeof(int) );
v = (double *) malloc( p * sizeof(double) );
v2 = (double *) malloc( p * sizeof(double) );
v_aux = (double *) malloc( p * sizeof(double) );
yb = (double *) malloc( n * sizeof(double) );
Xb = (double*) malloc( n * p * sizeof(double) );
x = (double **) malloc ( n * sizeof(double *) );
xb = (double **) malloc ( n * sizeof(double *) );
Fi = (double *) malloc ( n * sizeof(double) );
res = (double *) malloc ( n * sizeof(double) );
res_s = (double *) malloc ( n * sizeof(double) );
ww = (double *) malloc ( n * sizeof(double) );
w = (double *) malloc ( n * sizeof(double) );
x2 = (double **) malloc ( p * sizeof(double *) );
x3 = (double **) malloc ( p * sizeof(double *) );
x4 = (double **) malloc ( p * sizeof(double *) );
beta_aux = (double *) malloc( p * sizeof(double) );
beta_m = (double *) malloc( p * sizeof(double) );
beta_s = (double *) malloc( p * sizeof(double) );
for(i=0;i<n;i++) {
x[i] = (double*) malloc (p * sizeof(double) );
xb[i] = (double*) malloc ((p+1) * sizeof(double) );
};
for(i=0;i<p;i++) {
x2[i] = (double*) malloc (p * sizeof(double) );
x3[i] = (double*) malloc (p * sizeof(double) );
x4[i] = (double*) malloc (p * sizeof(double) );
};
/* copy X into x for easier handling */
for(i=0;i<n;i++)
for(j=0;j<p;j++)
x[i][j]=X[j*n+i];
/* calculate robust regression estimates */
for(i=0;i<m;i++)
for(j=0;j<nboot;j++)
boot_samp[i][j]=Boot_Samp[j*m+i]-1;
R_S_rlm(X, y, N, P, Nres, max_it, &scale, Beta_s, Beta_m,
converged_mm, &seed, &c,
Psi_c, groups, n_group, k_fast_s);
*Scale = scale;
/* get M-fitted values in Fi */
mat_vec(x,Beta_m,Fi,n,p);
/* get residuals of M-est in res */
dif_vec(y,Fi,res,n);
/* get S-fitted values in res_s */
mat_vec(x,Beta_s,res_s,n,p);
/* get residuals of S-est in res_s */
dif_vec(y,res_s,res_s,n);
/* set auxiliary matrices to zero */
initialize_mat(x3, p, p);
initialize_mat(x4, p, p);
initialize_vec(v, p);
u2 = 0.0;
/* calculate correction matrix */
for(i=0;i<n;i++) {
u = res[i]/scale ;
w[i] = Psi_reg(u,Psi_constant)/res[i];
matias_vec_vec(x2,x[i],x[i],p);
scalar_mat(x2,Psi_reg_prime(u,Psi_constant),
x2,p,p);
sum_mat(x3,x2,x3,p,p);
matias_vec_vec(x2,x[i],x[i],p);
scalar_mat(x2,w[i],x2,p,p);
sum_mat(x4,x2,x4,p,p);
scalar_vec(x[i],Psi_reg_prime(u,Psi_constant)*u,v_aux,p);
sum_vec(v,v_aux,v,p);
u2 += Chi_prime(u, c) * u;
};
/* scalar_vec(v, .5 * (double) (n-p) * scale / u2 , v, p); */
scalar_vec(v, .5 * (double) n * scale / u2 , v, p);
inverse(x3,x2,p);
mat_mat(x2,x4,x3,p,p,p);
mat_vec(x2,v,v2,p,p);
scalar_mat(x3,scale,x3,p,p);
/* the correction matrix is now in x3 */
/* the correction vector is now in v2 */
/* start the bootstrap replications */
for(i=0;i<m;i++) {
/* change the seed! */
++seed;
// sampler_i(n,nboot,indices);
// for(j=0;j<nboot; j++)
// indices[j]=boot_samp[i][j];
/* get pseudo observed y's */
for(j=0;j<nboot;j++) /* xb[j][p] = */
yb[j] = y[boot_samp[i][j]];
for(j=0;j<nboot;j++)
for(k=0;k<p;k++) {
// xb[j][k] = x[boot_samp[i][j]][k];
// Xb[k*nboot+j] = X[k*n + indices[j]];
Xb[k*nboot+j] = x[boot_samp[i][j]][k];
xb[j][k] = Xb[k*nboot+j];
};
/* calculate full bootstrap estimate */
if( *calc_full == 1 )
R_S_rlm(Xb,yb,&nboot,P,Nres,max_it,&dummyscale,
beta_s,beta_m,converged_mm,&seed,&c,
Psi_c, groups, n_group, k_fast_s);
/* void R_S_rlm(double *X, double *y, int *n, int *P,
int *nres, int *max_it,
double *SCale, double *beta_s, double *beta_m,
int *converged_mm,
int *seed_rand, double *C, double *Psi_c,
int *Groups, int *N_group, int *K_fast_s) */
/* double *C, double *Psi_c, int *max_it,
int *groups, int *n_group, int *k_fast_s); */
// HERE
/* disp_mat(xb, nboot,p); */
// disp_vec(yb,nboot);
// Rprintf("\nfull scale: %f", dummyscale);
/* calculate robust bootsrap */
scalar_vec(v,0.0,v,p); /* v <- 0 */
scalar_mat(x2,0.0,x2,p,p); /* x2 <- 0 */
s = 0.0;
for(j=0;j<nboot;j++) {
scalar_vec(xb[j],yb[j]*w[boot_samp[i][j]],v_aux,p);
sum_vec(v,v_aux,v,p);
matias_vec_vec(x4,xb[j],xb[j],p);
scalar_mat(x4,w[boot_samp[i][j]],x4,p,p);
sum_mat(x2,x4,x2,p,p);
s += Chi(res_s[boot_samp[i][j]] / scale , c);
};
/* s = s * scale / .5 / (double) (nboot - p) ; */
s = s * scale / .5 / (double) n;
inverse(x2,x4,p); /* x4 <- x2^-1 */
mat_vec(x4,v,v_aux,p,p); /* v_aux <- x4 * v */
dif_vec(v_aux,Beta_m,v_aux,p); /* v_aux <- v_aux - beta_m */
/* v has the robust bootstrapped vector, correct it */
mat_vec(x3,v_aux,v,p,p); /* v <- x3 * v_aux */
scalar_vec(v2,s-scale,v_aux,p);
sum_vec(v_aux,v,v,p);
/* store the betas (splus-wise!) */
for(j=0;j<p;j++) {
ours[j*m+i]=v[j];
if( *calc_full == 1 )
// full[j*m+i]=beta_m[j]-Beta_m[j];
full[j*m+i]=beta_m[j];
};
};
for(i=0;i<m;i++)
free(boot_samp[i]);
free(boot_samp);
for(i=0;i<n;i++) {
free(x[i]);
free(xb[i]);
};
for(i=0;i<p;i++) {
free(x2[i]);
free(x3[i]);
free(x4[i]);
};
free(x) ;free(x2);free(xb);
free(x3);free(x4);
free(beta_aux);free(beta_m);free(beta_s);
free(w);free(ww);free(Fi);free(res);
free(v);free(v2);free(v_aux);free(yb);
free(res_s);
free(Xb);
return(0);
}
