void rlc_cleanup(struct pci_dev *dev)
{
struct dev_drv_data *dd;
dd = pci_get_drvdata(dev);
if (dd->family == VERDE)
rng_free(&dd->vram.mng, dd->rlc.save_restore);
rng_free(&dd->vram.mng, dd->rlc.clr_restore);
}
int main (int argc, char * argv[]) {
bool restart;
unsigned long int particles_n;
if (argc < 2) {
particles_n = NUMBER_OF_PARTICLES;
} else {
errno = 0;
particles_n = strtoul(argv[1], NULL, 0);
if (errno) {
perror(__func__);
exit(EXIT_FAILURE);
}
}
n = particles_n;
px =
align_padded_malloc(ALIGN_BOUNDARY, n*sizeof(value), ALLOC_PADDING);
py =
align_padded_malloc(ALIGN_BOUNDARY, n*sizeof(value), ALLOC_PADDING);
vx =
align_padded_malloc(ALIGN_BOUNDARY, n*sizeof(value), ALLOC_PADDING);
vy =
align_padded_malloc(ALIGN_BOUNDARY, n*sizeof(value), ALLOC_PADDING);
m =
align_padded_malloc(ALIGN_BOUNDARY, n*sizeof(value), ALLOC_PADDING);
if (px == NULL || py == NULL ||
vx == NULL || vy == NULL || m == NULL) {
perror("main");
exit(EXIT_FAILURE);
}
draw_init(SCREEN_WIDTH, SCREEN_HEIGHT, FRAME_RATE, n);
physics_init(n);
rng_init();
do {
draw_reset(n);
physics_reset(n);
restart = main_loop();
} while (restart);
rng_free();
physics_free();
draw_free();
align_free(m);
align_free(vy);
align_free(vx);
align_free(py);
align_free(px);
exit(EXIT_SUCCESS);
}
void* pique_thread(void* arg)
{
pique_ctx_t* ctx = arg;
seq_t* seq = seq_create();
twobit_t* tb = twobit_alloc();
rng_t* rng = rng_alloc(1234);
bool r;
while (true) {
pthread_mutex_lock(ctx->f_mutex);
if (ctx->fmt == INPUT_FMT_FASTA) r = fasta_read(ctx->f, seq);
else if (ctx->fmt == INPUT_FMT_FASTQ) r = fastq_read(ctx->f, seq);
pthread_mutex_unlock(ctx->f_mutex);
if (!r) break;
/* TODO: remove sequences with Ns? */
twobit_copy_str_n(tb, seq->seq.s, seq->seq.n);
dbg_add_twobit_seq(ctx->G, rng, tb);
}
rng_free(rng);
seq_free(seq);
return NULL;
}
int main( int argc, /* number of arguments */ char *argv[] /* arguments */){
int j,k;
unsigned int p;
myuint *X=rng_alloc();
//myuint X[N+1];
fprintf(stderr,"Welcome to the Yerevan random number generator!\nThe curent matrix size is %u\n"
"(the actual matrix is not kept in memory in this new efficient implementation)",N);
fprintf(stderr,"\nWe will now generate a power of our matrix A^m\n ");
fprintf(stderr,"\nHow many iterations m do you want?\n(2^10=1024, 2^20=1048576, 2^30=1073741824)\nEnter m: "); scanf("%u", &p);
printf("\nHere is convenient Mathematica Input:\n AT={");
for (k=1; (k<=N); k++) {
seedvector(X,k);
for (j=0; (j<p); j++) {
IT(X);
}
printf("\n{");
printf("%llu", X[1] );
for (j=2; (j<=N); j++) {
printf(",\t%llu", X[j] );
}
printf("},");
}
printf("\b} \n A = Transpose[AT] \n d = Det[A] \n AM = Mod[A, 2^61-1] \n cp = CharacteristicPolynomial[AM, x] \n "
"N[Solve[cp == 0, x]] \n N[Eigenvalues[AM]] \n Abs[%%]\n"
"M1K = Mod[MatrixPower[A, 2^10], 2^61 - 1]\n ");
//\n Maxima input: eq:charpoly(N,x);\n p:radcan(eq); r:allroots(p);
printf("\n\nok\n");
rng_free(X);
return 1;
}
void test_diag_std() {
const int N = 25;
double_vector_type * data = double_vector_alloc( 0,0);
rng_type * rng = rng_alloc(MZRAN , INIT_DEV_URANDOM );
matrix_type * m = matrix_alloc( N , N );
double sum1 = 0;
double sum2 = 0;
int i;
for (i=0; i < N; i++) {
double R = rng_get_double( rng );
matrix_iset(m , i , i , R);
double_vector_iset( data , i , R );
sum1 += R;
sum2 += R*R;
}
{
double mean = sum1 / N;
double std = sqrt( sum2 / N - mean * mean );
test_assert_double_equal( std , matrix_diag_std( m , mean ));
test_assert_double_equal( statistics_std( data ) , matrix_diag_std( m , mean ));
test_assert_double_equal( statistics_mean( data ) , mean );
}
matrix_free( m );
rng_free( rng );
}
void test_det2() {
matrix_type * m = matrix_alloc(2,2);
rng_type * rng = rng_alloc(MZRAN , INIT_DEV_URANDOM );
matrix_random_init( m , rng );
{
double det2 = matrix_det2( m );
double det = matrix_det( m );
test_assert_double_equal( det , det2 );
}
matrix_free( m );
rng_free( rng );
}
/* -------------------------- Effectors ------------------ */
void free_partdeflect(PartDeflect *pd)
{
if(!pd)
return;
if(pd->tex)
pd->tex->id.us--;
if(pd->rng)
rng_free(pd->rng);
MEM_freeN(pd);
}
void ert_test_context_free( ert_test_context_type * test_context ) {
if (test_context->enkf_main)
enkf_main_free( test_context->enkf_main );
if (test_context->work_area)
test_work_area_free( test_context->work_area );
if (test_context->rng)
rng_free( test_context->rng );
free( test_context );
}
void test_det4() {
matrix_type * m = matrix_alloc(4 , 4 );
rng_type * rng = rng_alloc(MZRAN , INIT_DEV_URANDOM );
for (int i=0; i < 10; i++) {
matrix_random_init( m , rng );
{
double det4 = matrix_det4( m );
double det = matrix_det( m );
test_assert_double_equal( det , det4 );
}
}
matrix_free( m );
rng_free( rng );
}
int main(int argc , char ** argv) {
const int queue_timeout = 180;
const int submit_timeout = 180;
const int status_timeout = 180;
const int number_of_jobs = 250;
const int submit_threads = number_of_jobs / 10 ;
const int status_threads = number_of_jobs + 1;
const char * job = util_alloc_abs_path(argv[1]);
rng_type * rng = rng_alloc( MZRAN , INIT_CLOCK );
test_work_area_type * work_area = test_work_area_alloc("job_queue");
job_type **jobs = alloc_jobs( rng , number_of_jobs , job);
job_queue_type * queue = job_queue_alloc(number_of_jobs, "OK", "ERROR");
queue_driver_type * driver = queue_driver_alloc_local();
job_queue_manager_type * queue_manager = job_queue_manager_alloc( queue );
job_queue_set_driver(queue, driver);
job_queue_manager_start_queue(queue_manager, 0, false , true);
{
thread_pool_type * status_pool = thread_pool_alloc( status_threads , true );
thread_pool_type * submit_pool = thread_pool_alloc( submit_threads , true );
submit_jobs( queue , number_of_jobs , jobs , submit_pool );
status_jobs( queue , number_of_jobs , jobs , status_pool );
if (!thread_pool_try_join( submit_pool , submit_timeout ))
util_exit("Joining submit pool failed \n");
thread_pool_free( submit_pool );
job_queue_submit_complete(queue);
if (!thread_pool_try_join( status_pool , status_timeout))
util_exit("Joining status pool failed \n");
thread_pool_free( status_pool );
}
if (!job_queue_manager_try_wait(queue_manager , queue_timeout))
util_exit("job_queue never completed \n");
job_queue_manager_free(queue_manager);
job_queue_free(queue);
queue_driver_free(driver);
check_jobs( number_of_jobs , jobs );
test_work_area_free(work_area);
rng_free( rng );
}
void test_column_equal() {
matrix_type * m1 = matrix_alloc(5,5);
matrix_type * m2 = matrix_alloc(5,5);
matrix_type * m3 = matrix_alloc(6,5);
rng_type * rng = rng_alloc( MZRAN , INIT_DEFAULT );
matrix_random_init( m1 , rng );
matrix_assign( m2 , m1 );
test_assert_true( matrix_columns_equal( m1 , 2 , m2 , 2 ));
test_assert_false( matrix_columns_equal( m1 , 2 , m2 , 3 ));
test_assert_false( matrix_columns_equal( m1 , 2 , m3 , 3 ));
rng_free( rng );
matrix_free( m1 );
matrix_free( m2 );
matrix_free( m3 );
}
void test_resize() {
matrix_type * m1 = matrix_alloc(5,5);
matrix_type * m2 = matrix_alloc(5,5);
rng_type * rng = rng_alloc( MZRAN , INIT_DEFAULT );
matrix_random_init( m1 , rng );
matrix_assign( m2 , m1 );
test_assert_true( matrix_equal( m1 , m2 ));
matrix_resize( m1 , 5 , 5 , false );
test_assert_true( matrix_equal( m1 , m2 ));
matrix_resize( m1 , 5 , 5 , true );
test_assert_true( matrix_equal( m1 , m2 ));
rng_free( rng );
matrix_free( m1 );
matrix_free( m2 );
}
int main(int argc , char ** argv) {
rng_type * rng = rng_alloc( MZRAN , INIT_DEFAULT );
{
int val1 = rng_get_int( rng , MAX_INT);
int val2 = rng_get_int( rng , MAX_INT);
test_assert_int_not_equal( val1 , val2 );
rng_init( rng , INIT_DEFAULT );
val2 = rng_get_int( rng , MAX_INT );
test_assert_int_equal( val1 , val2 );
}
{
int val2 , val1;
int state_size = rng_state_size( rng );
char * buffer1 = util_calloc( state_size , sizeof * buffer1 );
char * buffer2 = util_calloc( state_size , sizeof * buffer2 );
test_assert_int_not_equal( state_size , 0 );
test_assert_int_equal( state_size , MZRAN_STATE_SIZE );
rng_init( rng , INIT_DEFAULT );
rng_get_state( rng , buffer1 );
val1 = rng_get_int( rng , MAX_INT);
val2 = rng_get_int( rng , MAX_INT);
test_assert_int_not_equal( val1 , val2 );
rng_set_state( rng , buffer1 );
val2 = rng_get_int( rng , MAX_INT);
test_assert_int_equal( val1 , val2 );
rng_init( rng , INIT_DEFAULT );
rng_get_state( rng , buffer2 );
test_assert_mem_equal( buffer1 , buffer2 , state_size );
val2 = rng_get_int( rng , MAX_INT);
rng_get_state( rng , buffer2 );
test_assert_mem_not_equal( buffer1 , buffer2 , state_size );
free( buffer1 );
free( buffer2 );
}
rng_free( rng );
exit(0);
}
void test_readwrite() {
test_work_area_type * test_area = test_work_area_alloc("matrix-test");
{
rng_type * rng = rng_alloc(MZRAN , INIT_DEV_URANDOM );
matrix_type * m1 = matrix_alloc(3 , 3);
matrix_type * m2 = matrix_alloc(3 , 3);
matrix_random_init( m1 , rng );
matrix_assign(m2 , m1);
test_assert_true( matrix_equal( m1 , m2 ) );
{
FILE * stream = util_fopen("m1" , "w");
matrix_fwrite( m1 , stream );
fclose( stream );
}
matrix_random_init( m1 , rng );
test_assert_false( matrix_equal( m1 , m2 ) );
{
FILE * stream = util_fopen("m1" , "r");
matrix_free( m1 );
m1 = matrix_alloc(1,1);
printf("-----------------------------------------------------------------\n");
matrix_fread( m1 , stream );
test_assert_int_equal( matrix_get_rows(m1) , matrix_get_rows( m2));
test_assert_int_equal( matrix_get_columns(m1) , matrix_get_columns( m2));
util_fseek( stream , 0 , SEEK_SET);
{
matrix_type * m3 = matrix_fread_alloc( stream );
test_assert_true( matrix_equal( m2 , m3 ));
matrix_free( m3 );
}
fclose( stream );
}
test_assert_true( matrix_equal( m1 , m2 ) );
matrix_free( m2 );
matrix_free( m1 );
rng_free( rng );
}
test_work_area_free( test_area );
}
long rlc_init(struct pci_dev *dev)
{
struct dev_drv_data *dd;
long r;
dd = pci_get_drvdata(dev);
if (dd->family == VERDE) {
r = verde_init(dev);
if (r == -SI_ERR)
return -SI_ERR;
}
r = clearstate_init(dev);
if (r == -SI_ERR)
goto err_free_save_restore;
return 0;
err_free_save_restore:
if (dd->family == VERDE)
rng_free(&dd->vram.mng, dd->rlc.save_restore);
return -SI_ERR;
}
int main( int argc, char ** argv) {
rng_type * rng = rng_alloc( MZRAN , INIT_DEV_RANDOM );
matrix_type * A = matrix_alloc( 12 , 12 );
matrix_type * B = matrix_alloc( 12 , 12 );
matrix_random_init( A , rng );
matrix_assign( B , A );
matrix_pretty_print( A , " A " , "%8.4f" );
#ifdef WITH_LAPACK
matrix_inv( B );
printf("\n");
matrix_pretty_print( B , "inv(A)" , "%8.4f" );
matrix_inplace_matmul( B , A );
printf("\n");
matrix_pretty_print( B , " I " , "%8.4f" );
{
matrix_type * A3 = matrix_alloc(3,3);
matrix_random_init( A3 , rng );
matrix_iset( A3 , 0 , 0 , sin(0.98));
printf("matrix_det3:%g ",matrix_det3( A3 ));
printf("matrix_det:%g \n",matrix_det( A3 ));
}
{
matrix_type * A4 = matrix_alloc(4,4);
matrix_random_init( A4 , rng );
matrix_iset( A4 , 0 , 0 , sin(0.98));
printf("matrix_det4:%g ",matrix_det4( A4 ));
printf("matrix_det:%g \n",matrix_det( A4 ));
}
#endif
matrix_free( A );
matrix_free( B );
rng_free( rng );
}
int main(int argc, char* argv[])
{
int i, j, c, iter, ITER=200;
unsigned long int seed=0;
int bcycle = 0;
float bstart = 0;
int acycle = 0;
float astart = 0;
int burnin = 0;
stable_t *ST = NULL;
int useN = DIM*2;
MAXN = 1;
MAXT = MAXSTAB;
/*
* default values for args
*/
while ( (c=getopt(argc, argv,"a:b:B:C:I:hH:I:N:P:S:s:T:v"))>=0 ) {
switch ( c ) {
case 'h':
usage(burnin?burnin:ITER/2, ITER, useN);
exit(0);
case 'b':
if ( !optarg || sscanf(optarg,"%f,%f",&bpar, &bstart)<1 )
yaps_quit("Need a valid 'b' argument\n");
break;
case 'a':
if ( !optarg || sscanf(optarg,"%f,%f",&apar,&astart)<1 )
yaps_quit("Need a valid 'a' argument\n");
break;
case 'H':
if ( !optarg || sscanf(optarg,"%d",&bcycle)!=1 )
yaps_quit("Need a valid 'G' argument\n");
break;
case 'T':
if ( !optarg || sscanf(optarg,"%d",&MAXT)!=1 )
yaps_quit("Need a valid 'T' argument\n");
break;
case 'I':
if ( !optarg || sscanf(optarg,"%d",&acycle)!=1 )
yaps_quit("Need a valid 'H' argument\n");
break;
case 'N':
if ( !optarg || sscanf(optarg,"%d",&useN)!=1 )
yaps_quit("Need a valid 'N' argument\n");
break;
case 'C':
if ( !optarg || sscanf(optarg,"%d",&ITER)!=1 )
yaps_quit("Need a valid 'C' argument\n");
break;
case 'B':
if ( !optarg || sscanf(optarg,"%d",&burnin)!=1 )
yaps_quit("Need a valid 'B' argument\n");
break;
case 's':
if ( !optarg || sscanf(optarg,"%lu",&seed)!=1 )
yaps_quit("Need a valid 's' argument\n");
break;
case 'v':
verbose++;
break;
#ifdef S_USE_THREADS
case 'P':
if ( !optarg || sscanf(optarg,"%u",&threads)!=1 )
yaps_quit("Need a valid 'P' argument\n");
break;
#endif
default:
yaps_message("Bad command line argument\n\n");
usage(burnin?burnin:ITER/2, ITER, useN);
exit(0);
}
}
if ( useN>=MAXDATA )
yaps_quit("N too large\n");
if ( burnin==0 )
burnin = ITER/2;
else
if ( burnin>=ITER-1 )
yaps_quit("Burnin %d too large for cycles %d\n", burnin, ITER);
yaps_message("Configuration details\n");
yaps_message("=====================\n");
/*
* set random number generator
*/
if ( seed ) {
rng_seed(rng,seed);
} else {
rng_time(rng,&seed);
}
yaps_message("Setting seed for data = %lu\n", seed);
if ( acycle && apar==0 )
apar = 0.5;
yaps_message("Setting a=%f, b=%f, N=%d, D=%d\n",
apar, bpar, useN, NUMMN);
yaps_message(" burnin=%d,", burnin);
yaps_message(" cycles=%d\n", ITER);
/*
* fix pointers
*/
for (j=0; j<NUMMN; j++) {
n[j] = &n_data[j*DIM];
t[j] = &t_data[j*DIM];
tave[j] = &tave_data[j*DIM];
}
/*
* initialise everything
*/
for (j=0; j<NUMMN; j++) {
N[j] = useN;
T[j] = 0;
Tave[j] = 0;
for (i=0; i<DIM; i++) {
n[j][i] = 0;
t[j][i] = 0;
tave[j][i] = 0;
}
}
/*
* fix base distribution, uniform
*/
{
for (i=0; i<DIM; i++) {
H[i] = 1.0/DIM;
}
}
/*
* create data using a CRP to get initialisation for n[]
*/
c = 0;
for (j=0; j<NUMMN; j++) {
int cc;
i = sampleH();
data[c++] = i;
// first entry always adds a table
n[j][i]++;
t[j][i]++;
T[j]++;
for (cc=1; cc<N[j]; cc++) {
float val = (cc+bpar)*rng_unit(rng);
val -= T[j]*apar+bpar;
if ( val<=0 ) {
// new table
i = sampleH();
t[j][i]++;
T[j]++;
} else {
for (i=0; i<DIM; i++) {
val -= n[j][i] - t[j][i]*apar;
if ( val<0 )
break;
}
}
assert(i<DIM);
n[j][i]++;
data[c++] = i;
}
}
binit = bpar;
/*
* record maximum entries in data
* do this where possible so that one can get the table
* sizes right
*
*/
MAXN = n[0][0]+1;
MAXT = 1;
for (j=0; j<NUMMN; j++) {
for (i=0; i<DIM; i++) {
if ( MAXN<=n[j][i] ) MAXN = n[j][i]+1;
if ( MAXT<t[j][i] ) MAXT = t[j][i]*1.1+1;
}
}
if ( MAXT>MAXN )
MAXT = MAXN;
yaps_message("Making S for N=%d M=%d a=%lf\n", MAXN,MAXT,apar);
ST = S_make(MAXN, MAXT, MAXN, MAXTAB,
apar, S_STABLE | S_UVTABLE);
if ( ST==NULL )
yaps_quit("Making S failed!\n");
S_report(ST,stdout);
/*
* the seed only sets the data/sample,
* the seed for the simulation/Gibbs is always random
*/
rng_free(rng);
rng_time(rng,&seed);
//yaps_message("Resetting seed = %lu\n", seed);
/*
* report on initial data statistics
*/
yaps_message("\nData sampled\n");
yaps_message("============\n");
for (j=0; j<NUMMN; j++) {
yaps_message("n[%d] =", j);
for (i=0; i<DIM; i++)
yaps_message(" %d", n[j][i]);
yaps_message(" = %d\n", N[j]);
yaps_message("t[%d] =",j);
for (i=0; i<DIM; i++)
yaps_message(" %d", t[j][i]);
yaps_message(" = %d\n", T[j]);
}
/*
* set the hyperparameters used in Gibbs,
* can be different to data
*/
if ( bstart==0 )
bstart = bpar;
if ( astart==0 )
astart = apar;
// initialise latent stats and reporting info
for (j=0; j<NUMMN; j++) {
T[j] = 0;
Tave[j] = 0;
}
tcnt = 0;
bave = 0;
bcnt = 0;
aave = 0;
acnt = 0;
bpar = bstart;
if ( verbose && bcycle!=0 )
yaps_message("Starting with initial b=%f\n", bpar);
apar = astart;
if ( verbose && acycle!=0 )
yaps_message("Starting with initial a=%f\n", apar);
for (j=0; j<NUMMN; j++) {
for (i=0; i<DIM; i++) {
tave[j][i] = 0;
t[j][i] = 0;
if ( n[j][i]>0 ) {
/*
* initialise to a single table
*/
t[j][i] = 1;
T[j]++;
}
}
}
for ( iter=0; iter<ITER; iter++) {
/*
* sampling with table indicators
*/
c = 0;
for (j=0; j<NUMMN; j++) {
int cc;
for (cc=0; cc<N[j]; cc++) {
float one;
i = data[c++];
assert(n[j][i]);
if ( n[j][i]==1 )
// this indicator must always be 1, no sampling
continue;
// sample whether it contributes to a table
if ( t[j][i]>1 &&
(n[j][i]-1)*rng_unit(rng)<(t[j][i]-1) ) {
t[j][i]--;
T[j]--;
}
assert(t[j][i]<n[j][i]);
// sample new table indicator
one = H[i] * (bpar + T[j]*apar) * (t[j][i]) / (n[j][i]-t[j][i]+1)
* S_V(ST, n[j][i],t[j][i]+1);
if ( rng_unit(rng) < one/(one+1.0) ) {
t[j][i]++;
T[j]++;
}
}
}
/*
* one major cycle of Gibbs sampler finished
*/
if ( verbose>1 ) {
for (j=0; j<NUMMN; j++) {
for (i=0; i<DIM; i++)
printf(" %d", t[j][i]);
printf(" = %d\n", T[j]);
}
}
/*
* sample & record b
*/
if ( bcycle!=0 && iter%bcycle==0 ) {
// Gibbs on bpar (concentration par) too
if ( bcycle<0 ) {
int bc = -bcycle;
for (bc-- ; bc>0; bc--)
bpar = sampleb(bpar, 1, PB_shape, PB_scale, N, T,
apar, rng, 1, 1);
}
bpar = sampleb(bpar, 1, PB_shape, PB_scale, N, T,
apar, rng, 1, 1);
if ( iter>=burnin ) {
bave += bpar;
bcnt ++;
}
}
/*
* sample & record a
*/
if ( acycle!=0 && iter%acycle==0 ) {
int dimI[NUMMN];
double dimb[NUMMN];
for (j=0; j<NUMMN; j++) {
dimI[j] = DIM;
dimb[j] = bpar;
}
// Gibbs on apar (discount par) too
if ( acycle<0 ) {
int bc = -acycle;
for (bc-- ; bc>0; bc--)
apar = samplea(apar, NUMMN, dimI, T, n, t, NULL, dimb, rng, 1, 1);
}
apar = samplea(apar, NUMMN, dimI, T, n, t, NULL, dimb, rng, 1, 1);
if ( iter>=burnin ) {
aave += apar;
acnt ++;
}
if ( verbose>1 )
yaps_message("Extending S for a=%lf\n", apar);
if ( S_remake(ST,apar) )
yaps_message("Extending S failed\n");
}
/*
* full statistics collection
*/
if ( iter>=burnin ) {
for (j=0; j<NUMMN; j++) {
for (i=0; i<DIM; i++) {
tave[j][i] += t[j][i];
}
Tave[j] += T[j];
}
tcnt ++;
}
}
/*
* report for this experiment
*/
yaps_message("\nEstimates\n");
yaps_message("=========\n");
for (j=0; j<NUMMN; j++) {
yaps_message("t[%d] = ", j);
for (i=0; i<DIM; i++)
yaps_message(" %.2f", tave[j][i]/tcnt);
yaps_message("\nT[%d]=%.2f\n", j, Tave[j]/tcnt);
}
if ( bcycle!=0 && bcnt>0 )
yaps_message("\nb=%.2f", bave/bcnt);
if ( acycle!=0 && acnt>0 )
yaps_message("\na=%.3f", aave/acnt);
yaps_message("\n");
S_free(ST);
rng_free(rng);
return 0;
}
int main(int argc, char* argv[])
{
size_t step_size = 1;
while (1) {
int opt = getopt(argc, argv, "hs:");
if (opt == -1) break;
switch (opt) {
case 's':
step_size = (size_t) strtoul(optarg, NULL, 10);
break;
case 'h':
print_usage(stdout);
return EXIT_SUCCESS;
case '?':
return EXIT_FAILURE;
default:
abort();
}
}
if (optind >= argc) {
print_usage(stderr);
return EXIT_FAILURE;
}
const char* fn = argv[optind];
FILE* f = fopen(fn, "r");
if (f == NULL) {
fprintf(stderr, "Can't open %s for reading.\n", fn);
return EXIT_FAILURE;
}
fprintf(stderr, "Reading adjacency matrix ... ");
/* some rather brittle parsing of matrix market files */
char buffer[512];
fgets(buffer, sizeof(buffer), f);
if (strcmp(buffer, "%%MatrixMarket matrix coordinate integer general\n") != 0) {
fprintf(stderr, "Error: Incorrectly formatted matrix market file.\n");
return EXIT_FAILURE;
}
unsigned int n, n2, m;
fscanf(f, "%u %u %u\n", &n, &n2, &m);
edge_array_t E;
E.size = m;
E.m = 0;
E.es = malloc_or_die(E.size * sizeof(edge_t));
edge_t e;
while (fgets(buffer, sizeof(buffer), f)) {
sscanf(buffer, "%u %u %u\n", &e.u, &e.v, &e.w);
e.u -= 1; e.v -= 1; /* make 0-based */
push_edge(&E, &e);
}
fclose(f);
rng_t* rng = rng_alloc();
shuffle(rng, E.es, E.m);
rng_free(rng);
qsort(E.es, E.m, sizeof(edge_t), edge_cmp);
fprintf(stderr, "done. (%zu edges)\n", E.m);
unsigned int* ds = malloc_or_die(n * sizeof(unsigned int));
size_t i;
for (i = 0; i < E.m; i += step_size) {
size_t c = count_components(E.es + i, n, E.m - i, ds);
printf("%zu\n", c);
fflush(stdout);
}
free(ds);
free(E.es);
return EXIT_SUCCESS;
}
/*==========================================
* main
*========================================== */
int main(int argc, char* argv[])
{
int c, iter, ITER=0, seed=0;
enum dataType data = LdaC;
enum dataType testdata = LdaC;
int dots = 0;
enum GibbsType fix_hold = GibbsNone;
char *stem;
char *resstem;
int topwords = 20;
int noerrorlog = 0;
int displayed = 0;
int load_vocab = 0;
int checkpoint = 0;
int restart = 0;
int dopmi = 0;
int restart_hca = 0;
int load_phi = 0;
int load_mu = 0;
int procs = 1;
int maxW = 0;
enum ScoreType score=ST_idf;
double BM0val=0, BM1val =0, BP0val=0, BP1val=0;
clock_t t1=0, t2=0, t3=0;
double tot_time = 0;
double psample_time = 0;
enum ParType par;
/*
* default values
*/
ddN.T = 10;
ITER = 100;
ddN.TEST = 0;
pctl_init();
while ( (c=getopt(argc, argv,"b:c:C:d:ef:F:g:G:h:K:l:L:N:o:pq:vr:s:S:t:T:vVW:"))>=0 ) {
switch ( c ) {
case 'b':
if ( !optarg || sscanf(optarg,"%d",&ddP.back)!=1 )
yap_quit("Need a valid 'b' argument\n");
break;
case 'c':
if ( !optarg || sscanf(optarg,"%d",&checkpoint)!=1 )
yap_quit("Need a valid 'c' argument\n");
break;
case 'C':
if ( !optarg || sscanf(optarg,"%d",&ITER)!=1 )
yap_quit("Need a valid 'C' argument\n");
break;
case 'd':
if ( !optarg || sscanf(optarg,"%d",&dots)!=1 )
yap_quit("Need a valid 'd' argument\n");
break;
case 'e':
noerrorlog++;
break;
case 'f':
if ( strcmp(optarg,"witdit")==0 )
data = WitDit;
else if ( strcmp(optarg,"docword")==0 )
data = Docword;
else if ( strcmp(optarg,"ldac")==0 )
data = LdaC;
else if ( strcmp(optarg,"bag")==0 )
data = TxtBag;
else if ( strcmp(optarg,"lst")==0 )
data = SeqTxtBag;
else
yap_quit("Illegal data type for -f\n");
break;
case 'F':
if ( strcmp(optarg,"all")==0 ) {
for (par=ParAM; par<=ParBB; par++)
ddT[par].fix = 1;
} else {
par = findpar(optarg);
if ( par==ParNone )
yap_quit("Illegal arg for -F\n");
ddT[par].fix = 1;
}
break;
case 'g':
{
char var[100];
int st=0;
if ( !optarg || sscanf(optarg,"%[^, ],%d", &var[0], &st)<1 )
yap_quit("Need a valid 'g' argument\n");
par = findpar(var);
if ( par==ParBP1 )
ddP.kbatch = st;
else
yap_quit("Illegal var for -g\n");
}
break;
case 'G':
{
char var[100];
int st=0, cy=0;
if ( !optarg || sscanf(optarg,"%[^, ],%d,%d",
&var[0], &cy, &st)<2 || st<0 || cy<0 )
yap_quit("Need a valid 'G' argument\n");
par = findpar(var);
if ( par==ParNone || par==ParB0P || par==ParB0M )
yap_quit("Illegal var for -G\n");
ddT[par].fix = 0;
ddT[par].start = st;
ddT[par].cycles = cy;
}
break;
case 'h':
{
fix_hold = GibbsHold;
if ( !optarg )
yap_quit("Need a valid 'h' argument\n");
if ( strncmp(optarg,"dict,",5)==0 ) {
if ( sscanf(&optarg[5],"%d",&ddP.hold_dict)<1 || ddP.hold_dict<2 )
yap_quit("Need a valid 'hdict' argument\n");
} else if ( strncmp(optarg,"fract,",6)==0 ) {
if ( sscanf(&optarg[6],"%lf",&ddP.hold_fraction)<1
|| ddP.hold_fraction<=0 || ddP.hold_fraction>=1 )
yap_quit("Need a valid 'hfract' argument\n");
} else if ( strncmp(optarg,"doc,",4)==0 ) {
if ( sscanf(&optarg[4],"%d",&ddP.hold_every)<1 || ddP.hold_every<2 )
yap_quit("Need a valid 'hdoc' argument\n");
} else
yap_quit("Need a valid 'h' argument\n");
}
break;
case 'K':
if ( !optarg || sscanf(optarg,"%d",&ddN.T)!=1 )
yap_quit("Need a valid 'K' argument\n");
break;
case 'l':
if ( !optarg )
yap_quit("Need a valid 'l ' argument\n");
if ( strncmp(optarg,"phi,",4)==0 ) {
if ( sscanf(&optarg[4],"%d,%d",&ddP.phiiter, &ddP.phiburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"theta,",6)==0 ) {
if ( sscanf(&optarg[6],"%d,%d",&ddP.thetaiter, &ddP.thetaburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"mu,",3)==0 ) {
if ( sscanf(&optarg[3],"%d,%d",&ddP.muiter, &ddP.muburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"prog,",5)==0 ) {
if ( sscanf(&optarg[5],"%d,%d",&ddP.progiter, &ddP.progburn)<2 )
yap_quit("Need a valid 'l prog,' argument\n");
} else
yap_quit("Need a valid DIAG code in 'l' argument\n");
break;
case 'L':
if ( !optarg )
yap_quit("Need a valid 'L ' argument\n");
if ( strncmp(optarg,"like,",5)==0 ) {
if ( sscanf(&optarg[5],"%d,%d",&ddP.mltiter, &ddP.mltburn)<1 )
yap_quit("Need a valid 'L like' argument\n");
} else
yap_quit("Need a valid DIAG code in 'L' argument\n");
break;
case 'N':
if ( !optarg || sscanf(optarg,"%d,%d", &ddP.maxN, &ddP.maxM)<1 )
yap_quit("Need a valid 'N' argument\n");
break;
case 'o':
{
char *ptr = strchr(optarg, ',');
int len = strlen(optarg);
if ( ptr )
len = ptr - optarg;
if ( strncmp(optarg,"idf",len)==0 )
score = ST_idf;
else if ( strncmp(optarg,"count",len)==0 )
score = ST_count;
else if ( strncmp(optarg,"Q",len)==0 )
score = ST_Q;
else if ( strncmp(optarg,"cost",len)==0 )
score = ST_cost;
else
yap_quit("Need a valid parameter for 'o' argument\n");
if ( ptr ) {
/* there was a second arg */
if ( sscanf(ptr+1, "%d", &topwords) != 1)
yap_quit("Need a valid second 'o' argument\n");
}
break;
}
break;
case 'p':
dopmi++;
break;
case 'q':
if(!optarg || sscanf(optarg, "%d", &procs) != 1)
yap_quit("Need a valid 'q' argument\n");
break;
case 'r':
if(!optarg )
yap_quit("Need a valid 'r' argument\n");
if ( strcmp(optarg,"tca")==0 )
restart++;
else if ( strcmp(optarg,"hca")==0 )
restart_hca++;
else if ( strcmp(optarg,"phi")==0 )
load_phi++;
else if ( strcmp(optarg,"mu")==0 )
load_mu++;
else
yap_quit("Need a valid 'r' argument\n");
break;
case 's':
if ( !optarg || sscanf(optarg,"%d",&seed)!=1 )
yap_quit("Need a valid 's' argument\n");
break;
case 'S':
{
char var[100];
double vin=0;
if ( !optarg || sscanf(optarg,"%[^=, ]=%lf",
&var[0], &vin)<2 )
yap_quit("Need a valid 'S' argument\n");
par = findpar(var);
if ( par==ParNone )
yap_quit("Illegal var for -S\n");
else if ( par==ParBM0 )
BM0val = vin;
else if ( par==ParBM1 )
BM1val = vin;
else if ( par==ParBP0 )
BP0val = vin;
else if ( par==ParBP1 )
BP1val = vin;
else
*(ddT[par].ptr) = vin;
}
break;
case 't':
if ( !optarg || sscanf(optarg,"%d",&ddP.training)!=1 )
yap_quit("Need a valid 't' argument\n");
break;
case 'T':
if ( !optarg )
yap_quit("Need a valid 'T' argument\n");
{
char *tname = data_name(optarg,data);
FILE *fp = fopen(tname,"r");
if ( fp==NULL ) {
free(tname);
tname = data_name(optarg,testdata);
fp = fopen(tname,"r");
} else {
testdata = data;
}
free(tname);
if ( fp!=NULL ) {
/* its a valid test filename */
ddP.teststem = optarg;
fclose(fp);
} else if ( sscanf(optarg,"%d",&ddN.TEST)!=1 )
yap_quit("Need a valid 'T' argument\n");
}
break;
case 'v':
verbose++;
break;
case 'V':
load_vocab = 1;
break;
case 'W':
if ( !optarg || sscanf(optarg,"%d",&maxW)<1 )
yap_quit("Need a valid 'W' argument\n");
break;
default:
yap_quit("Unknown option '%c'\n", c);
}
}
if (argc-optind != 2) {
usage();
exit(-1);
}
if ( optind>=argc ) {
yap_quit("No arguments given\n");
}
stem = strdup(argv[optind++]);
resstem = strdup(argv[optind++]);
if ( dopmi )
load_vocab = 1;
if ( dopmi && verbose !=2 ) {
/*
* due to the use of the ".top" file
* its really multi-purpose
*/
yap_quit("When computing PMI verbose must be exactly 2\n");
}
if ( noerrorlog==0 ) {
char *wname = yap_makename(resstem, ".log");
yap_file(wname);
free(wname);
}
yap_commandline(argc, argv);
#ifdef H_THREADS
yap_message(" Threads,");
#endif
if ( restart || restart_hca ) {
char *fname = yap_makename(resstem,".par");
FILE *fp = fopen(fname,"r");
char *buf;
if ( !fp )
yap_quit("Parameter file '%s' doesn't exist\n", fname);
fclose(fp);
free(fname);
buf = readpar(resstem,"T",50);
if ( !buf )
yap_quit("Parameter file '%s' has no T\n", fname);
ddN.T = atoi(buf);
free(buf);
if ( restart ) {
buf = readpar(resstem,"E",50);
if ( !buf )
yap_quit("Parameter file '%s' has no E\n", fname);
ddN.E = atoi(buf);
free(buf);
pctl_read(resstem);
}
if ( maxW==0 ) {
buf = readpar(resstem,"W",50);
if ( buf ) {
maxW = atoi(buf);
free(buf);
}
}
if ( ddP.training==0 ) {
buf = readpar(resstem,"TRAIN",50);
if ( buf ) {
ddP.training = atoi(buf);
free(buf);
}
}
if ( ddN.TEST==0 ) {
buf = readpar(resstem,"TEST",50);
if ( buf ) {
ddN.TEST = atoi(buf);
free(buf);
}
}
}
assert(ddN.T>0);
assert(ddN.TEST>=0);
assert(restart || restart_hca || ITER>0);
if ( load_phi && ddP.phiiter>0 )
yap_quit("Options '-l phi,...' and '-r phi' incompatible\n");
if ( load_mu && ddP.muiter>0 )
yap_quit("Options '-l mu,...' and '-r mu' incompatible\n");
/*
* set random number generator
*/
if ( seed ) {
rng_seed(rngp,seed);
} else {
rng_time(rngp,&seed);
}
yap_message("Setting seed = %lu\n", seed);
/*
* read data and get dimensions
*/
{
D_bag_t *dbp = data_read(stem, data);
int training = pctl_training(dbp->D);
if ( ddP.teststem ) {
D_bag_t *dbpt = data_read(ddP.teststem, testdata);
/* need to load a separate test set, strip to bare training */
data_shrink(dbp, training);
ddN.TEST = dbpt->D;
data_append(dbp, dbpt);
free(dbpt->w); free(dbpt->d); free(dbpt);
}
if ( maxW>0 ) {
if ( dbp->W <= maxW )
dbp->W = maxW;
if ( dbp->W > maxW )
data_vocabshrink(dbp, maxW);
}
/*
* transfer into system
*/
ddN.D = dbp->D;
ddN.W = dbp->W;
ddN.N = dbp->N;
ddN.NT = dbp->N;
ddN.DT = training;
ddD.w = dbp->w;
ddD.d = dbp->d;
free(dbp);
if ( ddN.DT<ddN.D ) {
/* recompute NT */
int i;
for (i=0; i<ddN.N; i++)
if ( ddD.d[i]>=ddN.DT )
break;
ddN.NT = i;
}
}
data_read_epoch(stem);
/*
* at this point, dimensions are fixed, so load phi and mu if needed
*/
if ( load_phi )
pctl_loadphi(resstem);
if ( load_mu )
pctl_loadmu(resstem);
/*
* correct parameters after command line
*/
pctl_fix(ITER);
if ( BM0val>0 ) {
ddP.b_mu[0] = BM0val;
}
if ( BM1val>0 ) {
int i;
for (i=1; i<ddN.E; i++)
ddP.b_mu[i] = BM1val;
}
if ( BP0val>0 ) {
int i;
for (i=0; i<ddN.T; i++)
ddP.b_phi[0][i] = BP0val;
}
if ( BP1val>0 ) {
int i;
if ( ddN.E==1 )
yap_quit("b_phi[1] invalid when epochs==1\n");
for (i=0; i<ddN.T; i++)
ddP.b_phi[1][i] = BP1val;
}
pctl_samplereport();
/*
* all data structures
*/
data_alloc();
if ( ddP.phiiter>0 )
phi_init(resstem);
else
ddS.phi = NULL;
if ( ddP.muiter>0 )
mu_init(resstem);
else
ddS.mu = NULL;
if ( ddP.thetaiter>0 )
theta_init(resstem);
else
ddS.theta = NULL;
tca_alloc();
if ( PCTL_BURSTY() )
dmi_init(&ddM, ddS.z, ddD.w, ddD.N_dTcum,
ddN.T, ddN.N, ddN.W, ddN.D, ddN.DT,
(fix_hold==GibbsHold)?pctl_hold:NULL);
if ( load_vocab ) {
data_vocab(stem);
}
cache_init();
/*
* yap some details
*/
data_report(ITER, seed);
pctl_report();
/*
* load/init topic assignments and prepare statistics
*/
if ( restart || restart_hca) {
tca_read_z(resstem, 0, ddN.DT);
tca_rand_z(ddN.T, ddN.DT, ddN.D);
} else {
tca_rand_z(ddN.T, 0, ddN.D);
}
tca_reset_stats(resstem, restart, 0);
if ( (restart || restart_hca ) && ITER )
yap_message("Initial log_2(perp)=%lf\n", -M_LOG2E * likelihood()/ddN.NT);
if ( ITER )
yap_report("cycles: ");
for (iter=0; iter<ITER; iter++) {
int pro;
double thislp = 0;
int thisNd = 0;
int doc;
#ifdef H_THREADS
pthread_t thread[procs];
#endif
D_pargs_p parg[procs];
#ifdef MU_CACHE
mu_side_fact_reinit();
#endif
#ifdef PHI_CACHE
phi_cache_reinit();
#endif
t1 = clock();
/*
* sampling
*/
#ifdef IND_STATS
ddP.doc_ind_stats = u32tri(ddN.T,ddN.E,ddN.E);
ddP.word_ind_stats = u32tri(ddN.T,ddN.E,ddN.E);
#endif
/* a bit complex if no threads! */
doc = 0;
for (pro = 0 ; pro < procs ; pro++){
parg[pro].dots=dots;
parg[pro].procs=procs;
parg[pro].doc = &doc;
#ifndef H_THREADS
sampling_p(&parg[pro]);
#else
if ( procs==1 )
sampling_p(&parg[pro]);
else if( pthread_create(&thread[pro],NULL,sampling_p,(void*) &parg[pro]) != 0){
yap_message("thread failed %d\n",pro+1 );
}
#endif
}
#ifdef H_THREADS
if ( procs>1 ) {
//waiting for threads to finish
for (pro = 0; pro < procs; pro++){
pthread_join(thread[pro], NULL);
}
}
#endif
// getting lp, Nd and clock
for(pro = 0; pro < procs; pro++){
thislp += parg[pro].thislp;
thisNd += parg[pro].thisNd;
tot_time += parg[pro].tot_time;
}
#ifdef H_THREADS
if ( procs>1 )
tca_reset_stats(NULL,1,1);
#endif
/*
* full check
*/
#ifndef NDEBUG
{
int e, d;
check_cp_et();
for (e=0; e<ddN.E; e++)
check_m_vte(e);
for (d=0; d<ddN.DT; d++)
check_n_dt(d);
}
#endif
#ifdef IND_STATS
{
char *fname = yap_makename(resstem,".istats");
FILE *ifp = fopen(fname,"a");
int e1, e2, kk;
fprintf(ifp,"Iteration %d\n", iter);
for (kk=0; kk<ddN.T; kk++) {
fprintf(ifp," Topic %d\n", kk);
for (e1=0; e1<ddN.E; e1++) {
fprintf(ifp," Epoch %d\n ", e1);
for (e2=0; e2<ddN.E; e2++)
fprintf(ifp," %u", (unsigned)ddP.doc_ind_stats[kk][e1][e2]);
fprintf(ifp,"\n ");
for (e2=0; e2<ddN.E; e2++)
fprintf(ifp," %u", (unsigned)ddP.word_ind_stats[kk][e1][e2]);
fprintf(ifp,"\n");
}
}
fclose(ifp);
free(ddP.doc_ind_stats[0][0]); free(ddP.doc_ind_stats[0]);
free(ddP.doc_ind_stats);
free(ddP.word_ind_stats[0][0]); free(ddP.word_ind_stats[0]);
free(ddP.word_ind_stats);
free(fname);
}
#endif
/*
* sample hyperparameters
*/
t3 = clock();
pctl_sample(iter, procs);
/*
* do time calcs here to remove diagnostics+reporting
*/
t2 = clock();
tot_time += (double)(t2 - t1) / CLOCKS_PER_SEC;
psample_time += (double)(t2 - t3) / CLOCKS_PER_SEC;
/*
* progress reports
*/
if ( ( iter>ddP.progburn && (iter%ddP.progiter)==0 ) || iter+1>=ITER ) {
yap_message(" %d\nlog_2(perp)=%lf,%lf", iter,
-M_LOG2E * likelihood()/ddN.NT, -M_LOG2E * thislp/thisNd);
pctl_update(iter);
if ( verbose && iter%10==0 )
yap_probs();
if ( iter>0 && verbose>1 ) {
if ( ddN.tokens ) {
tca_displaytopics(resstem,topwords,score);
displayed++;
}
}
if ( iter+1<ITER ) {
// yap_message("\n");
yap_report("cycles: ");
}
} else {
yap_message(" %d", iter);
if ( verbose>1) yap_message("\n");
}
if ( checkpoint>0 && iter>0 && iter%checkpoint==0 ) {
data_checkpoint(resstem, stem, iter+1);
yap_message(" checkpointed\n");
tca_report(resstem, stem, ITER, procs, fix_hold,
(dopmi&&displayed>0)?1:0);
}
if ( ddP.phiiter>0 && iter>ddP.phiburn && (iter%ddP.phiiter)==0 )
phi_update();
if ( ddP.thetaiter>0 && iter>ddP.thetaburn && (iter%ddP.thetaiter)==0 )
theta_update();
if ( ddP.muiter>0 && iter>ddP.muburn && (iter%ddP.muiter)==0 )
mu_update();
} // over iter
if ( ITER )
yap_report("Finished after %d cycles on average of %lf+%lf(s) per cycle\n",
iter, (tot_time-psample_time)/iter, psample_time/iter);
if ( ( verbose==1 || ((iter+1)%5!=0 && verbose>1) ) ) {
if ( ddN.tokens ) {
tca_displaytopics(resstem,topwords,score);
displayed++;
}
}
yap_probs();
if ( ITER>0 )
data_checkpoint(resstem, stem, ITER);
tca_report(resstem, stem, ITER, procs, fix_hold, (dopmi&&displayed>0)?1:0);
if ( ddP.phiiter>0 )
phi_save(resstem);
if ( ddP.thetaiter>0 )
theta_save(resstem);
if ( ddP.muiter>0 )
mu_save(resstem);
/*
* free
*/
phi_free();
theta_free();
mu_free();
cache_free();
pctl_free();
data_free();
dmi_free(&ddM);
tca_free();
free(stem);
free(resstem);
rng_free(rngp);
return 0;
}
