static void put_commands(int argc, char** argv) { if (argc != 2) { printf("usage: %s <filename>\n", argv[0]); ADLB_Fail(-1); } printf("command file is %s\n", argv[1]); FILE* fp = fopen(argv[1], "r"); if (fp == NULL) { printf("could not open command file\n"); ADLB_Fail(-1); } while (fgets(cmdbuffer,1024,fp) != NULL) { cmdbuffer[strlen(cmdbuffer)] = '\0'; if (!quiet) printf("command = %s\n", cmdbuffer); if (cmdbuffer[0] != '#') { /* put command into adlb here */ int rc = ADLB_Put(cmdbuffer, strlen(cmdbuffer)+1, ADLB_RANK_ANY, -1, CMDLINE, ADLB_DEFAULT_PUT_OPTS); printf("put cmd, rc = %d\n", rc); } } printf("\nall commands submitted\n"); fclose(fp); }
int main(int argc, char *argv[]) { int i, rc, done, rand_work_unit_len, total_work_units_handled, nworkthrds; int my_world_rank, nranks, num_work_units, provided; int work_prio, work_type, work_handle[ADLB_HANDLE_SIZE], work_len, answer_rank; int *num_handled_by_rank, num_handled_by_me; int dbgprintf_flag = 1, use_prio_for_reserve_flag = 0; int req_types[3]; int num_types = 1; int type_vect[2] = {WORK}; char thread_type[32]; double adlb_dbls_args[ADLB_NUM_INIT_ARGS]; double temptime, fake_work_time, fake_work_time_per_thrd; double start_job_time, end_put_time, start_work_time, end_work_time; double total_work_time, total_loop_time; double total_reserve_time, total_get_time; double min_work_unit_time, max_work_unit_time, min_work_unit_len, max_work_unit_len; double rand_work_unit_time, *work_unit_buf; total_work_units_handled = 0; total_work_time = 0.0; total_loop_time = 0.0; total_reserve_time = 0.0; total_get_time = 0.0; num_work_units = 0; min_work_unit_time = DEFAULT_NSECS_FAKE_WORK; max_work_unit_time = DEFAULT_NSECS_FAKE_WORK; min_work_unit_len = DEFAULT_WORK_UNIT_LEN; max_work_unit_len = DEFAULT_WORK_UNIT_LEN; for (i=1; i < argc; i++) { // printf("av %s\n",argv[i]); if (strcmp(argv[i],"-n") == 0) num_work_units = atoi(argv[++i]); else if (strcmp(argv[i],"-t") == 0) { min_work_unit_time = atof(argv[++i]); max_work_unit_time = atof(argv[++i]); } else if (strcmp(argv[i],"-l") == 0) { min_work_unit_len = atoi(argv[++i]); max_work_unit_len = atoi(argv[++i]); } else { printf("st4: unrecognized cmd-line arg at %d :%s:\n",i,argv[i]); exit(-1); } } if (min_work_unit_len < sizeof(double)) { printf("st4: min len must be at least size of double %d\n",sizeof(double)); exit(-1); } rc = MPI_Init_thread(NULL,NULL,MPI_THREAD_MULTIPLE,&provided); if (rc != MPI_SUCCESS) { printf("st4: MPI_Init_thread failed with rc=%d\n",rc); exit(-1); } switch (provided) { case MPI_THREAD_SINGLE: strcpy(thread_type,"MPI_THREAD_SINGLE"); break; case MPI_THREAD_FUNNELED: strcpy(thread_type,"MPI_THREAD_FUNNELED"); break; case MPI_THREAD_SERIALIZED: strcpy(thread_type,"MPI_THREAD_SERIALIZED"); break; case MPI_THREAD_MULTIPLE: strcpy(thread_type,"MPI_THREAD_MULTIPLE"); break; default: strcpy(thread_type,"UNKNOWN"); break; } printf("st4: MPI provides %s\n",thread_type); MPI_Comm_size(MPI_COMM_WORLD,&nranks); MPI_Comm_rank(MPI_COMM_WORLD,&my_world_rank); num_handled_by_me = 0; if (my_world_rank == 0) num_handled_by_rank = malloc(nranks * sizeof(int)); else num_handled_by_rank = NULL; work_unit_buf = malloc(max_work_unit_len); adlb_dbls_args[0] = (double) SRVR_MALLOC_AMT; adlb_dbls_args[1] = (double) dbgprintf_flag; adlb_dbls_args[2] = (double) use_prio_for_reserve_flag; adlb_dbls_args[3] = (double) num_types; for (i=0; i < num_types; i++) adlb_dbls_args[i+4] = (double) type_vect[i]; rc = ADLB_Init(adlb_dbls_args); rc = MPI_Barrier( MPI_COMM_WORLD ); start_job_time = MPI_Wtime(); end_work_time = MPI_Wtime(); /* dummy val until set below */ if ( my_world_rank == 0 ) /* if master app, put work */ { for (i=0; i < num_work_units; i++) { memset(work_unit_buf,'X',max_work_unit_len); rand_work_unit_len = random_int_in_range(min_work_unit_len,max_work_unit_len); rand_work_unit_time = random_dbl_in_range(min_work_unit_time,max_work_unit_time); work_unit_buf[0] = rand_work_unit_time; // dbgprintf( 1, "putting work_unit %d len %d time %f\n", // i, rand_work_unit_len,rand_work_unit_time); rc = ADLB_Put( work_unit_buf, rand_work_unit_len, -1, -1, WORK, 1 ); // dbgprintf( 1, "put work_unit %d rc %d\n", i, rc ); } dbgprintf(1,"st4: all work submitted after %f secs\n",MPI_Wtime()-start_job_time); } rc = MPI_Barrier( MPI_COMM_WORLD ); end_put_time = start_work_time = MPI_Wtime(); # ifdef USE_OMP nworkthrds = omp_get_max_threads() - 1; /* 1 for server */ # else nworkthrds = 1; /* default */ # endif done = 0; while ( !done ) { req_types[0] = -1; req_types[1] = -1; req_types[2] = -1; // dbgprintf( 1, "st4: reserving work\n" ); temptime = MPI_Wtime(); rc = ADLB_Reserve(req_types,&work_type,&work_prio,work_handle,&work_len,&answer_rank); // dbgprintf( 1, "st4: after reserve rc %d len %d type %d\n", rc, work_len, work_type ); if ( rc == ADLB_DONE_BY_EXHAUSTION ) { dbgprintf( 1, "st4: done by exhaustion\n" ); break; } else if ( rc == ADLB_NO_MORE_WORK ) { dbgprintf( 1, "st4: done by no more work\n" ); break; } else if (rc < 0) { dbgprintf( 1, "st4: ** reserve failed, rc = %d\n", rc ); ADLB_Abort(-1); } else if (work_type == WORK) { total_reserve_time += MPI_Wtime() - temptime; /* only count for work */ temptime = MPI_Wtime(); rc = ADLB_Get_reserved( work_unit_buf, work_handle ); total_get_time += MPI_Wtime() - temptime; if (rc == ADLB_NO_MORE_WORK) { dbgprintf( 1, "st4: no more work on get_reserved\n" ); break; } else /* got good work */ { /* do dummy/fake work */ num_handled_by_me++; fake_work_time = work_unit_buf[0]; if (fake_work_time == 0.0) { dbgprintf(1,"st4: fakeworktime 0.0\n"); } else { fake_work_time_per_thrd = fake_work_time / nworkthrds; // dbgprintf(1,"st4: fakeworktime %f %f\n", // fake_work_time,fake_work_time_per_thrd); temptime = MPI_Wtime(); # ifdef USE_OMP // #pragma omp parallel private(tid) num_threads(nworkthrds) #pragma omp parallel num_threads(nworkthrds) # endif { while (1) { for (i=0; i < 1000000; i++) ; if (MPI_Wtime()-temptime > fake_work_time_per_thrd) break; } } dbgprintf(1,"st4: fakeworktime %f perthrd %f looptime %f\n", fake_work_time,fake_work_time_per_thrd,MPI_Wtime()-temptime); } } end_work_time = MPI_Wtime(); /* chgs on each work unit */ } else { dbgprintf( 1, "st4: ** unexpected work type %d\n", work_type ); ADLB_Abort( -1 ); } } rc = MPI_Barrier( MPI_COMM_WORLD ); // total_loop_time can be misleading since we have to wait for exhaustion // total_loop_time = MPI_Wtime() - start_work_time; // dbgprintf(1,"st4: total loop time %f\n",total_loop_time); total_work_time = end_work_time - start_work_time; dbgprintf(1,"st4: num handled by me %d\n",num_handled_by_me); dbgprintf(1,"st4: last end_work_time %f\n",end_work_time); dbgprintf(1,"st4: total work_time %f ; avg work time %f\n", total_work_time,total_work_time/((float)num_handled_by_me)); dbgprintf(1,"st4: total reserve time %f ; avg reserve time %f\n", total_reserve_time,total_reserve_time/((float)num_handled_by_me)); dbgprintf(1,"st4: total get time %f ; avg get time %f\n", total_get_time,total_get_time/((float)num_handled_by_me)); MPI_Gather(&num_handled_by_me,1,MPI_INT, num_handled_by_rank,1,MPI_INT, 0,MPI_COMM_WORLD); if (my_world_rank == 0) { for (i=0; i < nranks; i++) { if (num_handled_by_rank[i] > 0) { dbgprintf(1,"st4: num handled by rank %d : total %d avg work time %f\n", i,num_handled_by_rank[i], (total_work_time/(float)num_handled_by_rank[i])); total_work_units_handled += num_handled_by_rank[i]; } else dbgprintf(1,"st4: num handled by rank %d : total 0 avg work time 0.0\n",i); } if (total_work_units_handled != num_work_units) { dbgprintf( 1, "st4: ** wrong number of work units handled: %d of %d\n", total_work_units_handled, num_work_units); ADLB_Abort( -1 ); } } ADLB_Finalize(); // printf("st4: calling mpi_finalize\n"); MPI_Finalize(); // printf("st4: past mpi_finalize\n"); return 0; }
int main(int argc, char *argv[]) { int i, j, rc, provided, use_exhaustion; int num_slaves, num_world_nodes, my_world_rank; int num_As_gend, num_As_to_gen_per_rank, num_As_to_gen_per_put; int req_types[4], work_type, answer_rank, work_len, num_iters_for_fake_work; int work_prio, work_handle[ADLB_HANDLE_SIZE], num_C_answers, num_C_answers_total; int work_A[20], work_C[20]; int total_num_puts, total_num_reserves, total_num_gets; double tempval; double adlb_dbls_args[ADLB_NUM_INIT_ARGS]; double time1, total_put_time, total_reserve_time, total_get_time; num_iters_for_fake_work = 1000000; use_exhaustion = 0; num_C_answers = 0; num_C_answers_total = 0; num_As_gend = 0; num_As_to_gen_per_put = 4; num_As_to_gen_per_rank = 10; // not nec a multiple of num_As_to_gen_per_put for (i=1; i < argc; i++) { if (strcmp(argv[i],"-nas") == 0) num_As_to_gen_per_rank = atoi(argv[i+1]); else if (strcmp(argv[i],"-niters") == 0) num_iters_for_fake_work = atoi(argv[i+1]); else if (strcmp(argv[i],"-exhaust") == 0) use_exhaustion = 1; } rc = MPI_Init_thread(NULL,NULL,MPI_THREAD_MULTIPLE,&provided); MPI_Comm_size(MPI_COMM_WORLD,&num_world_nodes); MPI_Comm_rank(MPI_COMM_WORLD,&my_world_rank); if (num_world_nodes < 2) { dbgprintf(1,"**** TOO FEW RANKS - pgm requires at least a master and one slave\n"); ADLB_Abort(-1); } adlb_dbls_args[0] = (double) SRVR_MAX_MALLOC_AMT; adlb_dbls_args[1] = (double) 1.0; // dbgprintf_flag adlb_dbls_args[2] = (double) 0.0; // use_prio_for_reserve_flag adlb_dbls_args[3] = (double) num_types; for (i=0; i < num_types; i++) adlb_dbls_args[i+4] = (double) type_vect[i]; rc = ADLB_Init(adlb_dbls_args); num_slaves = num_world_nodes - 1; // all but master_rank total_put_time = total_reserve_time = total_get_time = 0.0; total_num_puts = total_num_reserves = total_num_gets = 0; rc = MPI_Barrier(MPI_COMM_WORLD); start_time = MPI_Wtime(); if (my_world_rank == MASTER_RANK) /* if master */ { dbgprintf(1,"STAT nums to handle: As %d Bs %d Cs %d total %d\n", ( num_slaves * num_As_to_gen_per_rank ), ( num_slaves * num_As_to_gen_per_rank ), // one B per A ( num_slaves * num_As_to_gen_per_rank * NUM_CS_PER_B), ( num_slaves * num_As_to_gen_per_rank * 2 + // As + Bs num_slaves * num_As_to_gen_per_rank * NUM_CS_PER_B) // + Cs ); time1 = MPI_Wtime(); tempval = (double)work_A[0]; for (j=0; j < num_iters_for_fake_work; j++) tempval = sqrt(tempval + 5000000.0) + 1; dbgprintf(1,"STAT fake work time for %d iters is %f\n", num_iters_for_fake_work,MPI_Wtime()-time1); for (i=0; i < (num_As_to_gen_per_rank * num_slaves); i++) { req_types[0] = TYPE_HANDLED_B_AND_Cs; req_types[1] = req_types[2] = req_types[3] = -1; time1 = MPI_Wtime(); rc = ADLB_Reserve(req_types,&work_type,&work_prio,work_handle, &work_len,&answer_rank); total_reserve_time += MPI_Wtime() - time1; total_num_reserves++; if (rc < 0) { dbgprintf(1,"**** STAT RESERVE NEGATIVE rc %d\n",rc); if (use_exhaustion) break; else ADLB_Abort(-1); } time1 = MPI_Wtime(); rc = ADLB_Get_reserved(work_A,work_handle); total_get_time += MPI_Wtime() - time1; total_num_gets++; if (rc < 0) { dbgprintf(1,"**** STAT RESERVE NEGATIVE rc %d\n",rc); if (use_exhaustion) break; else ADLB_Abort(-1); } num_C_answers_total += work_A[0]; dbgprintf(1,"HANDLED b_and_cs %d of %d ; c_answers %d of %d\n", i+1, ( num_slaves * num_As_to_gen_per_rank ), num_C_answers_total, ( num_slaves * num_As_to_gen_per_rank * NUM_CS_PER_B) ); } dbgprintf(1,"STAT total Cs handled %d; should be %d\n", num_C_answers_total, ( num_slaves * num_As_to_gen_per_rank * NUM_CS_PER_B) ); dbgprintf(1,"********** SETTING NO MORE WORK ***********************************\n"); ADLB_Set_problem_done(); dbgprintf(1,"********** done\n"); } else /* slave */ { while (num_As_gend < num_As_to_gen_per_rank) { num_As_gend++; for (i=0; i < NUM_CS_PER_B; i++) { tempval = (double)work_C[0]; for (j=0; j < num_iters_for_fake_work; j++) tempval = sqrt(tempval + 5000000.0) + 1; num_C_answers = 4; time1 = MPI_Wtime(); rc = ADLB_Put(&num_C_answers,sizeof(int),0,0,TYPE_HANDLED_B_AND_Cs,0); total_put_time += MPI_Wtime() - time1; total_num_puts++; // dbgprintf(1,"PUT %d \n",i); } } } /* slave */ if (my_world_rank == MASTER_RANK) /* if master */ { end_time = MPI_Wtime(); dbgprintf(1,"STAT total time = %10.2f\n",end_time-start_time); } else { dbgprintf(1,"total put time %14.6f numputs %d avg %14.6f\n", total_put_time,total_num_puts,total_put_time/total_num_puts); dbgprintf(1,"total reserve time %14.6f numreserves %d avg %14.6f\n", total_reserve_time,total_num_reserves, total_reserve_time/total_num_reserves); dbgprintf(1,"total get time %14.6f numgets %d avg %14.6f\n", total_get_time,total_num_gets,total_get_time/total_num_gets); } dbgprintf(1,"AT ADLB_FINALIZE\n"); ADLB_Finalize(); dbgprintf(1,"AT MPI_FINALIZE\n"); MPI_Finalize(); printf("DONE\n"); return 0; }