void run_max_io(char *dev, uint32_t vlen, uint32_t LEN) { ARK *ark[MAX_IO_ARKS]; worker_t w[MAX_IO_ARKS][MAX_IO_PTHS]; uint32_t i = 0; uint32_t j = 0; uint32_t ops = 0; uint32_t ios = 0; uint32_t tops = 0; uint32_t tios = 0; uint32_t e_secs = 0; init_kv_db(MAX_IO_PTHS, KLEN, vlen, LEN); testcase_start = time(0); for (i=0; i<MAX_IO_ARKS; i++) { NEW_ARK(dev, &ark[i]); /* create all threads */ for (j=0; j<MAX_IO_PTHS; j++) { w[i][j].ark = ark[i]; w[i][j].db = dbs+(j % KV_PTHS); pthread_create(&(w[i][j].pth), NULL, fp, (void*)(&w[i][j])); } } /* wait for all threads to complete */ for (i=0; i<MAX_IO_ARKS; i++) { for (j=0; j<MAX_IO_PTHS; j++) { pthread_join(w[i][j].pth, NULL); } } testcase_stop = time(0); for (i=0; i<MAX_IO_ARKS; i++) { get_stats(ark[i], &ops, &ios); tops += ops; tios += ios; } e_secs = testcase_stop - testcase_start; ops = tops / e_secs; ios = tios / e_secs; printf("op/s:%7d io/s:%7d secs:%d\n", ops, ios, e_secs); free_kv_db(MAX_IO_PTHS, LEN); for (i=0; i<MAX_IO_ARKS; i++) { assert(0 == ark_delete(ark[i])); } }
/** ******************************************************************************** ** \brief ** create threads to run one of the IO functions, wait for the threads to ** complete, and print the iops *******************************************************************************/ void kv_sync_io(char *dev, uint32_t vlen, uint32_t len, uint32_t pths, uint32_t read) { worker_t w[KV_PTHS]; ARK *ark = NULL; uint32_t i = 0; uint32_t ops = 0; uint32_t ios = 0; uint32_t e_secs = 0; char *fp_txt = NULL; void* (*fp)(void*); if (read == 75) { fp = (void*(*)(void*))read_75_percent; fp_txt="read: 75%"; } else { fp = (void*(*)(void*))read_100_percent; fp_txt="read:100%"; } init_kv_db(pths, KLEN, vlen, len); NEW_ARK(dev, &ark); printf("SYNC: ctxt:%-2d QD:%-3d size:%2dk %s ", 1, pths, vlen/1000,fp_txt); fflush(stdout); testcase_start = time(0); /* start all threads */ for (i=0; i<pths; i++) { w[i].ark = ark; w[i].db = dbs+(i % pths); pthread_create(&(w[i].pth), NULL, fp, (void*)(w+i)); } /* wait for all threads to complete */ for (i=0; i<pths; i++) { pthread_join(w[i].pth, NULL); } testcase_stop = time(0); e_secs = testcase_stop - testcase_start; get_stats(ark, &ops, &ios); ops = ops / e_secs; ios = ios / e_secs; printf("op/s:%-7d io/s:%-7d secs:%d\n", ops, ios, e_secs); free_kv_db(pths, len); assert(0 == ark_delete(ark)); }
/** ******************************************************************************* * \brief ******************************************************************************/ void kv_async_job_perf(uint32_t jobs, uint32_t klen, uint32_t vlen,uint32_t len) { long int wr_us = 0; long int rd_us = 0; long int mil = 1000000; float wr_s = 0; float rd_s = 0; float wr_mb = 0; float rd_mb = 0; uint64_t mb64_1 = (uint64_t)KV_1M; uint64_t wr_bytes = 0; uint64_t rd_bytes = 0; uint64_t ops = 0; uint64_t post_ops = 0; uint64_t ios = 0; uint64_t post_ios = 0; float wr_ops = 0; float wr_ios = 0; float rd_ops = 0; float rd_ios = 0; uint32_t secs = 5; uint32_t job = 0; struct timeval stop, start; kv_async_init_ctxt(0, secs); for (job=0; job<jobs; job++) { kv_async_init_job(KV_ASYNC_CB_SGD|KV_ASYNC_CB_WRITE_PERF, ASYNC_SINGLE_CONTEXT, job, klen+job, vlen, len); } pCTs->flags |= KV_ASYNC_CT_PERF; /* do writes */ (void)ark_stats(kv_async_get_ark(ASYNC_SINGLE_CONTEXT), &ops, &ios); KV_TRC(pFT, "PERF wr: ops:%"PRIu64" ios:%"PRIu64"", ops, ios); gettimeofday(&start, NULL); kv_async_run_jobs(); /* run write jobs */ KV_TRC(pFT, "writes done"); gettimeofday(&stop, NULL); wr_us += (stop.tv_sec*mil + stop.tv_usec) - (start.tv_sec*mil + start.tv_usec); (void)ark_stats(kv_async_get_ark(ASYNC_SINGLE_CONTEXT),&post_ops,&post_ios); KV_TRC(pFT, "PERF wr: ops:%"PRIu64" ios:%"PRIu64"", post_ops, post_ios); wr_ops += post_ops - ops; wr_ios += post_ios - ios; /* calc bytes written */ for (job=0; job<jobs; job++) { wr_bytes += (klen+vlen+job)*len*(pCTs->pCBs+job)->perf_loops; } /* do reads */ for (job=0; job<jobs; job++) { if ((pCTs->pCBs+job)->perf_loops) { (pCTs->pCBs+job)->perf_loops = 0; (pCTs->pCBs+job)->flags = KV_ASYNC_CB_GET | KV_ASYNC_CB_QUEUED | KV_ASYNC_CB_READ_PERF; } } pCTs->flags |= KV_ASYNC_CT_RUNNING; (void)ark_stats(kv_async_get_ark(0), &ops, &ios); KV_TRC(pFT, "PERF rd: ops:%"PRIu64" ios:%"PRIu64"", ops, ios); gettimeofday(&start, NULL); kv_async_run_jobs(); /* run read jobs */ gettimeofday(&stop, NULL); KV_TRC(pFT, "reads done"); rd_us += (stop.tv_sec*mil + stop.tv_usec) - (start.tv_sec*mil + start.tv_usec); (void)ark_stats(kv_async_get_ark(0), &post_ops, &post_ios); KV_TRC(pFT, "PERF rd: ops:%"PRIu64" ios:%"PRIu64"", post_ops, post_ios); rd_ops += post_ops - ops; rd_ios += post_ios - ios; ASSERT_EQ(0, ark_delete(pCTs->ark)); /* calc bytes read */ for (job=0; job<jobs; job++) { rd_bytes += vlen*len*(pCTs->pCBs+job)->perf_loops; kv_db_destroy((pCTs->pCBs+job)->db, (pCTs->pCBs+job)->len); if ((pCTs->pCBs+job)->gvalue) free((pCTs->pCBs+job)->gvalue); } /* calc and print results */ wr_s = (float)((float)wr_us/(float)mil); wr_mb = (float)((double)wr_bytes / (double)mb64_1); rd_s = (float)((float)rd_us/(float)mil); rd_mb = (float)((double)rd_bytes / (double)mb64_1); printf("ASYNC %dx%dx%d writes: %.3d jobs %2.3f mb in %.1f secs at ", klen, vlen, len, jobs, wr_mb, wr_s); printf("%2.3f mbps, %6.0f op/s, %.0f io/s\n", wr_mb/wr_s, wr_ops/wr_s, wr_ios/wr_s); printf("ASYNC %dx%dx%d reads: %.3d jobs %2.3f mb in %.1f secs at ", klen, vlen, len, jobs, rd_mb, rd_s); printf("%2.3f mbps, %6.0f op/s, %.0f io/s\n", rd_mb/rd_s, rd_ops/rd_s, rd_ios/rd_s); }
/** ******************************************************************************* * \brief ******************************************************************************/ void kv_async_run_jobs(void) { async_CB_t *pCB = NULL; uint32_t ctxt_running = 0; uint32_t jobs_running = 0; uint32_t i = 0; uint32_t next = 0; uint32_t elapse = 0; uint32_t inject = 0; uint32_t secs = 0; uint32_t log_interval = 600; uint64_t ops = 0; uint64_t ios = 0; uint32_t tops = 0; uint32_t tios = 0; uint32_t perf = 0; KV_TRC(pFT, "ASYNC START: 0 minutes"); if (!(pCTs->pCBs->flags & KV_ASYNC_CB_RUNNING)) start = time(0); next = log_interval; do { ctxt_running = FALSE; if (elapse > next) { KV_TRC(pFT, "ASYNC RUNNING: %d elapsed minutes", elapse/60); next += log_interval; } for (i=0; i<KV_ASYNC_MAX_CONTEXTS; i++) { if (! (pCTs[i].flags & KV_ASYNC_CT_RUNNING)) continue; jobs_running = kv_async_dispatch_jobs(i); if (!jobs_running) { pCTs[i].flags &= ~KV_ASYNC_CT_RUNNING; pCTs[i].flags |= KV_ASYNC_CT_DONE; KV_TRC(pFT, "ASYNC DONE ctxt %d %x", i, pCTs[i].flags); continue; } else { ctxt_running = TRUE; } elapse = time(0) - start; if (elapse >= inject && pCTs[i].flags & KV_ASYNC_CT_ERROR_INJECT) { KV_TRC_FFDC(pFT, "FFDC: INJECT ERRORS"); FVT_KV_INJECT_READ_ERROR; FVT_KV_INJECT_WRITE_ERROR; FVT_KV_INJECT_ALLOC_ERROR; ++inject; } if (elapse >= pCTs[i].secs) { for (pCB=pCTs[i].pCBs;pCB<pCTs[i].pCBs+KV_ASYNC_JOB_Q;pCB++) { if ((pCB->flags & KV_ASYNC_CB_RUNNING || pCB->flags & KV_ASYNC_CB_QUEUED) && (!(pCB->flags & KV_ASYNC_CB_SHUTDOWN)) ) { pCB->flags |= KV_ASYNC_CB_SHUTDOWN; KV_TRC_IO(pFT, "SHUTDOWN pCB %p (%d >= %d)", pCB, elapse, pCTs[i].secs); } } } usleep(100); } } while (ctxt_running); stop = time(0); secs = stop - start; KV_TRC(pFT, "ASYNC RUNNING DONE: %d minutes", elapse/60); /* log cleanup, since the first ark_delete closes the log file */ for (i=0; i<KV_ASYNC_MAX_CONTEXTS; i++) { if (pCTs[i].flags & KV_ASYNC_CT_DONE) KV_TRC(pFT, "ASYNC CLEANUP: ctxt:%d ark:%p", i, pCTs[i].ark); } /* check for MULTI_CTXT_IO, destroy common kv dbs */ for (pCB=pCTs->pCBs;pCB<pCTs->pCBs+KV_ASYNC_JOB_Q;pCB++) { if (pCB->flags & KV_ASYNC_CB_MULTI_CTXT_IO) { kv_db_destroy(pCB->db, pCB->len); } } for (i=0; i<KV_ASYNC_MAX_CONTEXTS; i++) { /* if this context didn't run any I/O */ if (! (pCTs[i].flags & KV_ASYNC_CT_DONE)) continue; pCTs[i].flags &= ~KV_ASYNC_CT_DONE; /* if perf then don't delete the ark here */ if (pCTs[i].flags & KV_ASYNC_CT_PERF) { perf = TRUE; continue; } (void)ark_stats(pCTs[i].ark, &ops, &ios); tops += (uint32_t)ops; tios += (uint32_t)ios; KV_TRC(pFT, "PERF ark%p ops:%"PRIu64" ios:%"PRIu64"", pCTs[i].ark, ops, ios); EXPECT_EQ(0, ark_delete(pCTs[i].ark)); } if (!perf) { tops = tops / secs; tios = tios / secs; printf("op/s:%d io/s:%d secs:%d\n", tops, tios, secs); KV_TRC(pFT, "PERF op/s:%d io/s:%d secs:%d", tops, tios, secs); } }
/** ******************************************************************************** ** \brief ** create max arks, create max threads to run one of the IO functions, wait ** for the threads to complete, prints the iops, delete the arks *******************************************************************************/ void kv_max_sync_io(char *dev, uint32_t vlen, uint32_t len) { ARK *ark[MAX_IO_ARKS]; worker_t w[MAX_IO_ARKS][MAX_IO_PTHS]; uint32_t i = 0; uint32_t j = 0; uint32_t ops = 0; uint32_t ios = 0; uint32_t tops = 0; uint32_t tios = 0; uint32_t e_secs = 0; printf("SYNC: ctxt:%-2d QD:%-3d size: 4k read:100%% ", MAX_IO_ARKS, MAX_IO_PTHS); fflush(stdout); init_kv_db(MAX_IO_PTHS, KLEN, vlen, len); for (i=0; i<MAX_IO_ARKS; i++) { NEW_ARK(dev, ark+i); } testcase_start = time(0); for (i=0; i<MAX_IO_ARKS; i++) { /* create all threads */ for (j=0; j<MAX_IO_PTHS; j++) { w[i][j].ark = ark[i]; w[i][j].db = dbs+(j % KV_PTHS); pthread_create(&(w[i][j].pth), NULL, (void*(*)(void*))read_100_percent, (void*)(&w[i][j])); } } /* wait for all threads to complete */ for (i=0; i<MAX_IO_ARKS; i++) { for (j=0; j<MAX_IO_PTHS; j++) { pthread_join(w[i][j].pth, NULL); } } testcase_stop = time(0); for (i=0; i<MAX_IO_ARKS; i++) { get_stats(ark[i], &ops, &ios); tops += ops; tios += ios; } e_secs = testcase_stop - testcase_start; ops = tops / e_secs; ios = tios / e_secs; printf("op/s:%-7d io/s:%-7d secs:%d\n", ops, ios, e_secs); free_kv_db(MAX_IO_PTHS, len); for (i=0; i<MAX_IO_ARKS; i++) { assert(0 == ark_delete(ark[i])); } }
/** ******************************************************************************** ** \brief ** do an ark asynchronous IO performance run *******************************************************************************/ void kv_async_io(char *dev, uint32_t ctxts, uint32_t jobs, uint32_t vlen, uint32_t len, uint32_t read) { async_context_t *pCT = NULL; async_CB_t *pCB = NULL; uint32_t job = 0; uint32_t ctxt = 0; uint32_t ctxt_running = 0; uint32_t i = 0; uint32_t e_secs = 0; uint64_t ops = 0; uint64_t ios = 0; uint32_t tops = 0; uint32_t tios = 0; if (read == 75) read100 = FALSE; else read100 = TRUE; printf("ASYNC: ctxt:%-2d QD:%-3d size:%2dk read:%3d%% ", ctxts, jobs, vlen/1000, read); fflush(stdout); init_kv_db(jobs, KLEN, vlen, len); pCTs = (async_context_t*) malloc(ctxts*sizeof(async_context_t)); assert(0 != pCTs); for (ctxt=0; ctxt<ctxts; ctxt++) { ARK *ark = NULL; pCT = pCTs+ctxt; memset(pCT, 0, sizeof(async_context_t)); NEW_ARK(dev, &ark); pCT->ark = ark; for (job=0; job<jobs; job++) { pCB = pCT->pCBs+job; pCB->itag = SET_ITAG(ctxt,job); pCB->ark = pCT->ark; pCB->flags = KV_ASYNC_SET; pCB->db = dbs+job; pCB->len = len; } } testcase_start = time(0); /* start all jobs for all contexts */ for (ctxt=0; ctxt<ctxts; ctxt++) { for (pCB=pCTs[ctxt].pCBs; pCB<pCTs[ctxt].pCBs+jobs; pCB++) { pCB->flags |= KV_ASYNC_RUNNING; kv_async_SET_KEY(pCB); } } /* loop until all jobs are done or until time elapses */ do { ctxt_running = FALSE; /* loop through contexts(arks) and check if all jobs are done or * time has elapsed */ for (i=0; i<ctxts; i++) { /* check if all jobs are done */ for (pCB=pCTs[i].pCBs;pCB<pCTs[i].pCBs+jobs;pCB++) { if (pCB->flags & KV_ASYNC_RUNNING) { ctxt_running = TRUE; } } if (!ctxt_running) continue; /* check if time has elapsed */ if (time(0) - testcase_start < KV_MIN_SECS) continue; for (pCB=pCTs[i].pCBs;pCB<pCTs[i].pCBs+jobs;pCB++) { if (pCB->flags & KV_ASYNC_RUNNING && (!(pCB->flags & KV_ASYNC_SHUTDOWN)) ) { pCB->flags |= KV_ASYNC_SHUTDOWN; } } } } while (ctxt_running); testcase_stop = time(0); e_secs = testcase_stop - testcase_start; /* sum perf ops for all contexts/jobs and delete arks */ for (i=0; i<ctxts; i++) { (void)ark_stats(pCTs[i].ark, &ops, &ios); tops += (uint32_t)ops; tios += (uint32_t)ios; } tops = tops / e_secs; tios = tios / e_secs; printf("op/s:%-7d io/s:%-7d secs:%d\n", tops, tios, e_secs); /* sum perf ops for all contexts/jobs and delete arks */ for (i=0; i<ctxts; i++) { assert(0 == ark_delete(pCTs[i].ark)); } free_kv_db(jobs, len); }
/** ******************************************************************************* * \brief ******************************************************************************/ void Sync_pth::run_multi_ctxt_wr(uint32_t num_ctxt, uint32_t num_pth, uint32_t npool, uint32_t vlen, uint32_t LEN, uint32_t secs) { uint32_t i = 0; uint32_t ctxt_i = 0; uint32_t pth_i = 0; uint32_t klen = 16; uint32_t tot_pth = num_ctxt * num_pth; set_get_args_t pth_args[tot_pth]; ARK *ark[num_ctxt]; kv_t *db[num_pth]; struct timeval stop, start; uint64_t ops = 0; uint64_t ios = 0; uint32_t t_ops = 0; uint32_t t_ios = 0; if (num_pth > MAX_PTH_PER_CONTEXT) { printf("cannot exceed %d pthreads for sync ops\n", MAX_PTH_PER_CONTEXT); return; } memset(pth_args, 0, sizeof(set_get_args_t) * tot_pth); /* alloc one set of db's, to be used for each context */ for (i=0; i<num_pth; i++) { db[i] = (kv_t*)kv_db_create_fixed(LEN, klen+i, vlen+i); } /* alloc one ark for each context */ for (ctxt_i=0; ctxt_i<num_ctxt; ctxt_i++) { ASSERT_EQ(0, ark_create_verbose(getenv("FVT_DEV"), &ark[ctxt_i], 1048576, 4096, 1048576, npool, 256, 8*1024, ARK_KV_VIRTUAL_LUN)); ASSERT_TRUE(NULL != ark[ctxt_i]); } /* start timer */ gettimeofday(&start, NULL); /* init each pth per context and run the pth */ ctxt_i = 0; i = 0; while (i < tot_pth) { pth_args[i].ark = ark[ctxt_i]; pth_args[i].db = db[pth_i]; pth_args[i].vlen = vlen+pth_i; pth_args[i].LEN = LEN; pth_args[i].secs = secs; run_write_loop(pth_args+i); ++pth_i; ++i; if ((i%num_pth) == 0) { /* reset for next context */ ++ctxt_i; pth_i = 0; } } printf("SYNC %dx%dx%d ctxt:%d pth:%d npool:%d ", klen, vlen, LEN, num_ctxt, num_pth, npool); fflush(stdout); for (i=0; i<tot_pth; i++) { wait(pth_args+i); } /* stop timer */ gettimeofday(&stop, NULL); secs = stop.tv_sec - start.tv_sec; t_ops = 0; t_ios = 0; for (i=0; i<num_ctxt; i++) { (void)ark_stats(ark[i], &ops, &ios); t_ops += (uint32_t)ops; t_ios += (uint32_t)ios; EXPECT_EQ(0, ark_delete(ark[i])); } for (i=0; i<num_pth; i++) { kv_db_destroy(db[i], LEN); } t_ops = t_ops / secs; t_ios = t_ios / secs; printf("op/s:%d io/s:%d secs:%d\n", t_ops, t_ios, secs); }
void kv_async_wait_jobs(void) { async_CB_t *pCB = NULL; uint32_t ctxt_running = 0; uint32_t i = 0; uint32_t e_secs = 0; uint64_t ops = 0; uint64_t ios = 0; uint32_t tops = 0; uint32_t tios = 0; printf("ASYNC: "); fflush(stdout); /* loop until all jobs are done or until time elapses */ do { ctxt_running = FALSE; /* loop through contexts(arks) and check if all jobs are done or * time has elapsed */ for (i=0; i<KV_ASYNC_CONTEXTS; i++) { /* check if all jobs are done */ for (pCB=pCTs[i].pCBs;pCB<pCTs[i].pCBs+KV_ASYNC_JOBS_PER_CTXT;pCB++) { if (pCB->flags & KV_ASYNC_RUNNING) { ctxt_running = TRUE; } } if (!ctxt_running) continue; /* check if time has elapsed */ if (time(0) - start < KV_ASYNC_SECS) continue; for (pCB=pCTs[i].pCBs;pCB<pCTs[i].pCBs+KV_ASYNC_JOBS_PER_CTXT;pCB++) { if (pCB->flags & KV_ASYNC_RUNNING && (!(pCB->flags & KV_ASYNC_SHUTDOWN)) ) { pCB->flags |= KV_ASYNC_SHUTDOWN; } } } } while (ctxt_running); stop = time(0); e_secs = stop - start; /* sum perf ops for all contexts/jobs and delete arks */ for (i=0; i<KV_ASYNC_CONTEXTS; i++) { (void)ark_stats(pCTs[i].ark, &ops, &ios); tops += (uint32_t)ops; tios += (uint32_t)ios; } tops = tops / e_secs; tios = tios / e_secs; printf("op/s:%d io/s:%d secs:%d\n", tops, tios, e_secs); /* sum perf ops for all contexts/jobs and delete arks */ for (i=0; i<KV_ASYNC_CONTEXTS; i++) { assert(0 == ark_delete(pCTs[i].ark)); } }
int main(int argc, char **argv) { ARK *ark; int i=1; if (argv[1] == NULL) { printf("dev name required as parameter\n"); exit(-1); } if (0 == strncmp(argv[1], "-p", 7)) { printf("Attempting to run with physical lun\n"); ark_create_flag = ARK_KV_PERSIST_STORE; i = 2; } printf("%s: SYNC: npool:%d nasync:%d basync:%d\n", argv[i], KV_NPOOL, KV_NASYNC, KV_BASYNC); fflush(stdout); /* 1 ctxt, vlen 4k */ init_kv_db(KV_PTHS, KLEN, VLEN_4k, LEN_4k); NEW_ARK(argv[i], &ark); printf("ctxt:1 4k-k/v: 75%% read: "); fflush(stdout); fp = (void*(*)(void*))do_75_percent_read; run_io(ark, KV_PTHS); free_kv_db(KV_PTHS, LEN_4k); assert(0 == ark_delete(ark)); /* 1 ctxt, vlen 4k */ init_kv_db(KV_PTHS, KLEN, VLEN_4k, LEN_4k); NEW_ARK(argv[i], &ark); printf("ctxt:1 4k-k/v: 100%% read: "); fflush(stdout); fp = (void*(*)(void*))do_100_percent_read; run_io(ark, KV_PTHS); free_kv_db(KV_PTHS, LEN_4k); assert(0 == ark_delete(ark)); /* 1 ctxt, vlen 64k */ init_kv_db(KV_PTHS_BIG, KLEN, VLEN_64k, LEN_BIG); NEW_ARK(argv[i], &ark); printf("ctxt:1 64k-k/v: 100%% read: "); fflush(stdout); fp = (void*(*)(void*))do_100_percent_read; run_io(ark, KV_PTHS_BIG); free_kv_db(KV_PTHS_BIG, LEN_BIG); assert(0 == ark_delete(ark)); /* 1 ctxt, vlen 1mb */ init_kv_db(KV_PTHS_BIG, KLEN, VLEN_500k, LEN_BIG); NEW_ARK(argv[i], &ark); printf("ctxt:1 500k-k/v: 100%% read: "); fflush(stdout); fp = (void*(*)(void*))do_100_percent_read; run_io(ark, KV_PTHS_BIG); free_kv_db(KV_PTHS_BIG, LEN_BIG); assert(0 == ark_delete(ark)); /* if physical lun, cannot run multi-context, so exit */ if (ARK_KV_PERSIST_STORE == ark_create_flag) return 0; printf("ctxt:4 4k-k/v: 100%% read: "); fflush(stdout); fp = (void*(*)(void*))do_100_percent_read; run_max_io(argv[i], VLEN_4k, LEN_4k); printf("ctxt:4 64k-k/v: 100%% read: "); fflush(stdout); fp = (void*(*)(void*))do_100_percent_read; run_max_io(argv[i], VLEN_64k, LEN_BIG); return 0; }