/*
* Run a single instance of the throughput test. When attempting to determine
* the maximum packet rate this will be invoked multiple times with the only
* difference between runs being the target PPS rate.
*/
static int run_test_single(odp_cpumask_t *thd_mask_tx,
odp_cpumask_t *thd_mask_rx,
test_status_t *status)
{
odph_odpthread_t thd_tbl[MAX_WORKERS];
thread_args_t args_tx, args_rx;
uint64_t expected_tx_cnt;
int num_tx_workers, num_rx_workers;
odph_odpthread_params_t thr_params;
memset(&thr_params, 0, sizeof(thr_params));
thr_params.thr_type = ODP_THREAD_WORKER;
thr_params.instance = gbl_args->instance;
odp_atomic_store_u32(&shutdown, 0);
memset(thd_tbl, 0, sizeof(thd_tbl));
memset(gbl_args->rx_stats, 0, gbl_args->rx_stats_size);
memset(gbl_args->tx_stats, 0, gbl_args->tx_stats_size);
expected_tx_cnt = status->pps_curr * gbl_args->args.duration;
/* start receiver threads first */
thr_params.start = run_thread_rx;
thr_params.arg = &args_rx;
args_rx.batch_len = gbl_args->args.rx_batch_len;
odph_odpthreads_create(&thd_tbl[0], thd_mask_rx, &thr_params);
odp_barrier_wait(&gbl_args->rx_barrier);
num_rx_workers = odp_cpumask_count(thd_mask_rx);
/* then start transmitters */
thr_params.start = run_thread_tx;
thr_params.arg = &args_tx;
num_tx_workers = odp_cpumask_count(thd_mask_tx);
args_tx.pps = status->pps_curr / num_tx_workers;
args_tx.duration = gbl_args->args.duration;
args_tx.batch_len = gbl_args->args.tx_batch_len;
odph_odpthreads_create(&thd_tbl[num_rx_workers], thd_mask_tx,
&thr_params);
odp_barrier_wait(&gbl_args->tx_barrier);
/* wait for transmitter threads to terminate */
odph_odpthreads_join(&thd_tbl[num_rx_workers]);
/* delay to allow transmitted packets to reach the receivers */
odp_time_wait_ns(SHUTDOWN_DELAY_NS);
/* indicate to the receivers to exit */
odp_atomic_store_u32(&shutdown, 1);
/* wait for receivers */
odph_odpthreads_join(&thd_tbl[0]);
if (!status->warmup)
return process_results(expected_tx_cnt, status);
return 1;
}
/*
* Run a single instance of the throughput test. When attempting to determine
* the maximum packet rate this will be invoked multiple times with the only
* difference between runs being the target PPS rate.
*/
static int run_test_single(odp_cpumask_t *thd_mask_tx,
odp_cpumask_t *thd_mask_rx,
test_status_t *status)
{
odph_linux_pthread_t thd_tbl[MAX_WORKERS];
thread_args_t args_tx, args_rx;
uint64_t expected_tx_cnt;
int num_tx_workers, num_rx_workers;
odp_atomic_store_u32(&shutdown, 0);
memset(thd_tbl, 0, sizeof(thd_tbl));
memset(&gbl_args->rx_stats, 0, sizeof(gbl_args->rx_stats));
memset(&gbl_args->tx_stats, 0, sizeof(gbl_args->tx_stats));
expected_tx_cnt = status->pps_curr * gbl_args->args.duration;
/* start receiver threads first */
args_rx.batch_len = gbl_args->args.rx_batch_len;
odph_linux_pthread_create(&thd_tbl[0], thd_mask_rx,
run_thread_rx, &args_rx);
odp_barrier_wait(&gbl_args->rx_barrier);
num_rx_workers = odp_cpumask_count(thd_mask_rx);
/* then start transmitters */
num_tx_workers = odp_cpumask_count(thd_mask_tx);
args_tx.pps = status->pps_curr / num_tx_workers;
args_tx.duration = gbl_args->args.duration;
args_tx.batch_len = gbl_args->args.tx_batch_len;
odph_linux_pthread_create(&thd_tbl[num_rx_workers], thd_mask_tx,
run_thread_tx, &args_tx);
odp_barrier_wait(&gbl_args->tx_barrier);
/* wait for transmitter threads to terminate */
odph_linux_pthread_join(&thd_tbl[num_rx_workers],
num_tx_workers);
/* delay to allow transmitted packets to reach the receivers */
busy_loop_ns(SHUTDOWN_DELAY_NS);
/* indicate to the receivers to exit */
odp_atomic_store_u32(&shutdown, 1);
/* wait for receivers */
odph_linux_pthread_join(&thd_tbl[0], num_rx_workers);
return process_results(expected_tx_cnt, status);
}
static int end_suite(void)
{
odp_atomic_store_u32(&still_running, 0);
odph_linux_pthread_join(&pp_thread_handle, 1);
return 0;
}
static int init_suite(void)
{
odp_pool_param_t pool_params;
ofp_pkt_hook pkt_hook[OFP_HOOK_MAX];
odp_pool_t pool;
odph_linux_thr_params_t thr_params;
odp_instance_t instance;
/* Init ODP before calling anything else */
if (odp_init_global(&instance, NULL, NULL)) {
OFP_ERR("Error: ODP global init failed.\n");
return -1;
}
/* Init this thread */
if (odp_init_local(instance, ODP_THREAD_CONTROL)) {
OFP_ERR("Error: ODP local init failed.\n");
return -1;
}
memset(pkt_hook, 0, sizeof(pkt_hook));
pool_params.pkt.seg_len = SHM_PKT_POOL_BUFFER_SIZE;
pool_params.pkt.len = SHM_PKT_POOL_BUFFER_SIZE;
pool_params.pkt.num = SHM_PKT_POOL_SIZE / SHM_PKT_POOL_BUFFER_SIZE;
pool_params.type = ODP_POOL_PACKET;
(void) ofp_init_pre_global(pool_name, &pool_params, pkt_hook, &pool,
ARP_AGE_INTERVAL, ARP_ENTRY_TIMEOUT);
/*
* Start a packet processing thread to service timer events.
*/
odp_atomic_store_u32(&still_running, 1);
odp_cpumask_t cpumask;
odp_cpumask_zero(&cpumask);
odp_cpumask_set(&cpumask, 0x1);
thr_params.start = pp_thread;
thr_params.arg = NULL;
thr_params.thr_type = ODP_THREAD_WORKER;
thr_params.instance = instance;
odph_linux_pthread_create(&pp_thread_handle,
&cpumask,
&thr_params);
return 0;
}
odp_pool_t odp_pool_create(const char *name, odp_pool_param_t *params)
{
odp_pool_t pool_hdl = ODP_POOL_INVALID;
pool_entry_t *pool;
uint32_t i, headroom = 0, tailroom = 0;
odp_shm_t shm;
if (params == NULL)
return ODP_POOL_INVALID;
/* Default size and align for timeouts */
if (params->type == ODP_POOL_TIMEOUT) {
params->buf.size = 0; /* tmo.__res1 */
params->buf.align = 0; /* tmo.__res2 */
}
/* Default initialization parameters */
uint32_t p_udata_size = 0;
uint32_t udata_stride = 0;
/* Restriction for v1.0: All non-packet buffers are unsegmented */
int unseg = 1;
/* Restriction for v1.0: No zeroization support */
const int zeroized = 0;
uint32_t blk_size, buf_stride, buf_num, seg_len = 0;
uint32_t buf_align =
params->type == ODP_POOL_BUFFER ? params->buf.align : 0;
/* Validate requested buffer alignment */
if (buf_align > ODP_CONFIG_BUFFER_ALIGN_MAX ||
buf_align != ODP_ALIGN_ROUNDDOWN_POWER_2(buf_align, buf_align))
return ODP_POOL_INVALID;
/* Set correct alignment based on input request */
if (buf_align == 0)
buf_align = ODP_CACHE_LINE_SIZE;
else if (buf_align < ODP_CONFIG_BUFFER_ALIGN_MIN)
buf_align = ODP_CONFIG_BUFFER_ALIGN_MIN;
/* Calculate space needed for buffer blocks and metadata */
switch (params->type) {
case ODP_POOL_BUFFER:
buf_num = params->buf.num;
blk_size = params->buf.size;
/* Optimize small raw buffers */
if (blk_size > ODP_MAX_INLINE_BUF || params->buf.align != 0)
blk_size = ODP_ALIGN_ROUNDUP(blk_size, buf_align);
buf_stride = sizeof(odp_buffer_hdr_stride);
break;
case ODP_POOL_PACKET:
unseg = 0; /* Packets are always segmented */
headroom = ODP_CONFIG_PACKET_HEADROOM;
tailroom = ODP_CONFIG_PACKET_TAILROOM;
buf_num = params->pkt.num + 1; /* more one for pkt_ctx */
seg_len = params->pkt.seg_len <= ODP_CONFIG_PACKET_SEG_LEN_MIN ?
ODP_CONFIG_PACKET_SEG_LEN_MIN :
(params->pkt.seg_len <= ODP_CONFIG_PACKET_SEG_LEN_MAX ?
params->pkt.seg_len : ODP_CONFIG_PACKET_SEG_LEN_MAX);
seg_len = ODP_ALIGN_ROUNDUP(
headroom + seg_len + tailroom,
ODP_CONFIG_BUFFER_ALIGN_MIN);
blk_size = params->pkt.len <= seg_len ? seg_len :
ODP_ALIGN_ROUNDUP(params->pkt.len, seg_len);
/* Reject create if pkt.len needs too many segments */
if (blk_size / seg_len > ODP_BUFFER_MAX_SEG)
return ODP_POOL_INVALID;
p_udata_size = params->pkt.uarea_size;
udata_stride = ODP_ALIGN_ROUNDUP(p_udata_size,
sizeof(uint64_t));
buf_stride = sizeof(odp_packet_hdr_stride);
break;
case ODP_POOL_TIMEOUT:
blk_size = 0;
buf_num = params->tmo.num;
buf_stride = sizeof(odp_timeout_hdr_stride);
break;
default:
return ODP_POOL_INVALID;
}
/* Validate requested number of buffers against addressable limits */
if (buf_num >
(ODP_BUFFER_MAX_BUFFERS / (buf_stride / ODP_CACHE_LINE_SIZE)))
return ODP_POOL_INVALID;
/* Find an unused buffer pool slot and iniitalize it as requested */
for (i = 0; i < ODP_CONFIG_POOLS; i++) {
pool = get_pool_entry(i);
POOL_LOCK(&pool->s.lock);
if (pool->s.pool_shm != ODP_SHM_INVALID) {
POOL_UNLOCK(&pool->s.lock);
continue;
}
/* found free pool */
size_t block_size, pad_size, mdata_size, udata_size;
pool->s.flags.all = 0;
if (name == NULL) {
pool->s.name[0] = 0;
} else {
strncpy(pool->s.name, name,
ODP_POOL_NAME_LEN - 1);
pool->s.name[ODP_POOL_NAME_LEN - 1] = 0;
pool->s.flags.has_name = 1;
}
pool->s.params = *params;
pool->s.buf_align = buf_align;
/* Optimize for short buffers: Data stored in buffer hdr */
if (blk_size <= ODP_MAX_INLINE_BUF) {
block_size = 0;
pool->s.buf_align = blk_size == 0 ? 0 : sizeof(void *);
} else {
/* more 64bytes for storing hdr address*/
block_size = buf_num * (blk_size +
ODP_HDR_BACK_PTR_SIZE);
pool->s.buf_align = buf_align;
}
pad_size = ODP_CACHE_LINE_SIZE_ROUNDUP(block_size) - block_size;
mdata_size = buf_num * buf_stride;
udata_size = buf_num * udata_stride;
pool->s.buf_num = buf_num;
pool->s.pool_size = ODP_PAGE_SIZE_ROUNDUP(block_size +
pad_size +
mdata_size +
udata_size);
shm = odp_shm_reserve(pool->s.name, pool->s.pool_size,
ODP_PAGE_SIZE, ODP_SHM_MONOPOLIZE_CNTNUS_PHY);
if (shm == ODP_SHM_INVALID) {
POOL_UNLOCK(&pool->s.lock);
return ODP_POOL_INVALID;
}
pool->s.pool_base_addr = odp_shm_addr(shm);
pool->s.pool_shm = shm;
/* Now safe to unlock since pool entry has been allocated */
POOL_UNLOCK(&pool->s.lock);
pool->s.flags.unsegmented = unseg;
pool->s.flags.zeroized = zeroized;
pool->s.seg_size = unseg ? blk_size : seg_len;
pool->s.blk_size = blk_size;
uint8_t *block_base_addr = pool->s.pool_base_addr;
uint8_t *mdata_base_addr =
block_base_addr + block_size + pad_size;
uint8_t *udata_base_addr = mdata_base_addr + mdata_size;
uint64_t pool_base_phy = odp_v2p(pool->s.pool_base_addr);
pool->s.v_p_offset = (uint64_t)pool->s.pool_base_addr -
pool_base_phy;
/* Pool mdata addr is used for indexing buffer metadata */
pool->s.pool_mdata_addr = mdata_base_addr;
pool->s.udata_size = p_udata_size;
pool->s.buf_stride = buf_stride;
pool->s.buf_freelist = NULL;
pool->s.blk_freelist = NULL;
/* Initialization will increment these to their target vals */
odp_atomic_store_u32(&pool->s.bufcount, 0);
odp_atomic_store_u32(&pool->s.blkcount, 0);
uint8_t *buf = udata_base_addr - buf_stride;
uint8_t *udat = udata_stride == 0 ? NULL :
udata_base_addr + udata_size - udata_stride;
/* Init buffer common header and add to pool buffer freelist */
do {
odp_buffer_hdr_t *tmp =
(odp_buffer_hdr_t *)(void *)buf;
/* Iniitalize buffer metadata */
tmp->allocator = ODP_FREEBUF;
tmp->flags.all = 0;
tmp->flags.zeroized = zeroized;
tmp->size = 0;
odp_atomic_init_u32(&tmp->ref_count, 0);
tmp->type = params->type;
tmp->event_type = params->type;
tmp->pool_hdl = pool->s.pool_hdl;
tmp->uarea_addr = (void *)udat;
tmp->uarea_size = p_udata_size;
tmp->segcount = 0;
tmp->segsize = pool->s.seg_size;
tmp->handle.handle = odp_buffer_encode_handle(tmp);
/* Set 1st seg addr for zero-len buffers */
tmp->addr[0] = NULL;
/* Special case for short buffer data */
if (blk_size <= ODP_MAX_INLINE_BUF) {
tmp->flags.hdrdata = 1;
if (blk_size > 0) {
tmp->segcount = 1;
tmp->addr[0] = &tmp->addr[1];
tmp->size = blk_size;
}
}
/* Push buffer onto pool's freelist */
ret_buf(&pool->s, tmp);
buf -= buf_stride;
udat -= udata_stride;
} while (buf >= mdata_base_addr);
/* Make sure blocks is divided into size align to 8 bytes,
* as odp_packet_seg_t refers to address and segment count.
* pool->s.seg_size is align to 8 bytes before here
*/
pool->s.seg_size = ODP_ALIGN_ROUNDUP(pool->s.seg_size,
sizeof(uint64_t));
/* Form block freelist for pool */
uint8_t *blk =
block_base_addr + block_size - pool->s.seg_size -
ODP_HDR_BACK_PTR_SIZE;
if (blk_size > ODP_MAX_INLINE_BUF)
do {
ret_blk(&pool->s, blk + ODP_HDR_BACK_PTR_SIZE);
blk -= (pool->s.seg_size +
ODP_HDR_BACK_PTR_SIZE);
} while (blk >= block_base_addr);
/* For pkt pool, initiating packet hdr relative area is stored
* in the pool entry.
*/
if (params->type == ODP_POOL_PACKET) {
odp_buffer_hdr_t *bh = get_buf(&pool->s);
uint8_t *pkt_ctx = ((uint8_t *)bh +
ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr));
memset(pkt_ctx, 0, sizeof(odp_packet_hdr_t) -
ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr));
((odp_packet_hdr_t *)bh)->l3_offset =
ODP_PACKET_OFFSET_INVALID;
((odp_packet_hdr_t *)bh)->l4_offset =
ODP_PACKET_OFFSET_INVALID;
((odp_packet_hdr_t *)bh)->payload_offset =
ODP_PACKET_OFFSET_INVALID;
((odp_packet_hdr_t *)bh)->headroom = headroom;
pool->s.cache_pkt_hdr = pkt_ctx;
pool->s.buf_num -= 1;
}
/* Every kind of pool has max_size unit, as alloc, just need to
* compare with max_size here to check
*/
if (!unseg)
pool->s.max_size = pool->s.seg_size *
ODP_BUFFER_MAX_SEG -
headroom - tailroom;
else
pool->s.max_size = pool->s.seg_size;
/* Initialize pool statistics counters */
odp_atomic_store_u64(&pool->s.poolstats.bufallocs, 0);
odp_atomic_store_u64(&pool->s.poolstats.buffrees, 0);
odp_atomic_store_u64(&pool->s.poolstats.blkallocs, 0);
odp_atomic_store_u64(&pool->s.poolstats.blkfrees, 0);
odp_atomic_store_u64(&pool->s.poolstats.bufempty, 0);
odp_atomic_store_u64(&pool->s.poolstats.blkempty, 0);
odp_atomic_store_u64(&pool->s.poolstats.high_wm_count, 0);
odp_atomic_store_u64(&pool->s.poolstats.low_wm_count, 0);
/* Reset other pool globals to initial state */
pool->s.low_wm_assert = 0;
pool->s.quiesced = 0;
pool->s.headroom = headroom;
pool->s.tailroom = tailroom;
pool->s.room_size = headroom + tailroom;
/* Watermarks are hard-coded for now to control caching */
pool->s.high_wm = pool->s.buf_num / 2;
pool->s.low_wm = pool->s.buf_num / 4;
pool_hdl = pool->s.pool_hdl;
break;
}
return pool_hdl;
}
void test_atomic_store(void)
{
odp_atomic_store_u32(&a32u, U32_INIT_VAL);
odp_atomic_store_u64(&a64u, U64_INIT_VAL);
}
