/*
* Resize allocated memory.
*/
void *
rte_realloc(void *ptr, size_t size, unsigned align)
{
if (ptr == NULL)
return rte_malloc(NULL, size, align);
struct malloc_elem *elem = malloc_elem_from_data(ptr);
if (elem == NULL)
rte_panic("Fatal error: memory corruption detected\n");
size = CACHE_LINE_ROUNDUP(size), align = CACHE_LINE_ROUNDUP(align);
/* check alignment matches first, and if ok, see if we can resize block */
if (RTE_PTR_ALIGN(ptr,align) == ptr &&
malloc_elem_resize(elem, size) == 0)
return ptr;
/* either alignment is off, or we have no room to expand,
* so move data. */
void *new_ptr = rte_malloc(NULL, size, align);
if (new_ptr == NULL)
return NULL;
const unsigned old_size = elem->size - MALLOC_ELEM_OVERHEAD;
rte_memcpy(new_ptr, ptr, old_size < size ? old_size : size);
rte_free(ptr);
return new_ptr;
}
static int
test_align(void)
{
#define FAIL_ALIGN(x, i, p)\
{printf(x "() test failed: %u %u\n", i, p);\
return -1;}
#define ERROR_FLOOR(res, i, pow) \
(res % pow) || /* check if not aligned */ \
((res / pow) != (i / pow)) /* check if correct alignment */
#define ERROR_CEIL(res, i, pow) \
(res % pow) || /* check if not aligned */ \
((i % pow) == 0 ? /* check if ceiling is invoked */ \
val / pow != i / pow : /* if aligned */ \
val / pow != (i / pow) + 1) /* if not aligned, hence +1 */
uint32_t i, p, val;
for (i = 1, p = 1; i <= MAX_NUM; i ++) {
if (rte_align32pow2(i) != p)
FAIL_ALIGN("rte_align32pow2", i, p);
if (i == p)
p <<= 1;
}
for (p = 2; p <= MAX_NUM; p <<= 1) {
if (!rte_is_power_of_2(p))
FAIL("rte_is_power_of_2");
for (i = 1; i <= MAX_NUM; i++) {
/* align floor */
if (RTE_ALIGN_FLOOR((uintptr_t)i, p) % p)
FAIL_ALIGN("RTE_ALIGN_FLOOR", i, p);
val = RTE_PTR_ALIGN_FLOOR((uintptr_t) i, p);
if (ERROR_FLOOR(val, i, p))
FAIL_ALIGN("RTE_PTR_ALIGN_FLOOR", i, p);
val = RTE_ALIGN_FLOOR(i, p);
if (ERROR_FLOOR(val, i, p))
FAIL_ALIGN("RTE_ALIGN_FLOOR", i, p);
/* align ceiling */
val = RTE_PTR_ALIGN((uintptr_t) i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_PTR_ALIGN", i, p);
val = RTE_ALIGN(i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_ALIGN", i, p);
val = RTE_ALIGN_CEIL(i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_ALIGN_CEIL", i, p);
val = RTE_PTR_ALIGN_CEIL((uintptr_t)i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_PTR_ALIGN_CEIL", i, p);
/* by this point we know that val is aligned to p */
if (!rte_is_aligned((void*)(uintptr_t) val, p))
FAIL("rte_is_aligned");
}
}
return 0;
}
fm10k_rxq_vec_setup(struct fm10k_rx_queue *rxq)
{
uintptr_t p;
struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
mb_def.nb_segs = 1;
/* data_off will be ajusted after new mbuf allocated for 512-byte
* alignment.
*/
mb_def.data_off = RTE_PKTMBUF_HEADROOM;
mb_def.port = rxq->port_id;
rte_mbuf_refcnt_set(&mb_def, 1);
/* prevent compiler reordering: rearm_data covers previous fields */
rte_compiler_barrier();
p = (uintptr_t)&mb_def.rearm_data;
rxq->mbuf_initializer = *(uint64_t *)p;
return 0;
}
static inline void
fm10k_rxq_rearm(struct fm10k_rx_queue *rxq)
{
int i;
uint16_t rx_id;
volatile union fm10k_rx_desc *rxdp;
struct rte_mbuf **mb_alloc = &rxq->sw_ring[rxq->rxrearm_start];
struct rte_mbuf *mb0, *mb1;
__m128i head_off = _mm_set_epi64x(
RTE_PKTMBUF_HEADROOM + FM10K_RX_DATABUF_ALIGN - 1,
RTE_PKTMBUF_HEADROOM + FM10K_RX_DATABUF_ALIGN - 1);
__m128i dma_addr0, dma_addr1;
/* Rx buffer need to be aligned with 512 byte */
const __m128i hba_msk = _mm_set_epi64x(0,
UINT64_MAX - FM10K_RX_DATABUF_ALIGN + 1);
rxdp = rxq->hw_ring + rxq->rxrearm_start;
/* Pull 'n' more MBUFs into the software ring */
if (rte_mempool_get_bulk(rxq->mp,
(void *)mb_alloc,
RTE_FM10K_RXQ_REARM_THRESH) < 0) {
dma_addr0 = _mm_setzero_si128();
/* Clean up all the HW/SW ring content */
for (i = 0; i < RTE_FM10K_RXQ_REARM_THRESH; i++) {
mb_alloc[i] = &rxq->fake_mbuf;
_mm_store_si128((__m128i *)&rxdp[i].q,
dma_addr0);
}
rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
RTE_FM10K_RXQ_REARM_THRESH;
return;
}
/* Initialize the mbufs in vector, process 2 mbufs in one loop */
for (i = 0; i < RTE_FM10K_RXQ_REARM_THRESH; i += 2, mb_alloc += 2) {
__m128i vaddr0, vaddr1;
uintptr_t p0, p1;
mb0 = mb_alloc[0];
mb1 = mb_alloc[1];
/* Flush mbuf with pkt template.
* Data to be rearmed is 6 bytes long.
* Though, RX will overwrite ol_flags that are coming next
* anyway. So overwrite whole 8 bytes with one load:
* 6 bytes of rearm_data plus first 2 bytes of ol_flags.
*/
p0 = (uintptr_t)&mb0->rearm_data;
*(uint64_t *)p0 = rxq->mbuf_initializer;
p1 = (uintptr_t)&mb1->rearm_data;
*(uint64_t *)p1 = rxq->mbuf_initializer;
/* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */
vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
/* convert pa to dma_addr hdr/data */
dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);
/* add headroom to pa values */
dma_addr0 = _mm_add_epi64(dma_addr0, head_off);
dma_addr1 = _mm_add_epi64(dma_addr1, head_off);
/* Do 512 byte alignment to satisfy HW requirement, in the
* meanwhile, set Header Buffer Address to zero.
*/
dma_addr0 = _mm_and_si128(dma_addr0, hba_msk);
dma_addr1 = _mm_and_si128(dma_addr1, hba_msk);
/* flush desc with pa dma_addr */
_mm_store_si128((__m128i *)&rxdp++->q, dma_addr0);
_mm_store_si128((__m128i *)&rxdp++->q, dma_addr1);
/* enforce 512B alignment on default Rx virtual addresses */
mb0->data_off = (uint16_t)(RTE_PTR_ALIGN((char *)mb0->buf_addr
+ RTE_PKTMBUF_HEADROOM, FM10K_RX_DATABUF_ALIGN)
- (char *)mb0->buf_addr);
mb1->data_off = (uint16_t)(RTE_PTR_ALIGN((char *)mb1->buf_addr
+ RTE_PKTMBUF_HEADROOM, FM10K_RX_DATABUF_ALIGN)
- (char *)mb1->buf_addr);
}
rxq->rxrearm_start += RTE_FM10K_RXQ_REARM_THRESH;
if (rxq->rxrearm_start >= rxq->nb_desc)
rxq->rxrearm_start = 0;
rxq->rxrearm_nb -= RTE_FM10K_RXQ_REARM_THRESH;
rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
(rxq->nb_desc - 1) : (rxq->rxrearm_start - 1));
/* Update the tail pointer on the NIC */
FM10K_PCI_REG_WRITE(rxq->tail_ptr, rx_id);
}
void *
eal_get_virtual_area(void *requested_addr, size_t *size,
size_t page_sz, int flags, int mmap_flags)
{
bool addr_is_hint, allow_shrink, unmap, no_align;
uint64_t map_sz;
void *mapped_addr, *aligned_addr;
if (system_page_sz == 0)
system_page_sz = sysconf(_SC_PAGESIZE);
mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS;
RTE_LOG(DEBUG, EAL, "Ask a virtual area of 0x%zx bytes\n", *size);
addr_is_hint = (flags & EAL_VIRTUAL_AREA_ADDR_IS_HINT) > 0;
allow_shrink = (flags & EAL_VIRTUAL_AREA_ALLOW_SHRINK) > 0;
unmap = (flags & EAL_VIRTUAL_AREA_UNMAP) > 0;
if (next_baseaddr == NULL && internal_config.base_virtaddr != 0 &&
rte_eal_process_type() == RTE_PROC_PRIMARY)
next_baseaddr = (void *) internal_config.base_virtaddr;
if (requested_addr == NULL && next_baseaddr != NULL) {
requested_addr = next_baseaddr;
requested_addr = RTE_PTR_ALIGN(requested_addr, page_sz);
addr_is_hint = true;
}
/* we don't need alignment of resulting pointer in the following cases:
*
* 1. page size is equal to system size
* 2. we have a requested address, and it is page-aligned, and we will
* be discarding the address if we get a different one.
*
* for all other cases, alignment is potentially necessary.
*/
no_align = (requested_addr != NULL &&
requested_addr == RTE_PTR_ALIGN(requested_addr, page_sz) &&
!addr_is_hint) ||
page_sz == system_page_sz;
do {
map_sz = no_align ? *size : *size + page_sz;
if (map_sz > SIZE_MAX) {
RTE_LOG(ERR, EAL, "Map size too big\n");
rte_errno = E2BIG;
return NULL;
}
mapped_addr = mmap(requested_addr, (size_t)map_sz, PROT_READ,
mmap_flags, -1, 0);
if (mapped_addr == MAP_FAILED && allow_shrink)
*size -= page_sz;
} while (allow_shrink && mapped_addr == MAP_FAILED && *size > 0);
/* align resulting address - if map failed, we will ignore the value
* anyway, so no need to add additional checks.
*/
aligned_addr = no_align ? mapped_addr :
RTE_PTR_ALIGN(mapped_addr, page_sz);
if (*size == 0) {
RTE_LOG(ERR, EAL, "Cannot get a virtual area of any size: %s\n",
strerror(errno));
rte_errno = errno;
return NULL;
} else if (mapped_addr == MAP_FAILED) {
RTE_LOG(ERR, EAL, "Cannot get a virtual area: %s\n",
strerror(errno));
/* pass errno up the call chain */
rte_errno = errno;
return NULL;
} else if (requested_addr != NULL && !addr_is_hint &&
aligned_addr != requested_addr) {
RTE_LOG(ERR, EAL, "Cannot get a virtual area at requested address: %p (got %p)\n",
requested_addr, aligned_addr);
munmap(mapped_addr, map_sz);
rte_errno = EADDRNOTAVAIL;
return NULL;
} else if (requested_addr != NULL && addr_is_hint &&
aligned_addr != requested_addr) {
RTE_LOG(WARNING, EAL, "WARNING! Base virtual address hint (%p != %p) not respected!\n",
requested_addr, aligned_addr);
RTE_LOG(WARNING, EAL, " This may cause issues with mapping memory into secondary processes\n");
} else if (next_baseaddr != NULL) {
next_baseaddr = RTE_PTR_ADD(aligned_addr, *size);
}
RTE_LOG(DEBUG, EAL, "Virtual area found at %p (size = 0x%zx)\n",
aligned_addr, *size);
if (unmap) {
munmap(mapped_addr, map_sz);
} else if (!no_align) {
void *map_end, *aligned_end;
size_t before_len, after_len;
/* when we reserve space with alignment, we add alignment to
* mapping size. On 32-bit, if 1GB alignment was requested, this
* would waste 1GB of address space, which is a luxury we cannot
* afford. so, if alignment was performed, check if any unneeded
* address space can be unmapped back.
*/
map_end = RTE_PTR_ADD(mapped_addr, (size_t)map_sz);
aligned_end = RTE_PTR_ADD(aligned_addr, *size);
/* unmap space before aligned mmap address */
before_len = RTE_PTR_DIFF(aligned_addr, mapped_addr);
if (before_len > 0)
munmap(mapped_addr, before_len);
/* unmap space after aligned end mmap address */
after_len = RTE_PTR_DIFF(map_end, aligned_end);
if (after_len > 0)
munmap(aligned_end, after_len);
}
return aligned_addr;
}
