/*
* xge_hal_mempool_create
* @memblock_size:
* @items_initial:
* @items_max:
* @item_size:
* @item_func:
*
* This function will create memory pool object. Pool may grow but will
* never shrink. Pool consists of number of dynamically allocated blocks
* with size enough to hold %items_initial number of items. Memory is
* DMA-able but client must map/unmap before interoperating with the device.
* See also: xge_os_dma_map(), xge_hal_dma_unmap(), xge_hal_status_e{}.
*/
xge_hal_mempool_t*
__hal_mempool_create(pci_dev_h pdev, int memblock_size, int item_size,
int items_priv_size, int items_initial, int items_max,
xge_hal_mempool_item_f item_func_alloc,
xge_hal_mempool_item_f item_func_free, void *userdata)
{
xge_hal_status_e status;
int memblocks_to_allocate;
xge_hal_mempool_t *mempool;
int allocated;
if (memblock_size < item_size) {
xge_debug_mm(XGE_ERR,
"memblock_size %d < item_size %d: misconfiguration",
memblock_size, item_size);
return NULL;
}
mempool = (xge_hal_mempool_t *) \
xge_os_malloc(pdev, sizeof(xge_hal_mempool_t));
if (mempool == NULL) {
xge_debug_mm(XGE_ERR, "mempool allocation failure");
return NULL;
}
xge_os_memzero(mempool, sizeof(xge_hal_mempool_t));
mempool->pdev = pdev;
mempool->memblock_size = memblock_size;
mempool->items_max = items_max;
mempool->items_initial = items_initial;
mempool->item_size = item_size;
mempool->items_priv_size = items_priv_size;
mempool->item_func_alloc = item_func_alloc;
mempool->item_func_free = item_func_free;
mempool->userdata = userdata;
mempool->memblocks_allocated = 0;
mempool->items_per_memblock = memblock_size / item_size;
mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) /
mempool->items_per_memblock;
/* allocate array of memblocks */
mempool->memblocks_arr = (void ** ) xge_os_malloc(mempool->pdev,
sizeof(void*) * mempool->memblocks_max);
if (mempool->memblocks_arr == NULL) {
xge_debug_mm(XGE_ERR, "memblocks_arr allocation failure");
__hal_mempool_destroy(mempool);
return NULL;
}
xge_os_memzero(mempool->memblocks_arr,
sizeof(void*) * mempool->memblocks_max);
/* allocate array of private parts of items per memblocks */
mempool->memblocks_priv_arr = (void **) xge_os_malloc(mempool->pdev,
sizeof(void*) * mempool->memblocks_max);
if (mempool->memblocks_priv_arr == NULL) {
xge_debug_mm(XGE_ERR, "memblocks_priv_arr allocation failure");
__hal_mempool_destroy(mempool);
return NULL;
}
xge_os_memzero(mempool->memblocks_priv_arr,
sizeof(void*) * mempool->memblocks_max);
/* allocate array of memblocks DMA objects */
mempool->memblocks_dma_arr =
(xge_hal_mempool_dma_t *) xge_os_malloc(mempool->pdev,
sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max);
if (mempool->memblocks_dma_arr == NULL) {
xge_debug_mm(XGE_ERR, "memblocks_dma_arr allocation failure");
__hal_mempool_destroy(mempool);
return NULL;
}
xge_os_memzero(mempool->memblocks_dma_arr,
sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max);
/* allocate hash array of items */
mempool->items_arr = (void **) xge_os_malloc(mempool->pdev,
sizeof(void*) * mempool->items_max);
if (mempool->items_arr == NULL) {
xge_debug_mm(XGE_ERR, "items_arr allocation failure");
__hal_mempool_destroy(mempool);
return NULL;
}
xge_os_memzero(mempool->items_arr, sizeof(void *) * mempool->items_max);
mempool->shadow_items_arr = (void **) xge_os_malloc(mempool->pdev,
sizeof(void*) * mempool->items_max);
if (mempool->shadow_items_arr == NULL) {
xge_debug_mm(XGE_ERR, "shadow_items_arr allocation failure");
__hal_mempool_destroy(mempool);
return NULL;
}
xge_os_memzero(mempool->shadow_items_arr,
sizeof(void *) * mempool->items_max);
/* calculate initial number of memblocks */
memblocks_to_allocate = (mempool->items_initial +
mempool->items_per_memblock - 1) /
mempool->items_per_memblock;
xge_debug_mm(XGE_TRACE, "allocating %d memblocks, "
"%d items per memblock", memblocks_to_allocate,
mempool->items_per_memblock);
/* pre-allocate the mempool */
status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated);
xge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr,
sizeof(void*) * mempool->items_max);
if (status != XGE_HAL_OK) {
xge_debug_mm(XGE_ERR, "mempool_grow failure");
__hal_mempool_destroy(mempool);
return NULL;
}
xge_debug_mm(XGE_TRACE,
"total: allocated %dk of DMA-capable memory",
mempool->memblock_size * allocated / 1024);
return mempool;
}
/*
* __hal_mempool_grow
*
* Will resize mempool up to %num_allocate value.
*/
xge_hal_status_e
__hal_mempool_grow(xge_hal_mempool_t *mempool, int num_allocate,
int *num_allocated)
{
int i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0;
int n_items = mempool->items_per_memblock;
*num_allocated = 0;
if ((mempool->memblocks_allocated + num_allocate) >
mempool->memblocks_max) {
xge_debug_mm(XGE_ERR, "%s",
"__hal_mempool_grow: can grow anymore");
return XGE_HAL_ERR_OUT_OF_MEMORY;
}
for (i = mempool->memblocks_allocated;
i < mempool->memblocks_allocated + num_allocate; i++) {
int j;
int is_last =
((mempool->memblocks_allocated+num_allocate-1) == i);
xge_hal_mempool_dma_t *dma_object =
mempool->memblocks_dma_arr + i;
void *the_memblock;
int dma_flags;
dma_flags = XGE_OS_DMA_CACHELINE_ALIGNED;
#ifdef XGE_HAL_DMA_DTR_CONSISTENT
dma_flags |= XGE_OS_DMA_CONSISTENT;
#else
dma_flags |= XGE_OS_DMA_STREAMING;
#endif
/* allocate DMA-capable memblock */
mempool->memblocks_arr[i] = xge_os_dma_malloc(mempool->pdev,
mempool->memblock_size,
dma_flags,
&dma_object->handle,
&dma_object->acc_handle);
if (mempool->memblocks_arr[i] == NULL) {
xge_debug_mm(XGE_ERR,
"memblock[%d]: out of DMA memory", i);
return XGE_HAL_ERR_OUT_OF_MEMORY;
}
xge_os_memzero(mempool->memblocks_arr[i],
mempool->memblock_size);
the_memblock = mempool->memblocks_arr[i];
/* allocate memblock's private part. Each DMA memblock
* has a space allocated for item's private usage upon
* mempool's user request. Each time mempool grows, it will
* allocate new memblock and its private part at once.
* This helps to minimize memory usage a lot. */
mempool->memblocks_priv_arr[i] = xge_os_malloc(mempool->pdev,
mempool->items_priv_size * n_items);
if (mempool->memblocks_priv_arr[i] == NULL) {
xge_os_dma_free(mempool->pdev,
the_memblock,
mempool->memblock_size,
&dma_object->acc_handle,
&dma_object->handle);
xge_debug_mm(XGE_ERR,
"memblock_priv[%d]: out of virtual memory, "
"requested %d(%d:%d) bytes", i,
mempool->items_priv_size * n_items,
mempool->items_priv_size, n_items);
return XGE_HAL_ERR_OUT_OF_MEMORY;
}
xge_os_memzero(mempool->memblocks_priv_arr[i],
mempool->items_priv_size * n_items);
/* map memblock to physical memory */
dma_object->addr = xge_os_dma_map(mempool->pdev,
dma_object->handle,
the_memblock,
mempool->memblock_size,
XGE_OS_DMA_DIR_BIDIRECTIONAL,
#ifdef XGE_HAL_DMA_DTR_CONSISTENT
XGE_OS_DMA_CONSISTENT
#else
XGE_OS_DMA_STREAMING
#endif
);
if (dma_object->addr == XGE_OS_INVALID_DMA_ADDR) {
xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i],
mempool->items_priv_size *
n_items);
xge_os_dma_free(mempool->pdev,
the_memblock,
mempool->memblock_size,
&dma_object->acc_handle,
&dma_object->handle);
return XGE_HAL_ERR_OUT_OF_MAPPING;
}
/* fill the items hash array */
for (j=0; j<n_items; j++) {
int index = i*n_items + j;
if (first_time && index >= mempool->items_initial) {
break;
}
mempool->items_arr[index] =
((char *)the_memblock + j*mempool->item_size);
/* let caller to do more job on each item */
if (mempool->item_func_alloc != NULL) {
xge_hal_status_e status;
if ((status = mempool->item_func_alloc(
mempool,
the_memblock,
i,
dma_object,
mempool->items_arr[index],
index,
is_last,
mempool->userdata)) != XGE_HAL_OK) {
if (mempool->item_func_free != NULL) {
int k;
for (k=0; k<j; k++) {
index =i*n_items + k;
(void)mempool->item_func_free(
mempool, the_memblock,
i, dma_object,
mempool->items_arr[index],
index, is_last,
mempool->userdata);
}
}
xge_os_free(mempool->pdev,
mempool->memblocks_priv_arr[i],
mempool->items_priv_size *
n_items);
xge_os_dma_unmap(mempool->pdev,
dma_object->handle,
dma_object->addr,
mempool->memblock_size,
XGE_OS_DMA_DIR_BIDIRECTIONAL);
xge_os_dma_free(mempool->pdev,
the_memblock,
mempool->memblock_size,
&dma_object->acc_handle,
&dma_object->handle);
return status;
}
}
mempool->items_current = index + 1;
}
xge_debug_mm(XGE_TRACE,
"memblock%d: allocated %dk, vaddr 0x"XGE_OS_LLXFMT", "
"dma_addr 0x"XGE_OS_LLXFMT, i, mempool->memblock_size / 1024,
(unsigned long long)(ulong_t)mempool->memblocks_arr[i],
(unsigned long long)dma_object->addr);
(*num_allocated)++;
if (first_time && mempool->items_current ==
mempool->items_initial) {
break;
}
}
/* increment actual number of allocated memblocks */
mempool->memblocks_allocated += *num_allocated;
return XGE_HAL_OK;
}
static xge_hal_status_e
__hal_fifo_mempool_item_alloc(xge_hal_mempool_h mempoolh,
void *memblock,
int memblock_index,
xge_hal_mempool_dma_t *dma_object,
void *item,
int index,
int is_last,
void *userdata)
{
int memblock_item_idx;
xge_hal_fifo_txdl_priv_t *txdl_priv;
xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)item;
xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)userdata;
xge_assert(item);
txdl_priv = __hal_mempool_item_priv(mempoolh, memblock_index,
item, &memblock_item_idx);
xge_assert(txdl_priv);
/* pre-format HAL's TxDL's private */
txdl_priv->dma_offset = (char*)item - (char*)memblock;
txdl_priv->dma_addr = dma_object->addr + txdl_priv->dma_offset;
txdl_priv->dma_handle = dma_object->handle;
txdl_priv->memblock = memblock;
txdl_priv->first_txdp = (xge_hal_fifo_txd_t *)item;
txdl_priv->next_txdl_priv = NULL;
txdl_priv->dang_txdl = NULL;
txdl_priv->dang_frags = 0;
txdl_priv->alloc_frags = 0;
#ifdef XGE_DEBUG_ASSERT
txdl_priv->dma_object = dma_object;
#endif
txdp->host_control = (u64)(ulong_t)txdl_priv;
#ifdef XGE_HAL_ALIGN_XMIT
txdl_priv->align_vaddr = NULL;
txdl_priv->align_dma_addr = (dma_addr_t)0;
#ifndef XGE_HAL_ALIGN_XMIT_ALLOC_RT
{
xge_hal_status_e status;
if (fifo->config->alignment_size) {
status =__hal_fifo_dtr_align_alloc_map(fifo, txdp);
if (status != XGE_HAL_OK) {
xge_debug_mm(XGE_ERR,
"align buffer[%d] %d bytes, status %d",
index,
fifo->config->alignment_size *
fifo->config->max_aligned_frags,
status);
return status;
}
}
}
#endif
#endif
if (fifo->channel.dtr_init) {
fifo->channel.dtr_init(fifo, (xge_hal_dtr_h)txdp, index,
fifo->channel.userdata, XGE_HAL_CHANNEL_OC_NORMAL);
}
return XGE_HAL_OK;
}
