void os_mem_info(void)
{
MEM_DEBUG("struct heap_mem=%d, MEM_BLOCK_ALIGN_BYTE=%d", sizeof(struct heap_mem), MEM_BLOCK_ALIGN_BYTE);
MEM_DEBUG("total memory: %d", mem_size_aligned);
MEM_DEBUG("used memory : %d", used_mem);
MEM_DEBUG("maximum allocated memory: %d", max_mem);
}
void memWriteProcess()
{
uint8_t memId = p.data[0];
uint8_t writeLen;
uint32_t memAddr;
uint8_t status = 0;
memcpy(&memAddr, &p.data[1], 4);
writeLen = p.size - 5;
MEM_DEBUG("Packet is MEM WRITE\n");
p.header = CRTP_HEADER(CRTP_PORT_MEM, WRITE_CH);
// Dont' touch the first 5 bytes, they will be the same.
if (memId == EEPROM_ID)
{
if (memAddr + writeLen <= EEPROM_SIZE &&
eepromWriteBuffer(&p.data[5], memAddr, writeLen))
status = 0;
else
status = EIO;
}
else if(memId == LEDMEM_ID)
{
if ((memAddr + writeLen) <= sizeof(ledringmem))
{
memcpy(&(ledringmem[memAddr]), &p.data[5], writeLen);
MEM_DEBUG("LED write addr:%i, led:%i\n", memAddr, writeLen);
}
else
{
MEM_DEBUG("\LED write failed! addr:%i, led:%i\n", memAddr, writeLen);
}
}
else
{
/*****************************************************************************
* function:
* crypto_free_slab_list(qae_slab_pool *list, int memfd)
* @param[in] list, pointer to a slab list
* @param[in] memfd, file descriptor for the memory driver
*
* @description
* Free all slabs in the supplied slab list.
*
*****************************************************************************/
static void crypto_free_slab_list(qae_slab_pool *list, int memfd)
{
qae_slab *slb, *s_next_slab;
#ifdef USE_QAT_CONTIG_MEM
qat_contig_mem_config qmcfg;
#endif
/* cleanup all the empty slabs */
for (slb = list->next; list->slot_size > 0 ; slb = s_next_slab) {
/* need to save this off before unmapping. This is why we can't have
slb = slb->next_slab in the for loop above. */
s_next_slab = slb->next;
#ifdef USE_QAT_CONTIG_MEM
MEM_DEBUG("%s do munmap of %p\n", __func__, slb);
qmcfg = *((qat_contig_mem_config *) slb);
if (munmap(slb, SLAB_SIZE) == -1) {
perror("munmap");
exit(EXIT_FAILURE);
}
MEM_DEBUG("%s ioctl free of %p\n", __func__, slb);
if (ioctl(memfd, QAT_CONTIG_MEM_FREE, &qmcfg) == -1) {
perror("ioctl QAT_CONTIG_MEM_FREE");
exit(EXIT_FAILURE);
}
#endif
list->slot_size--;
}
MEM_DEBUG("%s done\n", __func__);
}
void qaeCryptoMemFree(void *ptr)
{
int rc;
MEM_DEBUG("%s: Address: %p\n", __func__, ptr);
if (NULL == ptr) {
MEM_WARN("qaeCryptoMemFree trying to free NULL pointer.\n");
return;
}
MEM_DEBUG("%s: pthread_mutex_lock\n", __func__);
if ((rc = pthread_mutex_lock(&mem_mutex)) != 0) {
MEM_ERROR("pthread_mutex_lock: %s\n", strerror(rc));
return;
}
qaeMemFreeNUMA(&ptr);
if ((rc = pthread_mutex_unlock(&mem_mutex)) != 0) {
MEM_ERROR("pthread_mutex_unlock: %s\n", strerror(rc));
return;
}
MEM_DEBUG("%s: pthread_mutex_unlock\n", __func__);
}
/**
* This function will release the previously allocated memory block by
* os_malloc. The released memory block is taken back to system heap.
*
* @rmem the address of memory which will be released
*/
void os_free(void *rmem)
{
struct heap_mem *mem;
/*RT_DEBUG_NOT_IN_INTERRUPT;*/
if (rmem == NULL)
{
return;
}
MEM_ASSERT((((uint32_t)rmem) & (MEM_ALIGN_BYTE - 1)) == 0);
MEM_ASSERT((uint8_t *)rmem >= (uint8_t *)heap_ptr &&
(uint8_t *)rmem < (uint8_t *)heap_end);
if ((uint8_t *)rmem < (uint8_t *)heap_ptr ||
(uint8_t *)rmem >= (uint8_t *)heap_end)
{
MEM_DEBUG("illegal memory");
return;
}
/* Get the corresponding struct heap_mem ... */
mem = (struct heap_mem *)((uint8_t *)rmem - MEM_BLOCK_SIZE);
MEM_DEBUG("release memory 0x%x, size: %d",
(uint32_t)rmem,
(uint32_t)(mem->next - ((uint8_t *)mem - heap_ptr)));
#ifdef MEM_SAFETY
/* protect the heap from concurrent access */
sem_wait(&heap_sem);
#endif
/* ... which has to be in a used state ... */
MEM_ASSERT(mem->used);
MEM_ASSERT(mem->magic == HEAP_MAGIC);
/* ... and is now unused. */
mem->used = 0;
mem->magic = 0;
if (mem < lfree)
{
/* the newly freed struct is now the lowest */
lfree = mem;
}
#ifdef MEM_STATISTIC
used_mem -= (mem->next - ((uint8_t *)mem - heap_ptr));
#endif
/* finally, see if prev or next are free also */
plug_holes(mem);
#ifdef MEM_SAFETY
sem_post(&heap_sem);
#endif
}
/**
* This function will initialize system heap memory.
*
* @begin_addr the beginning address of heap memory.
* @end_addr the end address of heap memory.
*/
void os_mem_init(void *begin_addr, void *end_addr)
{
struct heap_mem *mem;
uint32_t begin_align = MEM_ALIGN_UP((uint32_t)begin_addr, MEM_ALIGN_BYTE);
uint32_t end_align = MEM_ALIGN_DOWN((uint32_t)end_addr, MEM_ALIGN_BYTE);
/*RT_DEBUG_NOT_IN_INTERRUPT;*/
/* alignment addr */
if ((end_align > (2 * MEM_BLOCK_SIZE)) &&
((end_align - 2 * MEM_BLOCK_SIZE) >= begin_align))
{
/* calculate the aligned memory size */
mem_size_aligned = end_align - begin_align - 2 * MEM_BLOCK_SIZE;
}
else
{
MEM_DEBUG("mem init, error begin address 0x%x, and end address 0x%x",
(uint32_t)begin_addr, (uint32_t)end_addr);
return;
}
/* point to begin address of heap */
heap_ptr = (uint8_t *)begin_align;
MEM_DEBUG("mem init, heap begin address 0x%x, size %d",
(uint32_t)heap_ptr, mem_size_aligned);
/* initialize the start of the heap */
mem = (struct heap_mem *)heap_ptr;
mem->magic = HEAP_MAGIC;
mem->next = mem_size_aligned + MEM_BLOCK_SIZE;
mem->prev = 0;
mem->used = 0;
/* initialize the end of the heap */
heap_end = (struct heap_mem *)&heap_ptr[mem->next];
heap_end->magic = HEAP_MAGIC;
heap_end->used = 1;
heap_end->next = mem_size_aligned + MEM_BLOCK_SIZE;
heap_end->prev = mem_size_aligned + MEM_BLOCK_SIZE;
#ifdef MEM_SAFETY
if(sem_init(&heap_sem, 0, 1) == -1)
{
MEM_DEBUG("heap_sem intitialization failed\n");
MEM_ASSERT(0);
}
#endif
/* initialize the lowest-free pointer to the start of the heap */
lfree = (struct heap_mem *)heap_ptr;
}
/******************************************************************************
* function:
* qaeCryptoMemRealloc(void *ptr, size_t memsize, const char *file,
* int line)
*
* @param[in] ptr, address of start of usable memory for old allocation
* @param[in] memsize, size of new block required
* @param[in] file, the C source filename of the call site
* @param[in] line, the line number withing the C source file of the call
* site
*
* description:
* Change the size of usable memory in an allocated block. This may allocate
* a new block and copy the data to it.
*
******************************************************************************/
void *qaeCryptoMemRealloc(void *ptr, size_t memsize, const char *file,
int line)
{
int copy = crypto_slot_get_size(ptr);
void *n = crypto_alloc_from_slab(memsize, file, line);
MEM_DEBUG("%s: Alloc Address: %p Size: %d File: %s:%d\n", __func__, n,
memsize, file, line);
if (memsize < copy)
copy = memsize;
memcpy(n, ptr, copy);
MEM_DEBUG("%s: Free Address: %p\n", __func__, ptr);
crypto_free_to_slab(ptr);
return n;
}
/******************************************************************************
* function:
* qaeCryptoMemAlloc(size_t memsize, const char *file, int line)
*
* @param[in] memsize, size of usable memory requested
* @param[in] file, the C source filename of the call site
* @param[in] line, the line number within the C source file of the call
* site
*
* description:
* Allocate a block of pinned memory.
*
******************************************************************************/
void *qaeCryptoMemAlloc(size_t memsize, const char *file, int line)
{
void *pAddress = crypto_alloc_from_slab(memsize, file, line);
MEM_DEBUG("%s: Address: %p Size: %d File: %s:%d\n", __func__, pAddress,
memsize, file, line);
return pAddress;
}
void memReadProcess()
{
uint8_t memId = p.data[0];
uint8_t readLen = p.data[5];
uint32_t memAddr;
uint8_t status = 0;
memcpy(&memAddr, &p.data[1], 4);
MEM_DEBUG("Packet is MEM READ\n");
p.header = CRTP_HEADER(CRTP_PORT_MEM, READ_CH);
// Dont' touch the first 5 bytes, they will be the same.
if (memId == EEPROM_ID)
{
if (memAddr + readLen <= EEPROM_SIZE &&
eepromReadBuffer(&p.data[6], memAddr, readLen))
status = 0;
else
status = EIO;
}
else if (memId == LEDMEM_ID)
{
if (memAddr + readLen <= sizeof(ledringmem) &&
memcpy(&p.data[6], &(ledringmem[memAddr]), readLen))
status = 0;
else
status = EIO;
}
else
{
memId = memId - NBR_STATIC_MEM;
if (memAddr + readLen <= OW_MAX_SIZE &&
owRead(memId, memAddr, readLen, &p.data[6]))
status = 0;
else
status = EIO;
}
#if 0
{
int i;
for (i = 0; i < readLen; i++)
consolePrintf("%X ", p.data[i+6]);
consolePrintf("\nStatus %i\n", status);
}
#endif
p.data[5] = status;
if (status == 0)
p.size = 6 + readLen;
else
p.size = 6;
crtpSendPacket(&p);
}
/*****************************************************************************
* function:
* crypto_free_slab(qae_slab *slb, void *thread_key)
*
* @param[in] slb, pointer to the slab to be freed
* @param[in] thread_key, thread local key that points to the slab pools
*
* @description
* free a slab to kernel
*
*****************************************************************************/
static void crypto_free_slab(qae_slab *slb, void *thread_key)
{
qat_contig_mem_config qmcfg;
qae_slab_pools_local *tls_ptr = (qae_slab_pools_local *)thread_key;
#ifdef USE_QAT_CONTIG_MEM
MEM_DEBUG("%s do munmap of %p\n", __func__, slb);
qmcfg = *((qat_contig_mem_config *) slb);
if (munmap(slb, SLAB_SIZE) == -1) {
perror("munmap");
exit(EXIT_FAILURE);
}
MEM_DEBUG("%s ioctl free of %p\n", __func__, slb);
if (ioctl(tls_ptr->crypto_qat_contig_memfd, QAT_CONTIG_MEM_FREE, &qmcfg)
== -1) {
perror("ioctl QAT_CONTIG_MEM_FREE");
exit(EXIT_FAILURE);
}
#endif
}
void *qaeCryptoMemAlloc(size_t memsize, const char *file, int line)
{
CpaStatus status = CPA_STATUS_SUCCESS;
int rc;
void *pAddress = NULL;
MEM_DEBUG("%s: pthread_mutex_lock\n", __func__);
if ((rc = pthread_mutex_lock(&mem_mutex)) != 0) {
MEM_ERROR("pthread_mutex_lock: %s\n", strerror(rc));
return NULL;
}
pAddress = qaeMemAllocNUMA(memsize, 0, QAT_BYTE_ALIGNMENT);
MEM_DEBUG("%s: Address: %p Size: %d File: %s:%d\n", __func__, pAddress,
memsize, file, line);
if ((rc = pthread_mutex_unlock(&mem_mutex)) != 0) {
MEM_ERROR("pthread_mutex_unlock: %s\n", strerror(rc));
}
MEM_DEBUG("%s: pthread_mutex_unlock\n", __func__);
return pAddress;
}
/**
* This function will change the previously allocated memory block.
*
* @rmem pointer to memory allocated by os_malloc
* @newsize the required new size
*
* @return the changed memory block address
*/
void *os_realloc(void *rmem, uint32_t newsize)
{
uint32_t size;
uint32_t ptr, ptr2;
struct heap_mem *mem, *mem2;
void *nmem;
/*RT_DEBUG_NOT_IN_INTERRUPT;*/
/* alignment size */
newsize = MEM_ALIGN_UP(newsize, MEM_ALIGN_BYTE);
if (newsize > mem_size_aligned)
{
MEM_DEBUG("realloc: out of memory");
return NULL;
}
/* allocate a new memory block */
if (rmem == NULL)
{
return os_malloc(newsize);
}
#ifdef MEM_SAFETY
sem_wait(&heap_sem);
#endif
if ((uint8_t *)rmem < (uint8_t *)heap_ptr ||
(uint8_t *)rmem >= (uint8_t *)heap_end)
{
#ifdef MEM_SAFETY
/* illegal memory */
sem_post(&heap_sem);
#endif
return rmem;
}
mem = (struct heap_mem *)((uint8_t *)rmem - MEM_BLOCK_SIZE);
ptr = (uint8_t *)mem - heap_ptr;
size = mem->next - ptr - MEM_BLOCK_SIZE;
if (size == newsize)
{
#ifdef MEM_SAFETY
/* the size is the same as */
sem_post(&heap_sem);
#endif
return rmem;
}
if (newsize + MEM_BLOCK_SIZE + MEM_BLOCK_MIN_SIZE < size)
{
/* split memory block */
#ifdef MEM_STATISTIC
used_mem -= (size - newsize);
#endif
ptr2 = ptr + MEM_BLOCK_SIZE + newsize;
mem2 = (struct heap_mem *)&heap_ptr[ptr2];
mem2->magic = HEAP_MAGIC;
mem2->used = 0;
mem2->next = mem->next;
mem2->prev = ptr;
mem->next = ptr2;
if (mem2->next != mem_size_aligned + MEM_BLOCK_SIZE)
{
((struct heap_mem *)&heap_ptr[mem2->next])->prev = ptr2;
}
plug_holes(mem2);
#ifdef MEM_SAFETY
sem_post(&heap_sem);
#endif
return rmem;
}
#ifdef MEM_SAFETY
sem_post(&heap_sem);
#endif
/* expand memory */
nmem = os_malloc(newsize);
if (nmem != NULL) /* check memory */
{
memcpy(nmem, rmem, size < newsize ? size : newsize);
os_free(rmem);
}
return nmem;
}
/**
* Allocate a block of memory with a minimum of 'size' bytes.
*
* @size is the minimum size of the requested block in bytes.
*
* @return pointer to allocated memory or NULL if no free memory was found.
*/
void *os_malloc(uint32_t size)
{
uint32_t ptr, ptr2;
struct heap_mem *mem, *mem2;
/* Note:shall not used in ISR, I never do this. */
/*RT_DEBUG_NOT_IN_INTERRUPT;*/
if (size == 0)
{
return NULL;
}
if (size != MEM_ALIGN_UP(size, MEM_ALIGN_BYTE))
{
MEM_DEBUG("malloc size %d, but align to %d", size, MEM_ALIGN_UP(size, MEM_ALIGN_BYTE));
}
else
{
MEM_DEBUG("malloc size %d", size);
}
/* alignment size */
size = MEM_ALIGN_UP(size, MEM_ALIGN_BYTE);
if (size > mem_size_aligned)
{
MEM_DEBUG("no memory");
return NULL;
}
/* every data block must be at least MEM_BLOCK_ALIGN_BYTE long */
if (size < MEM_BLOCK_ALIGN_BYTE)
{
size = MEM_BLOCK_ALIGN_BYTE;
}
#ifdef MEM_SAFETY
/* take memory semaphore */
sem_wait(&heap_sem);
#endif
for (ptr = (uint8_t *)lfree - heap_ptr;
ptr < mem_size_aligned - size;
ptr = ((struct heap_mem *)&heap_ptr[ptr])->next)
{
mem = (struct heap_mem *)&heap_ptr[ptr];
if ((!mem->used) && (mem->next - (ptr + MEM_BLOCK_SIZE)) >= size)
{
/* mem is not used and at least perfect fit is possible:
* mem->next - (ptr + MEM_BLOCK_SIZE) gives us the 'user data size' of mem */
if (mem->next - (ptr + MEM_BLOCK_SIZE) >=
(size + MEM_BLOCK_SIZE + MEM_BLOCK_ALIGN_BYTE))
{
/* (in addition to the above, we test if another struct heap_mem (MEM_BLOCK_SIZE) containing
* at least MEM_BLOCK_ALIGN_BYTE of data also fits in the 'user data space' of 'mem')
* -> split large block, create empty remainder,
* remainder must be large enough to contain MEM_BLOCK_ALIGN_BYTE data: if
* mem->next - (ptr + (2*MEM_BLOCK_SIZE)) == size,
* struct heap_mem would fit in but no data between mem2 and mem2->next
* @todo we could leave out MEM_BLOCK_ALIGN_BYTE. We would create an empty
* region that couldn't hold data, but when mem->next gets freed,
* the 2 regions would be combined, resulting in more free memory
*/
ptr2 = ptr + MEM_BLOCK_SIZE + size;
/* create mem2 struct */
mem2 = (struct heap_mem *)&heap_ptr[ptr2];
mem2->used = 0;
mem2->next = mem->next;
mem2->prev = ptr;
/* and insert it between mem and mem->next */
mem->next = ptr2;
mem->used = 1;
if (mem2->next != mem_size_aligned + MEM_BLOCK_SIZE)
{
((struct heap_mem *)&heap_ptr[mem2->next])->prev = ptr2;
}
#ifdef MEM_STATISTIC
used_mem += (size + MEM_BLOCK_SIZE);
if (max_mem < used_mem)
{
max_mem = used_mem;
}
#endif
}
else
{
/* (a mem2 struct does no fit into the user data space of mem and mem->next will always
* be used at this point: if not we have 2 unused structs in a row, plug_holes should have
* take care of this).
* -> near fit or excact fit: do not split, no mem2 creation
* also can't move mem->next directly behind mem, since mem->next
* will always be used at this point!
*/
mem->used = 1;
#ifdef MEM_STATISTIC
used_mem += mem->next - ((uint8_t *)mem - heap_ptr);
if (max_mem < used_mem)
{
max_mem = used_mem;
}
#endif
}
/* set memory block magic */
mem->magic = HEAP_MAGIC;
if (mem == lfree)
{
/* Find next free block after mem and update lowest free pointer */
while (lfree->used && lfree != heap_end)
{
lfree = (struct heap_mem *)&heap_ptr[lfree->next];
}
MEM_ASSERT(((lfree == heap_end) || (!lfree->used)));
}
#ifdef MEM_SAFETY
sem_post(&heap_sem);
#endif
MEM_ASSERT((uint32_t)mem + MEM_BLOCK_SIZE + size <= (uint32_t)heap_end);
MEM_ASSERT((uint32_t)((uint8_t *)mem + MEM_BLOCK_SIZE) % MEM_ALIGN_BYTE == 0);
MEM_ASSERT((((uint32_t)mem) & (MEM_ALIGN_BYTE - 1)) == 0);
MEM_DEBUG("allocate memory at 0x%x, size: %d",
(uint32_t)((uint8_t *)mem + MEM_BLOCK_SIZE),
(uint32_t)(mem->next - ((uint8_t *)mem - heap_ptr)));
/* return the memory data except mem struct */
return (uint8_t *)mem + MEM_BLOCK_SIZE;
}
}
#ifdef MEM_SAFETY
sem_post(&heap_sem);
#endif
return NULL;
}
/******************************************************************************
* function:
* qaeCryptoMemFree(void *ptr)
*
* @param[in] ptr, address of start of usable memory
*
* description:
* Free a block of memory previously allocated by this allocator.
*
******************************************************************************/
void qaeCryptoMemFree(void *ptr)
{
MEM_DEBUG("%s: Address: %p\n", __func__, ptr);
if (NULL != ptr)
crypto_free_to_slab(ptr);
}
/*****************************************************************************
* function:
* crypto_create_slab(int size, int pool_index, int memfd)
*
* @param[in] size, the size of the slots within the slab. Note that this is
* not the size of the slab itself
* @param[in] pool_index, the index of the slot pool
* @param[in] memfd, the file descriptor of the memory driver
* @retval qae_slab*, a pointer to the new slab.
*
* @description
* create a new slab and add it to the linked list
* retval pointer to the new slab
*
*****************************************************************************/
static qae_slab *crypto_create_slab(int size, int pool_index, int memfd)
{
int i = 0;
int nslot = 0;
qat_contig_mem_config qmcfg = { 0, (uintptr_t) NULL, 0, (uintptr_t) NULL };
qae_slab *result = NULL;
qae_slab *slb = NULL;
qae_slot *slt = NULL;
QAE_UINT alignment;
qmcfg.length = SLAB_SIZE;
#ifdef USE_QAT_CONTIG_MEM
if (ioctl(memfd, QAT_CONTIG_MEM_MALLOC, &qmcfg) == -1) {
static char errmsg[LINE_MAX];
snprintf(errmsg, LINE_MAX, "ioctl QAT_CONTIG_MEM_MALLOC(%d)",
qmcfg.length);
perror(errmsg);
goto exit;
}
if ((slb =
mmap(NULL, qmcfg.length*QAT_CONTIG_MEM_MMAP_ADJUSTMENT,
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_LOCKED, memfd,
qmcfg.virtualAddress)) == MAP_FAILED) {
static char errmsg[LINE_MAX];
snprintf(errmsg, LINE_MAX, "mmap: %d %s", errno, strerror(errno));
perror(errmsg);
goto exit;
}
#endif
MEM_DEBUG("%s slot size %d\n", __func__, size);
slb->slot_size = size;
slb->next_slot = NULL;
slb->sig = SIG_ALLOC;
slb->used_slots = 0;
for (i = sizeof(qae_slab); SLAB_SIZE - sizeof(qat_contig_mem_config)
- i >= size; i += size) {
slt = (qae_slot *) ((unsigned char *)slb + i);
alignment =
QAE_BYTE_ALIGNMENT -
(((QAE_UINT) slt + sizeof(qae_slot)) % QAE_BYTE_ALIGNMENT);
slt = (qae_slot *) (((QAE_UINT) slt) + alignment);
slt->next = slb->next_slot;
slt->pool_index = pool_index;
slt->sig = SIG_FREE;
slt->file = NULL;
slt->line = 0;
slb->next_slot = slt;
nslot++;
slt->slab = slb;
}
slb->total_slots = nslot;
/*
* Make sure the update of the slab list is the last thing to be done.
* This means it is not necessary to lock against anyone iterating the
* list from the head
*/
result = slb;
MEM_DEBUG("%s slab %p last slot is %p, count is %d\n", __func__, slb, slt,
nslot);
exit:
return result;
}