static void cmdq_driver_process_read_address_request(struct cmdqReadAddressStruct *req_user)
{
/* create kernel-space buffer for working */
uint32_t *addrs = NULL;
uint32_t *values = NULL;
dma_addr_t pa = 0;
int i = 0;
CMDQ_LOG("[READ_PA] cmdq_driver_process_read_address_request()\n");
do {
if (NULL == req_user ||
0 == req_user->count || NULL == CMDQ_U32_PTR(req_user->values)
|| NULL == CMDQ_U32_PTR(req_user->dmaAddresses)) {
CMDQ_ERR("[READ_PA] invalid req_user\n");
break;
}
addrs = kcalloc(req_user->count, sizeof(uint32_t), GFP_KERNEL);
if (NULL == addrs) {
CMDQ_ERR("[READ_PA] fail to alloc addr buf\n");
break;
}
values = kcalloc(req_user->count, sizeof(uint32_t), GFP_KERNEL);
if (NULL == values) {
CMDQ_ERR("[READ_PA] fail to alloc value buf\n");
break;
}
/* copy from user */
if (copy_from_user
(addrs, CMDQ_U32_PTR(req_user->dmaAddresses),
req_user->count * sizeof(uint32_t))) {
CMDQ_ERR("[READ_PA] fail to copy user dmaAddresses\n");
break;
}
/* actually read these PA write buffers */
for (i = 0; i < req_user->count; ++i) {
pa = (0xFFFFFFFF & addrs[i]);
CMDQ_LOG("[READ_PA] req read dma address 0x%pa\n", &pa);
values[i] = cmdqCoreReadWriteAddress(pa);
}
/* copy value to user */
if (copy_to_user
(CMDQ_U32_PTR(req_user->values), values, req_user->count * sizeof(uint32_t))) {
CMDQ_ERR("[READ_PA] fail to copy to user value buf\n");
break;
}
} while (0);
kfree(addrs);
kfree(values);
}
int32_t cmdqRecReset(cmdqRecHandle handle)
{
if (NULL == handle) {
return -EFAULT;
}
if (NULL != handle->pRunningTask) {
cmdqRecStopLoop(handle);
}
handle->blockSize = 0;
handle->prefetchCount = 0;
handle->finalized = false;
/* reset secure path data */
handle->secData.isSecure = false;
handle->secData.enginesNeedDAPC = 0LL;
handle->secData.enginesNeedPortSecurity = 0LL;
if (handle->secData.addrMetadatas) {
kfree(CMDQ_U32_PTR(handle->secData.addrMetadatas));
handle->secData.addrMetadatas = (cmdqU32Ptr_t)(unsigned long)NULL;
handle->secData.addrMetadataMaxCount = 0;
handle->secData.addrMetadataCount = 0;
}
/* profile marker */
cmdq_rec_reset_profile_maker_data(handle);
return 0;
}
int32_t cmdq_rec_realloc_addr_metadata_buffer(cmdqRecHandle handle, const uint32_t size)
{
void *pNewBuf = NULL;
void *pOriginalBuf = (void*)CMDQ_U32_PTR(handle->secData.addrMetadatas);
const uint32_t originalSize = sizeof(cmdqSecAddrMetadataStruct) * (handle->secData.addrMetadataMaxCount);
if (size <= originalSize) {
return 0;
}
pNewBuf = kzalloc(size, GFP_KERNEL);
if (NULL == pNewBuf) {
CMDQ_ERR("REC: secAddrMetadata, kzalloc %d bytes addr_metadata buffer failed\n", size);
return -ENOMEM;
}
if (pOriginalBuf && originalSize > 0) {
memcpy(pNewBuf, pOriginalBuf, originalSize);
}
CMDQ_VERBOSE("REC: secAddrMetadata, realloc size from %d to %d bytes\n", originalSize, size);
kfree(pOriginalBuf);
handle->secData.addrMetadatas = (cmdqU32Ptr_t)(unsigned long)(pNewBuf);
handle->secData.addrMetadataMaxCount = size / sizeof(cmdqSecAddrMetadataStruct);
return 0;
}
static int cmdq_driver_create_reg_address_buffer(struct cmdqCommandStruct *pCommand)
{
int status = 0;
uint32_t totalRegCount = 0;
uint32_t *regAddrBuf = NULL;
uint32_t *kernelRegAddr = NULL;
uint32_t kernelRegCount = 0;
const uint32_t userRegCount = pCommand->regRequest.count;
if (0 != pCommand->debugRegDump) {
/* get kernel dump request count */
status = cmdqCoreDebugRegDumpBegin(pCommand->debugRegDump,
&kernelRegCount, &kernelRegAddr);
if (0 != status) {
CMDQ_ERR
("cmdqCoreDebugRegDumpBegin returns %d, ignore kernel reg dump request\n",
status);
kernelRegCount = 0;
kernelRegAddr = NULL;
}
}
/* how many register to dump? */
totalRegCount = kernelRegCount + userRegCount;
if (0 == totalRegCount) {
/* no need to dump register */
pCommand->regRequest.count = 0;
pCommand->regValue.count = 0;
pCommand->regRequest.regAddresses = (cmdqU32Ptr_t) (unsigned long)NULL;
pCommand->regValue.regValues = (cmdqU32Ptr_t) (unsigned long)NULL;
} else {
regAddrBuf = kcalloc(totalRegCount, sizeof(uint32_t), GFP_KERNEL);
if (NULL == regAddrBuf)
return -ENOMEM;
/* collect user space dump request */
if (userRegCount) {
if (copy_from_user
(regAddrBuf, CMDQ_U32_PTR(pCommand->regRequest.regAddresses),
userRegCount * sizeof(uint32_t))) {
kfree(regAddrBuf);
return -EFAULT;
}
}
/* collect kernel space dump request, concatnate after user space request */
if (kernelRegCount) {
memcpy(regAddrBuf + userRegCount, kernelRegAddr,
kernelRegCount * sizeof(uint32_t));
}
/* replace address buffer and value address buffer with kzalloc memory */
pCommand->regRequest.regAddresses = (cmdqU32Ptr_t) (unsigned long)(regAddrBuf);
pCommand->regRequest.count = totalRegCount;
}
return 0;
}
void cmdq_core_dump_secure_metadata(cmdqSecDataStruct *pSecData)
{
uint32_t i = 0;
cmdqSecAddrMetadataStruct *pAddr = NULL;
if (NULL == pSecData) {
return;
}
pAddr = (cmdqSecAddrMetadataStruct *)(CMDQ_U32_PTR(pSecData->addrMetadatas));
CMDQ_LOG("========= pSecData: %p dump =========\n", pSecData);
CMDQ_LOG("count:%d(%d), enginesNeedDAPC:0x%llx, enginesPortSecurity:0x%llx\n",
pSecData->addrMetadataCount, pSecData->addrMetadataMaxCount,
pSecData->enginesNeedDAPC, pSecData->enginesNeedPortSecurity);
if (NULL == pAddr) {
return;
}
for (i = 0; i < pSecData->addrMetadataCount; i++) {
CMDQ_LOG("idx:%d, type:%d, baseHandle:%x, offset:%d, size:%d, port:%d\n",
i, pAddr[i].type, pAddr[i].baseHandle, pAddr[i].offset, pAddr[i].size, pAddr[i].port);
}
}
static long cmdq_driver_destroy_secure_medadata(cmdqCommandStruct *pCommand)
{
if (pCommand->secData.addrMetadatas) {
kfree(CMDQ_U32_PTR(pCommand->secData.addrMetadatas));
pCommand->secData.addrMetadatas = (cmdqU32Ptr_t) (unsigned long)NULL;
}
return 0;
}
static long cmdq_driver_create_secure_medadata(cmdqCommandStruct *pCommand)
{
void *pAddrMetadatas = NULL;
const uint32_t length =
(pCommand->secData.addrMetadataCount) * sizeof(cmdqSecAddrMetadataStruct);
/* verify parameter */
if ((false == pCommand->secData.isSecure) && (0 != pCommand->secData.addrMetadataCount)) {
/* normal path with non-zero secure metadata */
CMDQ_ERR
("[secData]mismatch secData.isSecure(%d) and secData.addrMetadataCount(%d)\n",
pCommand->secData.isSecure, pCommand->secData.addrMetadataCount);
return -EFAULT;
}
/* revise max count field */
pCommand->secData.addrMetadataMaxCount = pCommand->secData.addrMetadataCount;
/* bypass 0 metadata case */
if (0 == pCommand->secData.addrMetadataCount) {
pCommand->secData.addrMetadatas = (cmdqU32Ptr_t) (unsigned long)NULL;
return 0;
}
/* create kernel-space buffer for working */
pAddrMetadatas = kzalloc(length, GFP_KERNEL);
if (NULL == pAddrMetadatas) {
CMDQ_ERR("[secData]kzalloc for addrMetadatas failed, count:%d, alloacted_size:%d\n",
pCommand->secData.addrMetadataCount, length);
return -ENOMEM;
}
/* copy from user */
if (copy_from_user(pAddrMetadatas, CMDQ_U32_PTR(pCommand->secData.addrMetadatas), length)) {
CMDQ_ERR("[secData]fail to copy user addrMetadatas\n");
/* replace buffer first to ensure that */
/* addrMetadatas is valid kernel space buffer address when free it */
pCommand->secData.addrMetadatas = (cmdqU32Ptr_t) (unsigned long)pAddrMetadatas;
/* free secure path metadata */
cmdq_driver_destroy_secure_medadata(pCommand);
return -EFAULT;
}
/* replace buffer */
pCommand->secData.addrMetadatas = (cmdqU32Ptr_t) (unsigned long)pAddrMetadatas;
#if 0
cmdq_core_dump_secure_metadata(&(pCommand->secData));
#endif
return 0;
}
int32_t cmdq_append_addr_metadata(cmdqRecHandle handle, const cmdqSecAddrMetadataStruct *pMetadata)
{
cmdqSecAddrMetadataStruct *pAddrs;
int32_t status;
/* element index of the New appended addr metadat */
const uint32_t index = handle->secData.addrMetadataCount;
pAddrs = NULL;
status = 0;
if (0 >= handle->secData.addrMetadataMaxCount) {
/* not init yet, initialize to allow max 8 addr metadata */
status = cmdq_rec_realloc_addr_metadata_buffer(handle,
sizeof(cmdqSecAddrMetadataStruct) * 8);
}
else if (handle->secData.addrMetadataCount >= (handle->secData.addrMetadataMaxCount)) {
/* enlarge metadata buffer to twice as */
status = cmdq_rec_realloc_addr_metadata_buffer(handle,
sizeof(cmdqSecAddrMetadataStruct) * (handle->secData.addrMetadataMaxCount) * 2);
}
if(0 > status) {
return -ENOMEM;
}
pAddrs = (cmdqSecAddrMetadataStruct*)(CMDQ_U32_PTR(handle->secData.addrMetadatas));
/* append meatadata*/
pAddrs[index].instrIndex = pMetadata->instrIndex;
pAddrs[index].baseHandle= pMetadata->baseHandle;
pAddrs[index].offset = pMetadata->offset;
pAddrs[index].size = pMetadata->size;
pAddrs[index].port = pMetadata->port;
pAddrs[index].type = pMetadata->type;
/* meatadata count ++*/
handle->secData.addrMetadataCount += 1;
return 0;
}
static long cmdq_ioctl(struct file *pFile, unsigned int code, unsigned long param)
{
struct cmdqCommandStruct command;
struct cmdqJobStruct job;
int count[CMDQ_MAX_ENGINE_COUNT];
struct TaskStruct *pTask;
int32_t status;
struct cmdqJobResultStruct jobResult;
uint32_t *userRegValue = NULL;
uint32_t userRegCount = 0;
/* backup value after task release */
uint32_t regCount = 0, regCountUserSpace = 0, regUserToken = 0;
switch (code) {
case CMDQ_IOCTL_EXEC_COMMAND:
if (copy_from_user(&command, (void *)param, sizeof(cmdqCommandStruct)))
return -EFAULT;
/* insert private_data for resource reclaim */
command.privateData = (cmdqU32Ptr_t) (unsigned long)(pFile->private_data);
if (cmdq_driver_process_command_request(&command))
return -EFAULT;
break;
case CMDQ_IOCTL_QUERY_USAGE:
if (cmdqCoreQueryUsage(count))
return -EFAULT;
if (copy_to_user((void *)param, count, sizeof(int32_t) * CMDQ_MAX_ENGINE_COUNT)) {
CMDQ_ERR("CMDQ_IOCTL_QUERY_USAGE copy_to_user failed\n");
return -EFAULT;
}
break;
case CMDQ_IOCTL_ASYNC_JOB_EXEC:
if (copy_from_user(&job, (void *)param, sizeof(cmdqJobStruct)))
return -EFAULT;
/* not support secure path for async ioctl yet */
if (true == job.command.secData.isSecure) {
CMDQ_ERR("not support secure path for CMDQ_IOCTL_ASYNC_JOB_EXEC\n");
return -EFAULT;
}
/* backup */
userRegCount = job.command.regRequest.count;
/* insert private_data for resource reclaim */
job.command.privateData = (cmdqU32Ptr_t) (unsigned long)(pFile->private_data);
/* create kernel-space address buffer */
status = cmdq_driver_create_reg_address_buffer(&job.command);
if (0 != status)
return status;
/* scenario id fixup */
cmdq_core_fix_command_scenario_for_user_space(&job.command);
status = cmdqCoreSubmitTaskAsync(&job.command, NULL, 0, &pTask);
/* store user space request count in TaskStruct */
/* for later retrieval */
if (pTask) {
pTask->regCountUserSpace = userRegCount;
pTask->regUserToken = job.command.debugRegDump;
}
/* we don't need regAddress anymore, free it now */
kfree(CMDQ_U32_PTR(job.command.regRequest.regAddresses));
job.command.regRequest.regAddresses = (cmdqU32Ptr_t) (unsigned long)(NULL);
if (status >= 0) {
job.hJob = (unsigned long)pTask;
if (copy_to_user((void *)param, (void *)&job, sizeof(cmdqJobStruct))) {
CMDQ_ERR("CMDQ_IOCTL_ASYNC_JOB_EXEC copy_to_user failed\n");
return -EFAULT;
}
} else {
job.hJob = (unsigned long)NULL;
return -EFAULT;
}
break;
case CMDQ_IOCTL_ASYNC_JOB_WAIT_AND_CLOSE:
if (copy_from_user(&jobResult, (void *)param, sizeof(jobResult))) {
CMDQ_ERR("copy_from_user jobResult fail\n");
return -EFAULT;
}
/* verify job handle */
if (!cmdqIsValidTaskPtr((TaskStruct *) (unsigned long)jobResult.hJob)) {
CMDQ_ERR("invalid task ptr = 0x%llx\n", jobResult.hJob);
return -EFAULT;
}
pTask = (TaskStruct *) (unsigned long)jobResult.hJob;
/* utility service, fill the engine flag. */
/* this is required by MDP. */
jobResult.engineFlag = pTask->engineFlag;
/* check if reg buffer suffices */
if (jobResult.regValue.count < pTask->regCountUserSpace) {
jobResult.regValue.count = pTask->regCountUserSpace;
if (copy_to_user((void *)param, (void *)&jobResult, sizeof(jobResult))) {
CMDQ_ERR("copy_to_user fail, line=%d\n", __LINE__);
return -EINVAL;
}
CMDQ_ERR("insufficient register buffer\n");
return -ENOMEM;
}
/* inform client the actual read register count */
jobResult.regValue.count = pTask->regCountUserSpace;
/* update user space before we replace the regValues pointer. */
if (copy_to_user((void *)param, (void *)&jobResult, sizeof(jobResult))) {
CMDQ_ERR("copy_to_user fail line=%d\n", __LINE__);
return -EINVAL;
}
/* allocate kernel space result buffer */
/* which contains kernel + user space requests */
userRegValue = CMDQ_U32_PTR(jobResult.regValue.regValues);
jobResult.regValue.regValues = (cmdqU32Ptr_t) (unsigned long)
(kzalloc(pTask->regCount * sizeof(uint32_t), GFP_KERNEL));
jobResult.regValue.count = pTask->regCount;
if (NULL == CMDQ_U32_PTR(jobResult.regValue.regValues)) {
CMDQ_ERR("no reg value buffer\n");
return -ENOMEM;
}
/* backup value after task release */
regCount = pTask->regCount;
regCountUserSpace = pTask->regCountUserSpace;
regUserToken = pTask->regUserToken;
/* make sure the task is running and wait for it */
status = cmdqCoreWaitResultAndReleaseTask(pTask,
&jobResult.regValue,
msecs_to_jiffies
(CMDQ_DEFAULT_TIMEOUT_MS));
if (status < 0) {
CMDQ_ERR("waitResultAndReleaseTask fail=%d\n", status);
/* free kernel space result buffer */
kfree(CMDQ_U32_PTR(jobResult.regValue.regValues));
return status;
}
/* pTask is released, do not access it any more */
pTask = NULL;
/* notify kernel space dump callback */
if (regCount > regCountUserSpace) {
CMDQ_VERBOSE("kernel space reg dump = %d, %d, %d\n", regCount,
regCountUserSpace, regUserToken);
status = cmdqCoreDebugRegDumpEnd(regUserToken,
regCount - regCountUserSpace,
CMDQ_U32_PTR(jobResult.regValue.regValues +
regCountUserSpace));
if (0 != status) {
/* Error status print */
CMDQ_ERR("cmdqCoreDebugRegDumpEnd returns %d\n", status);
}
}
/* copy result to user space */
if (copy_to_user
((void *)userRegValue, (void *)(unsigned long)jobResult.regValue.regValues,
regCountUserSpace * sizeof(uint32_t))) {
CMDQ_ERR("Copy REGVALUE to user space failed\n");
return -EFAULT;
}
if (jobResult.readAddress.count > 0)
cmdq_driver_process_read_address_request(&jobResult.readAddress);
/* free kernel space result buffer */
kfree(CMDQ_U32_PTR(jobResult.regValue.regValues));
break;
case CMDQ_IOCTL_ALLOC_WRITE_ADDRESS:
do {
cmdqWriteAddressStruct addrReq;
dma_addr_t paStart = 0;
CMDQ_LOG("CMDQ_IOCTL_ALLOC_WRITE_ADDRESS\n");
if (copy_from_user(&addrReq, (void *)param, sizeof(addrReq))) {
CMDQ_ERR("CMDQ_IOCTL_ALLOC_WRITE_ADDRESS copy_from_user failed\n");
return -EFAULT;
}
status = cmdqCoreAllocWriteAddress(addrReq.count, &paStart);
if (0 != status) {
CMDQ_ERR
("CMDQ_IOCTL_ALLOC_WRITE_ADDRESS cmdqCoreAllocWriteAddress() failed\n");
return status;
}
addrReq.startPA = (uint32_t) paStart;
CMDQ_LOG("CMDQ_IOCTL_ALLOC_WRITE_ADDRESS get 0x%08x\n", addrReq.startPA);
if (copy_to_user((void *)param, &addrReq, sizeof(addrReq))) {
CMDQ_ERR("CMDQ_IOCTL_ALLOC_WRITE_ADDRESS copy_to_user failed\n");
return -EFAULT;
}
status = 0;
} while (0);
break;
case CMDQ_IOCTL_FREE_WRITE_ADDRESS:
do {
cmdqWriteAddressStruct freeReq;
CMDQ_LOG("CMDQ_IOCTL_FREE_WRITE_ADDRESS\n");
if (copy_from_user(&freeReq, (void *)param, sizeof(freeReq))) {
CMDQ_ERR("CMDQ_IOCTL_FREE_WRITE_ADDRESS copy_from_user failed\n");
return -EFAULT;
}
status = cmdqCoreFreeWriteAddress(freeReq.startPA);
if (0 != status)
return status;
status = 0;
} while (0);
break;
case CMDQ_IOCTL_READ_ADDRESS_VALUE:
do {
cmdqReadAddressStruct readReq;
CMDQ_LOG("CMDQ_IOCTL_READ_ADDRESS_VALUE\n");
if (copy_from_user(&readReq, (void *)param, sizeof(readReq))) {
CMDQ_ERR("CMDQ_IOCTL_READ_ADDRESS_VALUE copy_from_user failed\n");
return -EFAULT;
}
/* this will copy result to readReq->values buffer */
cmdq_driver_process_read_address_request(&readReq);
status = 0;
} while (0);
break;
case CMDQ_IOCTL_QUERY_CAP_BITS:
do {
int capBits = 0;
if (cmdq_driver_support_wait_and_receive_event_in_same_tick())
capBits |= (1L << CMDQ_CAP_WFE);
else
capBits &= ~(1L << CMDQ_CAP_WFE);
if (copy_to_user((void *)param, &capBits, sizeof(int))) {
CMDQ_ERR("Copy capacity bits to user space failed\n");
return -EFAULT;
}
} while (0);
break;
case CMDQ_IOCTL_QUERY_DTS:
do {
cmdqDTSDataStruct *pDtsData;
pDtsData = cmdq_core_get_whole_DTS_Data();
if (copy_to_user((void *)param, pDtsData, sizeof(cmdqDTSDataStruct))) {
CMDQ_ERR("Copy device tree information to user space failed\n");
return -EFAULT;
}
} while (0);
break;
case CMDQ_IOCTL_NOTIFY_ENGINE:
do {
uint64_t engineFlag;
if (copy_from_user(&engineFlag, (void *)param, sizeof(uint64_t))) {
CMDQ_ERR("CMDQ_IOCTL_NOTIFY_ENGINE copy_from_user failed\n");
return -EFAULT;
}
cmdqCoreLockResource(engineFlag, true);
} while (0);
break;
default:
CMDQ_ERR("unrecognized ioctl 0x%08x\n", code);
return -ENOIOCTLCMD;
}
return 0;
}
static long cmdq_driver_process_command_request(cmdqCommandStruct *pCommand)
{
int32_t status = 0;
uint32_t *userRegValue = NULL;
uint32_t userRegCount = 0;
if (pCommand->regRequest.count != pCommand->regValue.count) {
CMDQ_ERR("mismatch regRequest and regValue\n");
return -EFAULT;
}
/* allocate secure medatata */
status = cmdq_driver_create_secure_medadata(pCommand);
if (0 != status)
return status;
/* backup since we are going to replace these */
userRegValue = CMDQ_U32_PTR(pCommand->regValue.regValues);
userRegCount = pCommand->regValue.count;
/* create kernel-space address buffer */
status = cmdq_driver_create_reg_address_buffer(pCommand);
if (0 != status) {
/* free secure path metadata */
cmdq_driver_destroy_secure_medadata(pCommand);
return status;
}
/* create kernel-space value buffer */
pCommand->regValue.regValues = (cmdqU32Ptr_t) (unsigned long)
kzalloc(pCommand->regRequest.count * sizeof(uint32_t), GFP_KERNEL);
pCommand->regValue.count = pCommand->regRequest.count;
if (NULL == CMDQ_U32_PTR(pCommand->regValue.regValues)) {
kfree(CMDQ_U32_PTR(pCommand->regRequest.regAddresses));
return -ENOMEM;
}
/* scenario id fixup */
cmdq_core_fix_command_scenario_for_user_space(pCommand);
status = cmdqCoreSubmitTask(pCommand);
if (0 > status) {
CMDQ_ERR("Submit user commands for execution failed = %d\n", status);
cmdq_driver_destroy_secure_medadata(pCommand);
kfree(CMDQ_U32_PTR(pCommand->regRequest.regAddresses));
kfree(CMDQ_U32_PTR(pCommand->regValue.regValues));
return -EFAULT;
}
/* notify kernel space dump callback */
if (0 != pCommand->debugRegDump) {
status = cmdqCoreDebugRegDumpEnd(pCommand->debugRegDump,
pCommand->regRequest.count - userRegCount,
CMDQ_U32_PTR(pCommand->regValue.regValues) +
userRegCount);
if (0 != status) {
/* Error status print */
CMDQ_ERR("cmdqCoreDebugRegDumpEnd returns %d\n", status);
}
}
/* copy back to user space buffer */
if (userRegValue && userRegCount) {
/* copy results back to user space */
CMDQ_VERBOSE("regValue[0] is %d\n", CMDQ_U32_PTR(pCommand->regValue.regValues)[0]);
if (copy_to_user
(userRegValue, CMDQ_U32_PTR(pCommand->regValue.regValues),
userRegCount * sizeof(uint32_t))) {
CMDQ_ERR("Copy REGVALUE to user space failed\n");
}
}
/* free allocated kernel buffers */
kfree(CMDQ_U32_PTR(pCommand->regRequest.regAddresses));
kfree(CMDQ_U32_PTR(pCommand->regValue.regValues));
if (pCommand->readAddress.count > 0)
cmdq_driver_process_read_address_request(&pCommand->readAddress);
/* free allocated secure metadata */
cmdq_driver_destroy_secure_medadata(pCommand);
return 0;
}
static long cmdq_ioctl(struct file *pFile, unsigned int code, unsigned long param)
{
int mutex;
struct cmdqCommandStruct command;
struct cmdqJobStruct job;
struct cmdqFileNodeStruct *pNode;
unsigned long flags;
int count[CMDQ_MAX_ENGINE_COUNT];
struct TaskStruct *pTask;
int32_t status;
struct cmdqJobResultStruct jobResult;
uint32_t *userRegValue = NULL;
uint32_t userRegCount = 0;
/* backup value after task release */
uint32_t regCount = 0, regCountUserSpace = 0, regUserToken = 0;
switch (code) {
case CMDQ_IOCTL_LOCK_MUTEX:
mutex = cmdqMutexAcquire();
if (-1 == mutex)
return -IOCTL_RET_LOCK_MUTEX_FAIL;
if (copy_to_user((void *)param, &mutex, sizeof(int))) {
CMDQ_ERR("Copy mutex number to user failed\n");
cmdqMutexRelease(mutex);
return -IOCTL_RET_COPY_MUTEX_NUM_TO_USER_FAIL;
}
/* register mutex into file node data */
pNode = (struct cmdqFileNodeStruct *)pFile->private_data;
if (pNode) {
spin_lock_irqsave(&pNode->nodeLock, flags);
pNode->mutexFlag |= (1 << mutex);
spin_unlock_irqrestore(&pNode->nodeLock, flags);
}
break;
case CMDQ_IOCTL_UNLOCK_MUTEX:
if (copy_from_user(&mutex, (void *)param, sizeof(int))) {
CMDQ_ERR("Copy mutex number from user failed\n");
return -IOCLT_RET_COPY_MUTEX_NUM_FROM_USER_FAIL;
}
status = cmdqMutexRelease(mutex);
if (status < 0)
return -IOCTL_RET_RELEASE_MUTEX_FAIL;
if (-1 != mutex) {
pNode = (struct cmdqFileNodeStruct *)pFile->private_data;
spin_lock_irqsave(&pNode->nodeLock, flags);
pNode->mutexFlag &= ~(1 << mutex);
spin_unlock_irqrestore(&pNode->nodeLock, flags);
}
break;
case CMDQ_IOCTL_EXEC_COMMAND:
if (copy_from_user(&command, (void *)param, sizeof(struct cmdqCommandStruct)))
return -IOCTL_RET_COPY_EXEC_CMD_FROM_USER_FAIL;
if (cmdqCoreIsEarlySuspended() && (CMDQ_SCENARIO_USER_MDP == command.scenario))
return -IOCTL_RET_IS_SUSPEND_WHEN_EXEC_CMD;
/* insert private_data for resource reclaim */
command.privateData = (cmdqU32Ptr_t) pFile->private_data;
if (cmdq_driver_process_command_request(&command))
return -IOCTL_RET_PROCESS_CMD_REQUEST_FAIL;
break;
case CMDQ_IOCTL_QUERY_USAGE:
if (cmdqCoreQueryUsage(count))
return -IOCLT_RET_QUERY_USAGE_FAIL;
if (copy_to_user((void *)param, count, sizeof(int32_t) * CMDQ_MAX_ENGINE_COUNT)) {
CMDQ_ERR("CMDQ_IOCTL_QUERY_USAGE copy_to_user failed\n");
return -IOCTL_RET_COPY_USAGE_TO_USER_FAIL;
}
break;
case CMDQ_IOCTL_ASYNC_JOB_EXEC:
if (copy_from_user(&job, (void *)param, sizeof(struct cmdqJobStruct)))
return -IOCTL_RET_COPY_ASYNC_JOB_EXEC_FROM_USER_FAIL;
if (cmdqCoreIsEarlySuspended() && (CMDQ_SCENARIO_USER_MDP == job.command.scenario)) {
CMDQ_ERR("CMDQ_IOCTL_ASYNC_JOB_EXEC suspended, return\n");
return -IOCTL_RET_IS_SUSPEND_WHEN_ASYNC_JOB_EXEC;
}
/* not support secure path for async ioctl yet */
if (true == job.command.secData.isSecure) {
CMDQ_ERR("not support secure path for CMDQ_IOCTL_ASYNC_JOB_EXEC\n");
return -IOCTL_RET_NOT_SUPPORT_SEC_PATH_FOR_ASYNC_JOB_EXEC;
}
/* backup */
userRegCount = job.command.regRequest.count;
/* insert private_data for resource reclaim */
job.command.privateData = (cmdqU32Ptr_t) (unsigned long)(pFile->private_data);
/* create kernel-space address buffer */
status = cmdq_driver_create_reg_address_buffer(&job.command);
if (0 != status)
return -IOCTL_RET_CREATE_REG_ADDR_BUF_FAIL;
/* scenario id fixup */
if ((CMDQ_SCENARIO_USER_DISP_COLOR == job.command.scenario)
|| (CMDQ_SCENARIO_USER_MDP == job.command.scenario)) {
CMDQ_VERBOSE("user space request, scenario:%d\n", job.command.scenario);
} else {
CMDQ_LOG("[WARNING]fix user space request to CMDQ_SCENARIO_USER_SPACE\n");
job.command.scenario = CMDQ_SCENARIO_USER_SPACE;
}
status = cmdqCoreSubmitTaskAsync(&job.command, NULL, 0, &pTask);
/* store user space request count in struct TaskStruct */
/* for later retrieval */
if (pTask) {
pTask->regCountUserSpace = userRegCount;
pTask->regUserToken = job.command.debugRegDump;
}
/* we don't need regAddress anymore, free it now */
kfree(CMDQ_U32_PTR(job.command.regRequest.regAddresses));
job.command.regRequest.regAddresses = (cmdqU32Ptr_t) (unsigned long)(NULL);
if (status >= 0) {
job.hJob = (unsigned long)pTask;
if (copy_to_user((void *)param, (void *)&job, sizeof(struct cmdqJobStruct))) {
CMDQ_ERR("CMDQ_IOCTL_ASYNC_JOB_EXEC copy_to_user failed\n");
return -IOCLT_RET_COPY_ASYNC_JOB_EXEC_TO_USER_FAIL;
}
} else {
job.hJob = (unsigned long)NULL;
return -IOCTL_RET_SUBMIT_TASK_ASYNC_FAILED;
}
break;
case CMDQ_IOCTL_ASYNC_JOB_WAIT_AND_CLOSE:
if (copy_from_user(&jobResult, (void *)param, sizeof(jobResult))) {
CMDQ_ERR("copy_from_user jobResult fail\n");
return -IOCTL_RET_COPY_ASYNC_JOB_WAIT_AND_CLOSE_FROM_USER_FAIL;
}
if (cmdqCoreIsEarlySuspended() && (CMDQ_SCENARIO_USER_MDP == job.command.scenario)) {
CMDQ_ERR("CMDQ_IOCTL_ASYNC_JOB_WAIT_AND_CLOSE suspended, return\n");
return -IOCTL_RET_IS_SUSPEND_WHEN_ASYNC_JOB_WAIT_AND_CLOSE;
}
/* verify job handle */
if (!cmdqIsValidTaskPtr((struct TaskStruct *)(unsigned long)jobResult.hJob)) {
CMDQ_ERR("invalid task ptr = 0x%llx\n", jobResult.hJob);
return -IOCTL_RET_INVALID_TASK_PTR;
}
pTask = (struct TaskStruct *)(unsigned long)jobResult.hJob;
/* utility service, fill the engine flag. */
/* this is required by MDP. */
jobResult.engineFlag = pTask->engineFlag;
/* check if reg buffer suffices */
if (jobResult.regValue.count < pTask->regCountUserSpace) {
jobResult.regValue.count = pTask->regCountUserSpace;
if (copy_to_user((void *)param, (void *)&jobResult, sizeof(jobResult))) {
CMDQ_ERR("copy_to_user fail, line=%d\n", __LINE__);
return -IOCTL_RET_COPY_JOB_RESULT_TO_USER1_FAIL;
}
CMDQ_ERR("insufficient register buffer\n");
return -IOCTL_RET_NOT_ENOUGH_REGISTER_BUFFER;
}
/* inform client the actual read register count */
jobResult.regValue.count = pTask->regCountUserSpace;
/* update user space before we replace the regValues pointer. */
if (copy_to_user((void *)param, (void *)&jobResult, sizeof(jobResult))) {
CMDQ_ERR("copy_to_user fail line=%d\n", __LINE__);
return -IOCTL_RET_COPY_JOB_RESULT_TO_USER2_FAIL;
}
/* allocate kernel space result buffer */
/* which contains kernel + user space requests */
userRegValue = CMDQ_U32_PTR(jobResult.regValue.regValues);
jobResult.regValue.regValues =
(cmdqU32Ptr_t) (unsigned
long)(kzalloc(pTask->regCount * sizeof(uint32_t), GFP_KERNEL));
jobResult.regValue.count = pTask->regCount;
if (NULL == CMDQ_U32_PTR(jobResult.regValue.regValues)) {
CMDQ_ERR("no reg value buffer\n");
return -IOCTL_RET_NO_REG_VAL_BUFFER;
}
/* backup value after task release */
regCount = pTask->regCount;
regCountUserSpace = pTask->regCountUserSpace;
regUserToken = pTask->regUserToken;
/* make sure the task is running and wait for it */
status = cmdqCoreWaitResultAndReleaseTask(pTask,
&jobResult.regValue,
msecs_to_jiffies
(CMDQ_DEFAULT_TIMEOUT_MS));
if (status < 0) {
CMDQ_ERR("waitResultAndReleaseTask fail=%d\n", status);
/* free kernel space result buffer */
kfree(CMDQ_U32_PTR(jobResult.regValue.regValues));
return -IOCTL_RET_WAIT_RESULT_AND_RELEASE_TASK_FAIL;
}
/* pTask is released, do not access it any more */
pTask = NULL;
/* notify kernel space dump callback */
if (regCount > regCountUserSpace) {
CMDQ_VERBOSE("kernel space reg dump = %d, %d, %d\n", regCount,
regCountUserSpace, regUserToken);
status = cmdqCoreDebugRegDumpEnd(regUserToken,
regCount - regCountUserSpace,
CMDQ_U32_PTR(jobResult.regValue.regValues +
regCountUserSpace));
if (0 != status)
CMDQ_ERR("cmdqCoreDebugRegDumpEnd returns %d\n", status);
}
/* copy result to user space */
if (copy_to_user
((void *)userRegValue, (void *)(unsigned long)jobResult.regValue.regValues,
regCountUserSpace * sizeof(uint32_t))) {
CMDQ_ERR("Copy REGVALUE to user space failed\n");
return -IOCLT_RET_COPY_REG_VALUE_TO_USER_FAIL;
}
if (jobResult.readAddress.count > 0)
cmdq_driver_process_read_address_request(&jobResult.readAddress);
/* free kernel space result buffer */
kfree(CMDQ_U32_PTR(jobResult.regValue.regValues));
break;
case CMDQ_IOCTL_ALLOC_WRITE_ADDRESS:
do {
struct cmdqWriteAddressStruct addrReq;
dma_addr_t paStart = 0;
CMDQ_LOG("CMDQ_IOCTL_ALLOC_WRITE_ADDRESS\n");
if (copy_from_user(&addrReq, (void *)param, sizeof(addrReq))) {
CMDQ_ERR("CMDQ_IOCTL_ALLOC_WRITE_ADDRESS copy_from_user failed\n");
return -IOCTL_RET_COPY_ALLOC_WRITE_ADDR_FROM_USER_FAIL;
}
status = cmdqCoreAllocWriteAddress(addrReq.count, &paStart);
if (0 != status) {
CMDQ_ERR
("CMDQ_IOCTL_ALLOC_WRITE_ADDRESS cmdqCoreAllocWriteAddress() failed\n");
return -IOCTL_RET_ALLOC_WRITE_ADDR_FAIL;
}
addrReq.startPA = (uint32_t) paStart;
CMDQ_LOG("CMDQ_IOCTL_ALLOC_WRITE_ADDRESS get 0x%08x\n", addrReq.startPA);
if (copy_to_user((void *)param, &addrReq, sizeof(addrReq))) {
CMDQ_ERR("CMDQ_IOCTL_ALLOC_WRITE_ADDRESS copy_to_user failed\n");
return -IOCTL_RET_COPY_ALLOC_WRITE_ADDR_TO_USER_FAIL;
}
status = 0;
} while (0);
break;
case CMDQ_IOCTL_FREE_WRITE_ADDRESS:
do {
struct cmdqWriteAddressStruct freeReq;
CMDQ_LOG("CMDQ_IOCTL_FREE_WRITE_ADDRESS\n");
if (copy_from_user(&freeReq, (void *)param, sizeof(freeReq))) {
CMDQ_ERR("CMDQ_IOCTL_FREE_WRITE_ADDRESS copy_from_user failed\n");
return -IOCTL_RET_COPY_FREE_WRITE_ADDR_FROM_USER_FAIL;
}
status = cmdqCoreFreeWriteAddress(freeReq.startPA);
if (0 != status)
return -IOCTL_RET_FREE_WRITE_ADDR_FAIL;
status = 0;
} while (0);
break;
case CMDQ_IOCTL_READ_ADDRESS_VALUE:
do {
struct cmdqReadAddressStruct readReq;
CMDQ_LOG("CMDQ_IOCTL_READ_ADDRESS_VALUE\n");
if (copy_from_user(&readReq, (void *)param, sizeof(readReq))) {
CMDQ_ERR("CMDQ_IOCTL_READ_ADDRESS_VALUE copy_from_user failed\n");
return -IOCTL_RET_COPY_READ_ADDR_VAL_FROM_USER_FAIL;
}
/* this will copy result to readReq->values buffer */
cmdq_driver_process_read_address_request(&readReq);
status = 0;
} while (0);
break;
case CMDQ_IOCTL_QUERY_CAP_BITS:
do {
int capBits = 0;
if (cmdq_core_support_wait_and_receive_event_in_same_tick())
capBits |= (1L << CMDQ_CAP_WFE);
else
capBits &= ~(1L << CMDQ_CAP_WFE);
if (copy_to_user((void *)param, &capBits, sizeof(int))) {
CMDQ_ERR("Copy capacity bits to user space failed\n");
return -IOCTL_RET_COPY_CAP_BITS_TO_USER_FAIL;
}
} while (0);
break;
case CMDQ_IOCTL_SYNC_BUF_HDCP_VERSION:
#ifdef CMDQ_SECURE_PATH_SUPPORT
do {
struct cmdqSyncHandleHdcpStruct syncHandle;
if (copy_from_user(&syncHandle, (void *)param, sizeof(syncHandle))) {
CMDQ_ERR
("CMDQ_IOCTL_SYNC_BUF_HDCP_VERSION copy_from_user failed\n");
return -IOCTL_RET_COPY_HDCP_VERSION_FROM_USER_FAIL;
}
status = cmdq_sec_sync_handle_hdcp_unlock(syncHandle);
if (0 != status)
CMDQ_ERR("cmdq_sec_sync_handle_hdcp_unlock returns %d\n", status);
} while (0);
#else
CMDQ_ERR("SVP not supported\n");
return -IOCTL_RET_SVP_NOT_SUPPORT;
#endif
break;
default:
CMDQ_ERR("unrecognized ioctl 0x%08x\n", code);
CMDQ_ERR("CMDQ_IOCTL_LOCK_MUTEX:0x%08lx sizeof(int) = %ld\n",
CMDQ_IOCTL_LOCK_MUTEX, sizeof(int));
CMDQ_ERR("CMDQ_IOCTL_UNLOCK_MUTEX:0x%08lx sizeof(int) = %ld\n",
CMDQ_IOCTL_UNLOCK_MUTEX, sizeof(int));
CMDQ_ERR("CMDQ_IOCTL_EXEC_COMMAND:0x%08lx sizeof(struct cmdqCommandStruct) = %ld\n",
CMDQ_IOCTL_EXEC_COMMAND, sizeof(struct cmdqCommandStruct));
CMDQ_ERR("CMDQ_IOCTL_QUERY_USAGE:0x%08lx sizeof(struct cmdqUsageInfoStruct) = %ld\n",
CMDQ_IOCTL_QUERY_USAGE, sizeof(struct cmdqUsageInfoStruct));
CMDQ_ERR("CMDQ_IOCTL_ASYNC_JOB_EXEC:0x%08lx sizeof(struct cmdqJobStruct) = %ld\n",
CMDQ_IOCTL_ASYNC_JOB_EXEC, sizeof(struct cmdqJobStruct));
CMDQ_ERR("CMDQ_IOCTL_ASYNC_JOB_WAIT_AND_CLOSE:0x%08lx sizeof(struct cmdqJobResultStruct) = %ld\n",
CMDQ_IOCTL_ASYNC_JOB_WAIT_AND_CLOSE, sizeof(struct cmdqJobResultStruct));
CMDQ_ERR("CMDQ_IOCTL_ALLOC_WRITE_ADDRESS:0x%08lx sizeof(struct cmdqWriteAddressStruct) = %ld\n",
CMDQ_IOCTL_ALLOC_WRITE_ADDRESS, sizeof(struct cmdqWriteAddressStruct));
CMDQ_ERR("CMDQ_IOCTL_FREE_WRITE_ADDRESS:0x%08lx sizeof(struct cmdqWriteAddressStruct) = %ld\n",
CMDQ_IOCTL_FREE_WRITE_ADDRESS, sizeof(struct cmdqWriteAddressStruct));
CMDQ_ERR("CMDQ_IOCTL_READ_ADDRESS_VALUE:0x%08lx sizeof(struct cmdqReadAddressStruct) = %ld\n",
CMDQ_IOCTL_READ_ADDRESS_VALUE, sizeof(struct cmdqReadAddressStruct));
CMDQ_ERR("CMDQ_IOCTL_QUERY_CAP_BITS:0x%08lx sizeof(int) = %ld\n",
CMDQ_IOCTL_QUERY_CAP_BITS, sizeof(int));
CMDQ_ERR("CMDQ_IOCTL_SYNC_BUF_HDCP_VERSION:0x%08lx sizeof(struct cmdqSyncHandleHdcpStruct) = %ld\n",
CMDQ_IOCTL_SYNC_BUF_HDCP_VERSION, sizeof(struct cmdqSyncHandleHdcpStruct));
return -IOCTL_RET_UNRECOGNIZED_IOCTL;
}
return IOCTL_RET_SUCCESS;
}
int32_t cmdq_sec_fill_iwc_command_msg_unlocked(int32_t iwcCommand, void *_pTask, int32_t thread, void *_pIwc)
{
int32_t status;
const TaskStruct *pTask = (TaskStruct *)_pTask;
iwcCmdqMessage_t *pIwc;
/* cmdqSecDr will insert some instr*/
const uint32_t reservedCommandSize = 4 * CMDQ_INST_SIZE;
status = 0;
pIwc = (iwcCmdqMessage_t *)_pIwc;
/* check command size first */
if (pTask && (CMDQ_TZ_CMD_BLOCK_SIZE < (pTask->commandSize + reservedCommandSize))) {
CMDQ_ERR("[SEC]SESSION_MSG: pTask %p commandSize %d > %d\n",
pTask, pTask->commandSize, CMDQ_TZ_CMD_BLOCK_SIZE);
return -EFAULT;
}
CMDQ_MSG("[SEC]-->SESSION_MSG: cmdId[%d]\n", iwcCommand);
/* fill message buffer for inter world communication */
memset(pIwc, 0x0, sizeof(iwcCmdqMessage_t));
pIwc->cmd = iwcCommand;
/* metadata */
pIwc->command.metadata.enginesNeedDAPC = pTask->secData.enginesNeedDAPC;
pIwc->command.metadata.enginesNeedPortSecurity = pTask->secData.enginesNeedPortSecurity;
if (NULL != pTask && CMDQ_INVALID_THREAD != thread) {
/* basic data */
pIwc->command.scenario = pTask->scenario;
pIwc->command.thread = thread;
pIwc->command.priority = pTask->priority;
pIwc->command.engineFlag = pTask->engineFlag;
pIwc->command.commandSize = pTask->commandSize;
pIwc->command.hNormalTask = 0LL | ((unsigned long) pTask);
memcpy((pIwc->command.pVABase), (pTask->pVABase), (pTask->commandSize));
/* cookie */
pIwc->command.waitCookie = pTask->secData.waitCookie;
pIwc->command.resetExecCnt = pTask->secData.resetExecCnt;
CMDQ_MSG("[SEC]SESSION_MSG: task 0x%p, thread: %d, size: %d, bufferSize: %d, scenario:%d, flag:0x%08llx ,hNormalTask:0x%llx\n",
pTask, thread, pTask->commandSize, pTask->bufferSize, pTask->scenario, pTask->engineFlag, pIwc->command.hNormalTask);
CMDQ_VERBOSE("[SEC]SESSION_MSG: addrList[%d][0x%p]\n",
pTask->secData.addrMetadataCount,CMDQ_U32_PTR(pTask->secData.addrMetadatas));
if (0 < pTask->secData.addrMetadataCount) {
pIwc->command.metadata.addrListLength = pTask->secData.addrMetadataCount;
memcpy((pIwc->command.metadata.addrList), (pTask->secData.addrMetadatas),
(pTask->secData.addrMetadataCount) * sizeof(iwcCmdqAddrMetadata_t));
}
/* medatada: debug config */
pIwc->debug.logLevel = (cmdq_core_should_print_msg()) ? (1) : (0);
pIwc->debug.enableProfile = cmdq_core_profile_enabled();
} else {
/* relase resource, or debug function will go here */
CMDQ_VERBOSE("[SEC]-->SESSION_MSG: no task, cmdId[%d]\n", iwcCommand);
pIwc->command.commandSize = 0;
pIwc->command.metadata.addrListLength = 0;
}
CMDQ_MSG("[SEC]<--SESSION_MSG[%d]\n", status);
return status;
}