static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) { struct kfd_process *process; process = kfd_get_process(current); if (IS_ERR(process)) return PTR_ERR(process); return kfd_doorbell_mmap(process, vma); }
static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) { struct kfd_process *process; process = kfd_get_process(current); if (IS_ERR(process)) return PTR_ERR(process); if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) == KFD_MMAP_DOORBELL_MASK) { vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK; return kfd_doorbell_mmap(process, vma); } else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) == KFD_MMAP_EVENTS_MASK) { vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK; return kfd_event_mmap(process, vma); } return -EFAULT; }
static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { struct kfd_process *process; amdkfd_ioctl_t *func; const struct amdkfd_ioctl_desc *ioctl = NULL; unsigned int nr = _IOC_NR(cmd); char stack_kdata[128]; char *kdata = NULL; unsigned int usize, asize; int retcode = -EINVAL; if (nr >= AMDKFD_CORE_IOCTL_COUNT) goto err_i1; if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) { u32 amdkfd_size; ioctl = &amdkfd_ioctls[nr]; amdkfd_size = _IOC_SIZE(ioctl->cmd); usize = asize = _IOC_SIZE(cmd); if (amdkfd_size > asize) asize = amdkfd_size; cmd = ioctl->cmd; } else goto err_i1; dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg); process = kfd_get_process(current); if (IS_ERR(process)) { dev_dbg(kfd_device, "no process\n"); goto err_i1; } /* Do not trust userspace, use our own definition */ func = ioctl->func; if (unlikely(!func)) { dev_dbg(kfd_device, "no function\n"); retcode = -EINVAL; goto err_i1; } if (cmd & (IOC_IN | IOC_OUT)) { if (asize <= sizeof(stack_kdata)) { kdata = stack_kdata; } else { kdata = kmalloc(asize, GFP_KERNEL); if (!kdata) { retcode = -ENOMEM; goto err_i1; } } if (asize > usize) memset(kdata + usize, 0, asize - usize); } if (cmd & IOC_IN) { if (copy_from_user(kdata, (void __user *)arg, usize) != 0) { retcode = -EFAULT; goto err_i1; } } else if (cmd & IOC_OUT) { memset(kdata, 0, usize); } retcode = func(filep, process, kdata); if (cmd & IOC_OUT) if (copy_to_user((void __user *)arg, kdata, usize) != 0) retcode = -EFAULT; err_i1: if (!ioctl) dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n", task_pid_nr(current), cmd, nr); if (kdata != stack_kdata) kfree(kdata); if (retcode) dev_dbg(kfd_device, "ret = %d\n", retcode); return retcode; }
static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev, enum kfd_queue_type type, unsigned int queue_size) { struct queue_properties prop; int retval; union PM4_MES_TYPE_3_HEADER nop; BUG_ON(!kq || !dev); BUG_ON(type != KFD_QUEUE_TYPE_DIQ && type != KFD_QUEUE_TYPE_HIQ); pr_debug("amdkfd: In func %s initializing queue type %d size %d\n", __func__, KFD_QUEUE_TYPE_HIQ, queue_size); memset(&prop, 0, sizeof(prop)); memset(&nop, 0, sizeof(nop)); nop.opcode = IT_NOP; nop.type = PM4_TYPE_3; nop.u32all |= PM4_COUNT_ZERO; kq->dev = dev; kq->nop_packet = nop.u32all; switch (type) { case KFD_QUEUE_TYPE_DIQ: case KFD_QUEUE_TYPE_HIQ: kq->mqd = dev->dqm->ops.get_mqd_manager(dev->dqm, KFD_MQD_TYPE_HIQ); break; default: BUG(); break; } if (kq->mqd == NULL) return false; prop.doorbell_ptr = kfd_get_kernel_doorbell(dev, &prop.doorbell_off); if (prop.doorbell_ptr == NULL) { pr_err("amdkfd: error init doorbell"); goto err_get_kernel_doorbell; } retval = kfd_gtt_sa_allocate(dev, queue_size, &kq->pq); if (retval != 0) { pr_err("amdkfd: error init pq queues size (%d)\n", queue_size); goto err_pq_allocate_vidmem; } kq->pq_kernel_addr = kq->pq->cpu_ptr; kq->pq_gpu_addr = kq->pq->gpu_addr; retval = kq->ops_asic_specific.initialize(kq, dev, type, queue_size); if (retval == false) goto err_eop_allocate_vidmem; retval = kfd_gtt_sa_allocate(dev, sizeof(*kq->rptr_kernel), &kq->rptr_mem); if (retval != 0) goto err_rptr_allocate_vidmem; kq->rptr_kernel = kq->rptr_mem->cpu_ptr; kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr; retval = kfd_gtt_sa_allocate(dev, sizeof(*kq->wptr_kernel), &kq->wptr_mem); if (retval != 0) goto err_wptr_allocate_vidmem; kq->wptr_kernel = kq->wptr_mem->cpu_ptr; kq->wptr_gpu_addr = kq->wptr_mem->gpu_addr; memset(kq->pq_kernel_addr, 0, queue_size); memset(kq->rptr_kernel, 0, sizeof(*kq->rptr_kernel)); memset(kq->wptr_kernel, 0, sizeof(*kq->wptr_kernel)); prop.queue_size = queue_size; prop.is_interop = false; prop.priority = 1; prop.queue_percent = 100; prop.type = type; prop.vmid = 0; prop.queue_address = kq->pq_gpu_addr; prop.read_ptr = (uint32_t *) kq->rptr_gpu_addr; prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr; prop.eop_ring_buffer_address = kq->eop_gpu_addr; prop.eop_ring_buffer_size = PAGE_SIZE; if (init_queue(&kq->queue, &prop) != 0) goto err_init_queue; kq->queue->device = dev; kq->queue->process = kfd_get_process(current); retval = kq->mqd->init_mqd(kq->mqd, &kq->queue->mqd, &kq->queue->mqd_mem_obj, &kq->queue->gart_mqd_addr, &kq->queue->properties); if (retval != 0) goto err_init_mqd; /* assign HIQ to HQD */ if (type == KFD_QUEUE_TYPE_HIQ) { pr_debug("assigning hiq to hqd\n"); kq->queue->pipe = KFD_CIK_HIQ_PIPE; kq->queue->queue = KFD_CIK_HIQ_QUEUE; kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe, kq->queue->queue, NULL, NULL); } else { /* allocate fence for DIQ */ retval = kfd_gtt_sa_allocate(dev, sizeof(uint32_t), &kq->fence_mem_obj); if (retval != 0) goto err_alloc_fence; kq->fence_kernel_address = kq->fence_mem_obj->cpu_ptr; kq->fence_gpu_addr = kq->fence_mem_obj->gpu_addr; } print_queue(kq->queue); return true; err_alloc_fence: err_init_mqd: uninit_queue(kq->queue); err_init_queue: kfd_gtt_sa_free(dev, kq->wptr_mem); err_wptr_allocate_vidmem: kfd_gtt_sa_free(dev, kq->rptr_mem); err_rptr_allocate_vidmem: kfd_gtt_sa_free(dev, kq->eop_mem); err_eop_allocate_vidmem: kfd_gtt_sa_free(dev, kq->pq); err_pq_allocate_vidmem: kfd_release_kernel_doorbell(dev, prop.doorbell_ptr); err_get_kernel_doorbell: return false; }
static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { struct kfd_process *process; long err = -EINVAL; dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, _IOC_NR(cmd), arg); process = kfd_get_process(current); if (IS_ERR(process)) return PTR_ERR(process); switch (cmd) { case KFD_IOC_GET_VERSION: err = kfd_ioctl_get_version(filep, process, (void __user *)arg); break; case KFD_IOC_CREATE_QUEUE: err = kfd_ioctl_create_queue(filep, process, (void __user *)arg); break; case KFD_IOC_DESTROY_QUEUE: err = kfd_ioctl_destroy_queue(filep, process, (void __user *)arg); break; case KFD_IOC_SET_MEMORY_POLICY: err = kfd_ioctl_set_memory_policy(filep, process, (void __user *)arg); break; case KFD_IOC_GET_CLOCK_COUNTERS: err = kfd_ioctl_get_clock_counters(filep, process, (void __user *)arg); break; case KFD_IOC_GET_PROCESS_APERTURES: err = kfd_ioctl_get_process_apertures(filep, process, (void __user *)arg); break; case KFD_IOC_UPDATE_QUEUE: err = kfd_ioctl_update_queue(filep, process, (void __user *)arg); break; default: dev_err(kfd_device, "unknown ioctl cmd 0x%x, arg 0x%lx)\n", cmd, arg); err = -EINVAL; break; } if (err < 0) dev_err(kfd_device, "ioctl error %ld for ioctl cmd 0x%x (#%d)\n", err, cmd, _IOC_NR(cmd)); return err; }