int kfd_init_apertures(struct kfd_process *process)
{
uint8_t id = 0;
struct kfd_dev *dev;
struct kfd_process_device *pdd;
mutex_lock(&process->mutex);
/*Iterating over all devices*/
while ((dev = kfd_topology_enum_kfd_devices(id)) != NULL &&
id < NUM_OF_SUPPORTED_GPUS) {
pdd = kfd_get_process_device_data(dev, process, 1);
/*
* For 64 bit process aperture will be statically reserved in
* the x86_64 non canonical process address space
* amdkfd doesn't currently support apertures for 32 bit process
*/
if (process->is_32bit_user_mode) {
pdd->lds_base = pdd->lds_limit = 0;
pdd->gpuvm_base = pdd->gpuvm_limit = 0;
pdd->scratch_base = pdd->scratch_limit = 0;
} else {
/*
* node id couldn't be 0 - the three MSB bits of
* aperture shoudn't be 0
*/
pdd->lds_base = MAKE_LDS_APP_BASE(id + 1);
pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
pdd->gpuvm_base = MAKE_GPUVM_APP_BASE(id + 1);
pdd->gpuvm_limit =
MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base);
pdd->scratch_base = MAKE_SCRATCH_APP_BASE(id + 1);
pdd->scratch_limit =
MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
}
dev_dbg(kfd_device, "node id %u\n", id);
dev_dbg(kfd_device, "gpu id %u\n", pdd->dev->id);
dev_dbg(kfd_device, "lds_base %llX\n", pdd->lds_base);
dev_dbg(kfd_device, "lds_limit %llX\n", pdd->lds_limit);
dev_dbg(kfd_device, "gpuvm_base %llX\n", pdd->gpuvm_base);
dev_dbg(kfd_device, "gpuvm_limit %llX\n", pdd->gpuvm_limit);
dev_dbg(kfd_device, "scratch_base %llX\n", pdd->scratch_base);
dev_dbg(kfd_device, "scratch_limit %llX\n", pdd->scratch_limit);
id++;
}
mutex_unlock(&process->mutex);
return 0;
}
int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma)
{
phys_addr_t address;
struct kfd_dev *dev;
/*
* For simplicitly we only allow mapping of the entire doorbell
* allocation of a single device & process.
*/
if (vma->vm_end - vma->vm_start != doorbell_process_allocation())
return -EINVAL;
/* Find kfd device according to gpu id */
dev = kfd_device_by_id(vma->vm_pgoff);
if (dev == NULL)
return -EINVAL;
/* Find if pdd exists for combination of process and gpu id */
if (!kfd_get_process_device_data(dev, process, 0))
return -EINVAL;
/* Calculate physical address of doorbell */
address = kfd_get_process_doorbells(dev, process);
vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
VM_DONTDUMP | VM_PFNMAP;
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
pr_debug("kfd: mapping doorbell page in kfd_doorbell_mmap\n"
" target user address == 0x%08llX\n"
" physical address == 0x%08llX\n"
" vm_flags == 0x%04lX\n"
" size == 0x%04lX\n",
(unsigned long long) vma->vm_start, address, vma->vm_flags,
doorbell_process_allocation());
return io_remap_pfn_range(vma,
vma->vm_start,
address >> PAGE_SHIFT,
doorbell_process_allocation(),
vma->vm_page_prot);
}
int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
{
int status = 0;
unsigned int vmid;
union SQ_CMD_BITS reg_sq_cmd;
union GRBM_GFX_INDEX_BITS reg_gfx_index;
struct kfd_process_device *pdd;
struct dbg_wave_control_info wac_info;
int temp;
int first_vmid_to_scan = 8;
int last_vmid_to_scan = 15;
first_vmid_to_scan = ffs(dev->shared_resources.compute_vmid_bitmap) - 1;
temp = dev->shared_resources.compute_vmid_bitmap >> first_vmid_to_scan;
last_vmid_to_scan = first_vmid_to_scan + ffz(temp);
reg_sq_cmd.u32All = 0;
status = 0;
wac_info.mode = HSA_DBG_WAVEMODE_BROADCAST_PROCESS;
wac_info.operand = HSA_DBG_WAVEOP_KILL;
pr_debug("Killing all process wavefronts\n");
/* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..
* ATC_VMID15_PASID_MAPPING
* to check which VMID the current process is mapped to. */
for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_valid
(dev->kgd, vmid)) {
if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_valid
(dev->kgd, vmid) == p->pasid) {
pr_debug("Killing wave fronts of vmid %d and pasid %d\n",
vmid, p->pasid);
break;
}
}
}
if (vmid > last_vmid_to_scan) {
pr_err("amdkfd: didn't found vmid for pasid (%d)\n", p->pasid);
return -EFAULT;
}
/* taking the VMID for that process on the safe way using PDD */
pdd = kfd_get_process_device_data(dev, p);
if (!pdd)
return -EFAULT;
status = dbgdev_wave_control_set_registers(&wac_info, ®_sq_cmd,
®_gfx_index);
if (status != 0)
return -EINVAL;
/* for non DIQ we need to patch the VMID: */
reg_sq_cmd.bits.vm_id = vmid;
dev->kfd2kgd->wave_control_execute(dev->kgd,
reg_gfx_index.u32All,
reg_sq_cmd.u32All);
return 0;
}
static int dbgdev_wave_control_nodiq(struct kfd_dbgdev *dbgdev,
struct dbg_wave_control_info *wac_info)
{
int status;
union SQ_CMD_BITS reg_sq_cmd;
union GRBM_GFX_INDEX_BITS reg_gfx_index;
struct kfd_process_device *pdd;
BUG_ON(!dbgdev || !dbgdev->dev || !wac_info);
reg_sq_cmd.u32All = 0;
/* taking the VMID for that process on the safe way using PDD */
pdd = kfd_get_process_device_data(dbgdev->dev, wac_info->process);
if (!pdd) {
pr_err("amdkfd: Failed to get pdd for wave control no DIQ\n");
return -EFAULT;
}
status = dbgdev_wave_control_set_registers(wac_info, ®_sq_cmd,
®_gfx_index);
if (status) {
pr_err("amdkfd: Failed to set wave control registers\n");
return status;
}
/* for non DIQ we need to patch the VMID: */
reg_sq_cmd.bits.vm_id = pdd->qpd.vmid;
pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
pr_debug("\t\t mode is: %u\n", wac_info->mode);
pr_debug("\t\t operand is: %u\n", wac_info->operand);
pr_debug("\t\t trap id is: %u\n", wac_info->trapId);
pr_debug("\t\t msg value is: %u\n",
wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
pr_debug("\t\t vmid is: %u\n", pdd->qpd.vmid);
pr_debug("\t\t chk_vmid is : %u\n", reg_sq_cmd.bitfields.check_vmid);
pr_debug("\t\t command is : %u\n", reg_sq_cmd.bitfields.cmd);
pr_debug("\t\t queue id is : %u\n", reg_sq_cmd.bitfields.queue_id);
pr_debug("\t\t simd id is : %u\n", reg_sq_cmd.bitfields.simd_id);
pr_debug("\t\t mode is : %u\n", reg_sq_cmd.bitfields.mode);
pr_debug("\t\t vm_id is : %u\n", reg_sq_cmd.bitfields.vm_id);
pr_debug("\t\t wave_id is : %u\n", reg_sq_cmd.bitfields.wave_id);
pr_debug("\t\t ibw is : %u\n",
reg_gfx_index.bitfields.instance_broadcast_writes);
pr_debug("\t\t ii is : %u\n",
reg_gfx_index.bitfields.instance_index);
pr_debug("\t\t sebw is : %u\n",
reg_gfx_index.bitfields.se_broadcast_writes);
pr_debug("\t\t se_ind is : %u\n", reg_gfx_index.bitfields.se_index);
pr_debug("\t\t sh_ind is : %u\n", reg_gfx_index.bitfields.sh_index);
pr_debug("\t\t sbw is : %u\n",
reg_gfx_index.bitfields.sh_broadcast_writes);
pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
return dbgdev->dev->kfd2kgd->wave_control_execute(dbgdev->dev->kgd,
reg_gfx_index.u32All,
reg_sq_cmd.u32All);
}
static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
struct dbg_address_watch_info *adw_info)
{
union TCP_WATCH_ADDR_H_BITS addrHi;
union TCP_WATCH_ADDR_L_BITS addrLo;
union TCP_WATCH_CNTL_BITS cntl;
struct kfd_process_device *pdd;
unsigned int i;
BUG_ON(!dbgdev || !dbgdev->dev || !adw_info);
/* taking the vmid for that process on the safe way using pdd */
pdd = kfd_get_process_device_data(dbgdev->dev,
adw_info->process);
if (!pdd) {
pr_err("amdkfd: Failed to get pdd for wave control no DIQ\n");
return -EFAULT;
}
addrHi.u32All = 0;
addrLo.u32All = 0;
cntl.u32All = 0;
if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) ||
(adw_info->num_watch_points == 0)) {
pr_err("amdkfd: num_watch_points is invalid\n");
return -EINVAL;
}
if ((adw_info->watch_mode == NULL) ||
(adw_info->watch_address == NULL)) {
pr_err("amdkfd: adw_info fields are not valid\n");
return -EINVAL;
}
for (i = 0 ; i < adw_info->num_watch_points ; i++) {
dbgdev_address_watch_set_registers(adw_info, &addrHi, &addrLo,
&cntl, i, pdd->qpd.vmid);
pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
pr_debug("\t\t%20s %08x\n", "register index :", i);
pr_debug("\t\t%20s %08x\n", "vmid is :", pdd->qpd.vmid);
pr_debug("\t\t%20s %08x\n", "Address Low is :",
addrLo.bitfields.addr);
pr_debug("\t\t%20s %08x\n", "Address high is :",
addrHi.bitfields.addr);
pr_debug("\t\t%20s %08x\n", "Address high is :",
addrHi.bitfields.addr);
pr_debug("\t\t%20s %08x\n", "Control Mask is :",
cntl.bitfields.mask);
pr_debug("\t\t%20s %08x\n", "Control Mode is :",
cntl.bitfields.mode);
pr_debug("\t\t%20s %08x\n", "Control Vmid is :",
cntl.bitfields.vmid);
pr_debug("\t\t%20s %08x\n", "Control atc is :",
cntl.bitfields.atc);
pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
pdd->dev->kfd2kgd->address_watch_execute(
dbgdev->dev->kgd,
i,
cntl.u32All,
addrHi.u32All,
addrLo.u32All);
}
return 0;
}
