static void vi_enable_doorbell_aperture(struct amdgpu_device *adev,
bool enable)
{
u32 tmp;
/* not necessary on CZ */
if (adev->flags & AMD_IS_APU)
return;
tmp = RREG32(mmBIF_DOORBELL_APER_EN);
if (enable)
tmp = REG_SET_FIELD(tmp, BIF_DOORBELL_APER_EN, BIF_DOORBELL_APER_EN, 1);
else
tmp = REG_SET_FIELD(tmp, BIF_DOORBELL_APER_EN, BIF_DOORBELL_APER_EN, 0);
WREG32(mmBIF_DOORBELL_APER_EN, tmp);
}
static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
uint32_t queue_id, uint32_t __user *wptr,
uint32_t wptr_shift, uint32_t wptr_mask,
struct mm_struct *mm)
{
struct amdgpu_device *adev = get_amdgpu_device(kgd);
struct cik_mqd *m;
uint32_t *mqd_hqd;
uint32_t reg, wptr_val, data;
bool valid_wptr = false;
m = get_mqd(mqd);
acquire_queue(kgd, pipe_id, queue_id);
/* HQD registers extend from CP_MQD_BASE_ADDR to CP_MQD_CONTROL. */
mqd_hqd = &m->cp_mqd_base_addr_lo;
for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++)
WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]);
/* Copy userspace write pointer value to register.
* Activate doorbell logic to monitor subsequent changes.
*/
data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
/* read_user_ptr may take the mm->mmap_sem.
* release srbm_mutex to avoid circular dependency between
* srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
*/
release_queue(kgd);
valid_wptr = read_user_wptr(mm, wptr, wptr_val);
acquire_queue(kgd, pipe_id, queue_id);
if (valid_wptr)
WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
WREG32(mmCP_HQD_ACTIVE, data);
release_queue(kgd);
return 0;
}
/**
* Reset Fan Speed Control to default mode.
* @param hwmgr the address of the powerplay hardware manager.
* @exception Should always succeed.
*/
int vega10_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr)
{
struct amdgpu_device *adev = hwmgr->adev;
if (!hwmgr->fan_ctrl_is_in_default_mode) {
WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2,
REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
CG_FDO_CTRL2, FDO_PWM_MODE,
hwmgr->fan_ctrl_default_mode));
WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2,
REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
CG_FDO_CTRL2, TMIN,
hwmgr->tmin << CG_FDO_CTRL2__TMIN__SHIFT));
hwmgr->fan_ctrl_is_in_default_mode = true;
}
return 0;
}
static uint32_t vega10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih)
{
u32 ih_doorbell_rtpr = 0;
if (ih->use_doorbell) {
ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
IH_DOORBELL_RPTR, OFFSET,
ih->doorbell_index);
ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
IH_DOORBELL_RPTR,
ENABLE, 1);
} else {
ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
IH_DOORBELL_RPTR,
ENABLE, 0);
}
return ih_doorbell_rtpr;
}
static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, bool val)
{
u32 reg;
reg = RREG32_NO_KIQ(mmMAILBOX_CONTROL);
reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
TRN_MSG_VALID, val ? 1 : 0);
WREG32_NO_KIQ(mmMAILBOX_CONTROL, reg);
}
/* Encrypt all flash data that should be encrypted */
static esp_err_t encrypt_flash_contents(uint32_t flash_crypt_cnt, bool flash_crypt_wr_dis)
{
esp_err_t err;
esp_partition_info_t partition_table[ESP_PARTITION_TABLE_MAX_ENTRIES];
int num_partitions;
/* If the last flash_crypt_cnt bit is burned or write-disabled, the
device can't re-encrypt itself. */
if (flash_crypt_wr_dis || flash_crypt_cnt == 0xFF) {
ESP_LOGE(TAG, "Cannot re-encrypt data (FLASH_CRYPT_CNT 0x%02x write disabled %d", flash_crypt_cnt, flash_crypt_wr_dis);
return ESP_FAIL;
}
if (flash_crypt_cnt == 0) {
/* Very first flash of encrypted data: generate keys, etc. */
err = initialise_flash_encryption();
if (err != ESP_OK) {
return err;
}
}
err = encrypt_bootloader();
if (err != ESP_OK) {
return err;
}
err = encrypt_and_load_partition_table(partition_table, &num_partitions);
if (err != ESP_OK) {
return err;
}
/* Now iterate the just-loaded partition table, looking for entries to encrypt
*/
/* Go through each partition and encrypt if necessary */
for (int i = 0; i < num_partitions; i++) {
err = encrypt_partition(i, &partition_table[i]);
if (err != ESP_OK) {
return err;
}
}
ESP_LOGD(TAG, "All flash regions checked for encryption pass");
/* Set least significant 0-bit in flash_crypt_cnt */
int ffs_inv = __builtin_ffs((~flash_crypt_cnt) & 0xFF);
/* ffs_inv shouldn't be zero, as zero implies flash_crypt_cnt == 0xFF */
uint32_t new_flash_crypt_cnt = flash_crypt_cnt + (1 << (ffs_inv - 1));
ESP_LOGD(TAG, "FLASH_CRYPT_CNT 0x%x -> 0x%x", flash_crypt_cnt, new_flash_crypt_cnt);
REG_SET_FIELD(EFUSE_BLK0_WDATA0_REG, EFUSE_FLASH_CRYPT_CNT, new_flash_crypt_cnt);
esp_efuse_burn_new_values();
ESP_LOGI(TAG, "Flash encryption completed");
return ESP_OK;
}
static void nbio_v7_4_init_registers(struct amdgpu_device *adev)
{
uint32_t def, data;
def = data = RREG32_PCIE(smnPCIE_CI_CNTL);
data = REG_SET_FIELD(data, PCIE_CI_CNTL, CI_SLV_ORDERING_DIS, 1);
if (def != data)
WREG32_PCIE(smnPCIE_CI_CNTL, data);
}
static void xgpu_ai_mailbox_set_valid(struct amdgpu_device *adev, bool val)
{
u32 reg;
reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
mmBIF_BX_PF0_MAILBOX_CONTROL));
reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_CONTROL,
TRN_MSG_VALID, val ? 1 : 0);
WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_CONTROL),
reg);
}
/**
* iceland_ih_get_wptr - get the IH ring buffer wptr
*
* @adev: amdgpu_device pointer
*
* Get the IH ring buffer wptr from either the register
* or the writeback memory buffer (VI). Also check for
* ring buffer overflow and deal with it.
* Used by cz_irq_process(VI).
* Returns the value of the wptr.
*/
static u32 iceland_ih_get_wptr(struct amdgpu_device *adev)
{
u32 wptr, tmp;
wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);
if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
/* When a ring buffer overflow happen start parsing interrupt
* from the last not overwritten vector (wptr + 16). Hopefully
* this should allow us to catchup.
*/
dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
wptr, adev->irq.ih.rptr, (wptr + 16) & adev->irq.ih.ptr_mask);
adev->irq.ih.rptr = (wptr + 16) & adev->irq.ih.ptr_mask;
tmp = RREG32(mmIH_RB_CNTL);
tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
WREG32(mmIH_RB_CNTL, tmp);
}
return (wptr & adev->irq.ih.ptr_mask);
}
static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
enum idh_request req)
{
u32 reg;
reg = RREG32_NO_KIQ(mmMAILBOX_MSGBUF_TRN_DW0);
reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
MSGBUF_DATA, req);
WREG32_NO_KIQ(mmMAILBOX_MSGBUF_TRN_DW0, reg);
xgpu_vi_mailbox_set_valid(adev, true);
}
static int xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device *adev,
struct amdgpu_irq_src *src,
unsigned type,
enum amdgpu_interrupt_state state)
{
u32 tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL));
tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_MAILBOX_INT_CNTL, VALID_INT_EN,
(state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL), tmp);
return 0;
}
static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
struct amdgpu_irq_src *src,
unsigned type,
enum amdgpu_interrupt_state state)
{
u32 tmp = RREG32_NO_KIQ(mmMAILBOX_INT_CNTL);
tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
(state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
WREG32_NO_KIQ(mmMAILBOX_INT_CNTL, tmp);
return 0;
}
void hw_i2c_init(HW_I2C_ID id, const i2c_config *cfg)
{
IRQn_Type irq_type = I2C_IRQn;
int enable_loop_cnt = 0;
if (id == HW_I2C2) {
irq_type = I2C2_IRQn;
}
else if (id != HW_I2C1) {
/* Requested ID must be one of HW_I2C1 or HW_I2C2 */
ASSERT_ERROR(0);
}
struct i2c *i2c = get_i2c(id);
memset(i2c, 0, sizeof(*i2c));
GLOBAL_INT_DISABLE();
uint32_t clk_per_reg_local = CRG_PER->CLK_PER_REG;
REG_SET_FIELD(CRG_PER, CLK_PER_REG, I2C_CLK_SEL, clk_per_reg_local, 0);
REG_SET_FIELD(CRG_PER, CLK_PER_REG, I2C_ENABLE, clk_per_reg_local, 1);
CRG_PER->CLK_PER_REG = clk_per_reg_local;
GLOBAL_INT_RESTORE();
hw_i2c_disable(id);
while (hw_i2c_get_enable_status(id) & I2C_I2C_ENABLE_STATUS_REG_IC_EN_Msk) {
hw_cpm_delay_usec(500);
enable_loop_cnt++;
/* we shouldn't get stuck here, the HW I2C block should eventually be enabled */
ASSERT_ERROR(enable_loop_cnt < I2C_ENABLE_LOOP_LIMIT);
}
IBA(id)->I2C_INTR_MASK_REG = 0x0000;
hw_i2c_configure(id, cfg);
NVIC_EnableIRQ(irq_type);
}
/**
* Set Fan Speed Control to static mode,
* so that the user can decide what speed to use.
* @param hwmgr the address of the powerplay hardware manager.
* mode the fan control mode, 0 default, 1 by percent, 5, by RPM
* @exception Should always succeed.
*/
int vega10_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
{
struct amdgpu_device *adev = hwmgr->adev;
if (hwmgr->fan_ctrl_is_in_default_mode) {
hwmgr->fan_ctrl_default_mode =
REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
CG_FDO_CTRL2, FDO_PWM_MODE);
hwmgr->tmin =
REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
CG_FDO_CTRL2, TMIN);
hwmgr->fan_ctrl_is_in_default_mode = false;
}
WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2,
REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
CG_FDO_CTRL2, TMIN, 0));
WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2,
REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
CG_FDO_CTRL2, FDO_PWM_MODE, mode));
return 0;
}
static void xgpu_ai_mailbox_trans_msg(struct amdgpu_device *adev,
enum idh_request req)
{
u32 reg;
reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0));
reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0,
MSGBUF_DATA, req);
WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0),
reg);
xgpu_ai_mailbox_set_valid(adev, true);
}
/**
* vega10_ih_disable_interrupts - Disable the interrupt ring buffer
*
* @adev: amdgpu_device pointer
*
* Disable the interrupt ring buffer (VEGA10).
*/
static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
{
u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
/* set rptr, wptr to 0 */
WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
adev->irq.ih.enabled = false;
adev->irq.ih.rptr = 0;
if (adev->irq.ih1.ring_size) {
ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
RB_ENABLE, 0);
WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
/* set rptr, wptr to 0 */
WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
adev->irq.ih1.enabled = false;
adev->irq.ih1.rptr = 0;
}
if (adev->irq.ih2.ring_size) {
ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2,
RB_ENABLE, 0);
WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
/* set rptr, wptr to 0 */
WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
adev->irq.ih2.enabled = false;
adev->irq.ih2.rptr = 0;
}
}
static int kgd_wave_control_execute(struct kgd_dev *kgd,
uint32_t gfx_index_val,
uint32_t sq_cmd)
{
struct amdgpu_device *adev = get_amdgpu_device(kgd);
uint32_t data = 0;
mutex_lock(&adev->grbm_idx_mutex);
WREG32(mmGRBM_GFX_INDEX, gfx_index_val);
WREG32(mmSQ_CMD, sq_cmd);
data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
INSTANCE_BROADCAST_WRITES, 1);
data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
SH_BROADCAST_WRITES, 1);
data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
SE_BROADCAST_WRITES, 1);
WREG32(mmGRBM_GFX_INDEX, data);
mutex_unlock(&adev->grbm_idx_mutex);
return 0;
}
static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid)
{
u32 req = 0;
/* invalidate using legacy mode on vmid*/
req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
PER_VMID_INVALIDATE_REQ, 1 << vmid);
req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0);
req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
return req;
}
uint32_t rtc_sleep_start(uint32_t wakeup_opt, uint32_t reject_opt)
{
REG_SET_FIELD(RTC_CNTL_WAKEUP_STATE_REG, RTC_CNTL_WAKEUP_ENA, wakeup_opt);
WRITE_PERI_REG(RTC_CNTL_SLP_REJECT_CONF_REG, reject_opt);
/* Start entry into sleep mode */
SET_PERI_REG_MASK(RTC_CNTL_STATE0_REG, RTC_CNTL_SLEEP_EN);
while (GET_PERI_REG_MASK(RTC_CNTL_INT_RAW_REG,
RTC_CNTL_SLP_REJECT_INT_RAW | RTC_CNTL_SLP_WAKEUP_INT_RAW) == 0) {
;
}
/* In deep sleep mode, we never get here */
uint32_t reject = REG_GET_FIELD(RTC_CNTL_INT_RAW_REG, RTC_CNTL_SLP_REJECT_INT_RAW);
SET_PERI_REG_MASK(RTC_CNTL_INT_CLR_REG,
RTC_CNTL_SLP_REJECT_INT_CLR | RTC_CNTL_SLP_WAKEUP_INT_CLR);
return reject;
}
static void gmc_v7_0_mc_stop(struct amdgpu_device *adev)
{
u32 blackout;
gmc_v7_0_wait_for_idle((void *)adev);
blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
/* Block CPU access */
WREG32(mmBIF_FB_EN, 0);
/* blackout the MC */
blackout = REG_SET_FIELD(blackout,
MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
}
/* wait for the MC to settle */
udelay(100);
}
static void xgpu_ai_mailbox_trans_msg (struct amdgpu_device *adev,
enum idh_request req, u32 data1, u32 data2, u32 data3) {
u32 reg;
int r;
uint8_t trn;
/* IMPORTANT:
* clear TRN_MSG_VALID valid to clear host's RCV_MSG_ACK
* and with host's RCV_MSG_ACK cleared hw automatically clear host's RCV_MSG_ACK
* which lead to VF's TRN_MSG_ACK cleared, otherwise below xgpu_ai_poll_ack()
* will return immediatly
*/
do {
xgpu_ai_mailbox_set_valid(adev, false);
trn = xgpu_ai_peek_ack(adev);
if (trn) {
pr_err("trn=%x ACK should not assert! wait again !\n", trn);
msleep(1);
}
} while(trn);
reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0));
reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0,
MSGBUF_DATA, req);
WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0),
reg);
WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1),
data1);
WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2),
data2);
WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3),
data3);
xgpu_ai_mailbox_set_valid(adev, true);
/* start to poll ack */
r = xgpu_ai_poll_ack(adev);
if (r)
pr_err("Doesn't get ack from pf, continue\n");
xgpu_ai_mailbox_set_valid(adev, false);
}
static bool uvd_v6_0_check_soft_reset(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
u32 srbm_soft_reset = 0;
u32 tmp = RREG32(mmSRBM_STATUS);
if (REG_GET_FIELD(tmp, SRBM_STATUS, UVD_RQ_PENDING) ||
REG_GET_FIELD(tmp, SRBM_STATUS, UVD_BUSY) ||
(RREG32(mmUVD_STATUS) & AMDGPU_UVD_STATUS_BUSY_MASK))
srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_UVD, 1);
if (srbm_soft_reset) {
adev->uvd.srbm_soft_reset = srbm_soft_reset;
return true;
} else {
adev->uvd.srbm_soft_reset = 0;
return false;
}
}
/**
* vega10_ih_irq_init - init and enable the interrupt ring
*
* @adev: amdgpu_device pointer
*
* Allocate a ring buffer for the interrupt controller,
* enable the RLC, disable interrupts, enable the IH
* ring buffer and enable it (VI).
* Called at device load and reume.
* Returns 0 for success, errors for failure.
*/
static int vega10_ih_irq_init(struct amdgpu_device *adev)
{
int ret = 0;
int rb_bufsz;
u32 ih_rb_cntl, ih_doorbell_rtpr;
u32 tmp;
u64 wptr_off;
/* disable irqs */
vega10_ih_disable_interrupts(adev);
adev->nbio_funcs->ih_control(adev);
ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
if (adev->irq.ih.use_bus_addr) {
WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8);
WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
} else {
static uint32_t vega10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)
{
int rb_bufsz = order_base_2(ih->ring_size / 4);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
MC_SPACE, ih->use_bus_addr ? 1 : 4);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
WPTR_OVERFLOW_CLEAR, 1);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
WPTR_OVERFLOW_ENABLE, 1);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register
* value is written to memory
*/
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
WPTR_WRITEBACK_ENABLE, 1);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
return ih_rb_cntl;
}
/*
* Mailbox communication between GPU hypervisor and VFs
*/
static void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev)
{
u32 reg;
int timeout = VI_MAILBOX_TIMEDOUT;
u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, RCV_MSG_VALID);
reg = RREG32_NO_KIQ(mmMAILBOX_CONTROL);
reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
WREG32_NO_KIQ(mmMAILBOX_CONTROL, reg);
/*Wait for RCV_MSG_VALID to be 0*/
reg = RREG32_NO_KIQ(mmMAILBOX_CONTROL);
while (reg & mask) {
if (timeout <= 0) {
pr_err("RCV_MSG_VALID is not cleared\n");
break;
}
mdelay(1);
timeout -=1;
reg = RREG32_NO_KIQ(mmMAILBOX_CONTROL);
}
}
/**
* gmc_v7_0_set_prt - set PRT VM fault
*
* @adev: amdgpu_device pointer
* @enable: enable/disable VM fault handling for PRT
*/
static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable)
{
uint32_t tmp;
if (enable && !adev->mc.prt_warning) {
dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");
adev->mc.prt_warning = true;
}
tmp = RREG32(mmVM_PRT_CNTL);
tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
CB_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
CB_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
TC_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
TC_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
L2_CACHE_STORE_INVALID_ENTRIES, enable);
tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
L1_TLB_STORE_INVALID_ENTRIES, enable);
tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
MASK_PDE0_FAULT, enable);
WREG32(mmVM_PRT_CNTL, tmp);
if (enable) {
uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
uint32_t high = adev->vm_manager.max_pfn;
WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);
WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);
WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);
WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);
WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);
WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);
} else {
void hw_aes_hash_enable(const hw_aes_hash_setup setup)
{
hw_aes_hash_check_data_size(&setup);
hw_aes_hash_enable_clock();
hw_aes_hash_set_mode(&setup);
uint32_t crypto_ctrl_reg = AES_HASH->CRYPTO_CTRL_REG;
REG_SET_FIELD(AES_HASH, CRYPTO_CTRL_REG, CRYPTO_MORE_IN, crypto_ctrl_reg,
setup.moreDataToCome);
REG_SET_FIELD(AES_HASH, CRYPTO_CTRL_REG, CRYPTO_HASH_OUT_LEN, crypto_ctrl_reg,
(setup.hashOutLength - 1));
REG_SET_FIELD(AES_HASH, CRYPTO_CTRL_REG, CRYPTO_ENCDEC, crypto_ctrl_reg,
setup.aesDirection);
REG_SET_FIELD(AES_HASH, CRYPTO_CTRL_REG, CRYPTO_AES_KEXP, crypto_ctrl_reg,
setup.aesKeyExpand);
REG_SET_FIELD(AES_HASH, CRYPTO_CTRL_REG, CRYPTO_AES_KEY_SZ, crypto_ctrl_reg,
setup.aesKeySize);
REG_SET_FIELD(AES_HASH, CRYPTO_CTRL_REG, CRYPTO_OUT_MD, crypto_ctrl_reg,
!setup.aesWriteBackAll);
AES_HASH->CRYPTO_CTRL_REG = crypto_ctrl_reg;
if (MODE_IS_AES(setup.mode)) {
hw_aes_hash_store_keys(setup.aesKeySize, (uint8 *)setup.aesKeys, !setup.aesKeyExpand);
}
hw_aes_hash_cfg_dma((const uint8 *)setup.sourceAddress, (uint8 *)setup.destinationAddress,
(unsigned int)setup.dataSize);
if (setup.enableInterrupt) {
hw_aes_hash_old_style_cb = setup.callback;
hw_aes_hash_enable_interrupt_source();
hw_crypto_enable_aes_hash_interrupt(hw_aes_hash_old_cb_style_support);
}
else {
hw_aes_hash_disable_interrupt_source();
hw_crypto_disable_aes_hash_interrupt();
}
hw_aes_hash_start();
}
/**
* gmc_v8_0_set_fault_enable_default - update VM fault handling
*
* @adev: amdgpu_device pointer
* @value: true redirects VM faults to the default page
*/
static void gmc_v7_0_set_fault_enable_default(struct amdgpu_device *adev,
bool value)
{
u32 tmp;
tmp = RREG32(mmVM_CONTEXT1_CNTL);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
WREG32(mmVM_CONTEXT1_CNTL, tmp);
}
/**
* uvd_v6_0_start - start UVD block
*
* @adev: amdgpu_device pointer
*
* Setup and start the UVD block
*/
static int uvd_v6_0_start(struct amdgpu_device *adev)
{
struct amdgpu_ring *ring = &adev->uvd.ring;
uint32_t rb_bufsz, tmp;
uint32_t lmi_swap_cntl;
uint32_t mp_swap_cntl;
int i, j, r;
/* disable DPG */
WREG32_P(mmUVD_POWER_STATUS, 0, ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
/* disable byte swapping */
lmi_swap_cntl = 0;
mp_swap_cntl = 0;
uvd_v6_0_mc_resume(adev);
/* disable clock gating */
WREG32_FIELD(UVD_CGC_CTRL, DYN_CLOCK_MODE, 0);
/* disable interupt */
WREG32_FIELD(UVD_MASTINT_EN, VCPU_EN, 0);
/* stall UMC and register bus before resetting VCPU */
WREG32_FIELD(UVD_LMI_CTRL2, STALL_ARB_UMC, 1);
mdelay(1);
/* put LMI, VCPU, RBC etc... into reset */
WREG32(mmUVD_SOFT_RESET,
UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
mdelay(5);
/* take UVD block out of reset */
WREG32_FIELD(SRBM_SOFT_RESET, SOFT_RESET_UVD, 0);
mdelay(5);
/* initialize UVD memory controller */
WREG32(mmUVD_LMI_CTRL,
(0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
UVD_LMI_CTRL__REQ_MODE_MASK |
UVD_LMI_CTRL__DISABLE_ON_FWV_FAIL_MASK);
#ifdef __BIG_ENDIAN
/* swap (8 in 32) RB and IB */
lmi_swap_cntl = 0xa;
mp_swap_cntl = 0;
#endif
WREG32(mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
WREG32(mmUVD_MP_SWAP_CNTL, mp_swap_cntl);
WREG32(mmUVD_MPC_SET_MUXA0, 0x40c2040);
WREG32(mmUVD_MPC_SET_MUXA1, 0x0);
WREG32(mmUVD_MPC_SET_MUXB0, 0x40c2040);
WREG32(mmUVD_MPC_SET_MUXB1, 0x0);
WREG32(mmUVD_MPC_SET_ALU, 0);
WREG32(mmUVD_MPC_SET_MUX, 0x88);
/* take all subblocks out of reset, except VCPU */
WREG32(mmUVD_SOFT_RESET, UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
mdelay(5);
/* enable VCPU clock */
WREG32(mmUVD_VCPU_CNTL, UVD_VCPU_CNTL__CLK_EN_MASK);
/* enable UMC */
WREG32_FIELD(UVD_LMI_CTRL2, STALL_ARB_UMC, 0);
/* boot up the VCPU */
WREG32(mmUVD_SOFT_RESET, 0);
mdelay(10);
for (i = 0; i < 10; ++i) {
uint32_t status;
for (j = 0; j < 100; ++j) {
status = RREG32(mmUVD_STATUS);
if (status & 2)
break;
mdelay(10);
}
r = 0;
if (status & 2)
break;
DRM_ERROR("UVD not responding, trying to reset the VCPU!!!\n");
WREG32_FIELD(UVD_SOFT_RESET, VCPU_SOFT_RESET, 1);
mdelay(10);
WREG32_FIELD(UVD_SOFT_RESET, VCPU_SOFT_RESET, 0);
mdelay(10);
r = -1;
}
if (r) {
DRM_ERROR("UVD not responding, giving up!!!\n");
return r;
}
/* enable master interrupt */
WREG32_P(mmUVD_MASTINT_EN,
(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
/* clear the bit 4 of UVD_STATUS */
WREG32_P(mmUVD_STATUS, 0, ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
/* force RBC into idle state */
rb_bufsz = order_base_2(ring->ring_size);
tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
WREG32(mmUVD_RBC_RB_CNTL, tmp);
/* set the write pointer delay */
WREG32(mmUVD_RBC_RB_WPTR_CNTL, 0);
/* set the wb address */
WREG32(mmUVD_RBC_RB_RPTR_ADDR, (upper_32_bits(ring->gpu_addr) >> 2));
/* programm the RB_BASE for ring buffer */
WREG32(mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
lower_32_bits(ring->gpu_addr));
WREG32(mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
upper_32_bits(ring->gpu_addr));
/* Initialize the ring buffer's read and write pointers */
WREG32(mmUVD_RBC_RB_RPTR, 0);
ring->wptr = RREG32(mmUVD_RBC_RB_RPTR);
WREG32(mmUVD_RBC_RB_WPTR, ring->wptr);
WREG32_FIELD(UVD_RBC_RB_CNTL, RB_NO_FETCH, 0);
return 0;
}
static void vi_gpu_pci_config_reset(struct amdgpu_device *adev)
{
struct amdgpu_mode_mc_save save;
u32 tmp, i;
dev_info(adev->dev, "GPU pci config reset\n");
/* disable dpm? */
/* disable cg/pg */
/* Disable GFX parsing/prefetching */
tmp = RREG32(mmCP_ME_CNTL);
tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1);
tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1);
tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1);
WREG32(mmCP_ME_CNTL, tmp);
/* Disable MEC parsing/prefetching */
tmp = RREG32(mmCP_MEC_CNTL);
tmp = REG_SET_FIELD(tmp, CP_MEC_CNTL, MEC_ME1_HALT, 1);
tmp = REG_SET_FIELD(tmp, CP_MEC_CNTL, MEC_ME2_HALT, 1);
WREG32(mmCP_MEC_CNTL, tmp);
/* Disable GFX parsing/prefetching */
WREG32(mmCP_ME_CNTL, CP_ME_CNTL__ME_HALT_MASK |
CP_ME_CNTL__PFP_HALT_MASK | CP_ME_CNTL__CE_HALT_MASK);
/* Disable MEC parsing/prefetching */
WREG32(mmCP_MEC_CNTL,
CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK);
/* sdma0 */
tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET);
tmp = REG_SET_FIELD(tmp, SDMA0_F32_CNTL, HALT, 1);
WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp);
/* sdma1 */
tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET);
tmp = REG_SET_FIELD(tmp, SDMA0_F32_CNTL, HALT, 1);
WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp);
/* XXX other engines? */
/* halt the rlc, disable cp internal ints */
//XXX
//gfx_v8_0_rlc_stop(adev);
udelay(50);
/* disable mem access */
gmc_v8_0_mc_stop(adev, &save);
if (amdgpu_asic_wait_for_mc_idle(adev)) {
dev_warn(adev->dev, "Wait for MC idle timed out !\n");
}
/* disable BM */
pci_clear_master(adev->pdev);
/* reset */
amdgpu_pci_config_reset(adev);
udelay(100);
/* wait for asic to come out of reset */
for (i = 0; i < adev->usec_timeout; i++) {
if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff)
break;
udelay(1);
}
}
