static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long va,
int psize, int ssize, int local)
{
unsigned long want_v;
unsigned long lpar_rc;
u64 dummy1, dummy2;
unsigned long flags;
DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
slot, va, psize, local);
want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
atomic_spin_lock_irqsave(&beat_htab_lock, flags);
dummy1 = beat_lpar_hpte_getword0(slot);
if ((dummy1 & ~0x7FUL) != (want_v & ~0x7FUL)) {
DBG_LOW("not found !\n");
atomic_spin_unlock_irqrestore(&beat_htab_lock, flags);
return;
}
lpar_rc = beat_write_htab_entry(0, slot, 0, 0, HPTE_V_VALID, 0,
&dummy1, &dummy2);
atomic_spin_unlock_irqrestore(&beat_htab_lock, flags);
BUG_ON(lpar_rc != 0);
}
/*
* NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
* the low 3 bits of flags happen to line up. So no transform is needed.
* We can probably optimize here and assume the high bits of newpp are
* already zero. For now I am paranoid.
*/
static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
unsigned long newpp,
unsigned long va,
int psize, int ssize, int local)
{
unsigned long lpar_rc;
unsigned long want_v;
unsigned long pss;
want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc;
DBG_LOW(" update: "
"avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
want_v & HPTE_V_AVPN, slot, psize, newpp);
lpar_rc = beat_update_htab_permission3(0, slot, want_v, pss, 7, newpp);
if (lpar_rc == 0xfffffff7) {
DBG_LOW("not found !\n");
return -1;
}
DBG_LOW("ok\n");
BUG_ON(lpar_rc != 0);
return 0;
}
static long pSeries_lpar_hpte_insert(unsigned long hpte_group,
unsigned long vpn, unsigned long pa,
unsigned long rflags, unsigned long vflags,
int psize, int apsize, int ssize)
{
unsigned long lpar_rc;
unsigned long flags;
unsigned long slot;
unsigned long hpte_v, hpte_r;
if (!(vflags & HPTE_V_BOLTED))
pr_devel("hpte_insert(group=%lx, vpn=%016lx, "
"pa=%016lx, rflags=%lx, vflags=%lx, psize=%d)\n",
hpte_group, vpn, pa, rflags, vflags, psize);
hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
if (!(vflags & HPTE_V_BOLTED))
pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
/* Now fill in the actual HPTE */
/* Set CEC cookie to 0 */
/* Zero page = 0 */
/* I-cache Invalidate = 0 */
/* I-cache synchronize = 0 */
/* Exact = 0 */
flags = 0;
/* Make pHyp happy */
if ((rflags & _PAGE_NO_CACHE) && !(rflags & _PAGE_WRITETHRU))
hpte_r &= ~_PAGE_COHERENT;
if (firmware_has_feature(FW_FEATURE_XCMO) && !(hpte_r & HPTE_R_N))
flags |= H_COALESCE_CAND;
lpar_rc = plpar_pte_enter(flags, hpte_group, hpte_v, hpte_r, &slot);
if (unlikely(lpar_rc == H_PTEG_FULL)) {
if (!(vflags & HPTE_V_BOLTED))
pr_devel(" full\n");
return -1;
}
/*
* Since we try and ioremap PHBs we don't own, the pte insert
* will fail. However we must catch the failure in hash_page
* or we will loop forever, so return -2 in this case.
*/
if (unlikely(lpar_rc != H_SUCCESS)) {
if (!(vflags & HPTE_V_BOLTED))
pr_devel(" lpar err %ld\n", lpar_rc);
return -2;
}
if (!(vflags & HPTE_V_BOLTED))
pr_devel(" -> slot: %lu\n", slot & 7);
/* Because of iSeries, we have to pass down the secondary
* bucket bit here as well
*/
return (slot & 7) | (!!(vflags & HPTE_V_SECONDARY) << 3);
}
static long beat_lpar_hpte_find(unsigned long va, int psize)
{
unsigned long hash;
unsigned long i, j;
long slot;
unsigned long want_v, hpte_v;
hash = hpt_hash(va, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M);
want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
for (j = 0; j < 2; j++) {
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
for (i = 0; i < HPTES_PER_GROUP; i++) {
hpte_v = beat_lpar_hpte_getword0(slot);
if (HPTE_V_COMPARE(hpte_v, want_v)
&& (hpte_v & HPTE_V_VALID)
&& (!!(hpte_v & HPTE_V_SECONDARY) == j)) {
/* HPTE matches */
if (j)
slot = -slot;
return slot;
}
++slot;
}
hash = ~hash;
}
return -1;
}
static long ps3_hpte_insert(unsigned long hpte_group, unsigned long vpn,
unsigned long pa, unsigned long rflags, unsigned long vflags,
int psize, int apsize, int ssize)
{
int result;
u64 hpte_v, hpte_r;
u64 inserted_index;
u64 evicted_v, evicted_r;
u64 hpte_v_array[4], hpte_rs;
unsigned long flags;
long ret = -1;
/*
* lv1_insert_htab_entry() will search for victim
* entry in both primary and secondary pte group
*/
vflags &= ~HPTE_V_SECONDARY;
hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
hpte_r = hpte_encode_r(ps3_mm_phys_to_lpar(pa), psize, apsize) | rflags;
spin_lock_irqsave(&ps3_htab_lock, flags);
/* talk hvc to replace entries BOLTED == 0 */
result = lv1_insert_htab_entry(PS3_LPAR_VAS_ID_CURRENT, hpte_group,
hpte_v, hpte_r,
HPTE_V_BOLTED, 0,
&inserted_index,
&evicted_v, &evicted_r);
if (result) {
/* all entries bolted !*/
pr_info("%s:result=%s vpn=%lx pa=%lx ix=%lx v=%llx r=%llx\n",
__func__, ps3_result(result), vpn, pa, hpte_group,
hpte_v, hpte_r);
BUG();
}
/*
* see if the entry is inserted into secondary pteg
*/
result = lv1_read_htab_entries(PS3_LPAR_VAS_ID_CURRENT,
inserted_index & ~0x3UL,
&hpte_v_array[0], &hpte_v_array[1],
&hpte_v_array[2], &hpte_v_array[3],
&hpte_rs);
BUG_ON(result);
if (hpte_v_array[inserted_index % 4] & HPTE_V_SECONDARY)
ret = (inserted_index & 7) | (1 << 3);
else
ret = inserted_index & 7;
spin_unlock_irqrestore(&ps3_htab_lock, flags);
return ret;
}
long iSeries_hpte_insert(unsigned long hpte_group, unsigned long va,
unsigned long pa, unsigned long rflags,
unsigned long vflags, int psize)
{
long slot;
struct hash_pte lhpte;
int secondary = 0;
BUG_ON(psize != MMU_PAGE_4K);
/*
* The hypervisor tries both primary and secondary.
* If we are being called to insert in the secondary,
* it means we have already tried both primary and secondary,
* so we return failure immediately.
*/
if (vflags & HPTE_V_SECONDARY)
return -1;
iSeries_hlock(hpte_group);
slot = HvCallHpt_findValid(&lhpte, va >> HW_PAGE_SHIFT);
if (unlikely(lhpte.v & HPTE_V_VALID)) {
if (vflags & HPTE_V_BOLTED) {
HvCallHpt_setSwBits(slot, 0x10, 0);
HvCallHpt_setPp(slot, PP_RWXX);
iSeries_hunlock(hpte_group);
if (slot < 0)
return 0x8 | (slot & 7);
else
return slot & 7;
}
BUG();
}
if (slot == -1) { /* No available entry found in either group */
iSeries_hunlock(hpte_group);
return -1;
}
if (slot < 0) { /* MSB set means secondary group */
vflags |= HPTE_V_SECONDARY;
secondary = 1;
slot &= 0x7fffffffffffffff;
}
lhpte.v = hpte_encode_v(va, MMU_PAGE_4K) | vflags | HPTE_V_VALID;
lhpte.r = hpte_encode_r(phys_to_abs(pa), MMU_PAGE_4K) | rflags;
/* Now fill in the actual HPTE */
HvCallHpt_addValidate(slot, secondary, &lhpte);
iSeries_hunlock(hpte_group);
return (secondary << 3) | (slot & 7);
}
static long beat_lpar_hpte_insert(unsigned long hpte_group,
unsigned long va, unsigned long pa,
unsigned long rflags, unsigned long vflags,
int psize, int ssize)
{
unsigned long lpar_rc;
unsigned long slot;
unsigned long hpte_v, hpte_r;
/* same as iseries */
if (vflags & HPTE_V_SECONDARY)
return -1;
if (!(vflags & HPTE_V_BOLTED))
DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, "
"rflags=%lx, vflags=%lx, psize=%d)\n",
hpte_group, va, pa, rflags, vflags, psize);
hpte_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) |
vflags | HPTE_V_VALID;
hpte_r = hpte_encode_r(pa, psize) | rflags;
if (!(vflags & HPTE_V_BOLTED))
DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
if (rflags & (_PAGE_GUARDED|_PAGE_NO_CACHE))
hpte_r &= ~_PAGE_COHERENT;
spin_lock(&beat_htab_lock);
lpar_rc = beat_read_mask(hpte_group);
if (lpar_rc == 0) {
if (!(vflags & HPTE_V_BOLTED))
DBG_LOW(" full\n");
spin_unlock(&beat_htab_lock);
return -1;
}
lpar_rc = beat_insert_htab_entry(0, hpte_group, lpar_rc << 48,
hpte_v, hpte_r, &slot);
spin_unlock(&beat_htab_lock);
/*
* Since we try and ioremap PHBs we don't own, the pte insert
* will fail. However we must catch the failure in hash_page
* or we will loop forever, so return -2 in this case.
*/
if (unlikely(lpar_rc != 0)) {
if (!(vflags & HPTE_V_BOLTED))
DBG_LOW(" lpar err %lx\n", lpar_rc);
return -2;
}
if (!(vflags & HPTE_V_BOLTED))
DBG_LOW(" -> slot: %lx\n", slot);
/* We have to pass down the secondary bucket bit here as well */
return (slot ^ hpte_group) & 15;
}
static long ps3_hpte_updatepp(unsigned long slot, unsigned long newpp,
unsigned long va, int psize, int ssize, int local)
{
int result;
u64 hpte_v, want_v, hpte_rs;
u64 hpte_v_array[4];
unsigned long flags;
long ret;
want_v = hpte_encode_v(va, psize, ssize);
spin_lock_irqsave(&ps3_htab_lock, flags);
result = lv1_read_htab_entries(PS3_LPAR_VAS_ID_CURRENT, slot & ~0x3UL,
&hpte_v_array[0], &hpte_v_array[1],
&hpte_v_array[2], &hpte_v_array[3],
&hpte_rs);
if (result) {
pr_info("%s: res=%d read va=%lx slot=%lx psize=%d\n",
__func__, result, va, slot, psize);
BUG();
}
hpte_v = hpte_v_array[slot % 4];
/*
* As lv1_read_htab_entries() does not give us the RPN, we can
* not synthesize the new hpte_r value here, and therefore can
* not update the hpte with lv1_insert_htab_entry(), so we
<<<<<<< HEAD
* instead invalidate it and ask the caller to update it via
=======
* insted invalidate it and ask the caller to update it via
>>>>>>> 296c66da8a02d52243f45b80521febece5ed498a
* ps3_hpte_insert() by returning a -1 value.
*/
if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
/* not found */
ret = -1;
} else {
/* entry found, just invalidate it */
result = lv1_write_htab_entry(PS3_LPAR_VAS_ID_CURRENT,
slot, 0, 0);
ret = -1;
}
spin_unlock_irqrestore(&ps3_htab_lock, flags);
return ret;
}
static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long va,
int psize, int ssize, int local)
{
unsigned long want_v;
unsigned long lpar_rc;
unsigned long pss;
DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
slot, va, psize, local);
want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc;
lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss);
/* E_busy can be valid output: page may be already replaced */
BUG_ON(lpar_rc != 0 && lpar_rc != 0xfffffff7);
}
static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
unsigned long vpn, unsigned long pa,
unsigned long rflags, unsigned long vflags,
int psize, int ssize)
{
unsigned long lpar_rc;
u64 hpte_v, hpte_r, slot;
if (vflags & HPTE_V_SECONDARY)
return -1;
if (!(vflags & HPTE_V_BOLTED))
DBG_LOW("hpte_insert(group=%lx, vpn=%016lx, pa=%016lx, "
"rflags=%lx, vflags=%lx, psize=%d)\n",
hpte_group, vpn, pa, rflags, vflags, psize);
hpte_v = hpte_encode_v(vpn, psize, MMU_SEGSIZE_256M) |
vflags | HPTE_V_VALID;
hpte_r = hpte_encode_r(pa, psize) | rflags;
if (!(vflags & HPTE_V_BOLTED))
DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
if (rflags & _PAGE_NO_CACHE)
hpte_r &= ~_PAGE_COHERENT;
/* insert into not-volted entry */
lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r,
HPTE_V_BOLTED, 0, &slot);
/*
* Since we try and ioremap PHBs we don't own, the pte insert
* will fail. However we must catch the failure in hash_page
* or we will loop forever, so return -2 in this case.
*/
if (unlikely(lpar_rc != 0)) {
if (!(vflags & HPTE_V_BOLTED))
DBG_LOW(" lpar err %lx\n", lpar_rc);
return -2;
}
if (!(vflags & HPTE_V_BOLTED))
DBG_LOW(" -> slot: %lx\n", slot);
/* We have to pass down the secondary bucket bit here as well */
return (slot ^ hpte_group) & 15;
}
/*
* NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
* the low 3 bits of flags happen to line up. So no transform is needed.
* We can probably optimize here and assume the high bits of newpp are
* already zero. For now I am paranoid.
*/
static long beat_lpar_hpte_updatepp(unsigned long slot,
unsigned long newpp,
unsigned long va,
int psize, int ssize, int local)
{
unsigned long lpar_rc;
u64 dummy0, dummy1;
unsigned long want_v;
want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
DBG_LOW(" update: "
"avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
want_v & HPTE_V_AVPN, slot, psize, newpp);
atomic_spin_lock(&beat_htab_lock);
dummy0 = beat_lpar_hpte_getword0(slot);
if ((dummy0 & ~0x7FUL) != (want_v & ~0x7FUL)) {
DBG_LOW("not found !\n");
atomic_spin_unlock(&beat_htab_lock);
return -1;
}
lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, &dummy0,
&dummy1);
atomic_spin_unlock(&beat_htab_lock);
if (lpar_rc != 0 || dummy0 == 0) {
DBG_LOW("not found !\n");
return -1;
}
DBG_LOW("ok %lx %lx\n", dummy0, dummy1);
BUG_ON(lpar_rc != 0);
return 0;
}
/*
* The HyperVisor expects the "flags" argument in this form:
* bits 0..59 : reserved
* bit 60 : N
* bits 61..63 : PP2,PP1,PP0
*/
static long iSeries_hpte_updatepp(unsigned long slot, unsigned long newpp,
unsigned long va, int psize, int local)
{
struct hash_pte hpte;
unsigned long want_v;
iSeries_hlock(slot);
HvCallHpt_get(&hpte, slot);
want_v = hpte_encode_v(va, MMU_PAGE_4K);
if (HPTE_V_COMPARE(hpte.v, want_v) && (hpte.v & HPTE_V_VALID)) {
/*
* Hypervisor expects bits as NPPP, which is
* different from how they are mapped in our PP.
*/
HvCallHpt_setPp(slot, (newpp & 0x3) | ((newpp & 0x4) << 1));
iSeries_hunlock(slot);
return 0;
}
iSeries_hunlock(slot);
return -1;
}