/*
* This wrapper function around hv_flush_remote() does several things:
*
* - Provides a return value error-checking panic path, since
* there's never any good reason for hv_flush_remote() to fail.
* - Accepts a 32-bit PFN rather than a 64-bit PA, which generally
* is the type that Linux wants to pass around anyway.
* - Canonicalizes that lengths of zero make cpumasks NULL.
* - Handles deferring TLB flushes for dataplane tiles.
* - Tracks remote interrupts in the per-cpu irq_cpustat_t.
*
* Note that we have to wait until the cache flush completes before
* updating the per-cpu last_cache_flush word, since otherwise another
* concurrent flush can race, conclude the flush has already
* completed, and start to use the page while it's still dirty
* remotely (running concurrently with the actual evict, presumably).
*/
void flush_remote(unsigned long cache_pfn, unsigned long cache_control,
const struct cpumask *cache_cpumask_orig,
HV_VirtAddr tlb_va, unsigned long tlb_length,
unsigned long tlb_pgsize,
const struct cpumask *tlb_cpumask_orig,
HV_Remote_ASID *asids, int asidcount)
{
int rc;
struct cpumask cache_cpumask_copy, tlb_cpumask_copy;
struct cpumask *cache_cpumask, *tlb_cpumask;
HV_PhysAddr cache_pa;
mb(); /* provided just to simplify "magic hypervisor" mode */
/*
* Canonicalize and copy the cpumasks.
*/
if (cache_cpumask_orig && cache_control) {
cpumask_copy(&cache_cpumask_copy, cache_cpumask_orig);
cache_cpumask = &cache_cpumask_copy;
} else {
cpumask_clear(&cache_cpumask_copy);
cache_cpumask = NULL;
}
if (cache_cpumask == NULL)
cache_control = 0;
if (tlb_cpumask_orig && tlb_length) {
cpumask_copy(&tlb_cpumask_copy, tlb_cpumask_orig);
tlb_cpumask = &tlb_cpumask_copy;
} else {
cpumask_clear(&tlb_cpumask_copy);
tlb_cpumask = NULL;
}
hv_flush_update(cache_cpumask, tlb_cpumask, tlb_va, tlb_length,
asids, asidcount);
cache_pa = (HV_PhysAddr)cache_pfn << PAGE_SHIFT;
rc = hv_flush_remote(cache_pa, cache_control,
cpumask_bits(cache_cpumask),
tlb_va, tlb_length, tlb_pgsize,
cpumask_bits(tlb_cpumask),
asids, asidcount);
if (rc == 0)
return;
pr_err("hv_flush_remote(%#llx, %#lx, %p [%*pb], %#lx, %#lx, %#lx, %p [%*pb], %p, %d) = %d\n",
cache_pa, cache_control, cache_cpumask,
cpumask_pr_args(&cache_cpumask_copy),
(unsigned long)tlb_va, tlb_length, tlb_pgsize, tlb_cpumask,
cpumask_pr_args(&tlb_cpumask_copy), asids, asidcount, rc);
panic("Unsafe to continue.");
}
/*
* This wrapper function around hv_flush_remote() does several things:
*
* - Provides a return value error-checking panic path, since
* there's never any good reason for hv_flush_remote() to fail.
* - Accepts a 32-bit PFN rather than a 64-bit PA, which generally
* is the type that Linux wants to pass around anyway.
* - Centralizes the mark_caches_evicted() handling.
* - Canonicalizes that lengths of zero make cpumasks NULL.
* - Handles deferring TLB flushes for dataplane tiles.
* - Tracks remote interrupts in the per-cpu irq_cpustat_t.
*
* Note that we have to wait until the cache flush completes before
* updating the per-cpu last_cache_flush word, since otherwise another
* concurrent flush can race, conclude the flush has already
* completed, and start to use the page while it's still dirty
* remotely (running concurrently with the actual evict, presumably).
*/
void flush_remote(unsigned long cache_pfn, unsigned long cache_control,
const struct cpumask *cache_cpumask_orig,
HV_VirtAddr tlb_va, unsigned long tlb_length,
unsigned long tlb_pgsize,
const struct cpumask *tlb_cpumask_orig,
HV_Remote_ASID *asids, int asidcount)
{
int rc;
int timestamp = 0; /* happy compiler */
struct cpumask cache_cpumask_copy, tlb_cpumask_copy;
struct cpumask *cache_cpumask, *tlb_cpumask;
HV_PhysAddr cache_pa;
char cache_buf[NR_CPUS*5], tlb_buf[NR_CPUS*5];
mb(); /* provided just to simplify "magic hypervisor" mode */
/*
* Canonicalize and copy the cpumasks.
*/
if (cache_cpumask_orig && cache_control) {
cpumask_copy(&cache_cpumask_copy, cache_cpumask_orig);
cache_cpumask = &cache_cpumask_copy;
} else {
cpumask_clear(&cache_cpumask_copy);
cache_cpumask = NULL;
}
if (cache_cpumask == NULL)
cache_control = 0;
if (tlb_cpumask_orig && tlb_length) {
cpumask_copy(&tlb_cpumask_copy, tlb_cpumask_orig);
tlb_cpumask = &tlb_cpumask_copy;
} else {
cpumask_clear(&tlb_cpumask_copy);
tlb_cpumask = NULL;
}
hv_flush_update(cache_cpumask, tlb_cpumask, tlb_va, tlb_length,
asids, asidcount);
cache_pa = (HV_PhysAddr)cache_pfn << PAGE_SHIFT;
if (cache_control & HV_FLUSH_EVICT_L2)
timestamp = mark_caches_evicted_start();
rc = hv_flush_remote(cache_pa, cache_control,
cpumask_bits(cache_cpumask),
tlb_va, tlb_length, tlb_pgsize,
cpumask_bits(tlb_cpumask),
asids, asidcount);
if (cache_control & HV_FLUSH_EVICT_L2)
mark_caches_evicted_finish(cache_cpumask, timestamp);
if (rc == 0)
return;
cpumask_scnprintf(cache_buf, sizeof(cache_buf), &cache_cpumask_copy);
cpumask_scnprintf(tlb_buf, sizeof(tlb_buf), &tlb_cpumask_copy);
pr_err("hv_flush_remote(%#llx, %#lx, %p [%s],"
" %#lx, %#lx, %#lx, %p [%s], %p, %d) = %d\n",
cache_pa, cache_control, cache_cpumask, cache_buf,
(unsigned long)tlb_va, tlb_length, tlb_pgsize,
tlb_cpumask, tlb_buf,
asids, asidcount, rc);
panic("Unsafe to continue.");
}
