/**
* amdgpu_vram_mgr_del - free ranges
*
* @man: TTM memory type manager
* @tbo: TTM BO we need this range for
* @place: placement flags and restrictions
* @mem: TTM memory object
*
* Free the allocated VRAM again.
*/
static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
struct ttm_mem_reg *mem)
{
struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
struct amdgpu_vram_mgr *mgr = man->priv;
struct drm_mm_node *nodes = mem->mm_node;
uint64_t usage = 0, vis_usage = 0;
unsigned pages = mem->num_pages;
if (!mem->mm_node)
return;
spin_lock(&mgr->lock);
while (pages) {
pages -= nodes->size;
drm_mm_remove_node(nodes);
usage += nodes->size << PAGE_SHIFT;
vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes);
++nodes;
}
spin_unlock(&mgr->lock);
atomic64_sub(usage, &mgr->usage);
atomic64_sub(vis_usage, &mgr->vis_usage);
kfree(mem->mm_node);
mem->mm_node = NULL;
}
static void amdgpu_update_memory_usage(struct amdgpu_device *adev,
struct ttm_mem_reg *old_mem,
struct ttm_mem_reg *new_mem)
{
u64 vis_size;
if (!adev)
return;
if (new_mem) {
switch (new_mem->mem_type) {
case TTM_PL_TT:
atomic64_add(new_mem->size, &adev->gtt_usage);
break;
case TTM_PL_VRAM:
atomic64_add(new_mem->size, &adev->vram_usage);
vis_size = amdgpu_get_vis_part_size(adev, new_mem);
atomic64_add(vis_size, &adev->vram_vis_usage);
break;
}
}
if (old_mem) {
switch (old_mem->mem_type) {
case TTM_PL_TT:
atomic64_sub(old_mem->size, &adev->gtt_usage);
break;
case TTM_PL_VRAM:
atomic64_sub(old_mem->size, &adev->vram_usage);
vis_size = amdgpu_get_vis_part_size(adev, old_mem);
atomic64_sub(vis_size, &adev->vram_vis_usage);
break;
}
}
}
/**
* amdgpu_bo_subtract_pin_size - Remove BO from pin_size accounting
*
* @bo: &amdgpu_bo buffer object
*
* This function is called when a BO stops being pinned, and updates the
* &amdgpu_device pin_size values accordingly.
*/
static void amdgpu_bo_subtract_pin_size(struct amdgpu_bo *bo)
{
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
&adev->visible_pin_size);
} else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
}
}
static int nft_quota_do_dump(struct sk_buff *skb, struct nft_quota *priv,
bool reset)
{
u64 consumed, consumed_cap;
u32 flags = priv->flags;
/* Since we inconditionally increment consumed quota for each packet
* that we see, don't go over the quota boundary in what we send to
* userspace.
*/
consumed = atomic64_read(&priv->consumed);
if (consumed >= priv->quota) {
consumed_cap = priv->quota;
flags |= NFT_QUOTA_F_DEPLETED;
} else {
consumed_cap = consumed;
}
if (nla_put_be64(skb, NFTA_QUOTA_BYTES, cpu_to_be64(priv->quota),
NFTA_QUOTA_PAD) ||
nla_put_be64(skb, NFTA_QUOTA_CONSUMED, cpu_to_be64(consumed_cap),
NFTA_QUOTA_PAD) ||
nla_put_be32(skb, NFTA_QUOTA_FLAGS, htonl(flags)))
goto nla_put_failure;
if (reset) {
atomic64_sub(consumed, &priv->consumed);
clear_bit(NFT_QUOTA_DEPLETED_BIT, &priv->flags);
}
return 0;
nla_put_failure:
return -1;
}
/*
when cache device is removed...
*/
int _lun_rmcdev(lun_t * lun, uint32_t devnum)
{
fragdesc_t fdesc;
uint32_t fragnum;
frag_t *frag;
uint32_t nr_miss;
int ret = SCE_ERROR;
if ((lun) && (lun->fragmap) && (devnum < SCE_MAXCDEV)) {
for (fragnum = 0; fragnum < lun->nr_frag; fragnum++) {
fdesc = lun->fragmap[fragnum];
if ((fdesc & FRAGDESC_MAPPED) == 0)
continue;
if (devnum != SCE_PFIDDEV(fdesc))
continue;
frag = _pfid2frag((sce_t *) lun->scehndl, fdesc, NULL);
if (!frag)
continue;
nr_miss = frag->nr_miss;
lun->fragmap[fragnum] = (nr_miss & FRAGDESC_DATAMASK);
atomic64_dec(&lun->stats.alloc_sctrs);
atomic64_sub(frag->nr_valid * SCE_SCTRPERPAGE,
&lun->stats.valid_sctrs);
}
ret = SCE_SUCCESS;
}
return ret;
}
static inline
void __brick_block_free(void *data, int order, int cline)
{
#ifdef CONFIG_MARS_DEBUG_MEM_STRONG
struct mem_block_info *inf = _find_block_info(data, true);
if (likely(inf)) {
int inf_len = inf->inf_len;
int inf_line = inf->inf_line;
kfree(inf);
if (unlikely(inf_len != (PAGE_SIZE << order))) {
BRICK_ERR("line %d: address %p: bad freeing size %d (correct should be %d, previous line = %d)\n", cline, data, (int)(PAGE_SIZE << order), inf_len, inf_line);
goto err;
}
} else {
BRICK_ERR("line %d: trying to free non-existent address %p (order = %d)\n", cline, data, order);
goto err;
}
#endif
#ifdef USE_KERNEL_PAGES
__free_pages(virt_to_page((unsigned long)data), order);
#else
vfree(data);
#endif
#ifdef CONFIG_MARS_DEBUG_MEM_STRONG
err:
#endif
#ifdef BRICK_DEBUG_MEM
atomic_dec(&phys_block_alloc);
atomic_dec(&raw_count[order]);
#endif
atomic64_sub((PAGE_SIZE/1024) << order, &brick_global_block_used);
}
void nilfs_inode_sub_blocks(struct inode *inode, int n)
{
struct nilfs_root *root = NILFS_I(inode)->i_root;
inode_sub_bytes(inode, (1 << inode->i_blkbits) * n);
if (root)
atomic64_sub(n, &root->blocks_count);
}
static void s390_dma_free(struct device *dev, size_t size,
void *pa, dma_addr_t dma_handle,
struct dma_attrs *attrs)
{
struct zpci_dev *zdev = get_zdev(to_pci_dev(dev));
size = PAGE_ALIGN(size);
atomic64_sub(size / PAGE_SIZE, &zdev->fmb->allocated_pages);
s390_dma_unmap_pages(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
free_pages((unsigned long) pa, get_order(size));
}
static void radeon_update_memory_usage(struct radeon_bo *bo,
unsigned mem_type, int sign)
{
struct radeon_device *rdev = bo->rdev;
u64 size = (u64)bo->tbo.num_pages << PAGE_SHIFT;
switch (mem_type) {
case TTM_PL_TT:
if (sign > 0)
atomic64_add(size, &rdev->gtt_usage);
else
atomic64_sub(size, &rdev->gtt_usage);
break;
case TTM_PL_VRAM:
if (sign > 0)
atomic64_add(size, &rdev->vram_usage);
else
atomic64_sub(size, &rdev->vram_usage);
break;
}
}
static __init int test_atomic64(void)
{
long long v0 = 0xaaa31337c001d00dLL;
long long v1 = 0xdeadbeefdeafcafeLL;
long long v2 = 0xfaceabadf00df001LL;
long long onestwos = 0x1111111122222222LL;
long long one = 1LL;
atomic64_t v = ATOMIC64_INIT(v0);
long long r = v0;
BUG_ON(v.counter != r);
atomic64_set(&v, v1);
r = v1;
BUG_ON(v.counter != r);
BUG_ON(atomic64_read(&v) != r);
INIT(v0);
atomic64_add(onestwos, &v);
r += onestwos;
BUG_ON(v.counter != r);
INIT(v0);
atomic64_add(-one, &v);
r += -one;
BUG_ON(v.counter != r);
INIT(v0);
r += onestwos;
BUG_ON(atomic64_add_return(onestwos, &v) != r);
BUG_ON(v.counter != r);
INIT(v0);
r += -one;
BUG_ON(atomic64_add_return(-one, &v) != r);
BUG_ON(v.counter != r);
INIT(v0);
atomic64_sub(onestwos, &v);
r -= onestwos;
BUG_ON(v.counter != r);
INIT(v0);
atomic64_sub(-one, &v);
r -= -one;
BUG_ON(v.counter != r);
INIT(v0);
r -= onestwos;
BUG_ON(atomic64_sub_return(onestwos, &v) != r);
BUG_ON(v.counter != r);
INIT(v0);
r -= -one;
BUG_ON(atomic64_sub_return(-one, &v) != r);
BUG_ON(v.counter != r);
INIT(v0);
atomic64_inc(&v);
r += one;
BUG_ON(v.counter != r);
INIT(v0);
r += one;
BUG_ON(atomic64_inc_return(&v) != r);
BUG_ON(v.counter != r);
INIT(v0);
atomic64_dec(&v);
r -= one;
BUG_ON(v.counter != r);
INIT(v0);
r -= one;
BUG_ON(atomic64_dec_return(&v) != r);
BUG_ON(v.counter != r);
INIT(v0);
BUG_ON(atomic64_xchg(&v, v1) != v0);
r = v1;
BUG_ON(v.counter != r);
INIT(v0);
BUG_ON(atomic64_cmpxchg(&v, v0, v1) != v0);
r = v1;
BUG_ON(v.counter != r);
INIT(v0);
BUG_ON(atomic64_cmpxchg(&v, v2, v1) != v0);
BUG_ON(v.counter != r);
INIT(v0);
BUG_ON(atomic64_add_unless(&v, one, v0));
BUG_ON(v.counter != r);
INIT(v0);
BUG_ON(!atomic64_add_unless(&v, one, v1));
r += one;
BUG_ON(v.counter != r);
#ifdef CONFIG_ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
INIT(onestwos);
BUG_ON(atomic64_dec_if_positive(&v) != (onestwos - 1));
r -= one;
BUG_ON(v.counter != r);
INIT(0);
BUG_ON(atomic64_dec_if_positive(&v) != -one);
BUG_ON(v.counter != r);
INIT(-one);
BUG_ON(atomic64_dec_if_positive(&v) != (-one - one));
BUG_ON(v.counter != r);
#else
#warning Please implement atomic64_dec_if_positive for your architecture and select the above Kconfig symbol
#endif
INIT(onestwos);
BUG_ON(!atomic64_inc_not_zero(&v));
r += one;
BUG_ON(v.counter != r);
INIT(0);
BUG_ON(atomic64_inc_not_zero(&v));
BUG_ON(v.counter != r);
INIT(-one);
BUG_ON(!atomic64_inc_not_zero(&v));
r += one;
BUG_ON(v.counter != r);
#ifdef CONFIG_X86
pr_info("passed for %s platform %s CX8 and %s SSE\n",
#ifdef CONFIG_X86_64
"x86-64",
#elif defined(CONFIG_X86_CMPXCHG64)
"i586+",
#else
"i386+",
#endif
boot_cpu_has(X86_FEATURE_CX8) ? "with" : "without",
boot_cpu_has(X86_FEATURE_XMM) ? "with" : "without");
#else
pr_info("passed\n");
#endif
return 0;
}
static int
pscnv_vram_from_host(struct pscnv_chunk* sysram)
{
struct pscnv_bo *bo = sysram->bo;
struct drm_device *dev = bo->dev;
struct drm_nouveau_private *dev_priv = dev->dev_private;
struct pscnv_chunk vram; /* temporarily on stack */
struct pscnv_mm_node *primary_node = bo->primary_node;
struct pscnv_vspace *vs = NULL;
int res;
int flags = 0;
if ((bo->flags & PSCNV_GEM_MEMTYPE_MASK) == PSCNV_GEM_VRAM_LARGE) {
flags |= PSCNV_MM_LP;
}
if (!(bo->flags & PSCNV_GEM_CONTIG)) {
flags |= PSCNV_MM_FRAGOK;
}
if (!dev_priv->dma) {
pscnv_dma_init(dev);
}
if (!dev_priv->dma) {
NV_ERROR(dev, "pscnv_vram_to_host: no DMA available\n");
return -EINVAL;
}
if (pscnv_chunk_expect_alloc_type(sysram, PSCNV_CHUNK_SYSRAM, "pscnv_vram_from_host")) {
return -EINVAL;
}
WARN_ON(!(sysram->flags & PSCNV_CHUNK_SWAPPED));
if (!primary_node && pscnv_swapping_debug >= 1) {
NV_INFO(dev, "pscnv_swapping_replace: BO %08x/%d-%u has no "
"primary node attached, Strange.\n",
bo->cookie, bo->serial, vram.idx);
}
if (primary_node)
vs = primary_node->vspace;
memset(&vram, 0, sizeof(struct pscnv_chunk));
vram.bo = bo;
vram.idx = sysram->idx;
vram.flags = sysram->flags & ~(PSCNV_CHUNK_SWAPPED);
res = pscnv_vram_alloc_chunk(&vram, flags);
if (res) {
NV_ERROR(dev, "pscnv_vram_from_host: pscnv_vram_alloc_chunk "
"failed on %08x/%d-%u (fragmentation?)\n",
bo->cookie, bo->serial, vram.idx);
goto fail_vram_alloc;
}
res = pscnv_dma_chunk_to_chunk(sysram, &vram, PSCNV_DMA_ASYNC);
if (res) {
NV_INFO(dev, "pscnv_vram_from_host: failed to DMA- Transfer!\n");
goto fail_dma;
}
//pscnv_swapping_memdump(sysram);
/* this overwrites existing PTE */
if (vs) {
dev_priv->vm->do_unmap(vs,
primary_node->start + sysram->idx * dev_priv->chunk_size,
pscnv_chunk_size(sysram));
res = dev_priv->vm->do_map_chunk(vs, &vram,
primary_node->start + vram.idx * dev_priv->chunk_size);
if (res) {
NV_INFO(dev, "pscnv_vram_from_host: failed to replace mapping\n");
goto fail_map_chunk;
}
}
/* update vram_swapped value */
pscnv_sysram_free_chunk(sysram);
/* vram chunk is unallocated now, replace its values with the sysram
* chunk */
sysram->alloc_type = vram.alloc_type;
sysram->flags = vram.flags;
sysram->vram_node = vram.vram_node;
return 0;
fail_map_chunk:
/* reset PTEs to old value, just to be safe */
if (vs) {
dev_priv->vm->do_unmap(vs,
primary_node->start + vram.idx * dev_priv->chunk_size,
pscnv_chunk_size(&vram));
dev_priv->vm->do_map_chunk(vs, sysram,
primary_node->start + sysram->idx * dev_priv->chunk_size);
}
fail_dma:
pscnv_vram_free_chunk(&vram);
fail_vram_alloc:
if (sysram->bo->client)
atomic64_sub(pscnv_chunk_size(sysram), &sysram->bo->client->vram_demand);
return res;
}
static __init int test_atomic64(void)
{
long long v0 = 0xaaa31337c001d00dLL;
long long v1 = 0xdeadbeefdeafcafeLL;
long long v2 = 0xfaceabadf00df001LL;
long long onestwos = 0x1111111122222222LL;
long long one = 1LL;
atomic64_t v = ATOMIC64_INIT(v0);
long long r = v0;
BUG_ON(v.counter != r);
atomic64_set(&v, v1);
r = v1;
BUG_ON(v.counter != r);
BUG_ON(atomic64_read(&v) != r);
INIT(v0);
atomic64_add(onestwos, &v);
r += onestwos;
BUG_ON(v.counter != r);
INIT(v0);
atomic64_add(-one, &v);
r += -one;
BUG_ON(v.counter != r);
INIT(v0);
r += onestwos;
BUG_ON(atomic64_add_return(onestwos, &v) != r);
BUG_ON(v.counter != r);
INIT(v0);
r += -one;
BUG_ON(atomic64_add_return(-one, &v) != r);
BUG_ON(v.counter != r);
INIT(v0);
atomic64_sub(onestwos, &v);
r -= onestwos;
BUG_ON(v.counter != r);
INIT(v0);
atomic64_sub(-one, &v);
r -= -one;
BUG_ON(v.counter != r);
INIT(v0);
r -= onestwos;
BUG_ON(atomic64_sub_return(onestwos, &v) != r);
BUG_ON(v.counter != r);
INIT(v0);
r -= -one;
BUG_ON(atomic64_sub_return(-one, &v) != r);
BUG_ON(v.counter != r);
INIT(v0);
atomic64_inc(&v);
r += one;
BUG_ON(v.counter != r);
INIT(v0);
r += one;
BUG_ON(atomic64_inc_return(&v) != r);
BUG_ON(v.counter != r);
INIT(v0);
atomic64_dec(&v);
r -= one;
BUG_ON(v.counter != r);
INIT(v0);
r -= one;
BUG_ON(atomic64_dec_return(&v) != r);
BUG_ON(v.counter != r);
INIT(v0);
BUG_ON(atomic64_xchg(&v, v1) != v0);
r = v1;
BUG_ON(v.counter != r);
INIT(v0);
BUG_ON(atomic64_cmpxchg(&v, v0, v1) != v0);
r = v1;
BUG_ON(v.counter != r);
INIT(v0);
BUG_ON(atomic64_cmpxchg(&v, v2, v1) != v0);
BUG_ON(v.counter != r);
INIT(v0);
BUG_ON(atomic64_add_unless(&v, one, v0));
BUG_ON(v.counter != r);
INIT(v0);
BUG_ON(!atomic64_add_unless(&v, one, v1));
r += one;
BUG_ON(v.counter != r);
#if defined(CONFIG_X86) || defined(CONFIG_MIPS) || defined(CONFIG_PPC) || \
defined(CONFIG_S390) || defined(_ASM_GENERIC_ATOMIC64_H) || defined(CONFIG_ARM)
INIT(onestwos);
BUG_ON(atomic64_dec_if_positive(&v) != (onestwos - 1));
r -= one;
BUG_ON(v.counter != r);
INIT(0);
BUG_ON(atomic64_dec_if_positive(&v) != -one);
BUG_ON(v.counter != r);
INIT(-one);
BUG_ON(atomic64_dec_if_positive(&v) != (-one - one));
BUG_ON(v.counter != r);
#else
#warning Please implement atomic64_dec_if_positive for your architecture, and add it to the IF above
#endif
INIT(onestwos);
BUG_ON(!atomic64_inc_not_zero(&v));
r += one;
BUG_ON(v.counter != r);
INIT(0);
BUG_ON(atomic64_inc_not_zero(&v));
BUG_ON(v.counter != r);
INIT(-one);
BUG_ON(!atomic64_inc_not_zero(&v));
r += one;
BUG_ON(v.counter != r);
#ifdef CONFIG_X86
#ifdef CONFIG_DEBUG_PRINTK
printk(KERN_INFO "atomic64 test passed for %s platform %s CX8 and %s SSE\n",
#ifdef CONFIG_X86_64
"x86-64",
#elif defined(CONFIG_X86_CMPXCHG64)
"i586+",
#else
"i386+",
#endif
boot_cpu_has(X86_FEATURE_CX8) ? "with" : "without",
boot_cpu_has(X86_FEATURE_XMM) ? "with" : "without");
#else
;
#endif
#else
#ifdef CONFIG_DEBUG_PRINTK
printk(KERN_INFO "atomic64 test passed\n");
#else
;
#endif
#endif
return 0;
}
int sce_invalidate(sce_lunhndl_t lunhndl, sector_t sctrnum, uint32_t nr_sctr)
{
unsigned long flags;
lun_t *lun;
sce_t *sce;
frag_t *frag;
sector_t end;
sector_t start;
uint32_t count;
uint32_t fragnum;
fragdesc_t fdesc;
uint32_t pgcnt;
uint32_t pgoff;
int wh;
int ret;
ret = SCE_ERROR;
if (!lunhndl)
goto out;
lun = (lun_t *)lunhndl;
sce = (sce_t *)lun->scehndl;
if (!sce)
goto out;
spin_lock_irqsave(&lun->lock, flags);
if ((!lun->fragmap) || (!nr_sctr) ||
((sctrnum + nr_sctr) > lun->nr_sctr))
goto unlock_and_out;
if (_lun_is_serviceable(lun) != SCE_SUCCESS)
goto unlock_and_out;
wh = 0;
atomic64_inc(&lun->stats.writes);
atomic64_add(nr_sctr, &lun->stats.write_sctrs);
/* sector number to page number */
start = sctrnum / SCE_SCTRPERPAGE;
end = (sctrnum + nr_sctr - 1) / SCE_SCTRPERPAGE;
count = (uint32_t)(end - start) + 1;
/* calculate fragment id and page offset within a fragment */
fragnum = (uint32_t) (start / SCE_PAGEPERFRAG);
pgoff = (uint32_t) (start % SCE_PAGEPERFRAG);
/* visit fragments one by one */
for (; count > 0; fragnum++) {
/* number of pages to check:
adjust the number of pages to check considering fragment boundary */
pgcnt = SCE_PAGEPERFRAG - pgoff;
if (pgcnt > count)
pgcnt = count;
/* get fragment descriptor */
fdesc = lun->fragmap[fragnum];
if (fdesc & FRAGDESC_MAPPED) {
uint32_t n = 0;
frag = _pfid2frag(sce, fdesc, NULL);
if (!frag)
break;
wh++;
n = _frag_invalidate(frag, pgoff, pgcnt);
atomic64_sub(n * SCE_SCTRPERPAGE, &lun->stats.valid_sctrs);
if (fdesc & FRAGDESC_VALID) {
_lun_gc(lun, frag);
}
}
/* move to the next fragment */
pgoff = 0;
count -= pgcnt;
}
ASSERT(0 == count);
ret = SCE_SUCCESS;
if (0 != wh)
atomic64_inc(&lun->stats.write_hits);
unlock_and_out:
spin_unlock_irqrestore(&lun->lock, flags);
out:
return ret;
}
