/* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */ boolean nouveau_buffer_migrate(struct nouveau_context *nv, struct nv04_resource *buf, const unsigned new_domain) { struct nouveau_screen *screen = nv->screen; struct nouveau_bo *bo; const unsigned old_domain = buf->domain; unsigned size = buf->base.width0; unsigned offset; int ret; assert(new_domain != old_domain); if (new_domain == NOUVEAU_BO_GART && old_domain == 0) { if (!nouveau_buffer_allocate(screen, buf, new_domain)) return FALSE; ret = nouveau_bo_map_range(buf->bo, buf->offset, size, NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC); if (ret) return ret; memcpy(buf->bo->map, buf->data, size); nouveau_bo_unmap(buf->bo); FREE(buf->data); } else if (old_domain != 0 && new_domain != 0) { struct nouveau_mm_allocation *mm = buf->mm; if (new_domain == NOUVEAU_BO_VRAM) { /* keep a system memory copy of our data in case we hit a fallback */ if (!nouveau_buffer_data_fetch(buf, buf->bo, buf->offset, size)) return FALSE; if (nouveau_mesa_debug) debug_printf("migrating %u KiB to VRAM\n", size / 1024); } offset = buf->offset; bo = buf->bo; buf->bo = NULL; buf->mm = NULL; nouveau_buffer_allocate(screen, buf, new_domain); nv->copy_data(nv, buf->bo, buf->offset, new_domain, bo, offset, old_domain, buf->base.width0); nouveau_bo_ref(NULL, &bo); if (mm) release_allocation(&mm, screen->fence.current); } else if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) { if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM)) return FALSE; if (!nouveau_buffer_upload(nv, buf, 0, buf->base.width0)) return FALSE; } else return FALSE; assert(buf->domain == new_domain); return TRUE; }
INLINE void nouveau_buffer_release_gpu_storage(struct nv04_resource *buf) { nouveau_bo_ref(NULL, &buf->bo); if (buf->mm) release_allocation(&buf->mm, buf->fence); buf->domain = 0; }
static INLINE void nouveau_buffer_transfer_del(struct nouveau_context *nv, struct nouveau_transfer *tx) { if (tx->map) { if (likely(tx->bo)) { nouveau_bo_ref(NULL, &tx->bo); if (tx->mm) release_allocation(&tx->mm, nv->screen->fence.current); } else { align_free(tx->map - (tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK)); } } }
INLINE void nouveau_buffer_release_gpu_storage(struct nv04_resource *buf) { nouveau_bo_ref(NULL, &buf->bo); if (buf->mm) release_allocation(&buf->mm, buf->fence); if (buf->domain == NOUVEAU_BO_VRAM) NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_vid, -(uint64_t)buf->base.width0); if (buf->domain == NOUVEAU_BO_GART) NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_sys, -(uint64_t)buf->base.width0); buf->domain = 0; }
inline void nouveau_buffer_release_gpu_storage(struct nv04_resource *buf) { if (buf->fence && buf->fence->state < NOUVEAU_FENCE_STATE_FLUSHED) { nouveau_fence_work(buf->fence, nouveau_fence_unref_bo, buf->bo); buf->bo = NULL; } else { nouveau_bo_ref(NULL, &buf->bo); } if (buf->mm) release_allocation(&buf->mm, buf->fence); if (buf->domain == NOUVEAU_BO_VRAM) NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_vid, -(uint64_t)buf->base.width0); if (buf->domain == NOUVEAU_BO_GART) NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_sys, -(uint64_t)buf->base.width0); buf->domain = 0; }
static boolean nouveau_buffer_upload(struct nouveau_context *nv, struct nv04_resource *buf, unsigned start, unsigned size) { struct nouveau_mm_allocation *mm; struct nouveau_bo *bounce = NULL; uint32_t offset; if (size <= 192) { if (buf->base.bind & PIPE_BIND_CONSTANT_BUFFER) nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0, start, size / 4, (const uint32_t *)(buf->data + start)); else nv->push_data(nv, buf->bo, buf->offset + start, buf->domain, size, buf->data + start); return TRUE; } mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset); if (!bounce) return FALSE; nouveau_bo_map_range(bounce, offset, size, NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC); memcpy(bounce->map, buf->data + start, size); nouveau_bo_unmap(bounce); nv->copy_data(nv, buf->bo, buf->offset + start, NOUVEAU_BO_VRAM, bounce, offset, NOUVEAU_BO_GART, size); nouveau_bo_ref(NULL, &bounce); if (mm) release_allocation(&mm, nv->screen->fence.current); if (start == 0 && size == buf->base.width0) buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING; return TRUE; }
/* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */ boolean nouveau_buffer_migrate(struct nouveau_context *nv, struct nv04_resource *buf, const unsigned new_domain) { struct nouveau_screen *screen = nv->screen; struct nouveau_bo *bo; const unsigned old_domain = buf->domain; unsigned size = buf->base.width0; unsigned offset; int ret; assert(new_domain != old_domain); if (new_domain == NOUVEAU_BO_GART && old_domain == 0) { if (!nouveau_buffer_allocate(screen, buf, new_domain)) return FALSE; ret = nouveau_bo_map(buf->bo, 0, nv->client); if (ret) return ret; memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size); align_free(buf->data); } else if (old_domain != 0 && new_domain != 0) { struct nouveau_mm_allocation *mm = buf->mm; if (new_domain == NOUVEAU_BO_VRAM) { /* keep a system memory copy of our data in case we hit a fallback */ if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size)) return FALSE; if (nouveau_mesa_debug) debug_printf("migrating %u KiB to VRAM\n", size / 1024); } offset = buf->offset; bo = buf->bo; buf->bo = NULL; buf->mm = NULL; nouveau_buffer_allocate(screen, buf, new_domain); nv->copy_data(nv, buf->bo, buf->offset, new_domain, bo, offset, old_domain, buf->base.width0); nouveau_bo_ref(NULL, &bo); if (mm) release_allocation(&mm, screen->fence.current); } else if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) { struct nouveau_transfer tx; if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM)) return FALSE; tx.base.resource = &buf->base; tx.base.box.x = 0; tx.base.box.width = buf->base.width0; tx.bo = NULL; if (!nouveau_transfer_staging(nv, &tx, FALSE)) return FALSE; nouveau_transfer_write(nv, &tx, 0, tx.base.box.width); nouveau_buffer_transfer_del(nv, &tx); } else return FALSE; assert(buf->domain == new_domain); return TRUE; }