bool os_wait_until_zero(volatile int *var, uint64_t timeout) { if (!p_atomic_read(var)) return true; if (!timeout) return false; if (timeout == PIPE_TIMEOUT_INFINITE) { while (p_atomic_read(var)) { #if defined(PIPE_OS_UNIX) sched_yield(); #endif } return true; } else { int64_t start_time = os_time_get_nano(); int64_t end_time = start_time + timeout; while (p_atomic_read(var)) { if (os_time_timeout(start_time, end_time, os_time_get_nano())) return false; #if defined(PIPE_OS_UNIX) sched_yield(); #endif } return true; } }
/** * End the current occlusion query. * This is a bin command put in all bins. * Called per thread. */ static void lp_rast_end_query(struct lp_rasterizer_task *task, const union lp_rast_cmd_arg arg) { struct llvmpipe_query *pq = arg.query_obj; assert(task->query[pq->type] == pq || pq->type == PIPE_QUERY_TIMESTAMP); switch (pq->type) { case PIPE_QUERY_OCCLUSION_COUNTER: pq->count[task->thread_index] += task->thread_data.vis_counter; break; case PIPE_QUERY_TIMESTAMP: pq->count[task->thread_index] = os_time_get_nano(); break; case PIPE_QUERY_PRIMITIVES_GENERATED: case PIPE_QUERY_PRIMITIVES_EMITTED: case PIPE_QUERY_SO_STATISTICS: case PIPE_QUERY_PIPELINE_STATISTICS: case PIPE_QUERY_SO_OVERFLOW_PREDICATE: break; default: assert(0); break; } if (task->query[pq->type] == pq) { task->query[pq->type] = NULL; } }
/** * End the current occlusion query. * This is a bin command put in all bins. * Called per thread. */ static void lp_rast_end_query(struct lp_rasterizer_task *task, const union lp_rast_cmd_arg arg) { struct llvmpipe_query *pq = arg.query_obj; switch (pq->type) { case PIPE_QUERY_OCCLUSION_COUNTER: case PIPE_QUERY_OCCLUSION_PREDICATE: pq->end[task->thread_index] += task->thread_data.vis_counter - pq->start[task->thread_index]; pq->start[task->thread_index] = 0; break; case PIPE_QUERY_TIMESTAMP: pq->end[task->thread_index] = os_time_get_nano(); break; case PIPE_QUERY_PIPELINE_STATISTICS: pq->end[task->thread_index] += task->ps_invocations - pq->start[task->thread_index]; pq->start[task->thread_index] = 0; break; default: assert(0); break; } }
static boolean r600_fence_finish(struct pipe_screen *screen, struct pipe_context *ctx, struct pipe_fence_handle *fence, uint64_t timeout) { struct radeon_winsys *rws = ((struct r600_common_screen*)screen)->ws; struct r600_multi_fence *rfence = (struct r600_multi_fence *)fence; struct r600_common_context *rctx; int64_t abs_timeout = os_time_get_absolute_timeout(timeout); ctx = threaded_context_unwrap_sync(ctx); rctx = ctx ? (struct r600_common_context*)ctx : NULL; if (rfence->sdma) { if (!rws->fence_wait(rws, rfence->sdma, timeout)) return false; /* Recompute the timeout after waiting. */ if (timeout && timeout != PIPE_TIMEOUT_INFINITE) { int64_t time = os_time_get_nano(); timeout = abs_timeout > time ? abs_timeout - time : 0; } } if (!rfence->gfx) return true; /* Flush the gfx IB if it hasn't been flushed yet. */ if (rctx && rfence->gfx_unflushed.ctx == rctx && rfence->gfx_unflushed.ib_index == rctx->num_gfx_cs_flushes) { rctx->gfx.flush(rctx, timeout ? 0 : RADEON_FLUSH_ASYNC, NULL); rfence->gfx_unflushed.ctx = NULL; if (!timeout) return false; /* Recompute the timeout after all that. */ if (timeout && timeout != PIPE_TIMEOUT_INFINITE) { int64_t time = os_time_get_nano(); timeout = abs_timeout > time ? abs_timeout - time : 0; } } return rws->fence_wait(rws, rfence->gfx, timeout); }
static boolean llvmpipe_get_query_result(struct pipe_context *pipe, struct pipe_query *q, boolean wait, union pipe_query_result *vresult) { struct llvmpipe_query *pq = llvmpipe_query(q); uint64_t *result = (uint64_t *)vresult; int i; if (!pq->fence) { /* no fence because there was no scene, so results is zero */ *result = 0; return TRUE; } if (!lp_fence_signalled(pq->fence)) { if (!lp_fence_issued(pq->fence)) llvmpipe_flush(pipe, NULL, __FUNCTION__); if (!wait) return FALSE; lp_fence_wait(pq->fence); } /* Sum the results from each of the threads: */ *result = 0; switch (pq->type) { case PIPE_QUERY_OCCLUSION_COUNTER: for (i = 0; i < LP_MAX_THREADS; i++) { *result += pq->count[i]; } break; case PIPE_QUERY_TIMESTAMP: for (i = 0; i < LP_MAX_THREADS; i++) { if (pq->count[i] > *result) { *result = pq->count[i]; } if (*result == 0) *result = os_time_get_nano(); } break; case PIPE_QUERY_PRIMITIVES_GENERATED: *result = pq->num_primitives_generated; break; case PIPE_QUERY_PRIMITIVES_EMITTED: *result = pq->num_primitives_written; break; default: assert(0); break; } return TRUE; }
static boolean softpipe_begin_query(struct pipe_context *pipe, struct pipe_query *q) { struct softpipe_context *softpipe = softpipe_context( pipe ); struct softpipe_query *sq = softpipe_query(q); switch (sq->type) { case PIPE_QUERY_OCCLUSION_COUNTER: case PIPE_QUERY_OCCLUSION_PREDICATE: sq->start = softpipe->occlusion_count; break; case PIPE_QUERY_TIME_ELAPSED: sq->start = os_time_get_nano(); break; case PIPE_QUERY_SO_STATISTICS: sq->so.num_primitives_written = softpipe->so_stats.num_primitives_written; sq->so.primitives_storage_needed = softpipe->so_stats.primitives_storage_needed; break; case PIPE_QUERY_SO_OVERFLOW_PREDICATE: sq->end = FALSE; break; case PIPE_QUERY_PRIMITIVES_EMITTED: sq->so.num_primitives_written = softpipe->so_stats.num_primitives_written; break; case PIPE_QUERY_PRIMITIVES_GENERATED: sq->so.primitives_storage_needed = softpipe->so_stats.primitives_storage_needed; break; case PIPE_QUERY_TIMESTAMP: case PIPE_QUERY_GPU_FINISHED: case PIPE_QUERY_TIMESTAMP_DISJOINT: break; case PIPE_QUERY_PIPELINE_STATISTICS: /* reset our cache */ if (softpipe->active_statistics_queries == 0) { memset(&softpipe->pipeline_statistics, 0, sizeof(softpipe->pipeline_statistics)); } memcpy(&sq->stats, &softpipe->pipeline_statistics, sizeof(sq->stats)); softpipe->active_statistics_queries++; break; default: assert(0); break; } softpipe->active_query_count++; softpipe->dirty |= SP_NEW_QUERY; return true; }
int64_t os_time_get_absolute_timeout(uint64_t timeout) { int64_t time, abs_timeout; /* Also check for the type upper bound. */ if (timeout == PIPE_TIMEOUT_INFINITE || timeout > INT64_MAX) return PIPE_TIMEOUT_INFINITE; time = os_time_get_nano(); abs_timeout = time + (int64_t)timeout; /* Check for overflow. */ if (abs_timeout < time) return PIPE_TIMEOUT_INFINITE; return abs_timeout; }
bool os_wait_until_zero_abs_timeout(volatile int *var, int64_t timeout) { if (!p_atomic_read(var)) return true; if (timeout == PIPE_TIMEOUT_INFINITE) return os_wait_until_zero(var, PIPE_TIMEOUT_INFINITE); while (p_atomic_read(var)) { if (os_time_get_nano() >= timeout) return false; #if defined(PIPE_OS_UNIX) sched_yield(); #endif } return true; }
/** * This is called just before issuing the buffer swap/present. * We query the current time and determine if we should sleep before * issuing the swap/present. * This is a bit of a hack and is certainly not very accurate but it * basically works. * This is for the WGL_ARB_swap_interval extension. */ static void wait_swap_interval(struct stw_framebuffer *fb) { /* Note: all time variables here are in units of microseconds */ int64_t cur_time = os_time_get_nano() / 1000; if (fb->prev_swap_time != 0) { /* Compute time since previous swap */ int64_t delta = cur_time - fb->prev_swap_time; int64_t min_swap_period = 1.0e6 / stw_dev->refresh_rate * stw_dev->swap_interval; /* if time since last swap is less than wait period, wait */ if (delta < min_swap_period) { float fudge = 1.75f; /* emperical fudge factor */ int64_t wait = (min_swap_period - delta) * fudge; os_time_sleep(wait); } } fb->prev_swap_time = cur_time; }
static boolean r600_fence_finish(struct pipe_screen *screen, struct pipe_fence_handle *fence, uint64_t timeout) { struct radeon_winsys *rws = ((struct r600_common_screen*)screen)->ws; struct r600_multi_fence *rfence = (struct r600_multi_fence *)fence; int64_t abs_timeout = os_time_get_absolute_timeout(timeout); if (rfence->sdma) { if (!rws->fence_wait(rws, rfence->sdma, timeout)) return false; /* Recompute the timeout after waiting. */ if (timeout && timeout != PIPE_TIMEOUT_INFINITE) { int64_t time = os_time_get_nano(); timeout = abs_timeout > time ? abs_timeout - time : 0; } } if (!rfence->gfx) return true; return rws->fence_wait(rws, rfence->gfx, timeout); }
/** * Put an EndQuery command into all bins. */ void lp_setup_end_query(struct lp_setup_context *setup, struct llvmpipe_query *pq) { set_scene_state(setup, SETUP_ACTIVE, "end_query"); assert(setup->scene); if (setup->scene) { /* pq->fence should be the fence of the *last* scene which * contributed to the query result. */ lp_fence_reference(&pq->fence, setup->scene->fence); if (pq->type == PIPE_QUERY_OCCLUSION_COUNTER || pq->type == PIPE_QUERY_OCCLUSION_PREDICATE || pq->type == PIPE_QUERY_PIPELINE_STATISTICS || pq->type == PIPE_QUERY_TIMESTAMP) { if (pq->type == PIPE_QUERY_TIMESTAMP && !(setup->scene->tiles_x | setup->scene->tiles_y)) { /* * If there's a zero width/height framebuffer, there's no bins and * hence no rast task is ever run. So fill in something here instead. */ pq->end[0] = os_time_get_nano(); } if (!lp_scene_bin_everywhere(setup->scene, LP_RAST_OP_END_QUERY, lp_rast_arg_query(pq))) { if (!lp_setup_flush_and_restart(setup)) goto fail; if (!lp_scene_bin_everywhere(setup->scene, LP_RAST_OP_END_QUERY, lp_rast_arg_query(pq))) { goto fail; } } setup->scene->had_queries |= TRUE; } } else { lp_fence_reference(&pq->fence, setup->last_fence); } fail: /* Need to do this now not earlier since it still needs to be marked as * active when binning it would cause a flush. */ if (pq->type == PIPE_QUERY_OCCLUSION_COUNTER || pq->type == PIPE_QUERY_OCCLUSION_PREDICATE || pq->type == PIPE_QUERY_PIPELINE_STATISTICS) { unsigned i; /* remove from active binned query list */ for (i = 0; i < setup->active_binned_queries; i++) { if (setup->active_queries[i] == pq) break; } assert(i < setup->active_binned_queries); if (i == setup->active_binned_queries) return; setup->active_binned_queries--; setup->active_queries[i] = setup->active_queries[setup->active_binned_queries]; setup->active_queries[setup->active_binned_queries] = NULL; } }
static uint64_t virgl_get_timestamp(struct pipe_screen *_screen) { return os_time_get_nano(); }
static uint64_t etna_screen_get_timestamp(struct pipe_screen *pscreen) { return os_time_get_nano(); }
static void softpipe_end_query(struct pipe_context *pipe, struct pipe_query *q) { struct softpipe_context *softpipe = softpipe_context( pipe ); struct softpipe_query *sq = softpipe_query(q); softpipe->active_query_count--; switch (sq->type) { case PIPE_QUERY_OCCLUSION_COUNTER: case PIPE_QUERY_OCCLUSION_PREDICATE: sq->end = softpipe->occlusion_count; break; case PIPE_QUERY_TIMESTAMP: sq->start = 0; /* fall through */ case PIPE_QUERY_TIME_ELAPSED: sq->end = os_time_get_nano(); break; case PIPE_QUERY_SO_OVERFLOW_PREDICATE: sq->so.num_primitives_written = softpipe->so_stats.num_primitives_written - sq->so.num_primitives_written; sq->so.primitives_storage_needed = softpipe->so_stats.primitives_storage_needed - sq->so.primitives_storage_needed; sq->end = sq->so.primitives_storage_needed > sq->so.num_primitives_written; break; case PIPE_QUERY_SO_STATISTICS: sq->so.num_primitives_written = softpipe->so_stats.num_primitives_written - sq->so.num_primitives_written; sq->so.primitives_storage_needed = softpipe->so_stats.primitives_storage_needed - sq->so.primitives_storage_needed; break; case PIPE_QUERY_PRIMITIVES_EMITTED: sq->so.num_primitives_written = softpipe->so_stats.num_primitives_written - sq->so.num_primitives_written; break; case PIPE_QUERY_PRIMITIVES_GENERATED: sq->so.primitives_storage_needed = softpipe->so_stats.primitives_storage_needed - sq->so.primitives_storage_needed; break; case PIPE_QUERY_GPU_FINISHED: case PIPE_QUERY_TIMESTAMP_DISJOINT: break; case PIPE_QUERY_PIPELINE_STATISTICS: sq->stats.ia_vertices = softpipe->pipeline_statistics.ia_vertices - sq->stats.ia_vertices; sq->stats.ia_primitives = softpipe->pipeline_statistics.ia_primitives - sq->stats.ia_primitives; sq->stats.vs_invocations = softpipe->pipeline_statistics.vs_invocations - sq->stats.vs_invocations; sq->stats.gs_invocations = softpipe->pipeline_statistics.gs_invocations - sq->stats.gs_invocations; sq->stats.gs_primitives = softpipe->pipeline_statistics.gs_primitives - sq->stats.gs_primitives; sq->stats.c_invocations = softpipe->pipeline_statistics.c_invocations - sq->stats.c_invocations; sq->stats.c_primitives = softpipe->pipeline_statistics.c_primitives - sq->stats.c_primitives; sq->stats.ps_invocations = softpipe->pipeline_statistics.ps_invocations - sq->stats.ps_invocations; softpipe->active_statistics_queries--; break; default: assert(0); break; } softpipe->dirty |= SP_NEW_QUERY; }
static void *amdgpu_bo_map(struct radeon_winsys_cs_handle *buf, struct radeon_winsys_cs *rcs, enum pipe_transfer_usage usage) { struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf; struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs; int r; void *cpu = NULL; /* If it's not unsynchronized bo_map, flush CS if needed and then wait. */ if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) { /* DONTBLOCK doesn't make sense with UNSYNCHRONIZED. */ if (usage & PIPE_TRANSFER_DONTBLOCK) { if (!(usage & PIPE_TRANSFER_WRITE)) { /* Mapping for read. * * Since we are mapping for read, we don't need to wait * if the GPU is using the buffer for read too * (neither one is changing it). * * Only check whether the buffer is being used for write. */ if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo, RADEON_USAGE_WRITE)) { cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL); return NULL; } if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0, RADEON_USAGE_WRITE)) { return NULL; } } else { if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo)) { cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL); return NULL; } if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0, RADEON_USAGE_READWRITE)) { return NULL; } } } else { uint64_t time = os_time_get_nano(); if (!(usage & PIPE_TRANSFER_WRITE)) { /* Mapping for read. * * Since we are mapping for read, we don't need to wait * if the GPU is using the buffer for read too * (neither one is changing it). * * Only check whether the buffer is being used for write. */ if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo, RADEON_USAGE_WRITE)) { cs->flush_cs(cs->flush_data, 0, NULL); } amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE, RADEON_USAGE_WRITE); } else { /* Mapping for write. */ if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo)) cs->flush_cs(cs->flush_data, 0, NULL); amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE, RADEON_USAGE_READWRITE); } bo->rws->buffer_wait_time += os_time_get_nano() - time; } } /* If the buffer is created from user memory, return the user pointer. */ if (bo->user_ptr) return bo->user_ptr; r = amdgpu_bo_cpu_map(bo->bo, &cpu); return r ? NULL : cpu; }