starpu_mem_chunk_t _starpu_memchunk_cache_lookup(uint32_t node, starpu_data_handle handle) { uint32_t footprint = _starpu_compute_data_footprint(handle); pthread_rwlock_wrlock(&mc_rwlock[node]); /* go through all buffers in the cache */ starpu_mem_chunk_t mc; for (mc = starpu_mem_chunk_list_begin(memchunk_cache[node]); mc != starpu_mem_chunk_list_end(memchunk_cache[node]); mc = starpu_mem_chunk_list_next(mc)) { if (mc->footprint == footprint) { /* Is that a false hit ? (this is _very_ unlikely) */ if (_starpu_data_interface_compare(&handle->interface[node], handle->ops, mc->interface, mc->ops)) continue; /* Cache hit */ /* Remove from the cache */ starpu_mem_chunk_list_erase(memchunk_cache[node], mc); pthread_rwlock_unlock(&mc_rwlock[node]); return mc; } } /* This is a cache miss */ pthread_rwlock_unlock(&mc_rwlock[node]); return NULL; }
size_t _starpu_allocate_interface(starpu_data_handle handle, void *interface, uint32_t dst_node) { unsigned attempts = 0; size_t allocated_memory; _starpu_data_allocation_inc_stats(dst_node); #ifdef STARPU_USE_ALLOCATION_CACHE /* perhaps we can directly reuse a buffer in the free-list */ uint32_t footprint = _starpu_compute_data_footprint(handle); STARPU_TRACE_START_ALLOC_REUSE(dst_node); if (try_to_find_reusable_mem_chunk(dst_node, handle, footprint)) { _starpu_allocation_cache_hit(dst_node); return 0; } STARPU_TRACE_END_ALLOC_REUSE(dst_node); #endif do { STARPU_ASSERT(handle->ops); STARPU_ASSERT(handle->ops->allocate_data_on_node); STARPU_TRACE_START_ALLOC(dst_node); allocated_memory = handle->ops->allocate_data_on_node(interface, dst_node); STARPU_TRACE_END_ALLOC(dst_node); if (!allocated_memory) { /* XXX perhaps we should find the proper granularity * not to waste our cache all the time */ size_t data_size = _starpu_data_get_size(handle); STARPU_TRACE_START_MEMRECLAIM(dst_node); reclaim_memory(dst_node, 2*data_size); STARPU_TRACE_END_MEMRECLAIM(dst_node); } } while(!allocated_memory && attempts++ < 2); return allocated_memory; }
starpu_mem_chunk_t _starpu_memchunk_init(starpu_data_handle handle, size_t size, void *interface, size_t interface_size, unsigned automatically_allocated) { starpu_mem_chunk_t mc = starpu_mem_chunk_new(); STARPU_ASSERT(handle); STARPU_ASSERT(handle->ops); mc->data = handle; mc->size = size; mc->footprint = _starpu_compute_data_footprint(handle); mc->ops = handle->ops; mc->data_was_deleted = 0; mc->automatically_allocated = automatically_allocated; /* Save a copy of the interface */ mc->interface = malloc(interface_size); STARPU_ASSERT(mc->interface); memcpy(mc->interface, interface, interface_size); return mc; }
static void _starpu_data_partition(starpu_data_handle_t initial_handle, starpu_data_handle_t *childrenp, unsigned nparts, struct starpu_data_filter *f, int inherit_state) { unsigned i; unsigned node; /* first take care to properly lock the data header */ _starpu_spin_lock(&initial_handle->header_lock); initial_handle->nplans++; STARPU_ASSERT_MSG(nparts > 0, "Partitioning data %p in 0 piece does not make sense", initial_handle); /* allocate the children */ if (inherit_state) { initial_handle->children = (struct _starpu_data_state *) calloc(nparts, sizeof(struct _starpu_data_state)); STARPU_ASSERT(initial_handle->children); /* this handle now has children */ initial_handle->nchildren = nparts; } unsigned nworkers = starpu_worker_get_count(); for (node = 0; node < STARPU_MAXNODES; node++) { if (initial_handle->per_node[node].state != STARPU_INVALID) break; } if (node == STARPU_MAXNODES) { /* This is lazy allocation, allocate it now in main RAM, so as * to have somewhere to gather pieces later */ /* FIXME: mark as unevictable! */ int ret = _starpu_allocate_memory_on_node(initial_handle, &initial_handle->per_node[STARPU_MAIN_RAM], 0); #ifdef STARPU_DEVEL #warning we should reclaim memory if allocation failed #endif STARPU_ASSERT(!ret); } for (i = 0; i < nparts; i++) { starpu_data_handle_t child; if (inherit_state) child = &initial_handle->children[i]; else child = childrenp[i]; STARPU_ASSERT(child); struct starpu_data_interface_ops *ops; /* each child may have his own interface type */ /* what's this child's interface ? */ if (f->get_child_ops) ops = f->get_child_ops(f, i); else ops = initial_handle->ops; _starpu_data_handle_init(child, ops, initial_handle->mf_node); child->nchildren = 0; child->nplans = 0; child->switch_cl = NULL; child->partitioned = 0; child->readonly = 0; child->mpi_data = initial_handle->mpi_data; child->root_handle = initial_handle->root_handle; child->father_handle = initial_handle; child->sibling_index = i; child->depth = initial_handle->depth + 1; child->is_not_important = initial_handle->is_not_important; child->wt_mask = initial_handle->wt_mask; child->home_node = initial_handle->home_node; child->is_readonly = initial_handle->is_readonly; /* initialize the chunk lock */ _starpu_data_requester_list_init(&child->req_list); _starpu_data_requester_list_init(&child->reduction_req_list); child->reduction_tmp_handles = NULL; child->write_invalidation_req = NULL; child->refcnt = 0; child->unlocking_reqs = 0; child->busy_count = 0; child->busy_waiting = 0; STARPU_PTHREAD_MUTEX_INIT(&child->busy_mutex, NULL); STARPU_PTHREAD_COND_INIT(&child->busy_cond, NULL); child->reduction_refcnt = 0; _starpu_spin_init(&child->header_lock); child->sequential_consistency = initial_handle->sequential_consistency; STARPU_PTHREAD_MUTEX_INIT(&child->sequential_consistency_mutex, NULL); child->last_submitted_mode = STARPU_R; child->last_sync_task = NULL; child->last_submitted_accessors.task = NULL; child->last_submitted_accessors.next = &child->last_submitted_accessors; child->last_submitted_accessors.prev = &child->last_submitted_accessors; child->post_sync_tasks = NULL; /* Tell helgrind that the race in _starpu_unlock_post_sync_tasks is fine */ STARPU_HG_DISABLE_CHECKING(child->post_sync_tasks_cnt); child->post_sync_tasks_cnt = 0; /* The methods used for reduction are propagated to the * children. */ child->redux_cl = initial_handle->redux_cl; child->init_cl = initial_handle->init_cl; #ifdef STARPU_USE_FXT child->last_submitted_ghost_sync_id_is_valid = 0; child->last_submitted_ghost_sync_id = 0; child->last_submitted_ghost_accessors_id = NULL; #endif if (_starpu_global_arbiter) /* Just for testing purpose */ starpu_data_assign_arbiter(child, _starpu_global_arbiter); else child->arbiter = NULL; _starpu_data_requester_list_init(&child->arbitered_req_list); for (node = 0; node < STARPU_MAXNODES; node++) { struct _starpu_data_replicate *initial_replicate; struct _starpu_data_replicate *child_replicate; initial_replicate = &initial_handle->per_node[node]; child_replicate = &child->per_node[node]; if (inherit_state) child_replicate->state = initial_replicate->state; else child_replicate->state = STARPU_INVALID; if (inherit_state || !initial_replicate->automatically_allocated) child_replicate->allocated = initial_replicate->allocated; else child_replicate->allocated = 0; /* Do not allow memory reclaiming within the child for parent bits */ child_replicate->automatically_allocated = 0; child_replicate->refcnt = 0; child_replicate->memory_node = node; child_replicate->relaxed_coherency = 0; if (inherit_state) child_replicate->initialized = initial_replicate->initialized; else child_replicate->initialized = 0; /* update the interface */ void *initial_interface = starpu_data_get_interface_on_node(initial_handle, node); void *child_interface = starpu_data_get_interface_on_node(child, node); STARPU_ASSERT_MSG(!(!inherit_state && child_replicate->automatically_allocated && child_replicate->allocated), "partition planning is currently not supported when handle has some automatically allocated buffers"); f->filter_func(initial_interface, child_interface, f, i, nparts); } unsigned worker; for (worker = 0; worker < nworkers; worker++) { struct _starpu_data_replicate *child_replicate; child_replicate = &child->per_worker[worker]; child_replicate->state = STARPU_INVALID; child_replicate->allocated = 0; child_replicate->automatically_allocated = 0; child_replicate->refcnt = 0; child_replicate->memory_node = starpu_worker_get_memory_node(worker); child_replicate->requested = 0; for (node = 0; node < STARPU_MAXNODES; node++) { child_replicate->request[node] = NULL; } child_replicate->relaxed_coherency = 1; child_replicate->initialized = 0; /* duplicate the content of the interface on node 0 */ memcpy(child_replicate->data_interface, child->per_node[0].data_interface, child->ops->interface_size); } /* We compute the size and the footprint of the child once and * store it in the handle */ child->footprint = _starpu_compute_data_footprint(child); void *ptr; ptr = starpu_data_handle_to_pointer(child, STARPU_MAIN_RAM); if (ptr != NULL) _starpu_data_register_ram_pointer(child, ptr); } /* now let the header */ _starpu_spin_unlock(&initial_handle->header_lock); }