static int prep_more_ios(struct submitter *s, int max_ios) { struct io_sq_ring *ring = &s->sq_ring; unsigned index, tail, next_tail, prepped = 0; next_tail = tail = *ring->tail; do { next_tail++; read_barrier(); if (next_tail == *ring->head) break; index = tail & sq_ring_mask; init_io(s, index); ring->array[index] = index; prepped++; tail = next_tail; } while (prepped < max_ios); if (*ring->tail != tail) { /* order tail store with writes to sqes above */ write_barrier(); *ring->tail = tail; write_barrier(); } return prepped; }
static void fio_gtod_update(void) { if (fio_tv) { struct timeval __tv; gettimeofday(&__tv, NULL); fio_tv->tv_sec = __tv.tv_sec; write_barrier(); fio_tv->tv_usec = __tv.tv_usec; write_barrier(); } }
/* ----------------------------------------------------------------------------- Evaluate a THUNK_SELECTOR if possible. p points to a THUNK_SELECTOR that we want to evaluate. The result of "evaluating" it will be evacuated and a pointer to the to-space closure will be returned. If the THUNK_SELECTOR could not be evaluated (its selectee is still a THUNK, for example), then the THUNK_SELECTOR itself will be evacuated. -------------------------------------------------------------------------- */ static void unchain_thunk_selectors(StgSelector *p, StgClosure *val) { StgSelector *prev; prev = NULL; while (p) { ASSERT(p->header.info == &stg_WHITEHOLE_info); // val must be in to-space. Not always: when we recursively // invoke eval_thunk_selector(), the recursive calls will not // evacuate the value (because we want to select on the value, // not evacuate it), so in this case val is in from-space. // ASSERT(!HEAP_ALLOCED_GC(val) || Bdescr((P_)val)->gen_no > N || (Bdescr((P_)val)->flags & BF_EVACUATED)); prev = (StgSelector*)((StgClosure *)p)->payload[0]; // Update the THUNK_SELECTOR with an indirection to the // value. The value is still in from-space at this stage. // // (old note: Why not do upd_evacuee(q,p)? Because we have an // invariant that an EVACUATED closure always points to an // object in the same or an older generation (required by // the short-cut test in the EVACUATED case, below). if ((StgClosure *)p == val) { // must be a loop; just leave a BLACKHOLE in place. This // can happen when we have a chain of selectors that // eventually loops back on itself. We can't leave an // indirection pointing to itself, and we want the program // to deadlock if it ever enters this closure, so // BLACKHOLE is correct. // XXX we do not have BLACKHOLEs any more; replace with // a THUNK_SELECTOR again. This will go into a loop if it is // entered, and should result in a NonTermination exception. ((StgThunk *)p)->payload[0] = val; write_barrier(); SET_INFO((StgClosure *)p, &stg_sel_0_upd_info); } else { ((StgInd *)p)->indirectee = val; write_barrier(); SET_INFO((StgClosure *)p, &stg_IND_info); } // For the purposes of LDV profiling, we have created an // indirection. LDV_RECORD_CREATE(p); p = prev; } }
/* * It is up to the caller to fill in the object's fields in a meaningful * fashion! */ object *factor_vm::allot_large_object(cell type, cell size) { /* If tenured space does not have enough room, collect and compact */ if(!data->tenured->can_allot_p(size)) { primitive_compact_gc(); /* If it still won't fit, grow the heap */ if(!data->tenured->can_allot_p(size)) { gc(collect_growing_heap_op, size, /* requested size */ true /* trace contexts? */); } } object *obj = data->tenured->allot(size); /* Allows initialization code to store old->new pointers without hitting the write barrier in the common case of a nursery allocation */ write_barrier(obj,size); obj->initialize(type); return obj; }
Object* Tuple::put(STATE, native_int idx, Object* val) { assert(idx >= 0 && idx < num_fields()); this->field[idx] = val; if(val->reference_p()) write_barrier(state, val); return val; }
STATIC_INLINE void copy_tag_nolock(StgClosure **p, const StgInfoTable *info, StgClosure *src, nat size, nat gen_no, StgWord tag) { StgPtr to, from; nat i; to = alloc_for_copy(size,gen_no); from = (StgPtr)src; to[0] = (W_)info; for (i = 1; i < size; i++) { // unroll for small i to[i] = from[i]; } // if somebody else reads the forwarding pointer, we better make // sure there's a closure at the end of it. write_barrier(); *p = TAG_CLOSURE(tag,(StgClosure*)to); src->header.info = (const StgInfoTable *)MK_FORWARDING_PTR(to); // if (to+size+2 < bd->start + BLOCK_SIZE_W) { // __builtin_prefetch(to + size + 2, 1); // } #ifdef PROFILING // We store the size of the just evacuated object in the LDV word so that // the profiler can guess the position of the next object later. SET_EVACUAEE_FOR_LDV(from, size); #endif }
/* Allocates memory */ string* factor_vm::reallot_string(string* str_, cell capacity) { data_root<string> str(str_, this); if (reallot_string_in_place_p(str.untagged(), capacity)) { str->length = tag_fixnum(capacity); if (to_boolean(str->aux)) { byte_array* aux = untag<byte_array>(str->aux); aux->capacity = tag_fixnum(capacity * 2); } return str.untagged(); } else { cell to_copy = string_capacity(str.untagged()); if (capacity < to_copy) to_copy = capacity; data_root<string> new_str(allot_string_internal(capacity), this); memcpy(new_str->data(), str->data(), to_copy); if (to_boolean(str->aux)) { byte_array* new_aux = allot_uninitialized_array<byte_array>(capacity * 2); new_str->aux = tag<byte_array>(new_aux); write_barrier(&new_str->aux); byte_array* aux = untag<byte_array>(str->aux); memcpy(new_aux->data<uint16_t>(), aux->data<uint16_t>(), to_copy * sizeof(uint16_t)); } fill_string(new_str.untagged(), to_copy, capacity, '\0'); return new_str.untagged(); } }
void fiber_manager_yield(fiber_manager_t* manager) { assert(fiber_manager_state == FIBER_MANAGER_STATE_STARTED); assert(manager); if(wsd_work_stealing_deque_size(manager->schedule_from) == 0) { wsd_work_stealing_deque_t* const temp = manager->schedule_from; manager->schedule_from = manager->store_to; manager->store_to = temp; } do { manager->yield_count += 1; //occasionally steal some work from threads with more load if((manager->yield_count & 1023) == 0) { fiber_load_balance(manager); } if(wsd_work_stealing_deque_size(manager->schedule_from) > 0) { fiber_t* const new_fiber = (fiber_t*)wsd_work_stealing_deque_pop_bottom(manager->schedule_from); if(new_fiber != WSD_EMPTY && new_fiber != WSD_ABORT) { fiber_t* const old_fiber = manager->current_fiber; if(old_fiber->state == FIBER_STATE_RUNNING) { old_fiber->state = FIBER_STATE_READY; manager->to_schedule = old_fiber;/* must schedule it *after* fiber_swap_context, else another thread can start executing an invalid context */ } manager->current_fiber = new_fiber; new_fiber->state = FIBER_STATE_RUNNING; write_barrier(); fiber_swap_context(&old_fiber->context, &new_fiber->context); fiber_manager_do_maintenance(); } } } while((manager = fiber_manager_get()) && FIBER_STATE_WAITING == manager->current_fiber->state && fiber_load_balance(manager)); }
void factor_vm::set_string_nth_slow(string *str_, cell index, cell ch) { data_root<string> str(str_,this); byte_array *aux; str->data()[index] = ((ch & 0x7f) | 0x80); if(to_boolean(str->aux)) aux = untag<byte_array>(str->aux); else { /* We don't need to pre-initialize the byte array with any data, since we only ever read from the aux vector if the most significant bit of a character is set. Initially all of the bits are clear. */ aux = allot_uninitialized_array<byte_array>(untag_fixnum(str->length) * sizeof(u16)); str->aux = tag<byte_array>(aux); write_barrier(&str->aux); } aux->data<u16>()[index] = (u16)((ch >> 7) ^ 1); }
void set_local(STATE, int pos, Object* val) { locals_[pos] = val; if(locals_ == heap_locals_) { write_barrier(state, val); } }
inline void factor_vm::set_array_nth(array* array, cell slot, cell value) { FACTOR_ASSERT(slot < array_capacity(array)); FACTOR_ASSERT(array->type() == ARRAY_TYPE); cell* slot_ptr = &array->data()[slot]; *slot_ptr = value; write_barrier(slot_ptr); }
void set_local(STATE, int pos, Object* val) { locals_[pos] = val; if(!stack_allocated_p()) { write_barrier(state, val); } }
Object* Regexp::match_region(STATE, String* string, Fixnum* start, Fixnum* end, Object* forward) { int beg, max; const UChar *str; OnigRegion *region; Object* md; if(unlikely(!onig_data)) { Exception::argument_error(state, "Not properly initialized Regexp"); } maybe_recompile(state); region = onig_region_new(); max = string->size(); str = (UChar*)string->c_str(state); int* back_match = onig_data->int_map_backward; if(!RTEST(forward)) { beg = onig_search(onig_data, str, str + max, str + end->to_native(), str + start->to_native(), region, ONIG_OPTION_NONE); } else { beg = onig_search(onig_data, str, str + max, str + start->to_native(), str + end->to_native(), region, ONIG_OPTION_NONE); } // Seems like onig must setup int_map_backward lazily, so we have to watch // for it to appear here. if(onig_data->int_map_backward != back_match) { native_int size = sizeof(int) * ONIG_CHAR_TABLE_SIZE; ByteArray* ba = ByteArray::create(state, size); memcpy(ba->raw_bytes(), onig_data->int_map_backward, size); // Dispose of the old one. free(onig_data->int_map_backward); onig_data->int_map_backward = reinterpret_cast<int*>(ba->raw_bytes()); write_barrier(state, ba); } if(beg == ONIG_MISMATCH) { onig_region_free(region, 1); return Qnil; } md = get_match_data(state, region, string, this, 0); onig_region_free(region, 1); return md; }
void factor_vm::primitive_set_slot() { fixnum slot = untag_fixnum(ctx->pop()); object* obj = untag<object>(ctx->pop()); cell value = ctx->pop(); cell* slot_ptr = &obj->slots()[slot]; *slot_ptr = value; write_barrier(slot_ptr); }
Object* Tuple::put(STATE, native_int idx, Object* val) { if(idx < 0 || idx >= num_fields()) { rubinius::bug("Invalid tuple index"); } field[idx] = val; write_barrier(state, val); return val; }
void VMMethod::SetCachedFrame(VMFrame* frame) { cachedFrame = frame; if (frame != nullptr) { frame->SetContext(nullptr); frame->SetBytecodeIndex(0); frame->ResetStackPointer(); write_barrier(this, cachedFrame); } }
void Encoding::make_managed(STATE, const char* name, OnigEncodingType* enc) { ByteArray* enc_ba = ByteArray::create(state, sizeof(OnigEncodingType)); memcpy(enc_ba->raw_bytes(), enc, sizeof(OnigEncodingType)); encoding_ = reinterpret_cast<OnigEncodingType*>(enc_ba->raw_bytes()); write_barrier(state, enc_ba); int size = strlen(name); if(size >= ENCODING_NAMELEN_MAX) size = ENCODING_NAMELEN_MAX-1; ByteArray* name_ba = ByteArray::create(state, size); memcpy(name_ba->raw_bytes(), name, size); name_ba->raw_bytes()[size] = 0; encoding_->name = reinterpret_cast<const char*>(name_ba->raw_bytes()); write_barrier(state, name_ba); managed_ = true; }
Object* Tuple::put(STATE, size_t idx, Object* val) { if(num_fields() <= idx) { Exception::object_bounds_exceeded_error(state, this, idx); } this->field[idx] = val; if(val->reference_p()) write_barrier(state, val); return val; }
/** Add endpoint to the list and queue. * * @param[in] instance List to use. * @param[in] endpoint Endpoint to add. * * The endpoint is added to the end of the list and queue. */ void endpoint_list_add_ep(endpoint_list_t *instance, ohci_endpoint_t *ep) { assert(instance); assert(ep); usb_log_debug2("Queue %s: Adding endpoint(%p).\n", instance->name, ep); fibril_mutex_lock(&instance->guard); ed_t *last_ed = NULL; /* Add to the hardware queue. */ if (list_empty(&instance->endpoint_list)) { /* There are no active EDs */ last_ed = instance->list_head; } else { /* There are active EDs, get the last one */ ohci_endpoint_t *last = list_get_instance( list_last(&instance->endpoint_list), ohci_endpoint_t, link); last_ed = last->ed; } /* Keep link */ ep->ed->next = last_ed->next; /* Make sure ED is written to the memory */ write_barrier(); /* Add ed to the hw queue */ ed_append_ed(last_ed, ep->ed); /* Make sure ED is updated */ write_barrier(); /* Add to the sw list */ list_append(&ep->link, &instance->endpoint_list); ohci_endpoint_t *first = list_get_instance( list_first(&instance->endpoint_list), ohci_endpoint_t, link); usb_log_debug("HCD EP(%p) added to list %s, first is %p(%p).\n", ep, instance->name, first, first->ed); if (last_ed == instance->list_head) { usb_log_debug2("%s head ED(%p-0x%0" PRIx32 "): %x:%x:%x:%x.\n", instance->name, last_ed, instance->list_head_pa, last_ed->status, last_ed->td_tail, last_ed->td_head, last_ed->next); } fibril_mutex_unlock(&instance->guard); }
//arrays.hpp inline void factorvm::set_array_nth(array *array, cell slot, cell value) { #ifdef FACTOR_DEBUG assert(slot < array_capacity(array)); assert(array->h.hi_tag() == ARRAY_TYPE); check_tagged_pointer(value); #endif array->data()[slot] = value; write_barrier(array); }
/* The Tuple#put primitive. */ Object* Tuple::put_prim(STATE, Fixnum* index, Object* val) { native_int idx = index->to_native(); if(idx < 0 || num_fields() <= idx) { return bounds_exceeded_error(state, "Tuple::put_prim", idx); } field[idx] = val; write_barrier(state, val); return val; }
Object* PackedObject::set_packed_ivar(STATE, Symbol* sym, Object* val) { LookupTable* tbl = this->reference_class()->packed_ivar_info(); bool found = false; Fixnum* which = try_as<Fixnum>(tbl->fetch(state, sym, &found)); if(!found) { return set_table_ivar(state, sym, val); } body_as_array()[which->to_native()] = val; if(val->reference_p()) write_barrier(state, val); return val; }
int fiber_rwlock_init(fiber_rwlock_t* rwlock) { assert(rwlock); if(!mpsc_fifo_init(&rwlock->write_waiters) || !mpsc_fifo_init(&rwlock->read_waiters)) { mpsc_fifo_destroy(&rwlock->write_waiters); mpsc_fifo_destroy(&rwlock->read_waiters); return FIBER_ERROR; } rwlock->state.blob = 0; write_barrier(); return FIBER_SUCCESS; }
/* Special version of copy() for when we only want to copy the info * pointer of an object, but reserve some padding after it. This is * used to optimise evacuation of TSOs. */ static bool copyPart(StgClosure **p, StgClosure *src, uint32_t size_to_reserve, uint32_t size_to_copy, uint32_t gen_no) { StgPtr to, from; uint32_t i; StgWord info; #if defined(PARALLEL_GC) spin: info = xchg((StgPtr)&src->header.info, (W_)&stg_WHITEHOLE_info); if (info == (W_)&stg_WHITEHOLE_info) { #if defined(PROF_SPIN) whitehole_gc_spin++; #endif busy_wait_nop(); goto spin; } if (IS_FORWARDING_PTR(info)) { src->header.info = (const StgInfoTable *)info; evacuate(p); // does the failed_to_evac stuff return false; } #else info = (W_)src->header.info; #endif to = alloc_for_copy(size_to_reserve, gen_no); from = (StgPtr)src; to[0] = info; for (i = 1; i < size_to_copy; i++) { // unroll for small i to[i] = from[i]; } write_barrier(); src->header.info = (const StgInfoTable*)MK_FORWARDING_PTR(to); *p = (StgClosure *)to; #if defined(PROFILING) // We store the size of the just evacuated object in the LDV word so that // the profiler can guess the position of the next object later. SET_EVACUAEE_FOR_LDV(from, size_to_reserve); // fill the slop if (size_to_reserve - size_to_copy > 0) LDV_FILL_SLOP(to + size_to_copy, (int)(size_to_reserve - size_to_copy)); #endif return true; }
/* * It is up to the caller to fill in the object's fields in a meaningful * fashion! */ inline object *factorvm::allot_object(header header, cell size) { #ifdef GC_DEBUG if(!gc_off) gc(); #endif object *obj; if(nursery.size - allot_buffer_zone > size) { /* If there is insufficient room, collect the nursery */ if(nursery.here + allot_buffer_zone + size > nursery.end) garbage_collection(data->nursery(),false,0); cell h = nursery.here; nursery.here = h + align8(size); obj = (object *)h; } /* If the object is bigger than the nursery, allocate it in tenured space */ else { zone *tenured = &data->generations[data->tenured()]; /* If tenured space does not have enough room, collect */ if(tenured->here + size > tenured->end) { gc(); tenured = &data->generations[data->tenured()]; } /* If it still won't fit, grow the heap */ if(tenured->here + size > tenured->end) { garbage_collection(data->tenured(),true,size); tenured = &data->generations[data->tenured()]; } obj = allot_zone(tenured,size); /* Allows initialization code to store old->new pointers without hitting the write barrier in the common case of a nursery allocation */ write_barrier(obj); } obj->h = header; return obj; }
static enum fio_q_status fio_ioring_queue(struct thread_data *td, struct io_u *io_u) { struct ioring_data *ld = td->io_ops_data; struct io_sq_ring *ring = &ld->sq_ring; unsigned tail, next_tail; fio_ro_check(td, io_u); if (ld->queued == ld->iodepth) return FIO_Q_BUSY; if (io_u->ddir == DDIR_TRIM) { if (ld->queued) return FIO_Q_BUSY; do_io_u_trim(td, io_u); io_u_mark_submit(td, 1); io_u_mark_complete(td, 1); return FIO_Q_COMPLETED; } tail = *ring->tail; next_tail = tail + 1; read_barrier(); if (next_tail == *ring->head) return FIO_Q_BUSY; /* ensure sqe stores are ordered with tail update */ write_barrier(); ring->array[tail & ld->sq_ring_mask] = io_u->index; *ring->tail = next_tail; write_barrier(); ld->queued++; return FIO_Q_QUEUED; }
void verify_async_exit(struct thread_data *td) { td->verify_thread_exit = 1; write_barrier(); pthread_cond_broadcast(&td->verify_cond); pthread_mutex_lock(&td->io_u_lock); while (td->nr_verify_threads) pthread_cond_wait(&td->free_cond, &td->io_u_lock); pthread_mutex_unlock(&td->io_u_lock); free(td->verify_threads); td->verify_threads = NULL; }
void wsd_work_stealing_deque_push_bottom(wsd_work_stealing_deque_t* d, void* p) { assert(d); const int64_t b = d->bottom; const int64_t t = d->top; wsd_circular_array_t* a = d->underlying_array; const int64_t size = b - t; if(size >= a->size_minus_one) { /* top is actually < bottom. the circular array API expects start < end */ a = wsd_circular_array_grow(a, t, b); /* NOTE: d->underlying_array is lost. memory leak. */ d->underlying_array = a; } wsd_circular_array_put(a, b, p); write_barrier(); d->bottom = b + 1; }
int fiber_event_init() { fiber_spinlock_lock(&fiber_loop_spinlock); assert("libev version mismatch" && ev_version_major () == EV_VERSION_MAJOR && ev_version_minor () >= EV_VERSION_MINOR); active_threads = fiber_manager_get_kernel_thread_count(); write_barrier();//needed so active_threads is set before fiber_loop (see fiber_poll_events_blocking - active_threads should never be decremented before it's been set) fiber_loop = ev_loop_new(EVFLAG_AUTO); assert(fiber_loop); fiber_spinlock_unlock(&fiber_loop_spinlock); return fiber_loop ? FIBER_SUCCESS : FIBER_ERROR; }
STATIC_INLINE StgInd * lockCAF (StgRegTable *reg, StgIndStatic *caf) { const StgInfoTable *orig_info; Capability *cap = regTableToCapability(reg); StgInd *bh; orig_info = caf->header.info; #ifdef THREADED_RTS const StgInfoTable *cur_info; if (orig_info == &stg_IND_STATIC_info || orig_info == &stg_WHITEHOLE_info) { // already claimed by another thread; re-enter the CAF return NULL; } cur_info = (const StgInfoTable *) cas((StgVolatilePtr)&caf->header.info, (StgWord)orig_info, (StgWord)&stg_WHITEHOLE_info); if (cur_info != orig_info) { // already claimed by another thread; re-enter the CAF return NULL; } // successfully claimed by us; overwrite with IND_STATIC #endif // For the benefit of revertCAFs(), save the original info pointer caf->saved_info = orig_info; // Allocate the blackhole indirection closure bh = (StgInd *)allocate(cap, sizeofW(*bh)); SET_HDR(bh, &stg_CAF_BLACKHOLE_info, caf->header.prof.ccs); bh->indirectee = (StgClosure *)cap->r.rCurrentTSO; caf->indirectee = (StgClosure *)bh; write_barrier(); SET_INFO((StgClosure*)caf,&stg_IND_STATIC_info); return bh; }