jl_array_t *jl_ptr_to_array(jl_type_t *atype, void *data, jl_tuple_t *dims, int own_buffer) { size_t i, elsz, nel=1; jl_array_t *a; size_t ndims = jl_tuple_len(dims); for(i=0; i < ndims; i++) { nel *= jl_unbox_long(jl_tupleref(dims, i)); } jl_type_t *el_type = (jl_type_t*)jl_tparam0(atype); int isunboxed = jl_is_bits_type(el_type); if (isunboxed) elsz = jl_bitstype_nbits(el_type)/8; else elsz = sizeof(void*); int ndimwords = jl_array_ndimwords(ndims); a = allocobj((sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16); a->type = atype; a->data = data; a->length = nel; a->elsize = elsz; a->ptrarray = !isunboxed; a->ndims = ndims; if (own_buffer) { a->ismalloc = 1; jl_array_data_owner(a) = (jl_value_t*)jl_gc_acquire_buffer(data,nel*elsz); } else { a->ismalloc = 0; jl_array_data_owner(a) = (jl_value_t*)a; } if (ndims == 1) { a->nrows = a->length; a->maxsize = a->length; a->offset = 0; } else { size_t *adims = &a->nrows; for(i=0; i < ndims; i++) { adims[i] = jl_unbox_long(jl_tupleref(dims, i)); } } return a; }
JL_DLLEXPORT jl_value_t *jl_array_to_string(jl_array_t *a) { size_t len = jl_array_len(a); if (a->flags.how == 3 && a->offset == 0 && a->elsize == 1 && (jl_array_ndims(a) != 1 || ((a->maxsize + sizeof(void*) + 1 <= GC_MAX_SZCLASS) == (len + sizeof(void*) + 1 <= GC_MAX_SZCLASS)))) { jl_value_t *o = jl_array_data_owner(a); if (jl_is_string(o)) { a->flags.isshared = 1; *(size_t*)o = len; a->nrows = 0; #ifdef STORE_ARRAY_LEN a->length = 0; #endif a->maxsize = 0; return o; } } a->nrows = 0; #ifdef STORE_ARRAY_LEN a->length = 0; #endif a->maxsize = 0; return jl_pchar_to_string((const char*)jl_array_data(a), len); }
JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str) { jl_ptls_t ptls = jl_get_ptls_states(); jl_array_t *a; int ndimwords = jl_array_ndimwords(1); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_alloc(ptls, tsz, jl_array_uint8_type); a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->flags.ndims = 1; a->offset = 0; a->data = jl_string_data(str); a->flags.isaligned = 0; a->elsize = 1; a->flags.ptrarray = 0; jl_array_data_owner(a) = str; a->flags.how = 3; a->flags.isshared = 1; size_t l = jl_string_len(str); #ifdef STORE_ARRAY_LEN a->length = l; #endif a->nrows = a->maxsize = l; return a; }
// hack to expose ios_stdout to julia. we could create a new iostream pointing // to stdout, but then there would be two buffers for one descriptor, and // ios_stdout is used before julia IOStream is available, creating a potential // mess. DLLEXPORT jl_value_t *jl_stdout_stream(void) { jl_array_t *a = jl_alloc_array_1d(jl_array_uint8_type, sizeof(ios_t)); a->data = (void*)ios_stdout; jl_array_data_owner(a) = (jl_value_t*)a; return (jl_value_t*)a; }
STATIC_INLINE jl_value_t *jl_array_owner(jl_array_t *a) { if (a->flags.how == 3) { a = (jl_array_t*)jl_array_data_owner(a); assert(a->flags.how != 3); } return (jl_value_t*)a; }
jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_tuple_t *dims) { size_t i; jl_array_t *a; size_t ndims = jl_tuple_len(dims); int ndimwords = jl_array_ndimwords(ndims); a = allocobj((sizeof(jl_array_t) + sizeof(void*) + ndimwords*sizeof(size_t) + 15)&-16); a->type = atype; a->ndims = ndims; a->offset = 0; a->data = NULL; a->isaligned = data->isaligned; jl_value_t *el_type = jl_tparam0(atype); if (store_unboxed(el_type)) { a->elsize = jl_datatype_size(el_type); a->ptrarray = 0; } else { a->elsize = sizeof(void*); a->ptrarray = 1; } JL_GC_PUSH1(&a); jl_array_data_owner(a) = (jl_value_t*)data; a->how = 3; a->data = data->data; a->isshared = 1; data->isshared = 1; if (ndims == 1) { size_t l = jl_unbox_long(jl_tupleref(dims,0)); #ifdef STORE_ARRAY_LEN a->length = l; #endif a->nrows = l; a->maxsize = l; } else { size_t *adims = &a->nrows; size_t l=1; wideint_t prod; for(i=0; i < ndims; i++) { adims[i] = jl_unbox_long(jl_tupleref(dims, i)); prod = (wideint_t)l * (wideint_t)adims[i]; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); l = prod; } #ifdef STORE_ARRAY_LEN a->length = l; #endif } JL_GC_POP(); return a; }
// own_buffer != 0 iff GC should call free() on this pointer eventually jl_array_t *jl_ptr_to_array_1d(jl_type_t *atype, void *data, size_t nel, int own_buffer) { size_t elsz; jl_array_t *a; jl_type_t *el_type = (jl_type_t*)jl_tparam0(atype); int isunboxed = jl_is_bits_type(el_type); if (isunboxed) elsz = jl_bitstype_nbits(el_type)/8; else elsz = sizeof(void*); a = allocobj((sizeof(jl_array_t)+jl_array_ndimwords(1)*sizeof(size_t)+15)&-16); a->type = atype; a->data = data; a->length = nel; a->elsize = elsz; a->ptrarray = !isunboxed; a->ndims = 1; if (own_buffer) { a->ismalloc = 1; jl_array_data_owner(a) = (jl_value_t*)jl_gc_acquire_buffer(data,nel*elsz); } else { a->ismalloc = 0; jl_array_data_owner(a) = (jl_value_t*)a; } a->nrows = a->length; a->maxsize = a->length; a->offset = 0; return a; }
void jl_arrayset(jl_array_t *a, jl_value_t *rhs, size_t i) { assert(i < jl_array_len(a)); jl_value_t *el_type = jl_tparam0(jl_typeof(a)); if (el_type != (jl_value_t*)jl_any_type) { if (!jl_subtype(rhs, el_type, 1)) jl_type_error("arrayset", el_type, rhs); } if (!a->ptrarray) { jl_assign_bits(&((char*)a->data)[i*a->elsize], rhs); } else { ((jl_value_t**)a->data)[i] = rhs; jl_value_t *owner = (jl_value_t*)a; if (a->how == 3) { owner = jl_array_data_owner(a); } jl_gc_wb(owner, rhs); } }
STATIC_INLINE int jl_has_implicit_byte(jl_array_t *a) { // * unshared: // * how: 0-2 // We own and allocated the data. // It should have the extra byte. // * shared: // * how: 0, 2 // The data might come from external source without implicit NUL byte. // There could be an entra byte for a `reinterpreted` array // but that should be unlikely for strings. // * how: 1 // We allocated the data with the extra byte. // * how: 3 // We should check the owner. if (a->flags.how == 3) { a = (jl_array_t*)jl_array_data_owner(a); return a->elsize == 1 && jl_has_implicit_byte_owned(a); } return jl_has_implicit_byte_owned(a); }
void jl_array_grow_beg(jl_array_t *a, size_t inc) { // designed to handle the case of growing and shrinking at both ends if (inc == 0) return; size_t es = a->elsize; size_t nb = inc*es; if (a->offset >= inc) { a->data = (char*)a->data - nb; a->offset -= inc; } else { size_t alen = a->length; size_t anb = alen*es; char *newdata; jl_mallocptr_t *mp = NULL; if (inc > (a->maxsize-alen)/2 - (a->maxsize-alen)/20) { size_t newlen = a->maxsize==0 ? 2*inc : a->maxsize*2; while (alen+2*inc > newlen-a->offset) newlen *= 2; mp = array_new_buffer(a, newlen); newdata = mp->ptr; size_t center = (newlen - (alen + inc))/2; newdata += (center*es); a->maxsize = newlen; a->offset = center; } else { size_t center = (a->maxsize - (alen + inc))/2; newdata = (char*)a->data - es*a->offset + es*center; a->offset = center; } memmove(&newdata[nb], a->data, anb); a->data = newdata; if (mp) { jl_array_data_owner(a) = (jl_value_t*)mp; a->ismalloc = 1; } } a->length += inc; a->nrows += inc; }
void jl_array_grow_end(jl_array_t *a, size_t inc) { // optimized for the case of only growing and shrinking at the end size_t alen = a->length; if ((alen + inc) > a->maxsize - a->offset) { size_t newlen = a->maxsize==0 ? (inc<4?4:inc) : a->maxsize*2; while ((alen + inc) > newlen - a->offset) newlen *= 2; jl_mallocptr_t *mp = array_new_buffer(a, newlen); char *newdata = mp->ptr; size_t es = a->elsize; newdata += (a->offset*es); size_t anb = alen*es; memcpy(newdata, (char*)a->data, anb); if (es == 1) { memset(newdata + anb, 0, (newlen-a->offset-alen)*es); } a->maxsize = newlen; a->data = newdata; jl_array_data_owner(a) = (jl_value_t*)mp; a->ismalloc = 1; } a->length += inc; a->nrows += inc; }
jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_value_t *dims) { size_t i; jl_array_t *a; size_t ndims = jl_nfields(dims); int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*), 16); a = (jl_array_t*)jl_gc_allocobj(tsz); jl_set_typeof(a, atype); a->pooled = tsz <= GC_MAX_SZCLASS; a->ndims = ndims; a->offset = 0; a->data = NULL; a->isaligned = data->isaligned; jl_value_t *el_type = jl_tparam0(atype); if (store_unboxed(el_type)) { a->elsize = jl_datatype_size(el_type); a->ptrarray = 0; } else { a->elsize = sizeof(void*); a->ptrarray = 1; } JL_GC_PUSH1(&a); jl_array_t *owner = data; // if data is itself a shared wrapper, // owner should point back to the original array if (owner->how == 3) { owner = (jl_array_t*)jl_array_data_owner(owner); } assert(owner->how != 3); jl_array_data_owner(a) = (jl_value_t*)owner; a->how = 3; a->data = data->data; a->isshared = 1; data->isshared = 1; if (ndims == 1) { size_t l = ((size_t*)jl_data_ptr(dims))[0]; #ifdef STORE_ARRAY_LEN a->length = l; #endif a->nrows = l; a->maxsize = l; } else { size_t *adims = &a->nrows; size_t l=1; wideint_t prod; for(i=0; i < ndims; i++) { adims[i] = ((size_t*)jl_data_ptr(dims))[i]; prod = (wideint_t)l * (wideint_t)adims[i]; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); l = prod; } #ifdef STORE_ARRAY_LEN a->length = l; #endif } JL_GC_POP(); return a; }
JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_value_t *_dims) { jl_ptls_t ptls = jl_get_ptls_states(); jl_array_t *a; size_t ndims = jl_nfields(_dims); assert(is_ntuple_long(_dims)); size_t *dims = (size_t*)_dims; int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->flags.ndims = ndims; a->offset = 0; a->data = NULL; a->flags.isaligned = data->flags.isaligned; jl_value_t *el_type = jl_tparam0(atype); assert(store_unboxed(el_type) == !data->flags.ptrarray); if (!data->flags.ptrarray) { a->elsize = jl_datatype_size(el_type); a->flags.ptrarray = 0; } else { a->elsize = sizeof(void*); a->flags.ptrarray = 1; } // if data is itself a shared wrapper, // owner should point back to the original array jl_array_data_owner(a) = jl_array_owner(data); a->flags.how = 3; a->data = data->data; a->flags.isshared = 1; data->flags.isshared = 1; if (ndims == 1) { size_t l = dims[0]; #ifdef STORE_ARRAY_LEN a->length = l; #endif a->nrows = l; a->maxsize = l; } else { size_t *adims = &a->nrows; size_t l = 1; wideint_t prod; for (size_t i = 0; i < ndims; i++) { adims[i] = dims[i]; prod = (wideint_t)l * (wideint_t)adims[i]; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); l = prod; } #ifdef STORE_ARRAY_LEN a->length = l; #endif } return a; }
JL_DLLEXPORT jl_value_t *(jl_array_data_owner)(jl_array_t *a) { return jl_array_data_owner(a); }
jl_array_t *jl_reshape_array(jl_type_t *atype, jl_array_t *data, jl_tuple_t *dims) { size_t i; jl_array_t *a; size_t ndims = jl_tuple_len(dims); int ndimwords = jl_array_ndimwords(ndims); a = allocobj((sizeof(jl_array_t) + ndimwords*sizeof(size_t) + 15)&-16); a->type = atype; a->ndims = ndims; a->data = NULL; JL_GC_PUSH(&a); char *d = data->data; if (data->ndims == 1) d -= data->offset*data->elsize; if (d == jl_array_inline_data_area(data)) { if (data->ndims == 1) { // data might resize, so switch it to shared representation. // problem: we'd like to do that, but it might not be valid, // since the buffer might be used from C in a way that it's // assumed not to move. for now, just copy the data (note this // case only happens for sizes <= ARRAY_INLINE_NBYTES) jl_mallocptr_t *mp = array_new_buffer(data, data->length); memcpy(mp->ptr, data->data, data->length * data->elsize); a->data = mp->ptr; jl_array_data_owner(a) = (jl_value_t*)mp; a->ismalloc = 1; //data->data = mp->ptr; //data->offset = 0; //data->maxsize = data->length; //jl_array_data_owner(data) = (jl_value_t*)mp; } else { a->ismalloc = 0; jl_array_data_owner(a) = (jl_value_t*)data; } } else { a->ismalloc = data->ismalloc; jl_array_data_owner(a) = jl_array_data_owner(data); } if (a->data == NULL) a->data = data->data; jl_type_t *el_type = (jl_type_t*)jl_tparam0(atype); if (jl_is_bits_type(el_type)) { a->elsize = jl_bitstype_nbits(el_type)/8; a->ptrarray = 0; } else { a->elsize = sizeof(void*); a->ptrarray = 1; } if (ndims == 1) { a->length = jl_unbox_long(jl_tupleref(dims,0)); a->nrows = a->length; a->maxsize = a->length; a->offset = 0; } else { size_t *adims = &a->nrows; size_t l=1; for(i=0; i < ndims; i++) { adims[i] = jl_unbox_long(jl_tupleref(dims, i)); l *= adims[i]; } a->length = l; } JL_GC_POP(); return a; }
static void gc_mark_all() { while (mark_sp > 0) { jl_value_t *v = mark_stack[--mark_sp]; jl_value_t *vt = (jl_value_t*)gc_typeof(v); // some values have special representations if (vt == (jl_value_t*)jl_tuple_type) { size_t l = jl_tuple_len(v); jl_value_t **data = ((jl_tuple_t*)v)->data; for(size_t i=0; i < l; i++) { jl_value_t *elt = data[i]; if (elt != NULL) gc_push_root(elt); } } else if (((jl_datatype_t*)(vt))->name == jl_array_typename) { jl_array_t *a = (jl_array_t*)v; char *data = a->data; if (data == NULL) continue; int ndims = jl_array_ndims(a); char *data0 = data; if (ndims == 1) data0 -= a->offset*a->elsize; if (!a->isinline) { jl_value_t *owner = jl_array_data_owner(a); if (a->ismalloc) { // jl_mallocptr_t if (gc_marked(owner)) continue; gc_setmark(owner); } else { // an array v = owner; if (v != (jl_value_t*)a) { gc_push_root(v); continue; } } } if (a->ptrarray) { size_t l = jl_array_len(a); for(size_t i=0; i < l; i++) { jl_value_t *elt = ((jl_value_t**)data)[i]; if (elt != NULL) gc_push_root(elt); } } } else if (vt == (jl_value_t*)jl_module_type) { gc_mark_module((jl_module_t*)v); } else if (vt == (jl_value_t*)jl_task_type) { jl_task_t *ta = (jl_task_t*)v; if (ta->on_exit) gc_push_root(ta->on_exit); gc_push_root(ta->last); gc_push_root(ta->tls); gc_push_root(ta->consumers); if (ta->start) gc_push_root(ta->start); if (ta->result) gc_push_root(ta->result); if (ta->stkbuf != NULL || ta == jl_current_task) { if (ta->stkbuf != NULL) gc_setmark_buf(ta->stkbuf); #ifdef COPY_STACKS ptrint_t offset; if (ta == jl_current_task) { offset = 0; gc_mark_stack(jl_pgcstack, offset); } else { offset = (char *)ta->stkbuf - ((char *)ta->stackbase - ta->ssize); gc_mark_stack(ta->gcstack, offset); } #else gc_mark_stack(ta->gcstack, 0); #endif } } else { jl_datatype_t *dt = (jl_datatype_t*)vt; int nf = (int)jl_tuple_len(dt->names); for(int i=0; i < nf; i++) { if (dt->fields[i].isptr) { jl_value_t *fld = *(jl_value_t**)((char*)v + dt->fields[i].offset + sizeof(void*)); if (fld) gc_push_root(fld); } } } } }
JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_value_t *_dims) { jl_ptls_t ptls = jl_get_ptls_states(); jl_array_t *a; size_t ndims = jl_nfields(_dims); assert(is_ntuple_long(_dims)); size_t *dims = (size_t*)_dims; assert(jl_types_equal(jl_tparam0(jl_typeof(data)), jl_tparam0(atype))); int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords * sizeof(size_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->flags.ndims = ndims; a->offset = 0; a->data = NULL; a->flags.isaligned = data->flags.isaligned; jl_array_t *owner = (jl_array_t*)jl_array_owner(data); jl_value_t *eltype = jl_tparam0(atype); size_t elsz = 0, align = 0; int isboxed = !jl_islayout_inline(eltype, &elsz, &align); assert(isboxed == data->flags.ptrarray); if (!isboxed) { a->elsize = elsz; jl_value_t *ownerty = jl_typeof(owner); size_t oldelsz = 0, oldalign = 0; if (ownerty == (jl_value_t*)jl_string_type) { oldalign = 1; } else { jl_islayout_inline(jl_tparam0(ownerty), &oldelsz, &oldalign); } if (oldalign < align) jl_exceptionf(jl_argumenterror_type, "reinterpret from alignment %d bytes to alignment %d bytes not allowed", (int) oldalign, (int) align); a->flags.ptrarray = 0; } else { a->elsize = sizeof(void*); a->flags.ptrarray = 1; } // if data is itself a shared wrapper, // owner should point back to the original array jl_array_data_owner(a) = (jl_value_t*)owner; a->flags.how = 3; a->data = data->data; a->flags.isshared = 1; data->flags.isshared = 1; if (ndims == 1) { size_t l = dims[0]; #ifdef STORE_ARRAY_LEN a->length = l; #endif a->nrows = l; a->maxsize = l; } else { size_t *adims = &a->nrows; size_t l = 1; wideint_t prod; for (size_t i = 0; i < ndims; i++) { adims[i] = dims[i]; prod = (wideint_t)l * (wideint_t)adims[i]; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); l = prod; } #ifdef STORE_ARRAY_LEN a->length = l; #endif } return a; }
static void push_root(jl_value_t *v, int d) { assert(v != NULL); jl_value_t *vt = (jl_value_t*)gc_typeof(v); #ifdef OBJPROFILE if (!gc_marked(v)) { void **bp = ptrhash_bp(&obj_counts, vt); if (*bp == HT_NOTFOUND) *bp = (void*)2; else (*((ptrint_t*)bp))++; } #endif gc_setmark(v); if (vt == (jl_value_t*)jl_weakref_type || (jl_is_datatype(vt) && ((jl_datatype_t*)vt)->pointerfree)) { return; } if (d >= MAX_MARK_DEPTH) goto queue_the_root; d++; // some values have special representations if (vt == (jl_value_t*)jl_tuple_type) { size_t l = jl_tuple_len(v); jl_value_t **data = ((jl_tuple_t*)v)->data; for(size_t i=0; i < l; i++) { jl_value_t *elt = data[i]; if (elt != NULL) gc_push_root(elt, d); } } else if (((jl_datatype_t*)(vt))->name == jl_array_typename) { jl_array_t *a = (jl_array_t*)v; if (a->how == 3) { jl_value_t *owner = jl_array_data_owner(a); gc_push_root(owner, d); return; } else if (a->how == 1) { gc_setmark_buf((char*)a->data - a->offset*a->elsize); } if (a->ptrarray && a->data!=NULL) { size_t l = jl_array_len(a); if (l > 100000 && d > MAX_MARK_DEPTH-10) { // don't mark long arrays at high depth, to try to avoid // copying the whole array into the mark queue goto queue_the_root; } else { void *data = a->data; for(size_t i=0; i < l; i++) { jl_value_t *elt = ((jl_value_t**)data)[i]; if (elt != NULL) gc_push_root(elt, d); } } } } else if (vt == (jl_value_t*)jl_module_type) { gc_mark_module((jl_module_t*)v, d); } else if (vt == (jl_value_t*)jl_task_type) { gc_mark_task((jl_task_t*)v, d); } else { jl_datatype_t *dt = (jl_datatype_t*)vt; int nf = (int)jl_tuple_len(dt->names); for(int i=0; i < nf; i++) { if (dt->fields[i].isptr) { jl_value_t *fld = *(jl_value_t**)((char*)v + dt->fields[i].offset + sizeof(void*)); if (fld) gc_push_root(fld, d); } } } return; queue_the_root: if (mark_sp >= mark_stack_size) { size_t newsz = mark_stack_size>0 ? mark_stack_size*2 : 32000; mark_stack = (jl_value_t**)realloc(mark_stack,newsz*sizeof(void*)); if (mark_stack == NULL) exit(1); mark_stack_size = newsz; } mark_stack[mark_sp++] = v; }