jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims, int own_buffer) { size_t i, elsz, nel=1; jl_array_t *a; size_t ndims = jl_tuple_len(dims); wideint_t prod; for(i=0; i < ndims; i++) { prod = (wideint_t)nel * (wideint_t)jl_unbox_long(jl_tupleref(dims, i)); if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); nel = prod; } jl_value_t *el_type = jl_tparam0(atype); int isunboxed = store_unboxed(el_type); if (isunboxed) elsz = jl_datatype_size(el_type); else elsz = sizeof(void*); int ndimwords = jl_array_ndimwords(ndims); a = allocobj((sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16); a->type = atype; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->ptrarray = !isunboxed; a->ndims = ndims; a->offset = 0; a->isshared = 1; a->isaligned = 0; if (own_buffer) { a->how = 2; jl_gc_track_malloced_array(a); } else { a->how = 0; } if (ndims == 1) { a->nrows = nel; a->maxsize = nel; } else { size_t *adims = &a->nrows; for(i=0; i < ndims; i++) { adims[i] = jl_unbox_long(jl_tupleref(dims, i)); } } return a; }
JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_value_t *_dims, int own_buffer) { jl_ptls_t ptls = jl_get_ptls_states(); size_t elsz, nel = 1; jl_array_t *a; size_t ndims = jl_nfields(_dims); wideint_t prod; assert(is_ntuple_long(_dims)); size_t *dims = (size_t*)_dims; for (size_t i = 0; i < ndims; i++) { prod = (wideint_t)nel * (wideint_t)dims[i]; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); nel = prod; } if (__unlikely(ndims == 1)) return jl_ptr_to_array_1d(atype, data, nel, own_buffer); jl_value_t *el_type = jl_tparam0(atype); int isunboxed = store_unboxed(el_type); if (isunboxed) elsz = jl_datatype_size(el_type); else elsz = sizeof(void*); int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->flags.ptrarray = !isunboxed; a->flags.ndims = ndims; a->offset = 0; a->flags.isshared = 1; a->flags.isaligned = 0; if (own_buffer) { a->flags.how = 2; jl_gc_track_malloced_array(ptls, a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { a->flags.how = 0; } assert(ndims != 1); // handled above memcpy(&a->nrows, dims, ndims * sizeof(size_t)); return a; }
// own_buffer != 0 iff GC should call free() on this pointer eventually JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int own_buffer) { jl_ptls_t ptls = jl_get_ptls_states(); jl_array_t *a; jl_value_t *eltype = jl_tparam0(atype); int isunboxed = jl_array_store_unboxed(eltype); size_t elsz; unsigned align; if (isunboxed && jl_is_uniontype(eltype)) jl_exceptionf(jl_argumenterror_type, "unsafe_wrap: unspecified layout for union element type"); if (isunboxed) { elsz = jl_datatype_size(eltype); align = jl_datatype_align(eltype); } else { align = elsz = sizeof(void*); } if (((uintptr_t)data) & (align - 1)) jl_exceptionf(jl_argumenterror_type, "unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align); int ndimwords = jl_array_ndimwords(1); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->flags.ptrarray = !isunboxed; a->flags.ndims = 1; a->flags.isshared = 1; a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers if (own_buffer) { a->flags.how = 2; jl_gc_track_malloced_array(ptls, a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { a->flags.how = 0; } a->nrows = nel; a->maxsize = nel; a->offset = 0; return a; }
// Resize the buffer to a max size of `newlen` // The buffer can either be newly allocated or realloc'd, the return // value is 1 if a new buffer is allocated and 0 if it is realloc'd. // the caller needs to take care of moving the data from the old buffer // to the new one if necessary. // When this function returns, the `->data` pointer always points to // the **beginning** of the new buffer. static int NOINLINE array_resize_buffer(jl_array_t *a, size_t newlen) { jl_ptls_t ptls = jl_get_ptls_states(); assert(!a->flags.isshared || a->flags.how == 3); size_t elsz = a->elsize; size_t nbytes = newlen * elsz; size_t oldnbytes = a->maxsize * elsz; size_t oldoffsnb = a->offset * elsz; size_t oldlen = a->nrows; assert(nbytes >= oldnbytes); if (elsz == 1) { nbytes++; oldnbytes++; } int newbuf = 0; if (a->flags.how == 2) { // already malloc'd - use realloc char *olddata = (char*)a->data - oldoffsnb; a->data = jl_gc_managed_realloc(olddata, nbytes, oldnbytes, a->flags.isaligned, (jl_value_t*)a); } else { newbuf = 1; if ( #ifdef _P64 nbytes >= MALLOC_THRESH #else elsz > 4 #endif ) { a->data = jl_gc_managed_malloc(nbytes); jl_gc_track_malloced_array(ptls, a); a->flags.how = 2; a->flags.isaligned = 1; } else { a->data = jl_gc_alloc_buf(ptls, nbytes); a->flags.how = 1; jl_gc_wb_buf(a, a->data, nbytes); } } if (JL_ARRAY_IMPL_NUL && elsz == 1) memset((char*)a->data + oldnbytes - 1, 0, nbytes - oldnbytes + 1); (void)oldlen; assert(oldlen == a->nrows && "Race condition detected: recursive resizing on the same array."); a->flags.isshared = 0; a->maxsize = newlen; return newbuf; }
// allocate buffer of newlen elements, placing old data at given offset (in #elts) // newlen: new length (#elts), including offset // oldlen: old length (#elts), excluding offset // offs: new offset static void array_resize_buffer(jl_array_t *a, size_t newlen, size_t oldlen, size_t offs) { size_t es = a->elsize; size_t nbytes = newlen * es; size_t offsnb = offs * es; size_t oldnbytes = oldlen * es; size_t oldoffsnb = a->offset * es; if (es == 1) nbytes++; assert(!a->isshared || a->how==3); char *newdata; if (a->how == 2) { // already malloc'd - use realloc newdata = (char*)jl_gc_managed_realloc((char*)a->data - oldoffsnb, nbytes, oldnbytes+oldoffsnb, a->isaligned, (jl_value_t*)a); if (offs != a->offset) { memmove(&newdata[offsnb], &newdata[oldoffsnb], oldnbytes); } } else { if ( #ifdef _P64 nbytes >= MALLOC_THRESH #else es > 4 #endif ) { newdata = (char*)jl_gc_managed_malloc(nbytes); jl_gc_track_malloced_array(a); a->how = 2; a->isaligned = 1; } else { newdata = (char*)allocb(nbytes); a->how = 1; } memcpy(newdata + offsnb, (char*)a->data, oldnbytes); } a->data = newdata + offsnb; a->isshared = 0; if (a->ptrarray || es==1) memset(newdata+offsnb+oldnbytes, 0, nbytes-oldnbytes-offsnb); if (a->how == 1) jl_gc_wb_buf(a, newdata); a->maxsize = newlen; }
// own_buffer != 0 iff GC should call free() on this pointer eventually JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int own_buffer) { jl_ptls_t ptls = jl_get_ptls_states(); size_t elsz; jl_array_t *a; jl_value_t *el_type = jl_tparam0(atype); int isunboxed = store_unboxed(el_type); if (isunboxed) elsz = jl_datatype_size(el_type); else elsz = sizeof(void*); int ndimwords = jl_array_ndimwords(1); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->flags.ptrarray = !isunboxed; a->flags.ndims = 1; a->flags.isshared = 1; a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers if (own_buffer) { a->flags.how = 2; jl_gc_track_malloced_array(ptls, a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { a->flags.how = 0; } a->nrows = nel; a->maxsize = nel; a->offset = 0; return a; }
// own_buffer != 0 iff GC should call free() on this pointer eventually jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int own_buffer) { size_t elsz; jl_array_t *a; jl_value_t *el_type = jl_tparam0(atype); int isunboxed = store_unboxed(el_type); if (isunboxed) elsz = jl_datatype_size(el_type); else elsz = sizeof(void*); int ndimwords = jl_array_ndimwords(1); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), 16); a = (jl_array_t*)jl_gc_allocobj(tsz); jl_set_typeof(a, atype); a->pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->ptrarray = !isunboxed; a->ndims = 1; a->isshared = 1; a->isaligned = 0; // TODO: allow passing memalign'd buffers if (own_buffer) { a->how = 2; jl_gc_track_malloced_array(a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { a->how = 0; } a->nrows = nel; a->maxsize = nel; a->offset = 0; return a; }
// own_buffer != 0 iff GC should call free() on this pointer eventually jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int own_buffer) { size_t elsz; jl_array_t *a; jl_value_t *el_type = jl_tparam0(atype); int isunboxed = store_unboxed(el_type); if (isunboxed) elsz = jl_datatype_size(el_type); else elsz = sizeof(void*); a = allocobj((sizeof(jl_array_t)+jl_array_ndimwords(1)*sizeof(size_t)+15)&-16); a->type = atype; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->ptrarray = !isunboxed; a->ndims = 1; a->isshared = 1; a->isaligned = 0; // TODO: allow passing memalign'd buffers if (own_buffer) { a->how = 2; jl_gc_track_malloced_array(a); } else { a->how = 0; } a->nrows = nel; a->maxsize = nel; a->offset = 0; return a; }
static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, int isunboxed, int elsz) { jl_ptls_t ptls = jl_get_ptls_states(); size_t i, tot, nel=1; void *data; jl_array_t *a; for(i=0; i < ndims; i++) { wideint_t prod = (wideint_t)nel * (wideint_t)dims[i]; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); nel = prod; } if (isunboxed) { wideint_t prod = (wideint_t)elsz * (wideint_t)nel; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array size"); tot = prod; if (elsz == 1) { // extra byte for all julia allocated byte arrays tot++; } } else { wideint_t prod = (wideint_t)sizeof(void*) * (wideint_t)nel; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array size"); tot = prod; } int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); if (tot <= ARRAY_INLINE_NBYTES) { if (isunboxed && elsz >= 4) tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); // align data area size_t doffs = tsz; tsz += tot; tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); // align whole object a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this a->flags.how = 0; data = (char*)a + doffs; if (tot > 0 && !isunboxed) { memset(data, 0, tot); } } else { tsz = JL_ARRAY_ALIGN(tsz, JL_CACHE_BYTE_ALIGNMENT); // align whole object data = jl_gc_managed_malloc(tot); // Allocate the Array **after** allocating the data // to make sure the array is still young a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this a->flags.how = 2; jl_gc_track_malloced_array(ptls, a); if (!isunboxed) memset(data, 0, tot); } a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; if (JL_ARRAY_IMPL_NUL && elsz == 1) ((char*)data)[tot - 1] = '\0'; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->flags.ndims = ndims; a->flags.ptrarray = !isunboxed; a->elsize = elsz; a->flags.isshared = 0; a->flags.isaligned = 1; a->offset = 0; if (ndims == 1) { a->nrows = nel; a->maxsize = nel; } else { size_t *adims = &a->nrows; for(i=0; i < ndims; i++) adims[i] = dims[i]; } return a; }
static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, int isunboxed, int elsz) { size_t i, tot, nel=1; wideint_t prod; void *data; jl_array_t *a; for(i=0; i < ndims; i++) { prod = (wideint_t)nel * (wideint_t)dims[i]; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); nel = prod; } if (isunboxed) { prod = (wideint_t)elsz * (wideint_t)nel; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array size"); tot = prod; if (elsz == 1) { // hidden 0 terminator for all byte arrays tot++; } } else { prod = (wideint_t)sizeof(void*) * (wideint_t)nel; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array size"); tot = prod; } int ndimwords = jl_array_ndimwords(ndims); size_t tsz = sizeof(jl_array_t); tsz += ndimwords*sizeof(size_t); if (tot <= ARRAY_INLINE_NBYTES) { if (isunboxed && elsz >= 4) tsz = (tsz+15)&-16; // align data area 16 size_t doffs = tsz; tsz += tot; tsz = (tsz+15)&-16; // align whole object 16 a = allocobj(tsz); a->type = atype; a->how = 0; data = (char*)a + doffs; if (tot > 0 && !isunboxed) { memset(data, 0, tot); } } else { tsz = (tsz+15)&-16; // align whole object size 16 a = allocobj(tsz); JL_GC_PUSH1(&a); a->type = atype; // temporarily initialize to make gc-safe a->data = NULL; a->how = 2; data = jl_gc_managed_malloc(tot); jl_gc_track_malloced_array(a); if (!isunboxed) memset(data, 0, tot); JL_GC_POP(); } a->data = data; if (elsz == 1) ((char*)data)[tot-1] = '\0'; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->ndims = ndims; a->ptrarray = !isunboxed; a->elsize = elsz; a->isshared = 0; a->isaligned = 1; a->offset = 0; if (ndims == 1) { a->nrows = nel; a->maxsize = nel; } else { size_t *adims = &a->nrows; for(i=0; i < ndims; i++) adims[i] = dims[i]; } return a; }
jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_value_t *dims, int own_buffer) { size_t i, elsz, nel=1; jl_array_t *a; size_t ndims = jl_nfields(dims); wideint_t prod; for(i=0; i < ndims; i++) { prod = (wideint_t)nel * (wideint_t)jl_unbox_long(jl_fieldref(dims, i)); if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); nel = prod; } jl_value_t *el_type = jl_tparam0(atype); int isunboxed = store_unboxed(el_type); if (isunboxed) elsz = jl_datatype_size(el_type); else elsz = sizeof(void*); int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), 16); a = (jl_array_t*)jl_gc_allocobj(tsz); jl_set_typeof(a, atype); a->pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->ptrarray = !isunboxed; a->ndims = ndims; a->offset = 0; a->isshared = 1; a->isaligned = 0; if (own_buffer) { a->how = 2; jl_gc_track_malloced_array(a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { a->how = 0; } if (ndims == 1) { a->nrows = nel; a->maxsize = nel; } else { size_t *adims = &a->nrows; // jl_fieldref can allocate JL_GC_PUSH1(&a); for(i=0; i < ndims; i++) { adims[i] = jl_unbox_long(jl_fieldref(dims, i)); } JL_GC_POP(); } return a; }