static jl_mallocptr_t *array_new_buffer(jl_array_t *a, size_t newlen)
{
size_t nbytes = newlen * a->elsize;
if (a->elsize == 1) {
nbytes++;
}
jl_mallocptr_t *mp = jl_gc_managed_malloc(nbytes);
char *newdata = mp->ptr;
if (a->ptrarray)
memset(newdata, 0, nbytes);
if (a->elsize == 1) newdata[nbytes-1] = '\0';
return mp;
}
// Resize the buffer to a max size of `newlen`
// The buffer can either be newly allocated or realloc'd, the return
// value is 1 if a new buffer is allocated and 0 if it is realloc'd.
// the caller needs to take care of moving the data from the old buffer
// to the new one if necessary.
// When this function returns, the `->data` pointer always points to
// the **beginning** of the new buffer.
static int NOINLINE array_resize_buffer(jl_array_t *a, size_t newlen)
{
jl_ptls_t ptls = jl_get_ptls_states();
assert(!a->flags.isshared || a->flags.how == 3);
size_t elsz = a->elsize;
size_t nbytes = newlen * elsz;
size_t oldnbytes = a->maxsize * elsz;
size_t oldoffsnb = a->offset * elsz;
size_t oldlen = a->nrows;
assert(nbytes >= oldnbytes);
if (elsz == 1) {
nbytes++;
oldnbytes++;
}
int newbuf = 0;
if (a->flags.how == 2) {
// already malloc'd - use realloc
char *olddata = (char*)a->data - oldoffsnb;
a->data = jl_gc_managed_realloc(olddata, nbytes, oldnbytes,
a->flags.isaligned, (jl_value_t*)a);
}
else {
newbuf = 1;
if (
#ifdef _P64
nbytes >= MALLOC_THRESH
#else
elsz > 4
#endif
) {
a->data = jl_gc_managed_malloc(nbytes);
jl_gc_track_malloced_array(ptls, a);
a->flags.how = 2;
a->flags.isaligned = 1;
}
else {
a->data = jl_gc_alloc_buf(ptls, nbytes);
a->flags.how = 1;
jl_gc_wb_buf(a, a->data, nbytes);
}
}
if (JL_ARRAY_IMPL_NUL && elsz == 1)
memset((char*)a->data + oldnbytes - 1, 0, nbytes - oldnbytes + 1);
(void)oldlen;
assert(oldlen == a->nrows &&
"Race condition detected: recursive resizing on the same array.");
a->flags.isshared = 0;
a->maxsize = newlen;
return newbuf;
}
// allocate buffer of newlen elements, placing old data at given offset (in #elts)
// newlen: new length (#elts), including offset
// oldlen: old length (#elts), excluding offset
// offs: new offset
static void array_resize_buffer(jl_array_t *a, size_t newlen, size_t oldlen, size_t offs)
{
size_t es = a->elsize;
size_t nbytes = newlen * es;
size_t offsnb = offs * es;
size_t oldnbytes = oldlen * es;
size_t oldoffsnb = a->offset * es;
if (es == 1)
nbytes++;
assert(!a->isshared || a->how==3);
char *newdata;
if (a->how == 2) {
// already malloc'd - use realloc
newdata = (char*)jl_gc_managed_realloc((char*)a->data - oldoffsnb, nbytes,
oldnbytes+oldoffsnb, a->isaligned, (jl_value_t*)a);
if (offs != a->offset) {
memmove(&newdata[offsnb], &newdata[oldoffsnb], oldnbytes);
}
}
else {
if (
#ifdef _P64
nbytes >= MALLOC_THRESH
#else
es > 4
#endif
) {
newdata = (char*)jl_gc_managed_malloc(nbytes);
jl_gc_track_malloced_array(a);
a->how = 2;
a->isaligned = 1;
}
else {
newdata = (char*)allocb(nbytes);
a->how = 1;
}
memcpy(newdata + offsnb, (char*)a->data, oldnbytes);
}
a->data = newdata + offsnb;
a->isshared = 0;
if (a->ptrarray || es==1)
memset(newdata+offsnb+oldnbytes, 0, nbytes-oldnbytes-offsnb);
if (a->how == 1)
jl_gc_wb_buf(a, newdata);
a->maxsize = newlen;
}
static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
int isunboxed, int elsz)
{
jl_ptls_t ptls = jl_get_ptls_states();
size_t i, tot, nel=1;
void *data;
jl_array_t *a;
for(i=0; i < ndims; i++) {
wideint_t prod = (wideint_t)nel * (wideint_t)dims[i];
if (prod > (wideint_t) MAXINTVAL)
jl_error("invalid Array dimensions");
nel = prod;
}
if (isunboxed) {
wideint_t prod = (wideint_t)elsz * (wideint_t)nel;
if (prod > (wideint_t) MAXINTVAL)
jl_error("invalid Array size");
tot = prod;
if (elsz == 1) {
// extra byte for all julia allocated byte arrays
tot++;
}
}
else {
wideint_t prod = (wideint_t)sizeof(void*) * (wideint_t)nel;
if (prod > (wideint_t) MAXINTVAL)
jl_error("invalid Array size");
tot = prod;
}
int ndimwords = jl_array_ndimwords(ndims);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
if (tot <= ARRAY_INLINE_NBYTES) {
if (isunboxed && elsz >= 4)
tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); // align data area
size_t doffs = tsz;
tsz += tot;
tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); // align whole object
a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.how = 0;
data = (char*)a + doffs;
if (tot > 0 && !isunboxed) {
memset(data, 0, tot);
}
}
else {
tsz = JL_ARRAY_ALIGN(tsz, JL_CACHE_BYTE_ALIGNMENT); // align whole object
data = jl_gc_managed_malloc(tot);
// Allocate the Array **after** allocating the data
// to make sure the array is still young
a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.how = 2;
jl_gc_track_malloced_array(ptls, a);
if (!isunboxed)
memset(data, 0, tot);
}
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
a->data = data;
if (JL_ARRAY_IMPL_NUL && elsz == 1)
((char*)data)[tot - 1] = '\0';
#ifdef STORE_ARRAY_LEN
a->length = nel;
#endif
a->flags.ndims = ndims;
a->flags.ptrarray = !isunboxed;
a->elsize = elsz;
a->flags.isshared = 0;
a->flags.isaligned = 1;
a->offset = 0;
if (ndims == 1) {
a->nrows = nel;
a->maxsize = nel;
}
else {
size_t *adims = &a->nrows;
for(i=0; i < ndims; i++)
adims[i] = dims[i];
}
return a;
}
static jl_array_t *_new_array(jl_type_t *atype,
uint32_t ndims, size_t *dims)
{
size_t i, tot, nel=1;
int isunboxed=0, elsz;
void *data;
jl_array_t *a;
for(i=0; i < ndims; i++) {
nel *= dims[i];
}
jl_type_t *el_type = (jl_type_t*)jl_tparam0(atype);
isunboxed = jl_is_bits_type(el_type);
if (isunboxed) {
elsz = jl_bitstype_nbits(el_type)/8;
tot = elsz * nel;
if (elsz == 1) {
// hidden 0 terminator for all byte arrays
tot++;
}
}
else {
elsz = sizeof(void*);
tot = sizeof(void*) * nel;
}
int ndimwords = jl_array_ndimwords(ndims);
if (tot <= ARRAY_INLINE_NBYTES) {
size_t tsz = tot>sizeof(size_t) ? tot-sizeof(size_t) : tot;
a = allocobj((sizeof(jl_array_t)+tsz+ndimwords*sizeof(size_t)+15)&-16);
a->type = atype;
a->ismalloc = 0;
data = (&a->_space[0] + ndimwords*sizeof(size_t));
if (tot > 0 && !isunboxed) {
memset(data, 0, tot);
}
}
else {
a = allocobj((sizeof(jl_array_t)+ndimwords*sizeof(size_t)+15)&-16);
JL_GC_PUSH(&a);
a->type = atype;
a->ismalloc = 1;
// temporarily initialize to make gc-safe
a->data = NULL;
jl_value_t **powner = (jl_value_t**)(&a->_space[0] + ndimwords*sizeof(size_t));
*powner = (jl_value_t*)jl_gc_managed_malloc(tot);
data = ((jl_mallocptr_t*)*powner)->ptr;
if (!isunboxed)
memset(data, 0, tot);
JL_GC_POP();
}
a->data = data;
if (elsz == 1) ((char*)data)[tot-1] = '\0';
a->length = nel;
a->ndims = ndims;
a->ptrarray = !isunboxed;
a->elsize = elsz;
if (ndims == 1) {
a->nrows = nel;
a->maxsize = nel;
a->offset = 0;
}
else {
size_t *adims = &a->nrows;
for(i=0; i < ndims; i++)
adims[i] = dims[i];
}
return a;
}
static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
int isunboxed, int elsz)
{
size_t i, tot, nel=1;
wideint_t prod;
void *data;
jl_array_t *a;
for(i=0; i < ndims; i++) {
prod = (wideint_t)nel * (wideint_t)dims[i];
if (prod > (wideint_t) MAXINTVAL)
jl_error("invalid Array dimensions");
nel = prod;
}
if (isunboxed) {
prod = (wideint_t)elsz * (wideint_t)nel;
if (prod > (wideint_t) MAXINTVAL)
jl_error("invalid Array size");
tot = prod;
if (elsz == 1) {
// hidden 0 terminator for all byte arrays
tot++;
}
}
else {
prod = (wideint_t)sizeof(void*) * (wideint_t)nel;
if (prod > (wideint_t) MAXINTVAL)
jl_error("invalid Array size");
tot = prod;
}
int ndimwords = jl_array_ndimwords(ndims);
size_t tsz = sizeof(jl_array_t);
tsz += ndimwords*sizeof(size_t);
if (tot <= ARRAY_INLINE_NBYTES) {
if (isunboxed && elsz >= 4)
tsz = (tsz+15)&-16; // align data area 16
size_t doffs = tsz;
tsz += tot;
tsz = (tsz+15)&-16; // align whole object 16
a = allocobj(tsz);
a->type = atype;
a->how = 0;
data = (char*)a + doffs;
if (tot > 0 && !isunboxed) {
memset(data, 0, tot);
}
}
else {
tsz = (tsz+15)&-16; // align whole object size 16
a = allocobj(tsz);
JL_GC_PUSH1(&a);
a->type = atype;
// temporarily initialize to make gc-safe
a->data = NULL;
a->how = 2;
data = jl_gc_managed_malloc(tot);
jl_gc_track_malloced_array(a);
if (!isunboxed)
memset(data, 0, tot);
JL_GC_POP();
}
a->data = data;
if (elsz == 1) ((char*)data)[tot-1] = '\0';
#ifdef STORE_ARRAY_LEN
a->length = nel;
#endif
a->ndims = ndims;
a->ptrarray = !isunboxed;
a->elsize = elsz;
a->isshared = 0;
a->isaligned = 1;
a->offset = 0;
if (ndims == 1) {
a->nrows = nel;
a->maxsize = nel;
}
else {
size_t *adims = &a->nrows;
for(i=0; i < ndims; i++)
adims[i] = dims[i];
}
return a;
}
