// Note that this function updates len
static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len)
{
jl_ptls_t ptls = jl_get_ptls_states();
assert(jl_is_datatype(dt));
jl_datatype_t *bt = (jl_datatype_t*)dt;
size_t nb = jl_datatype_size(bt);
if (nb == 0)
return jl_new_struct_uninit(bt);
*len = LLT_ALIGN(*len, jl_datatype_align(bt));
data = (char*)data + (*len);
*len += nb;
if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data);
if (bt == jl_int64_type) return jl_box_int64(*(int64_t*)data);
if (bt == jl_bool_type) return (*(int8_t*)data) ? jl_true:jl_false;
if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data);
if (bt == jl_float64_type) return jl_box_float64(*(double*)data);
jl_value_t *v = jl_gc_alloc(ptls, nb, bt);
switch (nb) {
case 1: *(int8_t*) jl_data_ptr(v) = *(int8_t*)data; break;
case 2: *(int16_t*) jl_data_ptr(v) = *(int16_t*)data; break;
case 4: *(int32_t*) jl_data_ptr(v) = *(int32_t*)data; break;
case 8: *(int64_t*) jl_data_ptr(v) = *(int64_t*)data; break;
case 16: *(bits128_t*)jl_data_ptr(v) = *(bits128_t*)data; break;
default: memcpy(jl_data_ptr(v), data, nb);
}
return v;
}
// own_buffer != 0 iff GC should call free() on this pointer eventually
JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
size_t nel, int own_buffer)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_array_t *a;
jl_value_t *eltype = jl_tparam0(atype);
int isunboxed = jl_array_store_unboxed(eltype);
size_t elsz;
unsigned align;
if (isunboxed && jl_is_uniontype(eltype))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: unspecified layout for union element type");
if (isunboxed) {
elsz = jl_datatype_size(eltype);
align = jl_datatype_align(eltype);
}
else {
align = elsz = sizeof(void*);
}
if (((uintptr_t)data) & (align - 1))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align);
int ndimwords = jl_array_ndimwords(1);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
a->data = data;
#ifdef STORE_ARRAY_LEN
a->length = nel;
#endif
a->elsize = elsz;
a->flags.ptrarray = !isunboxed;
a->flags.ndims = 1;
a->flags.isshared = 1;
a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers
if (own_buffer) {
a->flags.how = 2;
jl_gc_track_malloced_array(ptls, a);
jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
}
else {
a->flags.how = 0;
}
a->nrows = nel;
a->maxsize = nel;
a->offset = 0;
return a;
}
// run time version of pointerref intrinsic (warning: i is not rooted)
JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i, jl_value_t *align)
{
JL_TYPECHK(pointerref, pointer, p);
JL_TYPECHK(pointerref, long, i)
JL_TYPECHK(pointerref, long, align);
jl_value_t *ety = jl_tparam0(jl_typeof(p));
if (ety == (jl_value_t*)jl_any_type) {
jl_value_t **pp = (jl_value_t**)(jl_unbox_long(p) + (jl_unbox_long(i)-1)*sizeof(void*));
return *pp;
}
else {
if (!jl_is_datatype(ety))
jl_error("pointerref: invalid pointer");
size_t nb = LLT_ALIGN(jl_datatype_size(ety), jl_datatype_align(ety));
char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb;
return jl_new_bits(ety, pp);
}
}
Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) const override
{
// Arguments are either scalar or passed by value
size_t size = jl_datatype_size(dt);
// don't need to change bitstypes
if (!jl_datatype_nfields(dt))
return NULL;
// legalize this into [n x f32/f64]
jl_datatype_t *ty0 = NULL;
bool hva = false;
int hfa = isHFA(dt, &ty0, &hva);
if (hfa <= 8) {
if (ty0 == jl_float32_type) {
return ArrayType::get(T_float32, hfa);
}
else if (ty0 == jl_float64_type) {
return ArrayType::get(T_float64, hfa);
}
else {
jl_datatype_t *vecty = (jl_datatype_t*)jl_field_type(ty0, 0);
assert(jl_is_datatype(vecty) && vecty->name == jl_vecelement_typename);
jl_value_t *elemty = jl_tparam0(vecty);
assert(jl_is_primitivetype(elemty));
Type *ety = julia_type_to_llvm(elemty);
Type *vty = VectorType::get(ety, jl_datatype_nfields(ty0));
return ArrayType::get(vty, hfa);
}
}
// rewrite integer-sized (non-HFA) struct to an array
// the bitsize of the integer gives the desired alignment
if (size > 8) {
if (jl_datatype_align(dt) <= 8) {
return ArrayType::get(T_int64, (size + 7) / 8);
}
else {
Type *T_int128 = Type::getIntNTy(jl_LLVMContext, 128);
return ArrayType::get(T_int128, (size + 15) / 16);
}
}
return Type::getIntNTy(jl_LLVMContext, size * 8);
}
// run time version of pointerset intrinsic (warning: x is not gc-rooted)
JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *i, jl_value_t *align)
{
JL_TYPECHK(pointerset, pointer, p);
JL_TYPECHK(pointerset, long, i);
JL_TYPECHK(pointerref, long, align);
jl_value_t *ety = jl_tparam0(jl_typeof(p));
if (ety == (jl_value_t*)jl_any_type) {
jl_value_t **pp = (jl_value_t**)(jl_unbox_long(p) + (jl_unbox_long(i)-1)*sizeof(void*));
*pp = x;
}
else {
if (!jl_is_datatype(ety))
jl_error("pointerset: invalid pointer");
size_t elsz = jl_datatype_size(ety);
size_t nb = LLT_ALIGN(elsz, jl_datatype_align(ety));
char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb;
if (jl_typeof(x) != ety)
jl_error("pointerset: type mismatch in assign");
memcpy(pp, x, elsz);
}
return p;
}
static unsigned union_isbits(jl_value_t *ty, size_t *nbytes, size_t *align)
{
if (jl_is_uniontype(ty)) {
unsigned na = union_isbits(((jl_uniontype_t*)ty)->a, nbytes, align);
if (na == 0)
return 0;
unsigned nb = union_isbits(((jl_uniontype_t*)ty)->b, nbytes, align);
if (nb == 0)
return 0;
return na + nb;
}
if (jl_isbits(ty)) {
size_t sz = jl_datatype_size(ty);
size_t al = jl_datatype_align(ty);
if (*nbytes < sz)
*nbytes = sz;
if (*align < al)
*align = al;
return 1;
}
return 0;
}
JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
jl_value_t *_dims, int own_buffer)
{
jl_ptls_t ptls = jl_get_ptls_states();
size_t nel = 1;
jl_array_t *a;
size_t ndims = jl_nfields(_dims);
wideint_t prod;
assert(is_ntuple_long(_dims));
size_t *dims = (size_t*)_dims;
for (size_t i = 0; i < ndims; i++) {
prod = (wideint_t)nel * (wideint_t)dims[i];
if (prod > (wideint_t) MAXINTVAL)
jl_error("invalid Array dimensions");
nel = prod;
}
if (__unlikely(ndims == 1))
return jl_ptr_to_array_1d(atype, data, nel, own_buffer);
jl_value_t *eltype = jl_tparam0(atype);
int isunboxed = jl_array_store_unboxed(eltype);
size_t elsz;
unsigned align;
if (isunboxed && jl_is_uniontype(eltype))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: unspecified layout for union element type");
if (isunboxed) {
elsz = jl_datatype_size(eltype);
align = jl_datatype_align(eltype);
}
else {
align = elsz = sizeof(void*);
}
if (((uintptr_t)data) & (align - 1))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align);
int ndimwords = jl_array_ndimwords(ndims);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
a->data = data;
#ifdef STORE_ARRAY_LEN
a->length = nel;
#endif
a->elsize = elsz;
a->flags.ptrarray = !isunboxed;
a->flags.ndims = ndims;
a->offset = 0;
a->flags.isshared = 1;
a->flags.isaligned = 0;
if (own_buffer) {
a->flags.how = 2;
jl_gc_track_malloced_array(ptls, a);
jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
}
else {
a->flags.how = 0;
}
assert(ndims != 1); // handled above
memcpy(&a->nrows, dims, ndims * sizeof(size_t));
return a;
}
JL_DLLEXPORT size_t jl_get_alignment(jl_datatype_t *ty)
{
if (ty->layout == NULL)
jl_error("non-leaf type doesn't have an alignment");
return jl_datatype_align(ty);
}
void jl_compute_field_offsets(jl_datatype_t *st)
{
size_t sz = 0, alignm = 1;
int homogeneous = 1;
jl_value_t *lastty = NULL;
uint64_t max_offset = (((uint64_t)1) << 32) - 1;
uint64_t max_size = max_offset >> 1;
if (st->name->wrapper) {
// If layout doesn't depend on type parameters, it's stored in st->name->wrapper
// and reused by all subtypes.
jl_datatype_t *w = (jl_datatype_t*)jl_unwrap_unionall(st->name->wrapper);
if (st != w && // this check allows us to re-compute layout for some types during init
w->layout) {
st->layout = w->layout;
st->size = w->size;
return;
}
}
if (st->types == NULL)
return;
uint32_t nfields = jl_svec_len(st->types);
if (nfields == 0) {
if (st == jl_sym_type || st == jl_string_type) {
// opaque layout - heap-allocated blob
static const jl_datatype_layout_t opaque_byte_layout = {0, 1, 0, 1, 0};
st->layout = &opaque_byte_layout;
}
else if (st == jl_simplevector_type || st->name == jl_array_typename) {
static const jl_datatype_layout_t opaque_ptr_layout = {0, sizeof(void*), 0, 1, 0};
st->layout = &opaque_ptr_layout;
}
else {
// reuse the same layout for all singletons
static const jl_datatype_layout_t singleton_layout = {0, 1, 0, 0, 0};
st->layout = &singleton_layout;
}
return;
}
if (!jl_is_leaf_type((jl_value_t*)st)) {
// compute layout whenever field types have no free variables
for (size_t i = 0; i < nfields; i++) {
if (jl_has_free_typevars(jl_field_type(st, i)))
return;
}
}
size_t descsz = nfields * sizeof(jl_fielddesc32_t);
jl_fielddesc32_t *desc;
if (descsz < jl_page_size)
desc = (jl_fielddesc32_t*)alloca(descsz);
else
desc = (jl_fielddesc32_t*)malloc(descsz);
int haspadding = 0;
assert(st->name == jl_tuple_typename ||
st == jl_sym_type ||
st == jl_simplevector_type ||
nfields != 0);
for (size_t i = 0; i < nfields; i++) {
jl_value_t *ty = jl_field_type(st, i);
size_t fsz, al;
if (jl_isbits(ty) && jl_is_leaf_type(ty) && ((jl_datatype_t*)ty)->layout) {
fsz = jl_datatype_size(ty);
// Should never happen
if (__unlikely(fsz > max_size))
goto throw_ovf;
al = jl_datatype_align(ty);
desc[i].isptr = 0;
if (((jl_datatype_t*)ty)->layout->haspadding)
haspadding = 1;
}
else {
fsz = sizeof(void*);
if (fsz > MAX_ALIGN)
fsz = MAX_ALIGN;
al = fsz;
desc[i].isptr = 1;
}
assert(al <= JL_HEAP_ALIGNMENT && (JL_HEAP_ALIGNMENT % al) == 0);
if (al != 0) {
size_t alsz = LLT_ALIGN(sz, al);
if (sz & (al - 1))
haspadding = 1;
sz = alsz;
if (al > alignm)
alignm = al;
}
homogeneous &= lastty==NULL || lastty==ty;
lastty = ty;
desc[i].offset = sz;
desc[i].size = fsz;
if (__unlikely(max_offset - sz < fsz))
goto throw_ovf;
sz += fsz;
}
if (homogeneous && lastty!=NULL && jl_is_tuple_type(st)) {
// Some tuples become LLVM vectors with stronger alignment than what was calculated above.
unsigned al = jl_special_vector_alignment(nfields, lastty);
assert(al % alignm == 0);
// JL_HEAP_ALIGNMENT is the biggest alignment we can guarantee on the heap.
if (al > JL_HEAP_ALIGNMENT)
alignm = JL_HEAP_ALIGNMENT;
else if (al)
alignm = al;
}
st->size = LLT_ALIGN(sz, alignm);
if (st->size > sz)
haspadding = 1;
st->layout = jl_get_layout(nfields, alignm, haspadding, desc);
if (descsz >= jl_page_size) free(desc);
return;
throw_ovf:
if (descsz >= jl_page_size) free(desc);
jl_throw(jl_overflow_exception);
}
