// Determine if homogeneous tuple with fields of type t will have
// a special alignment beyond normal Julia rules.
// Return special alignment if one exists, 0 if normal alignment rules hold.
// A non-zero result *must* match the LLVM rules for a vector type <nfields x t>.
// For sake of Ahead-Of-Time (AOT) compilation, this routine has to work
// without LLVM being available.
unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) {
if (!is_vecelement_type(t))
return 0;
if (nfields>16 || (1<<nfields & 0x1157C) == 0)
// Number of fields is not 2, 3, 4, 5, 6, 8, 10, 12, or 16.
return 0;
assert(jl_datatype_nfields(t)==1);
jl_value_t *ty = jl_field_type(t, 0);
if( !jl_is_bitstype(ty) )
// LLVM requires that a vector element be a primitive type.
// LLVM allows pointer types as vector elements, but until a
// motivating use case comes up for Julia, we reject pointers.
return 0;
size_t elsz = jl_datatype_size(ty);
if (elsz>8 || (1<<elsz & 0x116) == 0)
// Element size is not 1, 2, 4, or 8.
return 0;
size_t size = nfields*elsz;
// LLVM's alignment rule for vectors seems to be to round up to
// a power of two, even if that's overkill for the target hardware.
size_t alignment=1;
for( ; size>alignment; alignment*=2 )
continue;
return alignment;
}
static Type *julia_type_to_llvm(jl_value_t *jt)
{
if (jt == (jl_value_t*)jl_bool_type) return T_int1;
if (jt == (jl_value_t*)jl_float32_type) return T_float32;
if (jt == (jl_value_t*)jl_float64_type) return T_float64;
if (jt == (jl_value_t*)jl_bottom_type) return T_void;
if (!jl_is_leaf_type(jt))
return jl_pvalue_llvmt;
if (jl_is_cpointer_type(jt)) {
Type *lt = julia_type_to_llvm(jl_tparam0(jt));
if (lt == NULL)
return NULL;
if (lt == T_void)
lt = T_int8;
return PointerType::get(lt, 0);
}
if (jl_is_bitstype(jt)) {
int nb = jl_datatype_size(jt)*8;
if (nb == 8) return T_int8;
if (nb == 16) return T_int16;
if (nb == 32) return T_int32;
if (nb == 64) return T_int64;
else return Type::getIntNTy(getGlobalContext(), nb);
}
if (jl_isbits(jt)) {
if (((jl_datatype_t*)jt)->size == 0) {
// TODO: come up with a representation for a 0-size value,
// and make this 0 size everywhere. as an argument, simply
// skip passing it.
return jl_pvalue_llvmt;
}
return julia_struct_to_llvm(jt);
}
return jl_pvalue_llvmt;
}
// Note that this function updates len
static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len)
{
assert(jl_is_datatype(dt));
jl_datatype_t *bt = (jl_datatype_t*)dt;
size_t nb = jl_datatype_size(bt);
if (nb == 0)
return jl_new_struct_uninit(bt);
*len = LLT_ALIGN(*len, bt->alignment);
data = (char*)data + (*len);
*len += nb;
if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data);
if (bt == jl_int64_type) return jl_box_int64(*(int64_t*)data);
if (bt == jl_bool_type) return (*(int8_t*)data) ? jl_true:jl_false;
if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data);
if (bt == jl_float64_type) return jl_box_float64(*(double*)data);
jl_value_t *v = (jl_value_t*)newobj((jl_value_t*)bt, NWORDS(nb));
switch (nb) {
case 1: *(int8_t*) jl_data_ptr(v) = *(int8_t*)data; break;
case 2: *(int16_t*) jl_data_ptr(v) = *(int16_t*)data; break;
case 4: *(int32_t*) jl_data_ptr(v) = *(int32_t*)data; break;
case 8: *(int64_t*) jl_data_ptr(v) = *(int64_t*)data; break;
case 16: *(bits128_t*)jl_data_ptr(v) = *(bits128_t*)data; break;
default: memcpy(jl_data_ptr(v), data, nb);
}
return v;
}
JL_DLLEXPORT jl_value_t *jl_new_bits(jl_value_t *dt, void *data)
{
// data may not have the alignment required by the size
// but will always have the alignment required by the datatype
jl_ptls_t ptls = jl_get_ptls_states();
assert(jl_is_datatype(dt));
jl_datatype_t *bt = (jl_datatype_t*)dt;
size_t nb = jl_datatype_size(bt);
// some types have special pools to minimize allocations
if (nb == 0) return jl_new_struct_uninit(bt); // returns bt->instance
if (bt == jl_bool_type) return (1 & *(int8_t*)data) ? jl_true : jl_false;
if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data);
if (bt == jl_int64_type) return jl_box_int64(*(int64_t*)data);
if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data);
if (bt == jl_int8_type) return jl_box_int8(*(int8_t*)data);
if (bt == jl_int16_type) return jl_box_int16(*(int16_t*)data);
if (bt == jl_uint64_type) return jl_box_uint64(*(uint64_t*)data);
if (bt == jl_uint32_type) return jl_box_uint32(*(uint32_t*)data);
if (bt == jl_uint16_type) return jl_box_uint16(*(uint16_t*)data);
if (bt == jl_char_type) return jl_box_char(*(uint32_t*)data);
jl_value_t *v = jl_gc_alloc(ptls, nb, bt);
switch (nb) {
case 1: *(uint8_t*) v = *(uint8_t*)data; break;
case 2: *(uint16_t*)v = jl_load_unaligned_i16(data); break;
case 4: *(uint32_t*)v = jl_load_unaligned_i32(data); break;
case 8: *(uint64_t*)v = jl_load_unaligned_i64(data); break;
case 16:
memcpy(jl_assume_aligned(v, 16), data, 16);
break;
default: memcpy(v, data, nb);
}
return v;
}
// Determine if homogeneous tuple with fields of type t will have
// a special alignment beyond normal Julia rules.
// Return special alignment if one exists, 0 if normal alignment rules hold.
// A non-zero result *must* match the LLVM rules for a vector type <nfields x t>.
// For sake of Ahead-Of-Time (AOT) compilation, this routine has to work
// without LLVM being available.
unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) {
if (!jl_is_vecelement_type(t))
return 0;
// LLVM 3.7 and 3.8 either crash or generate wrong code for many
// SIMD vector sizes N. It seems the rule is that N can have at
// most 2 non-zero bits. (This is true at least for N<=100.) See
// also <https://llvm.org/bugs/show_bug.cgi?id=27708>.
size_t mask = nfields;
// See e.g.
// <https://graphics.stanford.edu/%7Eseander/bithacks.html> for an
// explanation of this bit-counting algorithm.
mask &= mask-1; // clear least-significant 1 if present
mask &= mask-1; // clear another 1
if (mask)
return 0; // nfields has more than two 1s
assert(jl_datatype_nfields(t)==1);
jl_value_t *ty = jl_field_type(t, 0);
if (!jl_is_bitstype(ty))
// LLVM requires that a vector element be a primitive type.
// LLVM allows pointer types as vector elements, but until a
// motivating use case comes up for Julia, we reject pointers.
return 0;
size_t elsz = jl_datatype_size(ty);
if (elsz>8 || (1<<elsz & 0x116) == 0)
// Element size is not 1, 2, 4, or 8.
return 0;
size_t size = nfields*elsz;
// LLVM's alignment rule for vectors seems to be to round up to
// a power of two, even if that's overkill for the target hardware.
size_t alignment=1;
for( ; size>alignment; alignment*=2 )
continue;
return alignment;
}
JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args,
uint32_t na)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (type->instance != NULL) return type->instance;
size_t nf = jl_datatype_nfields(type);
jl_value_t *jv = jl_gc_alloc(ptls, jl_datatype_size(type), type);
JL_GC_PUSH1(&jv);
for (size_t i = 0; i < na; i++) {
jl_value_t *ft = jl_field_type(type, i);
if (!jl_isa(args[i], ft))
jl_type_error("new", ft, args[i]);
jl_set_nth_field(jv, i, args[i]);
}
for(size_t i=na; i < nf; i++) {
if (jl_field_isptr(type, i)) {
*(jl_value_t**)((char*)jl_data_ptr(jv)+jl_field_offset(type,i)) = NULL;
} else {
jl_value_t *ft = jl_field_type(type, i);
if (jl_is_uniontype(ft)) {
uint8_t *psel = &((uint8_t *)jv)[jl_field_offset(type, i) + jl_field_size(type, i) - 1];
*psel = 0;
}
}
}
JL_GC_POP();
return jv;
}
void needPassByRef(AbiState *state, jl_datatype_t *dt, bool *byRef, bool *inReg)
{
size_t size = jl_datatype_size(dt);
if (is_complex64(dt) || is_complex128(dt) || (jl_is_bitstype(dt) && size <= 8))
return;
*byRef = true;
}
JL_DLLEXPORT int jl_egal(jl_value_t *a, jl_value_t *b)
{
// warning: a,b may NOT have been gc-rooted by the caller
if (a == b)
return 1;
jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(a);
if (dt != (jl_datatype_t*)jl_typeof(b))
return 0;
if (dt == jl_simplevector_type)
return compare_svec((jl_svec_t*)a, (jl_svec_t*)b);
if (dt == jl_datatype_type) {
jl_datatype_t *dta = (jl_datatype_t*)a;
jl_datatype_t *dtb = (jl_datatype_t*)b;
return dta->name == dtb->name && compare_svec(dta->parameters, dtb->parameters);
}
if (dt == jl_string_type) {
size_t l = jl_string_len(a);
if (jl_string_len(b) != l)
return 0;
return !memcmp(jl_string_data(a), jl_string_data(b), l);
}
if (dt->mutabl)
return 0;
size_t sz = jl_datatype_size(dt);
if (sz == 0)
return 1;
size_t nf = jl_datatype_nfields(dt);
if (nf == 0)
return bits_equal(jl_data_ptr(a), jl_data_ptr(b), sz);
if (dt == jl_unionall_type)
return egal_types(a, b, NULL);
return compare_fields(a, b, dt);
}
void jl_compute_field_offsets(jl_datatype_t *st)
{
size_t sz = 0, alignm = 0;
int ptrfree = 1;
for(size_t i=0; i < jl_tuple_len(st->types); i++) {
jl_value_t *ty = jl_tupleref(st->types, i);
size_t fsz, al;
if (jl_isbits(ty) && (al=((jl_datatype_t*)ty)->alignment)!=0 &&
jl_is_leaf_type(ty)) {
fsz = jl_datatype_size(ty);
st->fields[i].isptr = 0;
}
else {
fsz = sizeof(void*);
al = fsz;
st->fields[i].isptr = 1;
ptrfree = 0;
}
sz = LLT_ALIGN(sz, al);
if (al > alignm)
alignm = al;
st->fields[i].offset = sz;
st->fields[i].size = fsz;
sz += fsz;
}
st->alignment = alignm;
st->size = LLT_ALIGN(sz, alignm);
st->pointerfree = ptrfree && !st->abstract;
}
Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) const override
{
size_t size = jl_datatype_size(dt);
if (size > 0 && win64_reg_size(size) && !jl_is_bitstype(dt))
return Type::getIntNTy(jl_LLVMContext, jl_datatype_nbits(dt));
return NULL;
}
bool use_sret(jl_datatype_t *dt) override
{
size_t size = jl_datatype_size(dt);
if (win64_reg_size(size) || is_native_simd_type(dt))
return false;
return true;
}
// this is a run-time function
// warning: cannot allocate memory except using alloc_temp_arg_space
extern "C" DLLEXPORT void *jl_value_to_pointer(jl_value_t *jt, jl_value_t *v, int argn,
int addressof)
{
jl_value_t *jvt = (jl_value_t*)jl_typeof(v);
if (addressof) {
if (jvt == jt) {
if (jl_is_bitstype(jvt)) {
size_t osz = jl_datatype_size(jt);
return alloc_temp_arg_copy(jl_data_ptr(v), osz);
}
else if (!jl_is_tuple(jvt) && jl_is_leaf_type(jvt) && !jl_is_array_type(jvt)) {
return v + 1;
}
}
goto value_to_pointer_error;
}
else {
if (jl_is_cpointer_type(jvt) && jl_tparam0(jvt) == jt) {
return (void*)jl_unbox_voidpointer(v);
}
}
if (((jl_value_t*)jl_uint8_type == jt ||
(jl_value_t*)jl_int8_type == jt) && jl_is_byte_string(v)) {
return jl_string_data(v);
}
if (jl_is_array_type(jvt)) {
if (jl_tparam0(jl_typeof(v)) == jt || jt==(jl_value_t*)jl_bottom_type)
return ((jl_array_t*)v)->data;
if (jl_is_cpointer_type(jt)) {
jl_array_t *ar = (jl_array_t*)v;
void **temp=(void**)alloc_temp_arg_space((1+jl_array_len(ar))*sizeof(void*));
size_t i;
for(i=0; i < jl_array_len(ar); i++) {
temp[i] = jl_value_to_pointer(jl_tparam0(jt),
jl_arrayref(ar, i), argn, 0);
}
temp[i] = 0;
return temp;
}
}
value_to_pointer_error:
std::map<int, std::string>::iterator it = argNumberStrings.find(argn);
if (it == argNumberStrings.end()) {
std::stringstream msg;
msg << "argument ";
msg << argn;
argNumberStrings[argn] = msg.str();
it = argNumberStrings.find(argn);
}
jl_value_t *targ=NULL, *pty=NULL;
JL_GC_PUSH2(&targ, &pty);
targ = (jl_value_t*)jl_tuple1(jt);
pty = (jl_value_t*)jl_apply_type((jl_value_t*)jl_pointer_type,
(jl_tuple_t*)targ);
jl_type_error_rt("ccall", (*it).second.c_str(), pty, v);
// doesn't return
return (jl_value_t*)jl_null;
}
bool use_sret(jl_datatype_t *dt) override
{
jl_datatype_t *ty0 = NULL;
bool hva = false;
if (jl_datatype_size(dt) > 16 && isHFA(dt, &ty0, &hva) > 8)
return true;
return false;
}
bool needPassByRef(jl_datatype_t *dt, AttrBuilder &ab) override
{
size_t size = jl_datatype_size(dt);
if (is_complex64(dt) || is_complex128(dt) || (jl_is_primitivetype(dt) && size <= 8))
return false;
ab.addAttribute(Attribute::ByVal);
return true;
}
JL_DLLEXPORT jl_value_t *jl_check_top_bit(jl_value_t *a)
{
jl_value_t *ty = jl_typeof(a);
if (!jl_is_bitstype(ty))
jl_error("check_top_bit: value is not a bitstype");
if (signbitbyte(jl_data_ptr(a), jl_datatype_size(ty)))
jl_throw(jl_inexact_exception);
return a;
}
bool use_sret(jl_datatype_t *dt) override
{
size_t size = jl_datatype_size(dt);
if (size == 0)
return false;
if (is_complex64(dt) || (jl_is_primitivetype(dt) && size <= 8))
return false;
return true;
}
bool use_sret(AbiState *state, jl_datatype_t *dt)
{
size_t size = jl_datatype_size(dt);
if (size == 0)
return false;
if (is_complex64(dt) || (jl_is_bitstype(dt) && size <= 8))
return false;
return true;
}
static uintptr_t jl_object_id_(jl_value_t *tv, jl_value_t *v)
{
if (tv == (jl_value_t*)jl_sym_type)
return ((jl_sym_t*)v)->hash;
if (tv == (jl_value_t*)jl_simplevector_type)
return hash_svec((jl_svec_t*)v);
jl_datatype_t *dt = (jl_datatype_t*)tv;
if (dt == jl_datatype_type) {
jl_datatype_t *dtv = (jl_datatype_t*)v;
// `name->wrapper` is cacheable even though it contains TypeVars
// that don't have stable IDs.
//if (jl_egal(dtv->name->wrapper, v))
// return bitmix(~dtv->name->hash, 0xaa5566aa);
return bitmix(~dtv->name->hash, hash_svec(dtv->parameters));
}
if (dt == jl_typename_type)
return ((jl_typename_t*)v)->hash;
#ifdef _P64
if (v == jl_ANY_flag) return 0x31c472f68ee30bddULL;
#else
if (v == jl_ANY_flag) return 0x8ee30bdd;
#endif
if (dt == jl_string_type) {
#ifdef _P64
return memhash_seed(jl_string_data(v), jl_string_len(v), 0xedc3b677);
#else
return memhash32_seed(jl_string_data(v), jl_string_len(v), 0xedc3b677);
#endif
}
if (dt->mutabl) return inthash((uintptr_t)v);
size_t sz = jl_datatype_size(tv);
uintptr_t h = jl_object_id(tv);
if (sz == 0) return ~h;
size_t nf = jl_datatype_nfields(dt);
if (nf == 0) {
return bits_hash(jl_data_ptr(v), sz) ^ h;
}
for (size_t f=0; f < nf; f++) {
size_t offs = jl_field_offset(dt, f);
char *vo = (char*)jl_data_ptr(v) + offs;
uintptr_t u;
if (jl_field_isptr(dt, f)) {
jl_value_t *f = *(jl_value_t**)vo;
u = f==NULL ? 0 : jl_object_id(f);
}
else {
jl_datatype_t *fieldtype = (jl_datatype_t*)jl_field_type(dt, f);
assert(jl_is_datatype(fieldtype) && !fieldtype->abstract && !fieldtype->mutabl);
if (fieldtype->layout->haspadding)
u = jl_object_id_((jl_value_t*)fieldtype, (jl_value_t*)vo);
else
u = bits_hash(vo, jl_field_size(dt, f));
}
h = bitmix(h, u);
}
return h;
}
static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *dims)
{
int isunboxed=0, elsz=sizeof(void*);
jl_value_t *el_type = jl_tparam0(atype);
isunboxed = store_unboxed(el_type);
if (isunboxed)
elsz = jl_datatype_size(el_type);
return _new_array_(atype, ndims, dims, isunboxed, elsz);
}
jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_tuple_t *dims)
{
size_t i;
jl_array_t *a;
size_t ndims = jl_tuple_len(dims);
int ndimwords = jl_array_ndimwords(ndims);
a = allocobj((sizeof(jl_array_t) + sizeof(void*) + ndimwords*sizeof(size_t) + 15)&-16);
a->type = atype;
a->ndims = ndims;
a->offset = 0;
a->data = NULL;
a->isaligned = data->isaligned;
jl_value_t *el_type = jl_tparam0(atype);
if (store_unboxed(el_type)) {
a->elsize = jl_datatype_size(el_type);
a->ptrarray = 0;
}
else {
a->elsize = sizeof(void*);
a->ptrarray = 1;
}
JL_GC_PUSH1(&a);
jl_array_data_owner(a) = (jl_value_t*)data;
a->how = 3;
a->data = data->data;
a->isshared = 1;
data->isshared = 1;
if (ndims == 1) {
size_t l = jl_unbox_long(jl_tupleref(dims,0));
#ifdef STORE_ARRAY_LEN
a->length = l;
#endif
a->nrows = l;
a->maxsize = l;
}
else {
size_t *adims = &a->nrows;
size_t l=1;
wideint_t prod;
for(i=0; i < ndims; i++) {
adims[i] = jl_unbox_long(jl_tupleref(dims, i));
prod = (wideint_t)l * (wideint_t)adims[i];
if (prod > (wideint_t) MAXINTVAL)
jl_error("invalid Array dimensions");
l = prod;
}
#ifdef STORE_ARRAY_LEN
a->length = l;
#endif
}
JL_GC_POP();
return a;
}
static inline jl_value_t *jl_intrinsic_cvt(jl_value_t *ty, jl_value_t *a, const char *name, intrinsic_cvt_t op, intrinsic_cvt_check_t check_op)
{
jl_value_t *aty = jl_typeof(a);
if (!jl_is_bitstype(aty))
jl_errorf("%s: value is not a bitstype", name);
if (!jl_is_bitstype(ty))
jl_errorf("%s: type is not a bitstype", name);
void *pa = jl_data_ptr(a);
unsigned isize = jl_datatype_size(aty);
unsigned osize = jl_datatype_size(ty);
if (check_op && check_op(isize, osize, pa))
jl_throw(jl_inexact_exception);
jl_value_t *newv = newstruct((jl_datatype_t*)ty);
op(aty == (jl_value_t*)jl_bool_type ? 1 : isize * host_char_bit, pa,
osize * host_char_bit, jl_data_ptr(newv));
if (ty == (jl_value_t*)jl_bool_type)
return *(uint8_t*)jl_data_ptr(newv) & 1 ? jl_true : jl_false;
return newv;
}
JL_DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (type->instance != NULL) return type->instance;
size_t size = jl_datatype_size(type);
jl_value_t *jv = jl_gc_alloc(ptls, size, type);
if (size > 0)
memset(jl_data_ptr(jv), 0, size);
return jv;
}
bool needPassByRef(jl_datatype_t *dt, AttrBuilder &ab) override
{
jl_datatype_t *ty0 = NULL;
bool hva = false;
if (jl_datatype_size(dt) > 64 && isHFA(dt, &ty0, &hva) > 8) {
ab.addAttribute(Attribute::ByVal);
return true;
}
return false;
}
jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims,
int own_buffer)
{
size_t i, elsz, nel=1;
jl_array_t *a;
size_t ndims = jl_tuple_len(dims);
wideint_t prod;
for(i=0; i < ndims; i++) {
prod = (wideint_t)nel * (wideint_t)jl_unbox_long(jl_tupleref(dims, i));
if (prod > (wideint_t) MAXINTVAL)
jl_error("invalid Array dimensions");
nel = prod;
}
jl_value_t *el_type = jl_tparam0(atype);
int isunboxed = store_unboxed(el_type);
if (isunboxed)
elsz = jl_datatype_size(el_type);
else
elsz = sizeof(void*);
int ndimwords = jl_array_ndimwords(ndims);
a = allocobj((sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16);
a->type = atype;
a->data = data;
#ifdef STORE_ARRAY_LEN
a->length = nel;
#endif
a->elsize = elsz;
a->ptrarray = !isunboxed;
a->ndims = ndims;
a->offset = 0;
a->isshared = 1;
a->isaligned = 0;
if (own_buffer) {
a->how = 2;
jl_gc_track_malloced_array(a);
}
else {
a->how = 0;
}
if (ndims == 1) {
a->nrows = nel;
a->maxsize = nel;
}
else {
size_t *adims = &a->nrows;
for(i=0; i < ndims; i++) {
adims[i] = jl_unbox_long(jl_tupleref(dims, i));
}
}
return a;
}
JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
jl_value_t *_dims, int own_buffer)
{
jl_ptls_t ptls = jl_get_ptls_states();
size_t elsz, nel = 1;
jl_array_t *a;
size_t ndims = jl_nfields(_dims);
wideint_t prod;
assert(is_ntuple_long(_dims));
size_t *dims = (size_t*)_dims;
for (size_t i = 0; i < ndims; i++) {
prod = (wideint_t)nel * (wideint_t)dims[i];
if (prod > (wideint_t) MAXINTVAL)
jl_error("invalid Array dimensions");
nel = prod;
}
if (__unlikely(ndims == 1))
return jl_ptr_to_array_1d(atype, data, nel, own_buffer);
jl_value_t *el_type = jl_tparam0(atype);
int isunboxed = store_unboxed(el_type);
if (isunboxed)
elsz = jl_datatype_size(el_type);
else
elsz = sizeof(void*);
int ndimwords = jl_array_ndimwords(ndims);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
a->data = data;
#ifdef STORE_ARRAY_LEN
a->length = nel;
#endif
a->elsize = elsz;
a->flags.ptrarray = !isunboxed;
a->flags.ndims = ndims;
a->offset = 0;
a->flags.isshared = 1;
a->flags.isaligned = 0;
if (own_buffer) {
a->flags.how = 2;
jl_gc_track_malloced_array(ptls, a);
jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
}
else {
a->flags.how = 0;
}
assert(ndims != 1); // handled above
memcpy(&a->nrows, dims, ndims * sizeof(size_t));
return a;
}
// run time version of pointerref intrinsic
DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i)
{
JL_TYPECHK(pointerref, pointer, p);
JL_TYPECHK(pointerref, long, i);
jl_value_t *ety = jl_tparam0(jl_typeof(p));
if (!jl_is_datatype(ety))
jl_error("pointerref: invalid pointer");
size_t nb = jl_datatype_size(ety);
char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb;
return jl_new_bits(ety, pp);
}
void jl_assign_bits(void *dest, jl_value_t *bits)
{
size_t nb = jl_datatype_size(jl_typeof(bits));
switch (nb) {
case 1: *(int8_t*)dest = *(int8_t*)jl_data_ptr(bits); break;
case 2: *(int16_t*)dest = *(int16_t*)jl_data_ptr(bits); break;
case 4: *(int32_t*)dest = *(int32_t*)jl_data_ptr(bits); break;
case 8: *(int64_t*)dest = *(int64_t*)jl_data_ptr(bits); break;
case 16: *(bits128_t*)dest = *(bits128_t*)jl_data_ptr(bits); break;
default: memcpy(dest, jl_data_ptr(bits), nb);
}
}
void jl_compute_field_offsets(jl_datatype_t *st)
{
size_t sz = 0, alignm = 1;
int ptrfree = 1;
assert(0 <= st->fielddesc_type && st->fielddesc_type <= 2);
uint64_t max_offset = (((uint64_t)1) <<
(1 << (3 + st->fielddesc_type))) - 1;
uint64_t max_size = max_offset >> 1;
for(size_t i=0; i < jl_datatype_nfields(st); i++) {
jl_value_t *ty = jl_field_type(st, i);
size_t fsz, al;
if (jl_isbits(ty) && jl_is_leaf_type(ty)) {
fsz = jl_datatype_size(ty);
// Should never happen
if (__unlikely(fsz > max_size))
jl_throw(jl_overflow_exception);
al = ((jl_datatype_t*)ty)->alignment;
jl_field_setisptr(st, i, 0);
if (((jl_datatype_t*)ty)->haspadding)
st->haspadding = 1;
}
else {
fsz = sizeof(void*);
if (fsz > MAX_ALIGN)
fsz = MAX_ALIGN;
al = fsz;
jl_field_setisptr(st, i, 1);
ptrfree = 0;
}
if (al != 0) {
size_t alsz = LLT_ALIGN(sz, al);
if (sz & (al - 1))
st->haspadding = 1;
sz = alsz;
if (al > alignm)
alignm = al;
}
jl_field_setoffset(st, i, sz);
jl_field_setsize(st, i, fsz);
if (__unlikely(max_offset - sz < fsz))
jl_throw(jl_overflow_exception);
sz += fsz;
}
st->alignment = alignm;
st->size = LLT_ALIGN(sz, alignm);
if (st->size > sz)
st->haspadding = 1;
st->pointerfree = ptrfree && !st->abstract;
}
// own_buffer != 0 iff GC should call free() on this pointer eventually
JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
size_t nel, int own_buffer)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_array_t *a;
jl_value_t *eltype = jl_tparam0(atype);
int isunboxed = jl_array_store_unboxed(eltype);
size_t elsz;
unsigned align;
if (isunboxed && jl_is_uniontype(eltype))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: unspecified layout for union element type");
if (isunboxed) {
elsz = jl_datatype_size(eltype);
align = jl_datatype_align(eltype);
}
else {
align = elsz = sizeof(void*);
}
if (((uintptr_t)data) & (align - 1))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align);
int ndimwords = jl_array_ndimwords(1);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
a->data = data;
#ifdef STORE_ARRAY_LEN
a->length = nel;
#endif
a->elsize = elsz;
a->flags.ptrarray = !isunboxed;
a->flags.ndims = 1;
a->flags.isshared = 1;
a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers
if (own_buffer) {
a->flags.how = 2;
jl_gc_track_malloced_array(ptls, a);
jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
}
else {
a->flags.how = 0;
}
a->nrows = nel;
a->maxsize = nel;
a->offset = 0;
return a;
}
// run time version of pointerset intrinsic
DLLEXPORT void jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *i)
{
JL_TYPECHK(pointerset, pointer, p);
JL_TYPECHK(pointerset, long, i);
jl_value_t *ety = jl_tparam0(jl_typeof(p));
if (!jl_is_datatype(ety))
jl_error("pointerset: invalid pointer");
size_t nb = jl_datatype_size(ety);
char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb;
if (jl_typeof(x) != ety)
jl_error("pointerset: type mismatch in assign");
jl_assign_bits(pp, x);
}
