// Determine if homogeneous tuple with fields of type t will have // a special alignment beyond normal Julia rules. // Return special alignment if one exists, 0 if normal alignment rules hold. // A non-zero result *must* match the LLVM rules for a vector type <nfields x t>. // For sake of Ahead-Of-Time (AOT) compilation, this routine has to work // without LLVM being available. unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) { if (!is_vecelement_type(t)) return 0; if (nfields>16 || (1<<nfields & 0x1157C) == 0) // Number of fields is not 2, 3, 4, 5, 6, 8, 10, 12, or 16. return 0; assert(jl_datatype_nfields(t)==1); jl_value_t *ty = jl_field_type(t, 0); if( !jl_is_bitstype(ty) ) // LLVM requires that a vector element be a primitive type. // LLVM allows pointer types as vector elements, but until a // motivating use case comes up for Julia, we reject pointers. return 0; size_t elsz = jl_datatype_size(ty); if (elsz>8 || (1<<elsz & 0x116) == 0) // Element size is not 1, 2, 4, or 8. return 0; size_t size = nfields*elsz; // LLVM's alignment rule for vectors seems to be to round up to // a power of two, even if that's overkill for the target hardware. size_t alignment=1; for( ; size>alignment; alignment*=2 ) continue; return alignment; }
static Type *julia_type_to_llvm(jl_value_t *jt) { if (jt == (jl_value_t*)jl_bool_type) return T_int1; if (jt == (jl_value_t*)jl_float32_type) return T_float32; if (jt == (jl_value_t*)jl_float64_type) return T_float64; if (jt == (jl_value_t*)jl_bottom_type) return T_void; if (!jl_is_leaf_type(jt)) return jl_pvalue_llvmt; if (jl_is_cpointer_type(jt)) { Type *lt = julia_type_to_llvm(jl_tparam0(jt)); if (lt == NULL) return NULL; if (lt == T_void) lt = T_int8; return PointerType::get(lt, 0); } if (jl_is_bitstype(jt)) { int nb = jl_datatype_size(jt)*8; if (nb == 8) return T_int8; if (nb == 16) return T_int16; if (nb == 32) return T_int32; if (nb == 64) return T_int64; else return Type::getIntNTy(getGlobalContext(), nb); } if (jl_isbits(jt)) { if (((jl_datatype_t*)jt)->size == 0) { // TODO: come up with a representation for a 0-size value, // and make this 0 size everywhere. as an argument, simply // skip passing it. return jl_pvalue_llvmt; } return julia_struct_to_llvm(jt); } return jl_pvalue_llvmt; }
// Note that this function updates len static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len) { assert(jl_is_datatype(dt)); jl_datatype_t *bt = (jl_datatype_t*)dt; size_t nb = jl_datatype_size(bt); if (nb == 0) return jl_new_struct_uninit(bt); *len = LLT_ALIGN(*len, bt->alignment); data = (char*)data + (*len); *len += nb; if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data); if (bt == jl_int64_type) return jl_box_int64(*(int64_t*)data); if (bt == jl_bool_type) return (*(int8_t*)data) ? jl_true:jl_false; if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data); if (bt == jl_float64_type) return jl_box_float64(*(double*)data); jl_value_t *v = (jl_value_t*)newobj((jl_value_t*)bt, NWORDS(nb)); switch (nb) { case 1: *(int8_t*) jl_data_ptr(v) = *(int8_t*)data; break; case 2: *(int16_t*) jl_data_ptr(v) = *(int16_t*)data; break; case 4: *(int32_t*) jl_data_ptr(v) = *(int32_t*)data; break; case 8: *(int64_t*) jl_data_ptr(v) = *(int64_t*)data; break; case 16: *(bits128_t*)jl_data_ptr(v) = *(bits128_t*)data; break; default: memcpy(jl_data_ptr(v), data, nb); } return v; }
JL_DLLEXPORT jl_value_t *jl_new_bits(jl_value_t *dt, void *data) { // data may not have the alignment required by the size // but will always have the alignment required by the datatype jl_ptls_t ptls = jl_get_ptls_states(); assert(jl_is_datatype(dt)); jl_datatype_t *bt = (jl_datatype_t*)dt; size_t nb = jl_datatype_size(bt); // some types have special pools to minimize allocations if (nb == 0) return jl_new_struct_uninit(bt); // returns bt->instance if (bt == jl_bool_type) return (1 & *(int8_t*)data) ? jl_true : jl_false; if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data); if (bt == jl_int64_type) return jl_box_int64(*(int64_t*)data); if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data); if (bt == jl_int8_type) return jl_box_int8(*(int8_t*)data); if (bt == jl_int16_type) return jl_box_int16(*(int16_t*)data); if (bt == jl_uint64_type) return jl_box_uint64(*(uint64_t*)data); if (bt == jl_uint32_type) return jl_box_uint32(*(uint32_t*)data); if (bt == jl_uint16_type) return jl_box_uint16(*(uint16_t*)data); if (bt == jl_char_type) return jl_box_char(*(uint32_t*)data); jl_value_t *v = jl_gc_alloc(ptls, nb, bt); switch (nb) { case 1: *(uint8_t*) v = *(uint8_t*)data; break; case 2: *(uint16_t*)v = jl_load_unaligned_i16(data); break; case 4: *(uint32_t*)v = jl_load_unaligned_i32(data); break; case 8: *(uint64_t*)v = jl_load_unaligned_i64(data); break; case 16: memcpy(jl_assume_aligned(v, 16), data, 16); break; default: memcpy(v, data, nb); } return v; }
// Determine if homogeneous tuple with fields of type t will have // a special alignment beyond normal Julia rules. // Return special alignment if one exists, 0 if normal alignment rules hold. // A non-zero result *must* match the LLVM rules for a vector type <nfields x t>. // For sake of Ahead-Of-Time (AOT) compilation, this routine has to work // without LLVM being available. unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) { if (!jl_is_vecelement_type(t)) return 0; // LLVM 3.7 and 3.8 either crash or generate wrong code for many // SIMD vector sizes N. It seems the rule is that N can have at // most 2 non-zero bits. (This is true at least for N<=100.) See // also <https://llvm.org/bugs/show_bug.cgi?id=27708>. size_t mask = nfields; // See e.g. // <https://graphics.stanford.edu/%7Eseander/bithacks.html> for an // explanation of this bit-counting algorithm. mask &= mask-1; // clear least-significant 1 if present mask &= mask-1; // clear another 1 if (mask) return 0; // nfields has more than two 1s assert(jl_datatype_nfields(t)==1); jl_value_t *ty = jl_field_type(t, 0); if (!jl_is_bitstype(ty)) // LLVM requires that a vector element be a primitive type. // LLVM allows pointer types as vector elements, but until a // motivating use case comes up for Julia, we reject pointers. return 0; size_t elsz = jl_datatype_size(ty); if (elsz>8 || (1<<elsz & 0x116) == 0) // Element size is not 1, 2, 4, or 8. return 0; size_t size = nfields*elsz; // LLVM's alignment rule for vectors seems to be to round up to // a power of two, even if that's overkill for the target hardware. size_t alignment=1; for( ; size>alignment; alignment*=2 ) continue; return alignment; }
JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, uint32_t na) { jl_ptls_t ptls = jl_get_ptls_states(); if (type->instance != NULL) return type->instance; size_t nf = jl_datatype_nfields(type); jl_value_t *jv = jl_gc_alloc(ptls, jl_datatype_size(type), type); JL_GC_PUSH1(&jv); for (size_t i = 0; i < na; i++) { jl_value_t *ft = jl_field_type(type, i); if (!jl_isa(args[i], ft)) jl_type_error("new", ft, args[i]); jl_set_nth_field(jv, i, args[i]); } for(size_t i=na; i < nf; i++) { if (jl_field_isptr(type, i)) { *(jl_value_t**)((char*)jl_data_ptr(jv)+jl_field_offset(type,i)) = NULL; } else { jl_value_t *ft = jl_field_type(type, i); if (jl_is_uniontype(ft)) { uint8_t *psel = &((uint8_t *)jv)[jl_field_offset(type, i) + jl_field_size(type, i) - 1]; *psel = 0; } } } JL_GC_POP(); return jv; }
void needPassByRef(AbiState *state, jl_datatype_t *dt, bool *byRef, bool *inReg) { size_t size = jl_datatype_size(dt); if (is_complex64(dt) || is_complex128(dt) || (jl_is_bitstype(dt) && size <= 8)) return; *byRef = true; }
JL_DLLEXPORT int jl_egal(jl_value_t *a, jl_value_t *b) { // warning: a,b may NOT have been gc-rooted by the caller if (a == b) return 1; jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(a); if (dt != (jl_datatype_t*)jl_typeof(b)) return 0; if (dt == jl_simplevector_type) return compare_svec((jl_svec_t*)a, (jl_svec_t*)b); if (dt == jl_datatype_type) { jl_datatype_t *dta = (jl_datatype_t*)a; jl_datatype_t *dtb = (jl_datatype_t*)b; return dta->name == dtb->name && compare_svec(dta->parameters, dtb->parameters); } if (dt == jl_string_type) { size_t l = jl_string_len(a); if (jl_string_len(b) != l) return 0; return !memcmp(jl_string_data(a), jl_string_data(b), l); } if (dt->mutabl) return 0; size_t sz = jl_datatype_size(dt); if (sz == 0) return 1; size_t nf = jl_datatype_nfields(dt); if (nf == 0) return bits_equal(jl_data_ptr(a), jl_data_ptr(b), sz); if (dt == jl_unionall_type) return egal_types(a, b, NULL); return compare_fields(a, b, dt); }
void jl_compute_field_offsets(jl_datatype_t *st) { size_t sz = 0, alignm = 0; int ptrfree = 1; for(size_t i=0; i < jl_tuple_len(st->types); i++) { jl_value_t *ty = jl_tupleref(st->types, i); size_t fsz, al; if (jl_isbits(ty) && (al=((jl_datatype_t*)ty)->alignment)!=0 && jl_is_leaf_type(ty)) { fsz = jl_datatype_size(ty); st->fields[i].isptr = 0; } else { fsz = sizeof(void*); al = fsz; st->fields[i].isptr = 1; ptrfree = 0; } sz = LLT_ALIGN(sz, al); if (al > alignm) alignm = al; st->fields[i].offset = sz; st->fields[i].size = fsz; sz += fsz; } st->alignment = alignm; st->size = LLT_ALIGN(sz, alignm); st->pointerfree = ptrfree && !st->abstract; }
Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) const override { size_t size = jl_datatype_size(dt); if (size > 0 && win64_reg_size(size) && !jl_is_bitstype(dt)) return Type::getIntNTy(jl_LLVMContext, jl_datatype_nbits(dt)); return NULL; }
bool use_sret(jl_datatype_t *dt) override { size_t size = jl_datatype_size(dt); if (win64_reg_size(size) || is_native_simd_type(dt)) return false; return true; }
// this is a run-time function // warning: cannot allocate memory except using alloc_temp_arg_space extern "C" DLLEXPORT void *jl_value_to_pointer(jl_value_t *jt, jl_value_t *v, int argn, int addressof) { jl_value_t *jvt = (jl_value_t*)jl_typeof(v); if (addressof) { if (jvt == jt) { if (jl_is_bitstype(jvt)) { size_t osz = jl_datatype_size(jt); return alloc_temp_arg_copy(jl_data_ptr(v), osz); } else if (!jl_is_tuple(jvt) && jl_is_leaf_type(jvt) && !jl_is_array_type(jvt)) { return v + 1; } } goto value_to_pointer_error; } else { if (jl_is_cpointer_type(jvt) && jl_tparam0(jvt) == jt) { return (void*)jl_unbox_voidpointer(v); } } if (((jl_value_t*)jl_uint8_type == jt || (jl_value_t*)jl_int8_type == jt) && jl_is_byte_string(v)) { return jl_string_data(v); } if (jl_is_array_type(jvt)) { if (jl_tparam0(jl_typeof(v)) == jt || jt==(jl_value_t*)jl_bottom_type) return ((jl_array_t*)v)->data; if (jl_is_cpointer_type(jt)) { jl_array_t *ar = (jl_array_t*)v; void **temp=(void**)alloc_temp_arg_space((1+jl_array_len(ar))*sizeof(void*)); size_t i; for(i=0; i < jl_array_len(ar); i++) { temp[i] = jl_value_to_pointer(jl_tparam0(jt), jl_arrayref(ar, i), argn, 0); } temp[i] = 0; return temp; } } value_to_pointer_error: std::map<int, std::string>::iterator it = argNumberStrings.find(argn); if (it == argNumberStrings.end()) { std::stringstream msg; msg << "argument "; msg << argn; argNumberStrings[argn] = msg.str(); it = argNumberStrings.find(argn); } jl_value_t *targ=NULL, *pty=NULL; JL_GC_PUSH2(&targ, &pty); targ = (jl_value_t*)jl_tuple1(jt); pty = (jl_value_t*)jl_apply_type((jl_value_t*)jl_pointer_type, (jl_tuple_t*)targ); jl_type_error_rt("ccall", (*it).second.c_str(), pty, v); // doesn't return return (jl_value_t*)jl_null; }
bool use_sret(jl_datatype_t *dt) override { jl_datatype_t *ty0 = NULL; bool hva = false; if (jl_datatype_size(dt) > 16 && isHFA(dt, &ty0, &hva) > 8) return true; return false; }
bool needPassByRef(jl_datatype_t *dt, AttrBuilder &ab) override { size_t size = jl_datatype_size(dt); if (is_complex64(dt) || is_complex128(dt) || (jl_is_primitivetype(dt) && size <= 8)) return false; ab.addAttribute(Attribute::ByVal); return true; }
JL_DLLEXPORT jl_value_t *jl_check_top_bit(jl_value_t *a) { jl_value_t *ty = jl_typeof(a); if (!jl_is_bitstype(ty)) jl_error("check_top_bit: value is not a bitstype"); if (signbitbyte(jl_data_ptr(a), jl_datatype_size(ty))) jl_throw(jl_inexact_exception); return a; }
bool use_sret(jl_datatype_t *dt) override { size_t size = jl_datatype_size(dt); if (size == 0) return false; if (is_complex64(dt) || (jl_is_primitivetype(dt) && size <= 8)) return false; return true; }
bool use_sret(AbiState *state, jl_datatype_t *dt) { size_t size = jl_datatype_size(dt); if (size == 0) return false; if (is_complex64(dt) || (jl_is_bitstype(dt) && size <= 8)) return false; return true; }
static uintptr_t jl_object_id_(jl_value_t *tv, jl_value_t *v) { if (tv == (jl_value_t*)jl_sym_type) return ((jl_sym_t*)v)->hash; if (tv == (jl_value_t*)jl_simplevector_type) return hash_svec((jl_svec_t*)v); jl_datatype_t *dt = (jl_datatype_t*)tv; if (dt == jl_datatype_type) { jl_datatype_t *dtv = (jl_datatype_t*)v; // `name->wrapper` is cacheable even though it contains TypeVars // that don't have stable IDs. //if (jl_egal(dtv->name->wrapper, v)) // return bitmix(~dtv->name->hash, 0xaa5566aa); return bitmix(~dtv->name->hash, hash_svec(dtv->parameters)); } if (dt == jl_typename_type) return ((jl_typename_t*)v)->hash; #ifdef _P64 if (v == jl_ANY_flag) return 0x31c472f68ee30bddULL; #else if (v == jl_ANY_flag) return 0x8ee30bdd; #endif if (dt == jl_string_type) { #ifdef _P64 return memhash_seed(jl_string_data(v), jl_string_len(v), 0xedc3b677); #else return memhash32_seed(jl_string_data(v), jl_string_len(v), 0xedc3b677); #endif } if (dt->mutabl) return inthash((uintptr_t)v); size_t sz = jl_datatype_size(tv); uintptr_t h = jl_object_id(tv); if (sz == 0) return ~h; size_t nf = jl_datatype_nfields(dt); if (nf == 0) { return bits_hash(jl_data_ptr(v), sz) ^ h; } for (size_t f=0; f < nf; f++) { size_t offs = jl_field_offset(dt, f); char *vo = (char*)jl_data_ptr(v) + offs; uintptr_t u; if (jl_field_isptr(dt, f)) { jl_value_t *f = *(jl_value_t**)vo; u = f==NULL ? 0 : jl_object_id(f); } else { jl_datatype_t *fieldtype = (jl_datatype_t*)jl_field_type(dt, f); assert(jl_is_datatype(fieldtype) && !fieldtype->abstract && !fieldtype->mutabl); if (fieldtype->layout->haspadding) u = jl_object_id_((jl_value_t*)fieldtype, (jl_value_t*)vo); else u = bits_hash(vo, jl_field_size(dt, f)); } h = bitmix(h, u); } return h; }
static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *dims) { int isunboxed=0, elsz=sizeof(void*); jl_value_t *el_type = jl_tparam0(atype); isunboxed = store_unboxed(el_type); if (isunboxed) elsz = jl_datatype_size(el_type); return _new_array_(atype, ndims, dims, isunboxed, elsz); }
jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_tuple_t *dims) { size_t i; jl_array_t *a; size_t ndims = jl_tuple_len(dims); int ndimwords = jl_array_ndimwords(ndims); a = allocobj((sizeof(jl_array_t) + sizeof(void*) + ndimwords*sizeof(size_t) + 15)&-16); a->type = atype; a->ndims = ndims; a->offset = 0; a->data = NULL; a->isaligned = data->isaligned; jl_value_t *el_type = jl_tparam0(atype); if (store_unboxed(el_type)) { a->elsize = jl_datatype_size(el_type); a->ptrarray = 0; } else { a->elsize = sizeof(void*); a->ptrarray = 1; } JL_GC_PUSH1(&a); jl_array_data_owner(a) = (jl_value_t*)data; a->how = 3; a->data = data->data; a->isshared = 1; data->isshared = 1; if (ndims == 1) { size_t l = jl_unbox_long(jl_tupleref(dims,0)); #ifdef STORE_ARRAY_LEN a->length = l; #endif a->nrows = l; a->maxsize = l; } else { size_t *adims = &a->nrows; size_t l=1; wideint_t prod; for(i=0; i < ndims; i++) { adims[i] = jl_unbox_long(jl_tupleref(dims, i)); prod = (wideint_t)l * (wideint_t)adims[i]; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); l = prod; } #ifdef STORE_ARRAY_LEN a->length = l; #endif } JL_GC_POP(); return a; }
static inline jl_value_t *jl_intrinsic_cvt(jl_value_t *ty, jl_value_t *a, const char *name, intrinsic_cvt_t op, intrinsic_cvt_check_t check_op) { jl_value_t *aty = jl_typeof(a); if (!jl_is_bitstype(aty)) jl_errorf("%s: value is not a bitstype", name); if (!jl_is_bitstype(ty)) jl_errorf("%s: type is not a bitstype", name); void *pa = jl_data_ptr(a); unsigned isize = jl_datatype_size(aty); unsigned osize = jl_datatype_size(ty); if (check_op && check_op(isize, osize, pa)) jl_throw(jl_inexact_exception); jl_value_t *newv = newstruct((jl_datatype_t*)ty); op(aty == (jl_value_t*)jl_bool_type ? 1 : isize * host_char_bit, pa, osize * host_char_bit, jl_data_ptr(newv)); if (ty == (jl_value_t*)jl_bool_type) return *(uint8_t*)jl_data_ptr(newv) & 1 ? jl_true : jl_false; return newv; }
JL_DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type) { jl_ptls_t ptls = jl_get_ptls_states(); if (type->instance != NULL) return type->instance; size_t size = jl_datatype_size(type); jl_value_t *jv = jl_gc_alloc(ptls, size, type); if (size > 0) memset(jl_data_ptr(jv), 0, size); return jv; }
bool needPassByRef(jl_datatype_t *dt, AttrBuilder &ab) override { jl_datatype_t *ty0 = NULL; bool hva = false; if (jl_datatype_size(dt) > 64 && isHFA(dt, &ty0, &hva) > 8) { ab.addAttribute(Attribute::ByVal); return true; } return false; }
jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims, int own_buffer) { size_t i, elsz, nel=1; jl_array_t *a; size_t ndims = jl_tuple_len(dims); wideint_t prod; for(i=0; i < ndims; i++) { prod = (wideint_t)nel * (wideint_t)jl_unbox_long(jl_tupleref(dims, i)); if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); nel = prod; } jl_value_t *el_type = jl_tparam0(atype); int isunboxed = store_unboxed(el_type); if (isunboxed) elsz = jl_datatype_size(el_type); else elsz = sizeof(void*); int ndimwords = jl_array_ndimwords(ndims); a = allocobj((sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16); a->type = atype; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->ptrarray = !isunboxed; a->ndims = ndims; a->offset = 0; a->isshared = 1; a->isaligned = 0; if (own_buffer) { a->how = 2; jl_gc_track_malloced_array(a); } else { a->how = 0; } if (ndims == 1) { a->nrows = nel; a->maxsize = nel; } else { size_t *adims = &a->nrows; for(i=0; i < ndims; i++) { adims[i] = jl_unbox_long(jl_tupleref(dims, i)); } } return a; }
JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_value_t *_dims, int own_buffer) { jl_ptls_t ptls = jl_get_ptls_states(); size_t elsz, nel = 1; jl_array_t *a; size_t ndims = jl_nfields(_dims); wideint_t prod; assert(is_ntuple_long(_dims)); size_t *dims = (size_t*)_dims; for (size_t i = 0; i < ndims; i++) { prod = (wideint_t)nel * (wideint_t)dims[i]; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); nel = prod; } if (__unlikely(ndims == 1)) return jl_ptr_to_array_1d(atype, data, nel, own_buffer); jl_value_t *el_type = jl_tparam0(atype); int isunboxed = store_unboxed(el_type); if (isunboxed) elsz = jl_datatype_size(el_type); else elsz = sizeof(void*); int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->flags.ptrarray = !isunboxed; a->flags.ndims = ndims; a->offset = 0; a->flags.isshared = 1; a->flags.isaligned = 0; if (own_buffer) { a->flags.how = 2; jl_gc_track_malloced_array(ptls, a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { a->flags.how = 0; } assert(ndims != 1); // handled above memcpy(&a->nrows, dims, ndims * sizeof(size_t)); return a; }
// run time version of pointerref intrinsic DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i) { JL_TYPECHK(pointerref, pointer, p); JL_TYPECHK(pointerref, long, i); jl_value_t *ety = jl_tparam0(jl_typeof(p)); if (!jl_is_datatype(ety)) jl_error("pointerref: invalid pointer"); size_t nb = jl_datatype_size(ety); char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb; return jl_new_bits(ety, pp); }
void jl_assign_bits(void *dest, jl_value_t *bits) { size_t nb = jl_datatype_size(jl_typeof(bits)); switch (nb) { case 1: *(int8_t*)dest = *(int8_t*)jl_data_ptr(bits); break; case 2: *(int16_t*)dest = *(int16_t*)jl_data_ptr(bits); break; case 4: *(int32_t*)dest = *(int32_t*)jl_data_ptr(bits); break; case 8: *(int64_t*)dest = *(int64_t*)jl_data_ptr(bits); break; case 16: *(bits128_t*)dest = *(bits128_t*)jl_data_ptr(bits); break; default: memcpy(dest, jl_data_ptr(bits), nb); } }
void jl_compute_field_offsets(jl_datatype_t *st) { size_t sz = 0, alignm = 1; int ptrfree = 1; assert(0 <= st->fielddesc_type && st->fielddesc_type <= 2); uint64_t max_offset = (((uint64_t)1) << (1 << (3 + st->fielddesc_type))) - 1; uint64_t max_size = max_offset >> 1; for(size_t i=0; i < jl_datatype_nfields(st); i++) { jl_value_t *ty = jl_field_type(st, i); size_t fsz, al; if (jl_isbits(ty) && jl_is_leaf_type(ty)) { fsz = jl_datatype_size(ty); // Should never happen if (__unlikely(fsz > max_size)) jl_throw(jl_overflow_exception); al = ((jl_datatype_t*)ty)->alignment; jl_field_setisptr(st, i, 0); if (((jl_datatype_t*)ty)->haspadding) st->haspadding = 1; } else { fsz = sizeof(void*); if (fsz > MAX_ALIGN) fsz = MAX_ALIGN; al = fsz; jl_field_setisptr(st, i, 1); ptrfree = 0; } if (al != 0) { size_t alsz = LLT_ALIGN(sz, al); if (sz & (al - 1)) st->haspadding = 1; sz = alsz; if (al > alignm) alignm = al; } jl_field_setoffset(st, i, sz); jl_field_setsize(st, i, fsz); if (__unlikely(max_offset - sz < fsz)) jl_throw(jl_overflow_exception); sz += fsz; } st->alignment = alignm; st->size = LLT_ALIGN(sz, alignm); if (st->size > sz) st->haspadding = 1; st->pointerfree = ptrfree && !st->abstract; }
// own_buffer != 0 iff GC should call free() on this pointer eventually JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int own_buffer) { jl_ptls_t ptls = jl_get_ptls_states(); jl_array_t *a; jl_value_t *eltype = jl_tparam0(atype); int isunboxed = jl_array_store_unboxed(eltype); size_t elsz; unsigned align; if (isunboxed && jl_is_uniontype(eltype)) jl_exceptionf(jl_argumenterror_type, "unsafe_wrap: unspecified layout for union element type"); if (isunboxed) { elsz = jl_datatype_size(eltype); align = jl_datatype_align(eltype); } else { align = elsz = sizeof(void*); } if (((uintptr_t)data) & (align - 1)) jl_exceptionf(jl_argumenterror_type, "unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align); int ndimwords = jl_array_ndimwords(1); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->flags.ptrarray = !isunboxed; a->flags.ndims = 1; a->flags.isshared = 1; a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers if (own_buffer) { a->flags.how = 2; jl_gc_track_malloced_array(ptls, a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { a->flags.how = 0; } a->nrows = nel; a->maxsize = nel; a->offset = 0; return a; }
// run time version of pointerset intrinsic DLLEXPORT void jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *i) { JL_TYPECHK(pointerset, pointer, p); JL_TYPECHK(pointerset, long, i); jl_value_t *ety = jl_tparam0(jl_typeof(p)); if (!jl_is_datatype(ety)) jl_error("pointerset: invalid pointer"); size_t nb = jl_datatype_size(ety); char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb; if (jl_typeof(x) != ety) jl_error("pointerset: type mismatch in assign"); jl_assign_bits(pp, x); }