// Note that this function updates len static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len) { jl_ptls_t ptls = jl_get_ptls_states(); assert(jl_is_datatype(dt)); jl_datatype_t *bt = (jl_datatype_t*)dt; size_t nb = jl_datatype_size(bt); if (nb == 0) return jl_new_struct_uninit(bt); *len = LLT_ALIGN(*len, jl_datatype_align(bt)); data = (char*)data + (*len); *len += nb; if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data); if (bt == jl_int64_type) return jl_box_int64(*(int64_t*)data); if (bt == jl_bool_type) return (*(int8_t*)data) ? jl_true:jl_false; if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data); if (bt == jl_float64_type) return jl_box_float64(*(double*)data); jl_value_t *v = jl_gc_alloc(ptls, nb, bt); switch (nb) { case 1: *(int8_t*) jl_data_ptr(v) = *(int8_t*)data; break; case 2: *(int16_t*) jl_data_ptr(v) = *(int16_t*)data; break; case 4: *(int32_t*) jl_data_ptr(v) = *(int32_t*)data; break; case 8: *(int64_t*) jl_data_ptr(v) = *(int64_t*)data; break; case 16: *(bits128_t*)jl_data_ptr(v) = *(bits128_t*)data; break; default: memcpy(jl_data_ptr(v), data, nb); } return v; }
// own_buffer != 0 iff GC should call free() on this pointer eventually JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int own_buffer) { jl_ptls_t ptls = jl_get_ptls_states(); jl_array_t *a; jl_value_t *eltype = jl_tparam0(atype); int isunboxed = jl_array_store_unboxed(eltype); size_t elsz; unsigned align; if (isunboxed && jl_is_uniontype(eltype)) jl_exceptionf(jl_argumenterror_type, "unsafe_wrap: unspecified layout for union element type"); if (isunboxed) { elsz = jl_datatype_size(eltype); align = jl_datatype_align(eltype); } else { align = elsz = sizeof(void*); } if (((uintptr_t)data) & (align - 1)) jl_exceptionf(jl_argumenterror_type, "unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align); int ndimwords = jl_array_ndimwords(1); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->flags.ptrarray = !isunboxed; a->flags.ndims = 1; a->flags.isshared = 1; a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers if (own_buffer) { a->flags.how = 2; jl_gc_track_malloced_array(ptls, a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { a->flags.how = 0; } a->nrows = nel; a->maxsize = nel; a->offset = 0; return a; }
// run time version of pointerref intrinsic (warning: i is not rooted) JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i, jl_value_t *align) { JL_TYPECHK(pointerref, pointer, p); JL_TYPECHK(pointerref, long, i) JL_TYPECHK(pointerref, long, align); jl_value_t *ety = jl_tparam0(jl_typeof(p)); if (ety == (jl_value_t*)jl_any_type) { jl_value_t **pp = (jl_value_t**)(jl_unbox_long(p) + (jl_unbox_long(i)-1)*sizeof(void*)); return *pp; } else { if (!jl_is_datatype(ety)) jl_error("pointerref: invalid pointer"); size_t nb = LLT_ALIGN(jl_datatype_size(ety), jl_datatype_align(ety)); char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb; return jl_new_bits(ety, pp); } }
Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) const override { // Arguments are either scalar or passed by value size_t size = jl_datatype_size(dt); // don't need to change bitstypes if (!jl_datatype_nfields(dt)) return NULL; // legalize this into [n x f32/f64] jl_datatype_t *ty0 = NULL; bool hva = false; int hfa = isHFA(dt, &ty0, &hva); if (hfa <= 8) { if (ty0 == jl_float32_type) { return ArrayType::get(T_float32, hfa); } else if (ty0 == jl_float64_type) { return ArrayType::get(T_float64, hfa); } else { jl_datatype_t *vecty = (jl_datatype_t*)jl_field_type(ty0, 0); assert(jl_is_datatype(vecty) && vecty->name == jl_vecelement_typename); jl_value_t *elemty = jl_tparam0(vecty); assert(jl_is_primitivetype(elemty)); Type *ety = julia_type_to_llvm(elemty); Type *vty = VectorType::get(ety, jl_datatype_nfields(ty0)); return ArrayType::get(vty, hfa); } } // rewrite integer-sized (non-HFA) struct to an array // the bitsize of the integer gives the desired alignment if (size > 8) { if (jl_datatype_align(dt) <= 8) { return ArrayType::get(T_int64, (size + 7) / 8); } else { Type *T_int128 = Type::getIntNTy(jl_LLVMContext, 128); return ArrayType::get(T_int128, (size + 15) / 16); } } return Type::getIntNTy(jl_LLVMContext, size * 8); }
// run time version of pointerset intrinsic (warning: x is not gc-rooted) JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *i, jl_value_t *align) { JL_TYPECHK(pointerset, pointer, p); JL_TYPECHK(pointerset, long, i); JL_TYPECHK(pointerref, long, align); jl_value_t *ety = jl_tparam0(jl_typeof(p)); if (ety == (jl_value_t*)jl_any_type) { jl_value_t **pp = (jl_value_t**)(jl_unbox_long(p) + (jl_unbox_long(i)-1)*sizeof(void*)); *pp = x; } else { if (!jl_is_datatype(ety)) jl_error("pointerset: invalid pointer"); size_t elsz = jl_datatype_size(ety); size_t nb = LLT_ALIGN(elsz, jl_datatype_align(ety)); char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb; if (jl_typeof(x) != ety) jl_error("pointerset: type mismatch in assign"); memcpy(pp, x, elsz); } return p; }
static unsigned union_isbits(jl_value_t *ty, size_t *nbytes, size_t *align) { if (jl_is_uniontype(ty)) { unsigned na = union_isbits(((jl_uniontype_t*)ty)->a, nbytes, align); if (na == 0) return 0; unsigned nb = union_isbits(((jl_uniontype_t*)ty)->b, nbytes, align); if (nb == 0) return 0; return na + nb; } if (jl_isbits(ty)) { size_t sz = jl_datatype_size(ty); size_t al = jl_datatype_align(ty); if (*nbytes < sz) *nbytes = sz; if (*align < al) *align = al; return 1; } return 0; }
JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_value_t *_dims, int own_buffer) { jl_ptls_t ptls = jl_get_ptls_states(); size_t nel = 1; jl_array_t *a; size_t ndims = jl_nfields(_dims); wideint_t prod; assert(is_ntuple_long(_dims)); size_t *dims = (size_t*)_dims; for (size_t i = 0; i < ndims; i++) { prod = (wideint_t)nel * (wideint_t)dims[i]; if (prod > (wideint_t) MAXINTVAL) jl_error("invalid Array dimensions"); nel = prod; } if (__unlikely(ndims == 1)) return jl_ptr_to_array_1d(atype, data, nel, own_buffer); jl_value_t *eltype = jl_tparam0(atype); int isunboxed = jl_array_store_unboxed(eltype); size_t elsz; unsigned align; if (isunboxed && jl_is_uniontype(eltype)) jl_exceptionf(jl_argumenterror_type, "unsafe_wrap: unspecified layout for union element type"); if (isunboxed) { elsz = jl_datatype_size(eltype); align = jl_datatype_align(eltype); } else { align = elsz = sizeof(void*); } if (((uintptr_t)data) & (align - 1)) jl_exceptionf(jl_argumenterror_type, "unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align); int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; #endif a->elsize = elsz; a->flags.ptrarray = !isunboxed; a->flags.ndims = ndims; a->offset = 0; a->flags.isshared = 1; a->flags.isaligned = 0; if (own_buffer) { a->flags.how = 2; jl_gc_track_malloced_array(ptls, a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { a->flags.how = 0; } assert(ndims != 1); // handled above memcpy(&a->nrows, dims, ndims * sizeof(size_t)); return a; }
JL_DLLEXPORT size_t jl_get_alignment(jl_datatype_t *ty) { if (ty->layout == NULL) jl_error("non-leaf type doesn't have an alignment"); return jl_datatype_align(ty); }
void jl_compute_field_offsets(jl_datatype_t *st) { size_t sz = 0, alignm = 1; int homogeneous = 1; jl_value_t *lastty = NULL; uint64_t max_offset = (((uint64_t)1) << 32) - 1; uint64_t max_size = max_offset >> 1; if (st->name->wrapper) { // If layout doesn't depend on type parameters, it's stored in st->name->wrapper // and reused by all subtypes. jl_datatype_t *w = (jl_datatype_t*)jl_unwrap_unionall(st->name->wrapper); if (st != w && // this check allows us to re-compute layout for some types during init w->layout) { st->layout = w->layout; st->size = w->size; return; } } if (st->types == NULL) return; uint32_t nfields = jl_svec_len(st->types); if (nfields == 0) { if (st == jl_sym_type || st == jl_string_type) { // opaque layout - heap-allocated blob static const jl_datatype_layout_t opaque_byte_layout = {0, 1, 0, 1, 0}; st->layout = &opaque_byte_layout; } else if (st == jl_simplevector_type || st->name == jl_array_typename) { static const jl_datatype_layout_t opaque_ptr_layout = {0, sizeof(void*), 0, 1, 0}; st->layout = &opaque_ptr_layout; } else { // reuse the same layout for all singletons static const jl_datatype_layout_t singleton_layout = {0, 1, 0, 0, 0}; st->layout = &singleton_layout; } return; } if (!jl_is_leaf_type((jl_value_t*)st)) { // compute layout whenever field types have no free variables for (size_t i = 0; i < nfields; i++) { if (jl_has_free_typevars(jl_field_type(st, i))) return; } } size_t descsz = nfields * sizeof(jl_fielddesc32_t); jl_fielddesc32_t *desc; if (descsz < jl_page_size) desc = (jl_fielddesc32_t*)alloca(descsz); else desc = (jl_fielddesc32_t*)malloc(descsz); int haspadding = 0; assert(st->name == jl_tuple_typename || st == jl_sym_type || st == jl_simplevector_type || nfields != 0); for (size_t i = 0; i < nfields; i++) { jl_value_t *ty = jl_field_type(st, i); size_t fsz, al; if (jl_isbits(ty) && jl_is_leaf_type(ty) && ((jl_datatype_t*)ty)->layout) { fsz = jl_datatype_size(ty); // Should never happen if (__unlikely(fsz > max_size)) goto throw_ovf; al = jl_datatype_align(ty); desc[i].isptr = 0; if (((jl_datatype_t*)ty)->layout->haspadding) haspadding = 1; } else { fsz = sizeof(void*); if (fsz > MAX_ALIGN) fsz = MAX_ALIGN; al = fsz; desc[i].isptr = 1; } assert(al <= JL_HEAP_ALIGNMENT && (JL_HEAP_ALIGNMENT % al) == 0); if (al != 0) { size_t alsz = LLT_ALIGN(sz, al); if (sz & (al - 1)) haspadding = 1; sz = alsz; if (al > alignm) alignm = al; } homogeneous &= lastty==NULL || lastty==ty; lastty = ty; desc[i].offset = sz; desc[i].size = fsz; if (__unlikely(max_offset - sz < fsz)) goto throw_ovf; sz += fsz; } if (homogeneous && lastty!=NULL && jl_is_tuple_type(st)) { // Some tuples become LLVM vectors with stronger alignment than what was calculated above. unsigned al = jl_special_vector_alignment(nfields, lastty); assert(al % alignm == 0); // JL_HEAP_ALIGNMENT is the biggest alignment we can guarantee on the heap. if (al > JL_HEAP_ALIGNMENT) alignm = JL_HEAP_ALIGNMENT; else if (al) alignm = al; } st->size = LLT_ALIGN(sz, alignm); if (st->size > sz) haspadding = 1; st->layout = jl_get_layout(nfields, alignm, haspadding, desc); if (descsz >= jl_page_size) free(desc); return; throw_ovf: if (descsz >= jl_page_size) free(desc); jl_throw(jl_overflow_exception); }