JL_DLLEXPORT jl_value_t *jl_alloc_string(size_t len)
{
jl_value_t *s = jl_gc_alloc(jl_get_ptls_states(), sizeof(size_t)+len+1, jl_string_type);
*(size_t*)s = len;
((char*)s + sizeof(size_t))[len] = 0;
return s;
}
JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_array_t *a;
int ndimwords = jl_array_ndimwords(1);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT);
a = (jl_array_t*)jl_gc_alloc(ptls, tsz, jl_array_uint8_type);
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
a->flags.ndims = 1;
a->offset = 0;
a->data = jl_string_data(str);
a->flags.isaligned = 0;
a->elsize = 1;
a->flags.ptrarray = 0;
jl_array_data_owner(a) = str;
a->flags.how = 3;
a->flags.isshared = 1;
size_t l = jl_string_len(str);
#ifdef STORE_ARRAY_LEN
a->length = l;
#endif
a->nrows = a->maxsize = l;
return a;
}
JL_DLLEXPORT jl_value_t *jl_new_bits(jl_value_t *dt, void *data)
{
// data may not have the alignment required by the size
// but will always have the alignment required by the datatype
jl_ptls_t ptls = jl_get_ptls_states();
assert(jl_is_datatype(dt));
jl_datatype_t *bt = (jl_datatype_t*)dt;
size_t nb = jl_datatype_size(bt);
// some types have special pools to minimize allocations
if (nb == 0) return jl_new_struct_uninit(bt); // returns bt->instance
if (bt == jl_bool_type) return (1 & *(int8_t*)data) ? jl_true : jl_false;
if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data);
if (bt == jl_int64_type) return jl_box_int64(*(int64_t*)data);
if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data);
if (bt == jl_int8_type) return jl_box_int8(*(int8_t*)data);
if (bt == jl_int16_type) return jl_box_int16(*(int16_t*)data);
if (bt == jl_uint64_type) return jl_box_uint64(*(uint64_t*)data);
if (bt == jl_uint32_type) return jl_box_uint32(*(uint32_t*)data);
if (bt == jl_uint16_type) return jl_box_uint16(*(uint16_t*)data);
if (bt == jl_char_type) return jl_box_char(*(uint32_t*)data);
jl_value_t *v = jl_gc_alloc(ptls, nb, bt);
switch (nb) {
case 1: *(uint8_t*) v = *(uint8_t*)data; break;
case 2: *(uint16_t*)v = jl_load_unaligned_i16(data); break;
case 4: *(uint32_t*)v = jl_load_unaligned_i32(data); break;
case 8: *(uint64_t*)v = jl_load_unaligned_i64(data); break;
case 16:
memcpy(jl_assume_aligned(v, 16), data, 16);
break;
default: memcpy(v, data, nb);
}
return v;
}
// used by boot.jl
JL_DLLEXPORT jl_value_t *jl_typemax_uint(jl_value_t *bt)
{
uint64_t data = 0xffffffffffffffffULL;
jl_value_t *v = jl_gc_alloc(jl_get_ptls_states(), sizeof(size_t), bt);
memcpy(v, &data, sizeof(size_t));
return v;
}
JL_DLLEXPORT jl_lambda_info_t *jl_new_lambda_info_uninit(void)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_lambda_info_t *li =
(jl_lambda_info_t*)jl_gc_alloc(ptls, sizeof(jl_lambda_info_t),
jl_lambda_info_type);
li->code = NULL;
li->slotnames = NULL;
li->slotflags = NULL;
li->slottypes = NULL;
li->ssavaluetypes = NULL;
li->rettype = (jl_value_t*)jl_any_type;
li->sparam_syms = jl_emptysvec;
li->sparam_vals = jl_emptysvec;
li->fptr = NULL;
li->jlcall_api = 0;
li->compile_traced = 0;
li->functionObjectsDecls.functionObject = NULL;
li->functionObjectsDecls.specFunctionObject = NULL;
li->specTypes = NULL;
li->unspecialized_ducttape = NULL;
li->inferred = 0;
li->inInference = 0;
li->inCompile = 0;
li->def = NULL;
li->constval = NULL;
li->pure = 0;
li->inlineable = 0;
return li;
}
STATIC_INLINE void jl_allocate_singleton_instance(jl_datatype_t *st)
{
if (jl_is_datatype_make_singleton(st)) {
st->instance = jl_gc_alloc(jl_get_ptls_states(), 0, st);
jl_gc_wb(st, st->instance);
}
}
JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_module_t *m = (jl_module_t*)jl_gc_alloc(ptls, sizeof(jl_module_t),
jl_module_type);
JL_GC_PUSH1(&m);
assert(jl_is_symbol(name));
m->name = name;
m->parent = NULL;
m->istopmod = 0;
static unsigned int mcounter; // simple counter backup, in case hrtime is not incrementing
m->uuid = jl_hrtime() + (++mcounter);
if (!m->uuid) m->uuid++; // uuid 0 is invalid
m->counter = 0;
htable_new(&m->bindings, 0);
arraylist_new(&m->usings, 0);
if (jl_core_module) {
jl_module_using(m, jl_core_module);
}
// export own name, so "using Foo" makes "Foo" itself visible
jl_set_const(m, name, (jl_value_t*)m);
jl_module_export(m, name);
JL_GC_POP();
return m;
}
JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args,
uint32_t na)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (type->instance != NULL) return type->instance;
size_t nf = jl_datatype_nfields(type);
jl_value_t *jv = jl_gc_alloc(ptls, jl_datatype_size(type), type);
JL_GC_PUSH1(&jv);
for (size_t i = 0; i < na; i++) {
jl_value_t *ft = jl_field_type(type, i);
if (!jl_isa(args[i], ft))
jl_type_error("new", ft, args[i]);
jl_set_nth_field(jv, i, args[i]);
}
for(size_t i=na; i < nf; i++) {
if (jl_field_isptr(type, i)) {
*(jl_value_t**)((char*)jl_data_ptr(jv)+jl_field_offset(type,i)) = NULL;
} else {
jl_value_t *ft = jl_field_type(type, i);
if (jl_is_uniontype(ft)) {
uint8_t *psel = &((uint8_t *)jv)[jl_field_offset(type, i) + jl_field_size(type, i) - 1];
*psel = 0;
}
}
}
JL_GC_POP();
return jv;
}
// Note that this function updates len
static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len)
{
jl_ptls_t ptls = jl_get_ptls_states();
assert(jl_is_datatype(dt));
jl_datatype_t *bt = (jl_datatype_t*)dt;
size_t nb = jl_datatype_size(bt);
if (nb == 0)
return jl_new_struct_uninit(bt);
*len = LLT_ALIGN(*len, bt->layout->alignment);
data = (char*)data + (*len);
*len += nb;
if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data);
if (bt == jl_int64_type) return jl_box_int64(*(int64_t*)data);
if (bt == jl_bool_type) return (*(int8_t*)data) ? jl_true:jl_false;
if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data);
if (bt == jl_float64_type) return jl_box_float64(*(double*)data);
jl_value_t *v = jl_gc_alloc(ptls, nb, bt);
switch (nb) {
case 1: *(int8_t*) jl_data_ptr(v) = *(int8_t*)data; break;
case 2: *(int16_t*) jl_data_ptr(v) = *(int16_t*)data; break;
case 4: *(int32_t*) jl_data_ptr(v) = *(int32_t*)data; break;
case 8: *(int64_t*) jl_data_ptr(v) = *(int64_t*)data; break;
case 16: *(bits128_t*)jl_data_ptr(v) = *(bits128_t*)data; break;
default: memcpy(jl_data_ptr(v), data, nb);
}
return v;
}
JL_DLLEXPORT jl_value_t *jl_pchar_to_string(const char *str, size_t len)
{
jl_value_t *s = jl_gc_alloc(jl_get_ptls_states(), sizeof(size_t)+len+1, jl_string_type);
*(size_t*)s = len;
memcpy((char*)s + sizeof(size_t), str, len);
((char*)s + sizeof(size_t))[len] = 0;
return s;
}
static inline jl_value_t *jl_intrinsiclambda_ty1(jl_value_t *ty, void *pa, unsigned osize, unsigned osize2, const void *voidlist)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_value_t *newv = jl_gc_alloc(ptls, ((jl_datatype_t*)ty)->size, ty);
intrinsic_1_t op = select_intrinsic_1(osize2, (const intrinsic_1_t*)voidlist);
op(osize * host_char_bit, pa, jl_data_ptr(newv));
return newv;
}
JL_DLLEXPORT jl_svec_t *jl_alloc_svec_uninit(size_t n)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (n == 0) return jl_emptysvec;
jl_svec_t *jv = (jl_svec_t*)jl_gc_alloc(ptls, (n + 1) * sizeof(void*),
jl_simplevector_type);
jl_svec_set_len_unsafe(jv, n);
return jv;
}
JL_DLLEXPORT jl_value_t *jl_box_char(uint32_t x)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (0 < (int32_t)x)
return boxed_char_cache[x >> 24];
jl_value_t *v = jl_gc_alloc(ptls, sizeof(void*), jl_char_type);
*(uint32_t*)jl_data_ptr(v) = x;
return v;
}
JL_DLLEXPORT jl_svec_t *jl_svec1(void *a)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_svec_t *v = (jl_svec_t*)jl_gc_alloc(ptls, sizeof(void*) * 2,
jl_simplevector_type);
jl_svec_set_len_unsafe(v, 1);
jl_svecset(v, 0, a);
return v;
}
JL_DLLEXPORT jl_value_t *jl_array_to_string(jl_array_t *a)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (!jl_typeis(a, jl_array_uint8_type))
jl_type_error("jl_array_to_string", (jl_value_t*)jl_array_uint8_type, (jl_value_t*)a);
jl_value_t *s = jl_gc_alloc(ptls, sizeof(void*), jl_string_type);
jl_set_nth_field(s, 0, (jl_value_t*)a);
return s;
}
JL_DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (type->instance != NULL) return type->instance;
size_t size = type->size;
jl_value_t *jv = jl_gc_alloc(ptls, size, type);
if (size > 0)
memset(jl_data_ptr(jv), 0, size);
return jv;
}
jl_datatype_t *jl_new_uninitialized_datatype(void)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc(ptls, sizeof(jl_datatype_t), jl_datatype_type);
t->depth = 0;
t->hasfreetypevars = 0;
t->isleaftype = 1;
t->layout = NULL;
return t;
}
static inline jl_value_t *jl_intrinsiclambda_2(jl_value_t *ty, void *pa, void *pb, unsigned sz, unsigned sz2, const void *voidlist)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_value_t *newv = jl_gc_alloc(ptls, ((jl_datatype_t*)ty)->size, ty);
intrinsic_2_t op = select_intrinsic_2(sz2, (const intrinsic_2_t*)voidlist);
op(sz * host_char_bit, pa, pb, jl_data_ptr(newv));
if (ty == (jl_value_t*)jl_bool_type)
return *(uint8_t*)jl_data_ptr(newv) & 1 ? jl_true : jl_false;
return newv;
}
JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
jl_value_t *_dims, int own_buffer)
{
jl_ptls_t ptls = jl_get_ptls_states();
size_t elsz, nel = 1;
jl_array_t *a;
size_t ndims = jl_nfields(_dims);
wideint_t prod;
assert(is_ntuple_long(_dims));
size_t *dims = (size_t*)_dims;
for (size_t i = 0; i < ndims; i++) {
prod = (wideint_t)nel * (wideint_t)dims[i];
if (prod > (wideint_t) MAXINTVAL)
jl_error("invalid Array dimensions");
nel = prod;
}
if (__unlikely(ndims == 1))
return jl_ptr_to_array_1d(atype, data, nel, own_buffer);
jl_value_t *el_type = jl_tparam0(atype);
int isunboxed = store_unboxed(el_type);
if (isunboxed)
elsz = jl_datatype_size(el_type);
else
elsz = sizeof(void*);
int ndimwords = jl_array_ndimwords(ndims);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
a->data = data;
#ifdef STORE_ARRAY_LEN
a->length = nel;
#endif
a->elsize = elsz;
a->flags.ptrarray = !isunboxed;
a->flags.ndims = ndims;
a->offset = 0;
a->flags.isshared = 1;
a->flags.isaligned = 0;
if (own_buffer) {
a->flags.how = 2;
jl_gc_track_malloced_array(ptls, a);
jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
}
else {
a->flags.how = 0;
}
assert(ndims != 1); // handled above
memcpy(&a->nrows, dims, ndims * sizeof(size_t));
return a;
}
// own_buffer != 0 iff GC should call free() on this pointer eventually
JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
size_t nel, int own_buffer)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_array_t *a;
jl_value_t *eltype = jl_tparam0(atype);
int isunboxed = jl_array_store_unboxed(eltype);
size_t elsz;
unsigned align;
if (isunboxed && jl_is_uniontype(eltype))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: unspecified layout for union element type");
if (isunboxed) {
elsz = jl_datatype_size(eltype);
align = jl_datatype_align(eltype);
}
else {
align = elsz = sizeof(void*);
}
if (((uintptr_t)data) & (align - 1))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align);
int ndimwords = jl_array_ndimwords(1);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
a->data = data;
#ifdef STORE_ARRAY_LEN
a->length = nel;
#endif
a->elsize = elsz;
a->flags.ptrarray = !isunboxed;
a->flags.ndims = 1;
a->flags.isshared = 1;
a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers
if (own_buffer) {
a->flags.how = 2;
jl_gc_track_malloced_array(ptls, a);
jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
}
else {
a->flags.how = 0;
}
a->nrows = nel;
a->maxsize = nel;
a->offset = 0;
return a;
}
jl_datatype_t *jl_new_uninitialized_datatype(void)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc(ptls, sizeof(jl_datatype_t), jl_datatype_type);
t->hasfreetypevars = 0;
t->isdispatchtuple = 0;
t->isbitstype = 0;
t->zeroinit = 0;
t->isinlinealloc = 0;
t->layout = NULL;
t->names = NULL;
return t;
}
jl_expr_t *jl_exprn(jl_sym_t *head, size_t n)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_array_t *ar = n==0 ? (jl_array_t*)jl_an_empty_vec_any : jl_alloc_vec_any(n);
JL_GC_PUSH1(&ar);
jl_expr_t *ex = (jl_expr_t*)jl_gc_alloc(ptls, sizeof(jl_expr_t),
jl_expr_type);
ex->head = head;
ex->args = ar;
ex->etype = (jl_value_t*)jl_any_type;
JL_GC_POP();
return ex;
}
JL_DLLEXPORT jl_value_t *jl_new_struct(jl_datatype_t *type, ...)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (type->instance != NULL) return type->instance;
va_list args;
size_t nf = jl_datatype_nfields(type);
va_start(args, type);
jl_value_t *jv = jl_gc_alloc(ptls, type->size, type);
for(size_t i=0; i < nf; i++) {
jl_set_nth_field(jv, i, va_arg(args, jl_value_t*));
}
va_end(args);
return jv;
}
static jl_typemap_level_t *jl_new_typemap_level(void)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_typemap_level_t *cache =
(jl_typemap_level_t*)jl_gc_alloc(ptls, sizeof(jl_typemap_level_t),
jl_typemap_level_type);
cache->key = NULL;
cache->linear = (jl_typemap_entry_t*)jl_nothing;
cache->any.unknown = jl_nothing;
cache->targ.indexes = (jl_array_t*)jl_nothing;
cache->targ.values = (jl_array_t*)jl_nothing;
cache->arg1.indexes = (jl_array_t*)jl_nothing;
cache->arg1.values = (jl_array_t*)jl_nothing;
return cache;
}
JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_methtable_t *mt =
(jl_methtable_t*)jl_gc_alloc(ptls, sizeof(jl_methtable_t),
jl_methtable_type);
mt->name = jl_demangle_typename(name);
mt->module = module;
mt->defs.unknown = jl_nothing;
mt->cache.unknown = jl_nothing;
mt->max_args = 0;
mt->kwsorter = NULL;
JL_MUTEX_INIT(&mt->writelock);
return mt;
}
static inline jl_value_t *jl_intrinsiclambda_u1(jl_value_t *ty, void *pa, unsigned osize, unsigned osize2, const void *voidlist)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_value_t *newv = jl_gc_alloc(ptls, ((jl_datatype_t*)ty)->size, ty);
intrinsic_u1_t op = select_intrinsic_u1(osize2, (const intrinsic_u1_t*)voidlist);
unsigned cnt = op(osize * host_char_bit, pa);
// TODO: the following memset/memcpy assumes little-endian
// for big-endian, need to copy from the other end of cnt
if (osize > sizeof(unsigned)) {
// perform zext, if needed
memset((char*)jl_data_ptr(newv) + sizeof(unsigned), 0, osize - sizeof(unsigned));
osize = sizeof(unsigned);
}
memcpy(jl_data_ptr(newv), &cnt, osize);
return newv;
}
JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *module)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_typename_t *tn =
(jl_typename_t*)jl_gc_alloc(ptls, sizeof(jl_typename_t),
jl_typename_type);
tn->name = name;
tn->module = module;
tn->wrapper = NULL;
tn->cache = jl_emptysvec;
tn->linearcache = jl_emptysvec;
tn->names = NULL;
tn->hash = bitmix(bitmix(module ? module->build_id : 0, name->hash), 0xa1ada1da);
tn->mt = NULL;
return tn;
}
JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args,
uint32_t na)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (type->instance != NULL) return type->instance;
size_t nf = jl_datatype_nfields(type);
jl_value_t *jv = jl_gc_alloc(ptls, type->size, type);
for(size_t i=0; i < na; i++) {
jl_set_nth_field(jv, i, args[i]);
}
for(size_t i=na; i < nf; i++) {
if (jl_field_isptr(type, i)) {
*(jl_value_t**)((char*)jl_data_ptr(jv)+jl_field_offset(type,i)) = NULL;
}
}
return jv;
}
JL_DLLEXPORT jl_value_t *jl_new_type_constructor(jl_svec_t *p, jl_value_t *body)
{
jl_ptls_t ptls = jl_get_ptls_states();
#ifndef NDEBUG
size_t i, np = jl_svec_len(p);
for (i = 0; i < np; i++) {
jl_tvar_t *tv = (jl_tvar_t*)jl_svecref(p, i);
assert(jl_is_typevar(tv) && !tv->bound);
}
#endif
jl_typector_t *tc =
(jl_typector_t*)jl_gc_alloc(ptls, sizeof(jl_typector_t),
jl_typector_type);
tc->parameters = p;
tc->body = body;
return (jl_value_t*)tc;
}
// own_buffer != 0 iff GC should call free() on this pointer eventually
JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
size_t nel, int own_buffer)
{
jl_ptls_t ptls = jl_get_ptls_states();
size_t elsz;
jl_array_t *a;
jl_value_t *el_type = jl_tparam0(atype);
int isunboxed = store_unboxed(el_type);
if (isunboxed)
elsz = jl_datatype_size(el_type);
else
elsz = sizeof(void*);
int ndimwords = jl_array_ndimwords(1);
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype);
// No allocation or safepoint allowed after this
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
a->data = data;
#ifdef STORE_ARRAY_LEN
a->length = nel;
#endif
a->elsize = elsz;
a->flags.ptrarray = !isunboxed;
a->flags.ndims = 1;
a->flags.isshared = 1;
a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers
if (own_buffer) {
a->flags.how = 2;
jl_gc_track_malloced_array(ptls, a);
jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
}
else {
a->flags.how = 0;
}
a->nrows = nel;
a->maxsize = nel;
a->offset = 0;
return a;
}
