jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t **args, size_t n, int8_t offs)
{
if (n > offs) {
jl_value_t *a1 = args[offs];
jl_value_t *ty = (jl_value_t*)jl_typeof(a1);
assert(jl_is_datatype(ty));
if (ty == (jl_value_t*)jl_datatype_type && cache->targ.values != (void*)jl_nothing) {
union jl_typemap_t ml_or_cache = mtcache_hash_lookup(&cache->targ, a1, 1, offs);
jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1);
if (ml) return ml;
}
if (cache->arg1.values != (void*)jl_nothing) {
union jl_typemap_t ml_or_cache = mtcache_hash_lookup(&cache->arg1, ty, 0, offs);
jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1);
if (ml) return ml;
}
}
if (cache->linear != (jl_typemap_entry_t*)jl_nothing) {
jl_typemap_entry_t *ml = jl_typemap_entry_assoc_exact(cache->linear, args, n);
if (ml) return ml;
}
if (cache->any.unknown != jl_nothing)
return jl_typemap_assoc_exact(cache->any, args, n, offs+1);
return NULL;
}
// this is the general entry point for looking up a type in the cache
// (as a subtype, or with typeseq)
jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_tupletype_t *types, jl_svec_t **penv,
int8_t subtype_inexact__sigseq_useenv, int8_t subtype, int8_t offs)
{
if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) {
jl_typemap_level_t *cache = ml_or_cache.node;
// called object is the primary key for constructors, otherwise first argument
jl_value_t *ty = NULL;
size_t l = jl_datatype_nfields(types);
// compute the type at offset `offs` into `types`, which may be a Vararg
if (l <= offs + 1) {
ty = jl_tparam(types, l - 1);
if (jl_is_vararg_type(ty))
ty = jl_tparam0(ty);
else if (l <= offs)
ty = NULL;
}
else if (l > offs) {
ty = jl_tparam(types, offs);
}
// If there is a type at offs, look in the optimized caches
if (ty) {
if (!subtype && jl_is_any(ty))
return jl_typemap_assoc_by_type(cache->any, types, penv, subtype_inexact__sigseq_useenv, subtype, offs+1);
if (cache->targ != (void*)jl_nothing && jl_is_type_type(ty)) {
jl_value_t *a0 = jl_tparam0(ty);
if (jl_is_datatype(a0)) {
union jl_typemap_t ml = mtcache_hash_lookup(cache->targ, a0, 1, offs);
if (ml.unknown != jl_nothing) {
jl_typemap_entry_t *li = jl_typemap_assoc_by_type(ml, types, penv,
subtype_inexact__sigseq_useenv, subtype, offs+1);
if (li) return li;
}
}
}
if (cache->arg1 != (void*)jl_nothing && jl_is_datatype(ty)) {
union jl_typemap_t ml = mtcache_hash_lookup(cache->arg1, ty, 0, offs);
if (ml.unknown != jl_nothing) {
jl_typemap_entry_t *li = jl_typemap_assoc_by_type(ml, types, penv,
subtype_inexact__sigseq_useenv, subtype, offs+1);
if (li) return li;
}
}
}
// Always check the list (since offs doesn't always start at 0)
if (subtype) {
jl_typemap_entry_t *li = jl_typemap_assoc_by_type_(cache->linear, types, subtype_inexact__sigseq_useenv, penv);
if (li) return li;
return jl_typemap_assoc_by_type(cache->any, types, penv, subtype_inexact__sigseq_useenv, subtype, offs+1);
}
else {
return jl_typemap_lookup_by_type_(cache->linear, types, subtype_inexact__sigseq_useenv);
}
}
else {
return subtype ?
jl_typemap_assoc_by_type_(ml_or_cache.leaf, types, subtype_inexact__sigseq_useenv, penv) :
jl_typemap_lookup_by_type_(ml_or_cache.leaf, types, subtype_inexact__sigseq_useenv);
}
}
int jl_typemap_intersection_visitor(union jl_typemap_t map, int offs,
struct typemap_intersection_env *closure)
{
if (jl_typeof(map.unknown) == (jl_value_t*)jl_typemap_level_type) {
jl_typemap_level_t *cache = map.node;
jl_value_t *ty = NULL;
size_t l = jl_datatype_nfields(closure->type);
if (closure->va && l <= offs + 1) {
ty = closure->va;
}
else if (l > offs) {
ty = jl_tparam(closure->type, offs);
}
if (ty) {
if (cache->targ != (void*)jl_nothing) {
jl_value_t *typetype = jl_is_type_type(ty) ? jl_tparam0(ty) : NULL;
if (typetype && !jl_has_typevars(typetype)) {
if (is_cache_leaf(typetype)) {
// direct lookup of leaf types
union jl_typemap_t ml = mtcache_hash_lookup(cache->targ, typetype, 1, offs);
if (ml.unknown != jl_nothing) {
if (!jl_typemap_intersection_visitor(ml, offs+1, closure)) return 0;
}
}
}
else {
// else an array scan is required to check subtypes
// first, fast-path: optimized pre-intersection test to see if `ty` could intersect with any Type
if (typetype || jl_type_intersection((jl_value_t*)jl_type_type, ty) != jl_bottom_type)
if (!jl_typemap_intersection_array_visitor(cache->targ, ty, 1, offs, closure)) return 0;
}
}
if (cache->arg1 != (void*)jl_nothing) {
if (is_cache_leaf(ty)) {
// direct lookup of leaf types
union jl_typemap_t ml = mtcache_hash_lookup(cache->arg1, ty, 0, offs);
if (ml.unknown != jl_nothing) {
if (!jl_typemap_intersection_visitor(ml, offs+1, closure)) return 0;
}
}
else {
// else an array scan is required to check subtypes
if (!jl_typemap_intersection_array_visitor(cache->arg1, ty, 0, offs, closure)) return 0;
}
}
}
if (!jl_typemap_intersection_node_visitor(map.node->linear, closure))
return 0;
if (ty)
return jl_typemap_intersection_visitor(map.node->any, offs+1, closure);
return 1;
}
else {
return jl_typemap_intersection_node_visitor(map.leaf, closure);
}
}
jl_typemap_entry_t *jl_typemap_assoc_exact(union jl_typemap_t ml_or_cache, jl_value_t **args, size_t n, int8_t offs)
{
// NOTE: This function is a huge performance hot spot!!
if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) {
jl_typemap_level_t *cache = ml_or_cache.node;
if (n > offs) {
jl_value_t *a1 = args[offs];
jl_value_t *ty = (jl_value_t*)jl_typeof(a1);
assert(jl_is_datatype(ty));
if (ty == (jl_value_t*)jl_datatype_type && cache->targ != (void*)jl_nothing) {
ml_or_cache = mtcache_hash_lookup(cache->targ, a1, 1, offs);
jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1);
if (ml) return ml;
}
if (cache->arg1 != (void*)jl_nothing) {
ml_or_cache = mtcache_hash_lookup(cache->arg1, ty, 0, offs);
if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_entry_type &&
ml_or_cache.leaf->simplesig == (void*)jl_nothing && offs < 2 && n > 1) {
jl_value_t *a0 = args[1-offs];
jl_value_t *t0 = (jl_value_t*)jl_typeof(a0);
if (ml_or_cache.leaf->next==(void*)jl_nothing && n==2 && jl_datatype_nfields(ml_or_cache.leaf->sig)==2 &&
jl_tparam(ml_or_cache.leaf->sig, 1 - offs) == t0)
return ml_or_cache.leaf;
if (n==3) {
// some manually-unrolled common special cases
jl_value_t *a2 = args[2];
if (!jl_is_tuple(a2)) { // issue #6426
jl_typemap_entry_t *mn = ml_or_cache.leaf;
if (jl_datatype_nfields(mn->sig)==3 &&
jl_tparam(mn->sig,1-offs)==t0 &&
jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2))
return mn;
mn = mn->next;
if (mn!=(void*)jl_nothing && jl_datatype_nfields(mn->sig)==3 &&
jl_tparam(mn->sig,1-offs)==t0 &&
jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2))
return mn;
}
}
}
jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1);
if (ml) return ml;
}
}
jl_typemap_entry_t *ml = jl_typemap_node_assoc_exact(cache->linear, args, n);
if (ml) return ml;
return jl_typemap_assoc_exact(cache->any, args, n, offs+1);
}
else {
return jl_typemap_node_assoc_exact(ml_or_cache.leaf, args, n);
}
}
// this is the general entry point for looking up a type in the cache
// (as a subtype, or with typeseq)
jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_tupletype_t *types, jl_svec_t **penv,
int8_t subtype_inexact__sigseq_useenv, int8_t subtype, int8_t offs)
{
if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) {
jl_typemap_level_t *cache = ml_or_cache.node;
// called object is the primary key for constructors, otherwise first argument
jl_value_t *ty = NULL;
if (jl_datatype_nfields(types) > offs) {
ty = jl_tparam(types, offs);
if (cache->targ != (void*)jl_nothing && jl_is_type_type(ty)) {
jl_value_t *a0 = jl_tparam0(ty);
if (jl_is_datatype(a0)) {
union jl_typemap_t ml = mtcache_hash_lookup(cache->targ, a0, 1, offs);
if (ml.unknown != jl_nothing) {
jl_typemap_entry_t *li = jl_typemap_assoc_by_type(ml, types, penv,
subtype_inexact__sigseq_useenv, subtype, offs+1);
if (li) return li;
}
}
}
if (cache->arg1 != (void*)jl_nothing && jl_is_datatype(ty)) {
union jl_typemap_t ml = mtcache_hash_lookup(cache->arg1, ty, 0, offs);
if (ml.unknown != jl_nothing) {
jl_typemap_entry_t *li = jl_typemap_assoc_by_type(ml, types, penv,
subtype_inexact__sigseq_useenv, subtype, offs+1);
if (li) return li;
}
}
}
if (subtype) {
jl_typemap_entry_t *li = jl_typemap_assoc_by_type_(cache->linear, types, subtype_inexact__sigseq_useenv, penv);
if (li) return li;
return jl_typemap_assoc_by_type(cache->any, types, penv, subtype_inexact__sigseq_useenv, subtype, offs+1);
}
else {
if (ty && jl_is_any(ty))
return jl_typemap_assoc_by_type(cache->any, types, penv, subtype_inexact__sigseq_useenv, subtype, offs+1);
else
return jl_typemap_lookup_by_type_(cache->linear, types, subtype_inexact__sigseq_useenv);
}
}
else {
return subtype ?
jl_typemap_assoc_by_type_(ml_or_cache.leaf, types, subtype_inexact__sigseq_useenv, penv) :
jl_typemap_lookup_by_type_(ml_or_cache.leaf, types, subtype_inexact__sigseq_useenv);
}
}
// this is the general entry point for looking up a type in the cache
// as a subtype, or with type_equal
jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_value_t *types, jl_svec_t **penv,
int8_t subtype, int8_t offs, size_t world, size_t max_world_mask)
{
if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) {
jl_typemap_level_t *cache = ml_or_cache.node;
// called object is the primary key for constructors, otherwise first argument
jl_value_t *ty = NULL;
jl_value_t *ttypes = jl_unwrap_unionall((jl_value_t*)types);
assert(jl_is_datatype(ttypes));
size_t l = jl_field_count(ttypes);
int isva = 0;
// compute the type at offset `offs` into `types`, which may be a Vararg
if (l <= offs + 1) {
ty = jl_tparam(ttypes, l - 1);
if (jl_is_vararg_type(ty)) {
ty = jl_unwrap_vararg(ty);
isva = 1;
}
else if (l <= offs) {
ty = NULL;
}
}
else if (l > offs) {
ty = jl_tparam(ttypes, offs);
}
// If there is a type at offs, look in the optimized caches
if (!subtype) {
if (ty && jl_is_any(ty))
return jl_typemap_assoc_by_type(cache->any, types, penv, subtype, offs + 1, world, max_world_mask);
if (isva) // in lookup mode, want to match Vararg exactly, not as a subtype
ty = NULL;
}
if (ty) {
if (jl_is_type_type(ty)) {
jl_value_t *a0 = jl_tparam0(ty);
if (cache->targ.values != (void*)jl_nothing && jl_is_datatype(a0)) {
union jl_typemap_t ml = mtcache_hash_lookup(&cache->targ, a0, 1, offs);
if (ml.unknown != jl_nothing) {
jl_typemap_entry_t *li =
jl_typemap_assoc_by_type(ml, types, penv, subtype, offs + 1, world, max_world_mask);
if (li) return li;
}
}
if (!subtype && is_cache_leaf(a0)) return NULL;
}
if (cache->arg1.values != (void*)jl_nothing && jl_is_datatype(ty)) {
union jl_typemap_t ml = mtcache_hash_lookup(&cache->arg1, ty, 0, offs);
if (ml.unknown != jl_nothing) {
jl_typemap_entry_t *li =
jl_typemap_assoc_by_type(ml, types, penv, subtype, offs + 1, world, max_world_mask);
if (li) return li;
}
}
if (!subtype && is_cache_leaf(ty)) return NULL;
}
// Always check the list (since offs doesn't always start at 0)
if (subtype) {
jl_typemap_entry_t *li = jl_typemap_assoc_by_type_(cache->linear, types, penv, world, max_world_mask);
if (li) return li;
return jl_typemap_assoc_by_type(cache->any, types, penv, subtype, offs + 1, world, max_world_mask);
}
else {
return jl_typemap_lookup_by_type_(cache->linear, types, world, max_world_mask);
}
}
else {
return subtype ?
jl_typemap_assoc_by_type_(ml_or_cache.leaf, types, penv, world, max_world_mask) :
jl_typemap_lookup_by_type_(ml_or_cache.leaf, types, world, max_world_mask);
}
}
jl_typemap_entry_t *jl_typemap_assoc_exact(union jl_typemap_t ml_or_cache, jl_value_t **args, size_t n, int8_t offs)
{
// NOTE: This function is a huge performance hot spot!!
jl_typemap_entry_t *ml;
if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) {
jl_typemap_level_t *cache = ml_or_cache.node;
if (n > offs) {
jl_value_t *a1 = args[offs];
jl_value_t *ty = (jl_value_t*)jl_typeof(a1);
assert(jl_is_datatype(ty));
if (ty == (jl_value_t*)jl_datatype_type && cache->targ != (void*)jl_nothing) {
ml_or_cache = mtcache_hash_lookup(cache->targ, a1, 1, offs);
ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1);
if (ml)
return ml;
}
if (cache->arg1 != (void*)jl_nothing) {
ml_or_cache = mtcache_hash_lookup(cache->arg1, ty, 0, offs);
if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_entry_type &&
ml_or_cache.leaf->simplesig == (void*)jl_nothing && offs < 2 && n > 1) {
jl_value_t *a0 = args[1-offs];
jl_value_t *t0 = (jl_value_t*)jl_typeof(a0);
if (ml_or_cache.leaf->next==(void*)jl_nothing && n==2 && jl_datatype_nfields(ml_or_cache.leaf->sig)==2 &&
jl_tparam(ml_or_cache.leaf->sig, 1 - offs) == t0)
return ml_or_cache.leaf;
if (n==3) {
// some manually-unrolled common special cases
jl_value_t *a2 = args[2];
if (!jl_is_tuple(a2)) { // issue #6426
jl_typemap_entry_t *mn = ml_or_cache.leaf;
if (jl_datatype_nfields(mn->sig)==3 &&
jl_tparam(mn->sig,1-offs)==t0 &&
jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2))
return mn;
mn = mn->next;
if (mn!=(void*)jl_nothing && jl_datatype_nfields(mn->sig)==3 &&
jl_tparam(mn->sig,1-offs)==t0 &&
jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2))
return mn;
}
}
}
ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1);
if (ml)
return ml;
}
}
ml = cache->linear;
}
else {
ml = ml_or_cache.leaf;
}
while (ml != (void*)jl_nothing) {
size_t lensig = jl_datatype_nfields(ml->sig);
if (lensig == n || (ml->va && lensig <= n+1)) {
int ismatch;
if (ml->simplesig != (void*)jl_nothing &&
!sig_match_simple(args, n, jl_svec_data(ml->simplesig->parameters), 0,
jl_datatype_nfields(ml->simplesig)))
ismatch = 0;
else if (ml->isleafsig)
ismatch = sig_match_leaf(args, jl_svec_data(ml->sig->parameters), n);
else if (ml->issimplesig)
ismatch = sig_match_simple(args, n, jl_svec_data(ml->sig->parameters), ml->va, lensig);
else
ismatch = jl_tuple_subtype(args, n, ml->sig, 1);
if (ismatch) {
size_t i, l;
for (i = 0, l = jl_svec_len(ml->guardsigs); i < l; i++) {
// checking guard entries require a more
// expensive subtype check, since guard entries added for ANY might be
// abstract. this fixed issue #12967.
if (jl_tuple_subtype(args, n, (jl_tupletype_t*)jl_svecref(ml->guardsigs, i), 1)) {
break;
}
}
if (i == l)
return ml;
}
}
ml = ml->next;
}
return NULL;
}
// calls fptr on each jl_typemap_entry_t in cache in sort order
// for which type ∩ ml->type != Union{}, until fptr return false
int jl_typemap_intersection_visitor(union jl_typemap_t map, int offs,
struct typemap_intersection_env *closure)
{
jl_typemap_entry_t *ml;
if (jl_typeof(map.unknown) == (jl_value_t*)jl_typemap_level_type) {
jl_typemap_level_t *cache = map.node;
jl_value_t *ty = NULL;
size_t l = jl_datatype_nfields(closure->type);
if (closure->va && l == offs - 1) {
ty = closure->va;
}
else if (l > offs) {
ty = jl_tparam(closure->type, offs);
}
if (ty) {
if (cache->targ != (void*)jl_nothing) {
if (jl_is_type_type(ty) && is_cache_leaf(jl_tparam0(ty))) {
// direct lookup of leaf types
union jl_typemap_t ml = mtcache_hash_lookup(cache->targ, jl_tparam0(ty), 1, offs);
if (ml.unknown != jl_nothing) {
if (!jl_typemap_intersection_visitor(ml, offs+1, closure)) return 0;
}
}
else {
// else an array scan is required to check subtypes
// TODO: fast-path: optimized pre-intersection test
if (!jl_typemap_intersection_array_visitor(cache->targ, ty, 1, offs, closure)) return 0;
}
}
if (cache->arg1 != (void*)jl_nothing) {
if (is_cache_leaf(ty)) {
// direct lookup of leaf types
union jl_typemap_t ml = mtcache_hash_lookup(cache->arg1, ty, 0, offs);
if (ml.unknown != jl_nothing) {
if (!jl_typemap_intersection_visitor(ml, offs+1, closure)) return 0;
}
}
else {
// else an array scan is required to check subtypes
if (!jl_typemap_intersection_array_visitor(cache->arg1, ty, 0, offs, closure)) return 0;
}
}
}
ml = map.node->linear;
}
else {
ml = map.leaf;
}
// slow-path scan everything else
// mark this `register` because (for branch prediction)
// that can be absolutely critical for speed
register jl_typemap_intersection_visitor_fptr fptr = closure->fptr;
while (ml != (void*)jl_nothing) {
// TODO: optimize intersection test
if (closure->type == (jl_value_t*)ml->sig) {
// fast-path for the intersection of a type with itself
if (closure->env)
closure->env = ml->tvars;
closure->ti = closure->type;
if (!fptr(ml, closure))
return 0;
}
else {
jl_value_t *ti;
if (closure->env) {
closure->env = jl_emptysvec;
ti = jl_lookup_match(closure->type, (jl_value_t*)ml->sig, &closure->env, ml->tvars);
}
else {
ti = jl_type_intersection(closure->type, (jl_value_t*)ml->sig);
}
if (ti != (jl_value_t*)jl_bottom_type) {
closure->ti = ti;
if (!fptr(ml, closure))
return 0;
}
}
ml = ml->next;
}
return 1;
}
