static inline bool broadcast_tagged_dims_from_type(intptr_t ndim, ndt::type tp, const intptr_t *tagged_dims,
intptr_t *out_tagged_dims) {
tp = tp.without_memory_type();
for (intptr_t i = 0; i < ndim; ++i) {
intptr_t tagged_dim = tagged_dims[i], dim_size;
switch (tp.get_id()) {
case fixed_dim_kind_id: {
if (tagged_dim < 0) {
out_tagged_dims[i] = -2;
}
} break;
case fixed_dim_id: {
dim_size = tp.extended<ndt::fixed_dim_type>()->get_fixed_dim_size();
if (tagged_dim < 0 || tagged_dim == 1) {
out_tagged_dims[i] = dim_size;
} else if (tagged_dim != dim_size && dim_size != 1) {
return false;
}
} break;
case var_dim_id:
// All broadcasting is done dynamically for var
break;
default: {
stringstream ss;
ss << "dim_fragment_type failed to get shape from type " << tp;
throw type_error(ss.str());
}
}
tp = tp.extended<ndt::base_dim_type>()->get_element_type();
}
return true;
}
static void array_getbuffer_pep3118_bytes(const ndt::type &tp, const char *arrmeta, char *data, Py_buffer *buffer,
int flags)
{
buffer->itemsize = 1;
if (flags & PyBUF_FORMAT) {
buffer->format = (char *)"c";
}
else {
buffer->format = NULL;
}
buffer->ndim = 1;
#if PY_VERSION_HEX == 0x02070000
buffer->internal = NULL;
buffer->shape = &buffer->smalltable[0];
buffer->strides = &buffer->smalltable[1];
#else
buffer->internal = malloc(2 * sizeof(intptr_t));
buffer->shape = reinterpret_cast<Py_ssize_t *>(buffer->internal);
buffer->strides = buffer->shape + 1;
#endif
buffer->strides[0] = 1;
if (tp.get_id() == bytes_id) {
// Variable-length bytes type
buffer->buf = reinterpret_cast<bytes *>(data)->begin();
buffer->len = reinterpret_cast<bytes *>(data)->size();
}
else {
// Fixed-length bytes type
buffer->len = tp.get_data_size();
}
buffer->shape[0] = buffer->len;
}
size_t pydynd::get_nonragged_dim_count(const ndt::type &tp, size_t max_count)
{
if (tp.is_symbolic()) {
if (tp.is_scalar()) {
return 0;
}
}
if (!tp.is_scalar()) {
if (max_count <= 1) {
return max_count;
}
else {
return min(max_count,
1 + get_nonragged_dim_count(
static_cast<const ndt::base_dim_type *>(tp.extended())
->get_element_type(),
max_count - 1));
}
}
switch (tp.get_id()) {
case struct_id:
case tuple_id:
if (max_count <= 1) {
return max_count;
}
else {
auto bsd = tp.extended<ndt::tuple_type>();
size_t field_count = bsd->get_field_count();
for (size_t i = 0; i != field_count; ++i) {
size_t candidate =
1 + get_nonragged_dim_count(bsd->get_field_type(i), max_count - 1);
if (candidate < max_count) {
max_count = candidate;
if (max_count <= 1) {
return max_count;
}
}
}
return max_count;
}
default:
return 0;
}
}
static void append_pep3118_format(intptr_t &out_itemsize, const ndt::type &tp, const char *arrmeta,
std::stringstream &o)
{
switch (tp.get_id()) {
case bool_id:
o << "?";
out_itemsize = 1;
return;
case int8_id:
o << "b";
out_itemsize = 1;
return;
case int16_id:
o << "h";
out_itemsize = 2;
return;
case int32_id:
o << "i";
out_itemsize = 4;
return;
case int64_id:
o << "q";
out_itemsize = 8;
return;
case uint8_id:
o << "B";
out_itemsize = 1;
return;
case uint16_id:
o << "H";
out_itemsize = 2;
return;
case uint32_id:
o << "I";
out_itemsize = 4;
return;
case uint64_id:
o << "Q";
out_itemsize = 8;
return;
case float32_id:
o << "f";
out_itemsize = 4;
return;
case float64_id:
o << "d";
out_itemsize = 8;
return;
case complex_float32_id:
o << "Zf";
out_itemsize = 8;
return;
case complex_float64_id:
o << "Zd";
out_itemsize = 16;
return;
case fixed_string_id:
switch (tp.extended<ndt::fixed_string_type>()->get_encoding()) {
case string_encoding_ascii: {
intptr_t element_size = tp.get_data_size();
o << element_size << "s";
out_itemsize = element_size;
return;
}
// TODO: Couldn't find documentation for UCS-2 character code?
case string_encoding_utf_32: {
intptr_t element_size = tp.get_data_size();
o << (element_size / 4) << "w";
out_itemsize = element_size;
return;
}
default:
break;
}
// Pass through to error
break;
case fixed_dim_id: {
ndt::type child_tp = tp;
o << "(";
do {
const ndt::fixed_dim_type *tdt = child_tp.extended<ndt::fixed_dim_type>();
intptr_t dim_size = tdt->get_fixed_dim_size();
o << dim_size;
if (child_tp.get_data_size() != tdt->get_element_type().get_data_size() * dim_size) {
stringstream ss;
ss << "Cannot convert dynd type " << tp << " into a PEP 3118 format because it is not C-order";
throw dynd::type_error(ss.str());
}
o << ")";
child_tp = tdt->get_element_type();
} while (child_tp.get_id() == fixed_dim_id && (o << ","));
append_pep3118_format(out_itemsize, child_tp, arrmeta, o);
out_itemsize = tp.get_data_size();
return;
}
case struct_id: {
o << "T{";
const ndt::struct_type *tdt = tp.extended<ndt::struct_type>();
size_t num_fields = tdt->get_field_count();
const uintptr_t *offsets = reinterpret_cast<const uintptr_t *>(arrmeta);
const uintptr_t *arrmeta_offsets = tdt->get_arrmeta_offsets_raw();
size_t format_offset = 0;
for (size_t i = 0; i != num_fields; ++i) {
size_t offset = offsets[i];
// Add padding bytes
while (offset > format_offset) {
o << "x";
++format_offset;
}
if (offset < format_offset) {
// DyND allows the order of fields in memory to differ from the logical
// order, something not supported by PEP 3118
stringstream ss;
ss << "Cannot convert dynd type " << tp << " with out of order data layout into a PEP 3118 format string";
throw type_error(ss.str());
}
// The field's type
append_pep3118_format(out_itemsize, tdt->get_field_type(i), arrmeta ? (arrmeta + arrmeta_offsets[i]) : NULL, o);
format_offset += out_itemsize;
// Append the name
o << ":" << tdt->get_field_name(i) << ":";
}
out_itemsize = format_offset;
o << "}";
return;
}
default:
break;
}
stringstream ss;
ss << "Cannot convert dynd type " << tp << " into a PEP 3118 format string";
throw dynd::type_error(ss.str());
}
void overload(const ndt::type &dst_tp, intptr_t DYND_UNUSED(nsrc), const ndt::type *DYND_UNUSED(src_tp),
const callable &value) {
m_dispatcher.insert({{dst_tp.get_id()}, value});
}
ndt::type resolve(base_callable *DYND_UNUSED(caller), char *data, call_graph &cg, const ndt::type &dst_tp,
size_t nsrc, const ndt::type *src_tp, size_t nkwd, const array *kwds,
const std::map<std::string, ndt::type> &tp_vars) {
const ndt::callable_type *child_tp =
reinterpret_cast<data_type *>(data)->child->get_type().template extended<ndt::callable_type>();
bool first = reinterpret_cast<data_type *>(data)->first;
reinterpret_cast<data_type *>(data)->first = false;
bool state = reinterpret_cast<data_type *>(data)->state;
bool res_ignore = reinterpret_cast<data_type *>(data)->res_ignore;
bool dst_variadic = dst_tp.is_variadic();
// Do a pass through the src types to classify them
bool src_all_strided = true, src_all_strided_or_var = true;
for (size_t i = 0; i < nsrc; ++i) {
intptr_t src_ndim = src_tp[i].get_ndim() - child_tp->get_pos_type(i).get_ndim();
switch (src_tp[i].get_id()) {
case fixed_dim_id:
break;
case var_dim_id:
src_all_strided = false;
break;
default:
// If it's a scalar, allow it to broadcast like
// a strided dimension
if (src_ndim > 0) {
src_all_strided_or_var = false;
}
break;
}
}
bool var_broadcast = !src_all_strided;
for (size_t i = 0; i < nsrc; ++i) {
var_broadcast &= src_tp[i].get_id() == var_dim_id ||
(src_tp[i].get_id() == fixed_dim_id &&
src_tp[i].extended<ndt::fixed_dim_type>()->get_fixed_dim_size() == 1);
}
if ((dst_variadic || (dst_tp.get_id() == fixed_dim_id || res_ignore)) && src_all_strided) {
static callable f = make_callable<elwise_callable<fixed_dim_id, fixed_dim_id, no_traits, N>>();
static callable g = make_callable<elwise_callable<fixed_dim_id, fixed_dim_id, state_traits, N>>();
if (!first && state) {
return g->resolve(this, data, cg, dst_tp, nsrc, src_tp, nkwd, kwds, tp_vars);
} else {
return f->resolve(this, data, cg, dst_tp, nsrc, src_tp, nkwd, kwds, tp_vars);
}
} else if (((dst_variadic) || dst_tp.get_id() == var_dim_id) && (var_broadcast || src_all_strided)) {
static callable f = make_callable<elwise_callable<var_dim_id, fixed_dim_id, no_traits, N>>();
return f->resolve(this, data, cg, dst_tp, nsrc, src_tp, nkwd, kwds, tp_vars);
} else if (src_all_strided_or_var) {
static callable f = make_callable<elwise_callable<fixed_dim_id, var_dim_id, no_traits, N>>();
return f->resolve(this, data, cg, dst_tp, nsrc, src_tp, nkwd, kwds, tp_vars);
}
std::stringstream ss;
ss << "Cannot process lifted elwise expression from (";
for (size_t i = 0; i < nsrc; ++i) {
ss << src_tp[i];
if (i != nsrc - 1) {
ss << ", ";
}
}
ss << ") to " << dst_tp;
throw std::runtime_error(ss.str());
}
static void make_numpy_dtype_for_copy(pyobject_ownref *out_numpy_dtype, intptr_t ndim, const ndt::type &dt,
const char *arrmeta)
{
// DyND builtin types
if (dt.is_builtin()) {
out_numpy_dtype->reset((PyObject *)PyArray_DescrFromType(dynd_to_numpy_id(dt.get_id())));
return;
}
switch (dt.get_id()) {
case fixed_string_id: {
const ndt::fixed_string_type *fsd = dt.extended<ndt::fixed_string_type>();
PyArray_Descr *result;
switch (fsd->get_encoding()) {
case string_encoding_ascii:
result = PyArray_DescrNewFromType(NPY_STRING);
result->elsize = (int)fsd->get_data_size();
out_numpy_dtype->reset((PyObject *)result);
return;
case string_encoding_utf_32:
result = PyArray_DescrNewFromType(NPY_UNICODE);
result->elsize = (int)fsd->get_data_size();
out_numpy_dtype->reset((PyObject *)result);
return;
default:
// If it's not one of the encodings NumPy supports,
// use Unicode
result = PyArray_DescrNewFromType(NPY_UNICODE);
result->elsize = (int)fsd->get_data_size() * 4 / string_encoding_char_size_table[fsd->get_encoding()];
out_numpy_dtype->reset((PyObject *)result);
return;
}
break;
}
case string_id: {
// Convert variable-length strings into NumPy object arrays
PyArray_Descr *dtype = PyArray_DescrNewFromType(NPY_OBJECT);
// Add metadata to the string type being created so that
// it can round-trip. This metadata is compatible with h5py.
out_numpy_dtype->reset((PyObject *)dtype);
if (dtype->metadata == NULL) {
dtype->metadata = PyDict_New();
}
PyDict_SetItemString(dtype->metadata, "vlen", (PyObject *)&PyUnicode_Type);
return;
}
case fixed_dim_id: {
if (ndim > 0) {
const ndt::base_dim_type *bdt = dt.extended<ndt::base_dim_type>();
make_numpy_dtype_for_copy(out_numpy_dtype, ndim - 1, bdt->get_element_type(),
arrmeta + sizeof(fixed_dim_type_arrmeta));
return;
}
else {
// If this isn't one of the array dimensions, it maps into
// a numpy dtype with a shape
// Build up the shape of the array for NumPy
pyobject_ownref shape(PyList_New(0));
ndt::type element_tp = dt;
while (ndim > 0) {
const fixed_dim_type_arrmeta *am = reinterpret_cast<const fixed_dim_type_arrmeta *>(arrmeta);
intptr_t dim_size = am->dim_size;
element_tp = dt.extended<ndt::base_dim_type>()->get_element_type();
arrmeta += sizeof(fixed_dim_type_arrmeta);
--ndim;
if (PyList_Append(shape.get(), PyLong_FromSize_t(dim_size)) < 0) {
throw runtime_error("propagating python error");
}
}
// Get the numpy dtype of the element
pyobject_ownref child_numpy_dtype;
make_numpy_dtype_for_copy(&child_numpy_dtype, 0, element_tp, arrmeta);
// Create the result numpy dtype
pyobject_ownref tuple_obj(PyTuple_New(2));
PyTuple_SET_ITEM(tuple_obj.get(), 0, child_numpy_dtype.release());
PyTuple_SET_ITEM(tuple_obj.get(), 1, shape.release());
PyArray_Descr *result = NULL;
if (!PyArray_DescrConverter(tuple_obj, &result)) {
throw dynd::type_error("failed to convert dynd type into numpy subarray dtype");
}
// Put the final numpy dtype reference in the output
out_numpy_dtype->reset((PyObject *)result);
return;
}
break;
}
case struct_id: {
const ndt::struct_type *bs = dt.extended<ndt::struct_type>();
size_t field_count = bs->get_field_count();
pyobject_ownref names_obj(PyList_New(field_count));
for (size_t i = 0; i < field_count; ++i) {
const dynd::string &fn = bs->get_field_name(i);
#if PY_VERSION_HEX >= 0x03000000
pyobject_ownref name_str(PyUnicode_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#else
pyobject_ownref name_str(PyString_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#endif
PyList_SET_ITEM(names_obj.get(), i, name_str.release());
}
pyobject_ownref formats_obj(PyList_New(field_count));
pyobject_ownref offsets_obj(PyList_New(field_count));
size_t standard_offset = 0, standard_alignment = 1;
for (size_t i = 0; i < field_count; ++i) {
// Get the numpy dtype of the element
pyobject_ownref field_numpy_dtype;
make_numpy_dtype_for_copy(&field_numpy_dtype, 0, bs->get_field_type(i), arrmeta);
size_t field_alignment = ((PyArray_Descr *)field_numpy_dtype.get())->alignment;
size_t field_size = ((PyArray_Descr *)field_numpy_dtype.get())->elsize;
standard_offset = inc_to_alignment(standard_offset, field_alignment);
standard_alignment = max(standard_alignment, field_alignment);
PyList_SET_ITEM(formats_obj.get(), i, field_numpy_dtype.release());
PyList_SET_ITEM((PyObject *)offsets_obj, i, PyLong_FromSize_t(standard_offset));
standard_offset += field_size;
}
// Get the full element size
standard_offset = inc_to_alignment(standard_offset, standard_alignment);
pyobject_ownref itemsize_obj(PyLong_FromSize_t(standard_offset));
pyobject_ownref dict_obj(PyDict_New());
PyDict_SetItemString(dict_obj, "names", names_obj);
PyDict_SetItemString(dict_obj, "formats", formats_obj);
PyDict_SetItemString(dict_obj, "offsets", offsets_obj);
PyDict_SetItemString(dict_obj, "itemsize", itemsize_obj);
PyArray_Descr *result = NULL;
if (!PyArray_DescrAlignConverter(dict_obj, &result)) {
stringstream ss;
ss << "failed to convert dynd type " << dt << " into numpy dtype via dict";
throw dynd::type_error(ss.str());
}
out_numpy_dtype->reset((PyObject *)result);
return;
}
default: {
break;
}
}
if (dt.get_base_id() == expr_kind_id) {
// Convert the value type for the copy
make_numpy_dtype_for_copy(out_numpy_dtype, ndim, dt.value_type(), NULL);
return;
}
// Anything which fell through is an error
stringstream ss;
ss << "dynd as_numpy could not convert dynd type ";
ss << dt;
ss << " to a numpy dtype";
throw dynd::type_error(ss.str());
}
static void as_numpy_analysis(pyobject_ownref *out_numpy_dtype, bool *out_requires_copy, intptr_t ndim,
const ndt::type &dt, const char *arrmeta)
{
if (dt.is_builtin()) {
// DyND builtin types
out_numpy_dtype->reset((PyObject *)PyArray_DescrFromType(dynd_to_numpy_id(dt.get_id())));
return;
}
switch (dt.get_id()) {
case fixed_string_id: {
const ndt::fixed_string_type *fsd = dt.extended<ndt::fixed_string_type>();
PyArray_Descr *result;
switch (fsd->get_encoding()) {
case string_encoding_ascii:
result = PyArray_DescrNewFromType(NPY_STRING);
result->elsize = (int)fsd->get_data_size();
out_numpy_dtype->reset((PyObject *)result);
return;
case string_encoding_utf_32:
result = PyArray_DescrNewFromType(NPY_UNICODE);
result->elsize = (int)fsd->get_data_size();
out_numpy_dtype->reset((PyObject *)result);
return;
default:
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
break;
}
case string_id: {
// Convert to numpy object type, requires copy
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
case fixed_dim_id: {
const ndt::base_dim_type *bdt = dt.extended<ndt::base_dim_type>();
if (ndim > 0) {
// If this is one of the array dimensions, it simply
// becomes one of the numpy _array dimensions
as_numpy_analysis(out_numpy_dtype, out_requires_copy, ndim - 1, bdt->get_element_type(),
arrmeta + sizeof(fixed_dim_type_arrmeta));
return;
}
else {
// If this isn't one of the array dimensions, it maps into
// a numpy dtype with a shape
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
break;
}
/*
case cfixed_dim_id: {
const cfixed_dim_type *fad = dt.extended<cfixed_dim_type>();
if (ndim > 0) {
// If this is one of the array dimensions, it simply
// becomes one of the numpy _array dimensions
as_numpy_analysis(out_numpy_dtype, out_requires_copy, ndim - 1,
fad->get_element_type(),
arrmeta + sizeof(cfixed_dim_type_arrmeta));
return;
} else {
// If this isn't one of the array dimensions, it maps into
// a numpy dtype with a shape
// Build up the shape of the array for NumPy
pyobject_ownref shape(PyList_New(0));
ndt::type element_tp = dt;
while (ndim > 0) {
size_t dim_size = 0;
if (dt.get_id() == cfixed_dim_id) {
const cfixed_dim_type *cfd =
element_tp.extended<cfixed_dim_type>();
element_tp = cfd->get_element_type();
if (cfd->get_data_size() != element_tp.get_data_size() * dim_size)
{
// If it's not C-order, a copy is required
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
} else {
stringstream ss;
ss << "dynd as_numpy could not convert dynd type ";
ss << dt;
ss << " to a numpy dtype";
throw dynd::type_error(ss.str());
}
--ndim;
if (PyList_Append(shape.get(), PyLong_FromSize_t(dim_size)) < 0) {
throw runtime_error("propagating python error");
}
}
// Get the numpy dtype of the element
pyobject_ownref child_numpy_dtype;
as_numpy_analysis(&child_numpy_dtype, out_requires_copy, 0,
element_tp,
arrmeta);
if (*out_requires_copy) {
// If the child required a copy, stop right away
out_numpy_dtype->clear();
return;
}
// Create the result numpy dtype
pyobject_ownref tuple_obj(PyTuple_New(2));
PyTuple_SET_ITEM(tuple_obj.get(), 0, child_numpy_dtype.release());
PyTuple_SET_ITEM(tuple_obj.get(), 1, shape.release());
PyArray_Descr *result = NULL;
if (!PyArray_DescrConverter(tuple_obj, &result)) {
throw dynd::type_error(
"failed to convert dynd type into numpy subarray dtype");
}
// Put the final numpy dtype reference in the output
out_numpy_dtype->reset((PyObject *)result);
return;
}
break;
}
*/
case struct_id: {
if (dt.get_id() == struct_id && arrmeta == NULL) {
// If it's a struct type with no arrmeta, a copy is required
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
const ndt::struct_type *bs = dt.extended<ndt::struct_type>();
const uintptr_t *offsets = reinterpret_cast<const uintptr_t *>(arrmeta);
size_t field_count = bs->get_field_count();
pyobject_ownref names_obj(PyList_New(field_count));
for (size_t i = 0; i < field_count; ++i) {
const dynd::string &fn = bs->get_field_name(i);
#if PY_VERSION_HEX >= 0x03000000
pyobject_ownref name_str(PyUnicode_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#else
pyobject_ownref name_str(PyString_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#endif
PyList_SET_ITEM(names_obj.get(), i, name_str.release());
}
pyobject_ownref formats_obj(PyList_New(field_count));
for (size_t i = 0; i < field_count; ++i) {
// Get the numpy dtype of the element
pyobject_ownref field_numpy_dtype;
as_numpy_analysis(&field_numpy_dtype, out_requires_copy, 0, bs->get_field_type(i), arrmeta);
if (*out_requires_copy) {
// If the field required a copy, stop right away
out_numpy_dtype->clear();
return;
}
PyList_SET_ITEM(formats_obj.get(), i, field_numpy_dtype.release());
}
pyobject_ownref offsets_obj(PyList_New(field_count));
for (size_t i = 0; i < field_count; ++i) {
PyList_SET_ITEM((PyObject *)offsets_obj, i, PyLong_FromSize_t(offsets[i]));
}
pyobject_ownref dict_obj(PyDict_New());
PyDict_SetItemString(dict_obj, "names", names_obj);
PyDict_SetItemString(dict_obj, "formats", formats_obj);
PyDict_SetItemString(dict_obj, "offsets", offsets_obj);
if (dt.get_data_size() > 0) {
pyobject_ownref itemsize_obj(PyLong_FromSize_t(dt.get_data_size()));
PyDict_SetItemString(dict_obj, "itemsize", itemsize_obj);
}
PyArray_Descr *result = NULL;
if (!PyArray_DescrConverter(dict_obj, &result)) {
stringstream ss;
ss << "failed to convert dynd type " << dt << " into numpy dtype via dict";
throw dynd::type_error(ss.str());
}
out_numpy_dtype->reset((PyObject *)result);
return;
}
default: {
break;
}
}
if (dt.get_base_id() == expr_kind_id) {
// If none of the prior checks caught this expression,
// a copy is required.
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
// Anything which fell through is an error
stringstream ss;
ss << "dynd as_numpy could not convert dynd type ";
ss << dt;
ss << " to a numpy dtype";
throw dynd::type_error(ss.str());
}
ndt::type ndt::detail::internal_substitute(const ndt::type &pattern, const std::map<std::string, ndt::type> &typevars,
bool concrete) {
// This function assumes that ``pattern`` is symbolic, so does not
// have to check types that are always concrete
switch (pattern.get_id()) {
#ifdef DYND_CUDA
case cuda_device_id:
return ndt::make_cuda_device(
ndt::substitute(pattern.extended<base_memory_type>()->get_element_type(), typevars, concrete));
#endif
case pointer_id:
return ndt::make_type<ndt::pointer_type>(
ndt::substitute(pattern.extended<pointer_type>()->get_target_type(), typevars, concrete));
case fixed_dim_kind_id:
if (!concrete) {
return ndt::make_type<ndt::fixed_dim_kind_type>(
ndt::substitute(pattern.extended<base_dim_type>()->get_element_type(), typevars, concrete));
} else {
throw invalid_argument("The dynd pattern type includes a symbolic "
"'fixed' dimension, which is not concrete as "
"requested");
}
case fixed_dim_id:
return ndt::make_fixed_dim(
pattern.extended<fixed_dim_type>()->get_fixed_dim_size(),
ndt::substitute(pattern.extended<fixed_dim_type>()->get_element_type(), typevars, concrete));
case var_dim_id:
return ndt::make_type<ndt::var_dim_type>(
ndt::substitute(pattern.extended<var_dim_type>()->get_element_type(), typevars, concrete));
case struct_id:
return ndt::make_type<ndt::struct_type>(
pattern.extended<struct_type>()->get_field_names(),
substitute_type_array(pattern.extended<struct_type>()->get_field_types(), typevars, concrete));
case tuple_id: {
const std::vector<ndt::type> &element_tp =
substitute_type_array(pattern.extended<tuple_type>()->get_field_types(), typevars, concrete);
return ndt::make_type<ndt::tuple_type>(element_tp.size(), element_tp.data());
}
case option_id:
return ndt::make_type<ndt::option_type>(
ndt::substitute(pattern.extended<option_type>()->get_value_type(), typevars, concrete));
case callable_id:
return ndt::make_type<ndt::callable_type>(
substitute(pattern.extended<callable_type>()->get_return_type(), typevars, concrete),
substitute(pattern.extended<callable_type>()->get_pos_tuple(), typevars, concrete),
substitute(pattern.extended<callable_type>()->get_kwd_struct(), typevars, concrete));
case typevar_constructed_id: {
map<std::string, ndt::type>::const_iterator it =
typevars.find(pattern.extended<typevar_constructed_type>()->get_name());
if (it->second.get_id() == void_id) {
return substitute(pattern.extended<typevar_constructed_type>()->get_arg(), typevars, concrete);
}
#ifdef DYND_CUDA
if (it->second.get_id() == cuda_device_id) {
return ndt::make_cuda_device(
substitute(pattern.extended<typevar_constructed_type>()->get_arg(), typevars, concrete));
}
#endif
}
case typevar_id: {
map<std::string, ndt::type>::const_iterator it = typevars.find(pattern.extended<typevar_type>()->get_name());
if (it != typevars.end()) {
if (it->second.get_ndim() != 0) {
stringstream ss;
ss << "The substitution for dynd typevar " << pattern << ", " << it->second
<< ", is a dimension, expected a dtype";
throw invalid_argument(ss.str());
}
if (!concrete || !it->second.is_symbolic()) {
return it->second;
} else {
stringstream ss;
ss << "The substitution for dynd typevar " << pattern << ", " << it->second << ", is not concrete as required";
throw invalid_argument(ss.str());
}
} else {
if (concrete) {
stringstream ss;
ss << "No substitution type for dynd type var " << pattern << " was available";
throw invalid_argument(ss.str());
} else {
return pattern;
}
}
}
case typevar_dim_id: {
map<std::string, ndt::type>::const_iterator it = typevars.find(pattern.extended<typevar_dim_type>()->get_name());
if (it != typevars.end()) {
if (it->second.get_ndim() == 0) {
stringstream ss;
ss << "The substitution for dynd typevar " << pattern << ", " << it->second
<< ", is a dtype, expected a dimension";
throw invalid_argument(ss.str());
}
if (!concrete || !it->second.is_symbolic()) {
switch (it->second.get_id()) {
case fixed_dim_kind_id:
return ndt::make_type<ndt::fixed_dim_kind_type>(
ndt::substitute(pattern.extended<typevar_dim_type>()->get_element_type(), typevars, concrete));
case fixed_dim_id:
return ndt::make_fixed_dim(
it->second.extended<fixed_dim_type>()->get_fixed_dim_size(),
ndt::substitute(pattern.extended<typevar_dim_type>()->get_element_type(), typevars, concrete));
case var_dim_id:
return ndt::make_type<ndt::var_dim_type>(
ndt::substitute(pattern.extended<typevar_dim_type>()->get_element_type(), typevars, concrete));
default: {
stringstream ss;
ss << "The substitution for dynd typevar " << pattern << ", " << it->second
<< ", is not a substitutable dimension type";
throw invalid_argument(ss.str());
}
}
} else {
stringstream ss;
ss << "The substitution for dynd typevar " << pattern << ", " << it->second << ", is not concrete as required";
throw invalid_argument(ss.str());
}
} else {
if (concrete) {
stringstream ss;
ss << "No substitution type for dynd typevar " << pattern << " was available";
throw invalid_argument(ss.str());
} else {
return ndt::make_type<ndt::typevar_dim_type>(
pattern.extended<typevar_dim_type>()->get_name(),
ndt::substitute(pattern.extended<typevar_dim_type>()->get_element_type(), typevars, concrete));
}
}
}
case pow_dimsym_id: {
// Look up to the exponent typevar
std::string exponent_name = pattern.extended<pow_dimsym_type>()->get_exponent();
map<std::string, ndt::type>::const_iterator tv_type = typevars.find(exponent_name);
intptr_t exponent = -1;
if (tv_type != typevars.end()) {
if (tv_type->second.get_id() == fixed_dim_id) {
exponent = tv_type->second.extended<fixed_dim_type>()->get_fixed_dim_size();
} else if (tv_type->second.get_id() == typevar_dim_id) {
// If it's a typevar, substitute the new name in
exponent_name = tv_type->second.extended<typevar_dim_type>()->get_name();
if (concrete) {
stringstream ss;
ss << "The substitution for dynd typevar " << exponent_name << ", " << tv_type->second
<< ", is not concrete as required";
throw invalid_argument(ss.str());
}
} else {
stringstream ss;
ss << "The substitution for dynd typevar " << exponent_name << ", " << tv_type->second
<< ", is not a fixed_dim integer as required";
throw invalid_argument(ss.str());
}
}
// If the exponent is zero, just substitute the rest of the type
if (exponent == 0) {
return ndt::substitute(pattern.extended<pow_dimsym_type>()->get_element_type(), typevars, concrete);
}
// Get the base type
ndt::type base_tp = pattern.extended<pow_dimsym_type>()->get_base_type();
if (base_tp.get_id() == typevar_dim_id) {
map<std::string, ndt::type>::const_iterator btv_type =
typevars.find(base_tp.extended<typevar_dim_type>()->get_name());
if (btv_type == typevars.end()) {
// We haven't seen this typevar yet, check if concrete
// is required
if (concrete) {
stringstream ss;
ss << "No substitution type for dynd typevar " << base_tp << " was available";
throw invalid_argument(ss.str());
}
} else if (btv_type->second.get_ndim() > 0 && btv_type->second.get_id() != dim_fragment_id) {
// Swap in for the base type
base_tp = btv_type->second;
} else {
stringstream ss;
ss << "The substitution for dynd typevar " << base_tp << ", " << btv_type->second
<< ", is not a substitutable dimension type";
throw invalid_argument(ss.str());
}
}
// Substitute the element type, then apply the exponent
ndt::type result = ndt::substitute(pattern.extended<pow_dimsym_type>()->get_element_type(), typevars, concrete);
if (exponent == 0) {
return result;
} else if (exponent < 0) {
return ndt::make_type<ndt::pow_dimsym_type>(base_tp, exponent_name, result);
} else {
switch (base_tp.get_id()) {
case fixed_dim_kind_id: {
if (concrete) {
stringstream ss;
ss << "The base for a dimensional power type, 'Fixed ** " << exponent << "', is not concrete as required";
throw invalid_argument(ss.str());
}
for (intptr_t i = 0; i < exponent; ++i) {
result = ndt::make_type<ndt::fixed_dim_kind_type>(result);
}
return result;
}
case fixed_dim_id: {
intptr_t dim_size = base_tp.extended<fixed_dim_type>()->get_fixed_dim_size();
for (intptr_t i = 0; i < exponent; ++i) {
result = ndt::make_fixed_dim(dim_size, result);
}
return result;
}
case var_dim_id:
for (intptr_t i = 0; i < exponent; ++i) {
result = ndt::make_type<ndt::var_dim_type>(result);
}
return result;
case typevar_dim_id: {
const std::string &tvname = base_tp.extended<typevar_dim_type>()->get_name();
for (intptr_t i = 0; i < exponent; ++i) {
result = ndt::make_type<ndt::typevar_dim_type>(tvname, result);
}
return result;
}
default: {
stringstream ss;
ss << "Cannot substitute " << base_tp << " as the base of a dynd dimensional power type";
throw invalid_argument(ss.str());
}
}
}
}
case ellipsis_dim_id: {
const std::string &name = pattern.extended<ellipsis_dim_type>()->get_name();
if (!name.empty()) {
map<std::string, ndt::type>::const_iterator it = typevars.find(pattern.extended<typevar_dim_type>()->get_name());
if (it != typevars.end()) {
if (it->second.get_id() == dim_fragment_id) {
return it->second.extended<dim_fragment_type>()->apply_to_dtype(
ndt::substitute(pattern.extended<ellipsis_dim_type>()->get_element_type(), typevars, concrete));
} else {
stringstream ss;
ss << "The substitution for dynd typevar " << pattern << ", " << it->second
<< ", is not a dim fragment as required";
throw invalid_argument(ss.str());
}
} else {
if (concrete) {
stringstream ss;
ss << "No substitution type for dynd typevar " << pattern << " was available";
throw invalid_argument(ss.str());
} else {
return ndt::make_type<ellipsis_dim_type>(
pattern.extended<ellipsis_dim_type>()->get_name(),
ndt::substitute(pattern.extended<ellipsis_dim_type>()->get_element_type(), typevars, concrete));
}
}
} else {
throw invalid_argument("Cannot substitute into an unnamed ellipsis typevar");
}
}
case any_kind_id: {
if (concrete) {
stringstream ss;
ss << "The dynd type " << pattern << " is not concrete as required";
throw invalid_argument(ss.str());
} else {
return pattern;
}
}
case scalar_kind_id: {
if (concrete) {
stringstream ss;
ss << "The dynd type " << pattern << " is not concrete as required";
throw invalid_argument(ss.str());
} else {
return pattern;
}
}
default:
break;
}
stringstream ss;
ss << "Unsupported dynd type \"" << pattern << "\" encountered for substituting typevars";
throw invalid_argument(ss.str());
}
void pydynd::deduce_pyseq_shape_using_dtype(PyObject *obj, const ndt::type &tp,
std::vector<intptr_t> &shape,
bool initial_pass,
size_t current_axis)
{
bool is_sequence = (PySequence_Check(obj) != 0 && !PyUnicode_Check(obj) &&
!PyDict_Check(obj));
#if PY_VERSION_HEX < 0x03000000
is_sequence = is_sequence && !PyString_Check(obj);
#endif
Py_ssize_t size = 0;
if (is_sequence) {
size = PySequence_Size(obj);
if (size == -1 && PyErr_Occurred()) {
PyErr_Clear();
is_sequence = false;
}
}
if (is_sequence) {
if (shape.size() == current_axis) {
if (initial_pass) {
shape.push_back(size);
}
else if (tp.get_id() == struct_id || tp.get_id() == tuple_id) {
// Signal that this is a dimension which is sometimes scalar, to allow
// for
// raggedness in the struct type's fields
shape.push_back(pydynd_shape_deduction_ragged);
}
else {
throw runtime_error(
"dynd array doesn't support dimensions"
" which are sometimes scalars and sometimes arrays");
}
}
else {
if (shape[current_axis] != size && shape[current_axis] >= 0) {
// A variable-sized dimension
shape[current_axis] = pydynd_shape_deduction_var;
}
}
for (Py_ssize_t i = 0; i < size; ++i) {
pyobject_ownref item(PySequence_GetItem(obj, i));
deduce_pyseq_shape_using_dtype(item.get(), tp, shape,
i == 0 && initial_pass, current_axis + 1);
}
}
else {
if (PyDict_Check(obj) && tp.get_id() == struct_id) {
if (shape.size() == current_axis) {
shape.push_back(pydynd_shape_deduction_dict);
}
else if (shape[current_axis] != pydynd_shape_deduction_ragged) {
shape[current_axis] = pydynd_shape_deduction_dict;
}
}
else if (shape.size() != current_axis) {
if (tp.get_id() == struct_id || tp.get_id() == tuple_id) {
shape[current_axis] = pydynd_shape_deduction_ragged;
}
else {
throw runtime_error(
"dynd array doesn't support dimensions"
" which are sometimes scalars and sometimes arrays");
}
}
}
}
