static void make_numpy_dtype_for_copy(pyobject_ownref *out_numpy_dtype, intptr_t ndim, const ndt::type &dt,
const char *arrmeta)
{
// DyND builtin types
if (dt.is_builtin()) {
out_numpy_dtype->reset((PyObject *)PyArray_DescrFromType(dynd_to_numpy_id(dt.get_id())));
return;
}
switch (dt.get_id()) {
case fixed_string_id: {
const ndt::fixed_string_type *fsd = dt.extended<ndt::fixed_string_type>();
PyArray_Descr *result;
switch (fsd->get_encoding()) {
case string_encoding_ascii:
result = PyArray_DescrNewFromType(NPY_STRING);
result->elsize = (int)fsd->get_data_size();
out_numpy_dtype->reset((PyObject *)result);
return;
case string_encoding_utf_32:
result = PyArray_DescrNewFromType(NPY_UNICODE);
result->elsize = (int)fsd->get_data_size();
out_numpy_dtype->reset((PyObject *)result);
return;
default:
// If it's not one of the encodings NumPy supports,
// use Unicode
result = PyArray_DescrNewFromType(NPY_UNICODE);
result->elsize = (int)fsd->get_data_size() * 4 / string_encoding_char_size_table[fsd->get_encoding()];
out_numpy_dtype->reset((PyObject *)result);
return;
}
break;
}
case string_id: {
// Convert variable-length strings into NumPy object arrays
PyArray_Descr *dtype = PyArray_DescrNewFromType(NPY_OBJECT);
// Add metadata to the string type being created so that
// it can round-trip. This metadata is compatible with h5py.
out_numpy_dtype->reset((PyObject *)dtype);
if (dtype->metadata == NULL) {
dtype->metadata = PyDict_New();
}
PyDict_SetItemString(dtype->metadata, "vlen", (PyObject *)&PyUnicode_Type);
return;
}
case fixed_dim_id: {
if (ndim > 0) {
const ndt::base_dim_type *bdt = dt.extended<ndt::base_dim_type>();
make_numpy_dtype_for_copy(out_numpy_dtype, ndim - 1, bdt->get_element_type(),
arrmeta + sizeof(fixed_dim_type_arrmeta));
return;
}
else {
// If this isn't one of the array dimensions, it maps into
// a numpy dtype with a shape
// Build up the shape of the array for NumPy
pyobject_ownref shape(PyList_New(0));
ndt::type element_tp = dt;
while (ndim > 0) {
const fixed_dim_type_arrmeta *am = reinterpret_cast<const fixed_dim_type_arrmeta *>(arrmeta);
intptr_t dim_size = am->dim_size;
element_tp = dt.extended<ndt::base_dim_type>()->get_element_type();
arrmeta += sizeof(fixed_dim_type_arrmeta);
--ndim;
if (PyList_Append(shape.get(), PyLong_FromSize_t(dim_size)) < 0) {
throw runtime_error("propagating python error");
}
}
// Get the numpy dtype of the element
pyobject_ownref child_numpy_dtype;
make_numpy_dtype_for_copy(&child_numpy_dtype, 0, element_tp, arrmeta);
// Create the result numpy dtype
pyobject_ownref tuple_obj(PyTuple_New(2));
PyTuple_SET_ITEM(tuple_obj.get(), 0, child_numpy_dtype.release());
PyTuple_SET_ITEM(tuple_obj.get(), 1, shape.release());
PyArray_Descr *result = NULL;
if (!PyArray_DescrConverter(tuple_obj, &result)) {
throw dynd::type_error("failed to convert dynd type into numpy subarray dtype");
}
// Put the final numpy dtype reference in the output
out_numpy_dtype->reset((PyObject *)result);
return;
}
break;
}
case struct_id: {
const ndt::struct_type *bs = dt.extended<ndt::struct_type>();
size_t field_count = bs->get_field_count();
pyobject_ownref names_obj(PyList_New(field_count));
for (size_t i = 0; i < field_count; ++i) {
const dynd::string &fn = bs->get_field_name(i);
#if PY_VERSION_HEX >= 0x03000000
pyobject_ownref name_str(PyUnicode_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#else
pyobject_ownref name_str(PyString_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#endif
PyList_SET_ITEM(names_obj.get(), i, name_str.release());
}
pyobject_ownref formats_obj(PyList_New(field_count));
pyobject_ownref offsets_obj(PyList_New(field_count));
size_t standard_offset = 0, standard_alignment = 1;
for (size_t i = 0; i < field_count; ++i) {
// Get the numpy dtype of the element
pyobject_ownref field_numpy_dtype;
make_numpy_dtype_for_copy(&field_numpy_dtype, 0, bs->get_field_type(i), arrmeta);
size_t field_alignment = ((PyArray_Descr *)field_numpy_dtype.get())->alignment;
size_t field_size = ((PyArray_Descr *)field_numpy_dtype.get())->elsize;
standard_offset = inc_to_alignment(standard_offset, field_alignment);
standard_alignment = max(standard_alignment, field_alignment);
PyList_SET_ITEM(formats_obj.get(), i, field_numpy_dtype.release());
PyList_SET_ITEM((PyObject *)offsets_obj, i, PyLong_FromSize_t(standard_offset));
standard_offset += field_size;
}
// Get the full element size
standard_offset = inc_to_alignment(standard_offset, standard_alignment);
pyobject_ownref itemsize_obj(PyLong_FromSize_t(standard_offset));
pyobject_ownref dict_obj(PyDict_New());
PyDict_SetItemString(dict_obj, "names", names_obj);
PyDict_SetItemString(dict_obj, "formats", formats_obj);
PyDict_SetItemString(dict_obj, "offsets", offsets_obj);
PyDict_SetItemString(dict_obj, "itemsize", itemsize_obj);
PyArray_Descr *result = NULL;
if (!PyArray_DescrAlignConverter(dict_obj, &result)) {
stringstream ss;
ss << "failed to convert dynd type " << dt << " into numpy dtype via dict";
throw dynd::type_error(ss.str());
}
out_numpy_dtype->reset((PyObject *)result);
return;
}
default: {
break;
}
}
if (dt.get_base_id() == expr_kind_id) {
// Convert the value type for the copy
make_numpy_dtype_for_copy(out_numpy_dtype, ndim, dt.value_type(), NULL);
return;
}
// Anything which fell through is an error
stringstream ss;
ss << "dynd as_numpy could not convert dynd type ";
ss << dt;
ss << " to a numpy dtype";
throw dynd::type_error(ss.str());
}
static void as_numpy_analysis(pyobject_ownref *out_numpy_dtype, bool *out_requires_copy, intptr_t ndim,
const ndt::type &dt, const char *arrmeta)
{
if (dt.is_builtin()) {
// DyND builtin types
out_numpy_dtype->reset((PyObject *)PyArray_DescrFromType(dynd_to_numpy_id(dt.get_id())));
return;
}
switch (dt.get_id()) {
case fixed_string_id: {
const ndt::fixed_string_type *fsd = dt.extended<ndt::fixed_string_type>();
PyArray_Descr *result;
switch (fsd->get_encoding()) {
case string_encoding_ascii:
result = PyArray_DescrNewFromType(NPY_STRING);
result->elsize = (int)fsd->get_data_size();
out_numpy_dtype->reset((PyObject *)result);
return;
case string_encoding_utf_32:
result = PyArray_DescrNewFromType(NPY_UNICODE);
result->elsize = (int)fsd->get_data_size();
out_numpy_dtype->reset((PyObject *)result);
return;
default:
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
break;
}
case string_id: {
// Convert to numpy object type, requires copy
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
case fixed_dim_id: {
const ndt::base_dim_type *bdt = dt.extended<ndt::base_dim_type>();
if (ndim > 0) {
// If this is one of the array dimensions, it simply
// becomes one of the numpy _array dimensions
as_numpy_analysis(out_numpy_dtype, out_requires_copy, ndim - 1, bdt->get_element_type(),
arrmeta + sizeof(fixed_dim_type_arrmeta));
return;
}
else {
// If this isn't one of the array dimensions, it maps into
// a numpy dtype with a shape
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
break;
}
/*
case cfixed_dim_id: {
const cfixed_dim_type *fad = dt.extended<cfixed_dim_type>();
if (ndim > 0) {
// If this is one of the array dimensions, it simply
// becomes one of the numpy _array dimensions
as_numpy_analysis(out_numpy_dtype, out_requires_copy, ndim - 1,
fad->get_element_type(),
arrmeta + sizeof(cfixed_dim_type_arrmeta));
return;
} else {
// If this isn't one of the array dimensions, it maps into
// a numpy dtype with a shape
// Build up the shape of the array for NumPy
pyobject_ownref shape(PyList_New(0));
ndt::type element_tp = dt;
while (ndim > 0) {
size_t dim_size = 0;
if (dt.get_id() == cfixed_dim_id) {
const cfixed_dim_type *cfd =
element_tp.extended<cfixed_dim_type>();
element_tp = cfd->get_element_type();
if (cfd->get_data_size() != element_tp.get_data_size() * dim_size)
{
// If it's not C-order, a copy is required
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
} else {
stringstream ss;
ss << "dynd as_numpy could not convert dynd type ";
ss << dt;
ss << " to a numpy dtype";
throw dynd::type_error(ss.str());
}
--ndim;
if (PyList_Append(shape.get(), PyLong_FromSize_t(dim_size)) < 0) {
throw runtime_error("propagating python error");
}
}
// Get the numpy dtype of the element
pyobject_ownref child_numpy_dtype;
as_numpy_analysis(&child_numpy_dtype, out_requires_copy, 0,
element_tp,
arrmeta);
if (*out_requires_copy) {
// If the child required a copy, stop right away
out_numpy_dtype->clear();
return;
}
// Create the result numpy dtype
pyobject_ownref tuple_obj(PyTuple_New(2));
PyTuple_SET_ITEM(tuple_obj.get(), 0, child_numpy_dtype.release());
PyTuple_SET_ITEM(tuple_obj.get(), 1, shape.release());
PyArray_Descr *result = NULL;
if (!PyArray_DescrConverter(tuple_obj, &result)) {
throw dynd::type_error(
"failed to convert dynd type into numpy subarray dtype");
}
// Put the final numpy dtype reference in the output
out_numpy_dtype->reset((PyObject *)result);
return;
}
break;
}
*/
case struct_id: {
if (dt.get_id() == struct_id && arrmeta == NULL) {
// If it's a struct type with no arrmeta, a copy is required
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
const ndt::struct_type *bs = dt.extended<ndt::struct_type>();
const uintptr_t *offsets = reinterpret_cast<const uintptr_t *>(arrmeta);
size_t field_count = bs->get_field_count();
pyobject_ownref names_obj(PyList_New(field_count));
for (size_t i = 0; i < field_count; ++i) {
const dynd::string &fn = bs->get_field_name(i);
#if PY_VERSION_HEX >= 0x03000000
pyobject_ownref name_str(PyUnicode_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#else
pyobject_ownref name_str(PyString_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#endif
PyList_SET_ITEM(names_obj.get(), i, name_str.release());
}
pyobject_ownref formats_obj(PyList_New(field_count));
for (size_t i = 0; i < field_count; ++i) {
// Get the numpy dtype of the element
pyobject_ownref field_numpy_dtype;
as_numpy_analysis(&field_numpy_dtype, out_requires_copy, 0, bs->get_field_type(i), arrmeta);
if (*out_requires_copy) {
// If the field required a copy, stop right away
out_numpy_dtype->clear();
return;
}
PyList_SET_ITEM(formats_obj.get(), i, field_numpy_dtype.release());
}
pyobject_ownref offsets_obj(PyList_New(field_count));
for (size_t i = 0; i < field_count; ++i) {
PyList_SET_ITEM((PyObject *)offsets_obj, i, PyLong_FromSize_t(offsets[i]));
}
pyobject_ownref dict_obj(PyDict_New());
PyDict_SetItemString(dict_obj, "names", names_obj);
PyDict_SetItemString(dict_obj, "formats", formats_obj);
PyDict_SetItemString(dict_obj, "offsets", offsets_obj);
if (dt.get_data_size() > 0) {
pyobject_ownref itemsize_obj(PyLong_FromSize_t(dt.get_data_size()));
PyDict_SetItemString(dict_obj, "itemsize", itemsize_obj);
}
PyArray_Descr *result = NULL;
if (!PyArray_DescrConverter(dict_obj, &result)) {
stringstream ss;
ss << "failed to convert dynd type " << dt << " into numpy dtype via dict";
throw dynd::type_error(ss.str());
}
out_numpy_dtype->reset((PyObject *)result);
return;
}
default: {
break;
}
}
if (dt.get_base_id() == expr_kind_id) {
// If none of the prior checks caught this expression,
// a copy is required.
out_numpy_dtype->clear();
*out_requires_copy = true;
return;
}
// Anything which fell through is an error
stringstream ss;
ss << "dynd as_numpy could not convert dynd type ";
ss << dt;
ss << " to a numpy dtype";
throw dynd::type_error(ss.str());
}
