static void make_numpy_dtype_for_copy(pyobject_ownref *out_numpy_dtype, intptr_t ndim, const ndt::type &dt, const char *arrmeta) { // DyND builtin types if (dt.is_builtin()) { out_numpy_dtype->reset((PyObject *)PyArray_DescrFromType(dynd_to_numpy_id(dt.get_id()))); return; } switch (dt.get_id()) { case fixed_string_id: { const ndt::fixed_string_type *fsd = dt.extended<ndt::fixed_string_type>(); PyArray_Descr *result; switch (fsd->get_encoding()) { case string_encoding_ascii: result = PyArray_DescrNewFromType(NPY_STRING); result->elsize = (int)fsd->get_data_size(); out_numpy_dtype->reset((PyObject *)result); return; case string_encoding_utf_32: result = PyArray_DescrNewFromType(NPY_UNICODE); result->elsize = (int)fsd->get_data_size(); out_numpy_dtype->reset((PyObject *)result); return; default: // If it's not one of the encodings NumPy supports, // use Unicode result = PyArray_DescrNewFromType(NPY_UNICODE); result->elsize = (int)fsd->get_data_size() * 4 / string_encoding_char_size_table[fsd->get_encoding()]; out_numpy_dtype->reset((PyObject *)result); return; } break; } case string_id: { // Convert variable-length strings into NumPy object arrays PyArray_Descr *dtype = PyArray_DescrNewFromType(NPY_OBJECT); // Add metadata to the string type being created so that // it can round-trip. This metadata is compatible with h5py. out_numpy_dtype->reset((PyObject *)dtype); if (dtype->metadata == NULL) { dtype->metadata = PyDict_New(); } PyDict_SetItemString(dtype->metadata, "vlen", (PyObject *)&PyUnicode_Type); return; } case fixed_dim_id: { if (ndim > 0) { const ndt::base_dim_type *bdt = dt.extended<ndt::base_dim_type>(); make_numpy_dtype_for_copy(out_numpy_dtype, ndim - 1, bdt->get_element_type(), arrmeta + sizeof(fixed_dim_type_arrmeta)); return; } else { // If this isn't one of the array dimensions, it maps into // a numpy dtype with a shape // Build up the shape of the array for NumPy pyobject_ownref shape(PyList_New(0)); ndt::type element_tp = dt; while (ndim > 0) { const fixed_dim_type_arrmeta *am = reinterpret_cast<const fixed_dim_type_arrmeta *>(arrmeta); intptr_t dim_size = am->dim_size; element_tp = dt.extended<ndt::base_dim_type>()->get_element_type(); arrmeta += sizeof(fixed_dim_type_arrmeta); --ndim; if (PyList_Append(shape.get(), PyLong_FromSize_t(dim_size)) < 0) { throw runtime_error("propagating python error"); } } // Get the numpy dtype of the element pyobject_ownref child_numpy_dtype; make_numpy_dtype_for_copy(&child_numpy_dtype, 0, element_tp, arrmeta); // Create the result numpy dtype pyobject_ownref tuple_obj(PyTuple_New(2)); PyTuple_SET_ITEM(tuple_obj.get(), 0, child_numpy_dtype.release()); PyTuple_SET_ITEM(tuple_obj.get(), 1, shape.release()); PyArray_Descr *result = NULL; if (!PyArray_DescrConverter(tuple_obj, &result)) { throw dynd::type_error("failed to convert dynd type into numpy subarray dtype"); } // Put the final numpy dtype reference in the output out_numpy_dtype->reset((PyObject *)result); return; } break; } case struct_id: { const ndt::struct_type *bs = dt.extended<ndt::struct_type>(); size_t field_count = bs->get_field_count(); pyobject_ownref names_obj(PyList_New(field_count)); for (size_t i = 0; i < field_count; ++i) { const dynd::string &fn = bs->get_field_name(i); #if PY_VERSION_HEX >= 0x03000000 pyobject_ownref name_str(PyUnicode_FromStringAndSize(fn.begin(), fn.end() - fn.begin())); #else pyobject_ownref name_str(PyString_FromStringAndSize(fn.begin(), fn.end() - fn.begin())); #endif PyList_SET_ITEM(names_obj.get(), i, name_str.release()); } pyobject_ownref formats_obj(PyList_New(field_count)); pyobject_ownref offsets_obj(PyList_New(field_count)); size_t standard_offset = 0, standard_alignment = 1; for (size_t i = 0; i < field_count; ++i) { // Get the numpy dtype of the element pyobject_ownref field_numpy_dtype; make_numpy_dtype_for_copy(&field_numpy_dtype, 0, bs->get_field_type(i), arrmeta); size_t field_alignment = ((PyArray_Descr *)field_numpy_dtype.get())->alignment; size_t field_size = ((PyArray_Descr *)field_numpy_dtype.get())->elsize; standard_offset = inc_to_alignment(standard_offset, field_alignment); standard_alignment = max(standard_alignment, field_alignment); PyList_SET_ITEM(formats_obj.get(), i, field_numpy_dtype.release()); PyList_SET_ITEM((PyObject *)offsets_obj, i, PyLong_FromSize_t(standard_offset)); standard_offset += field_size; } // Get the full element size standard_offset = inc_to_alignment(standard_offset, standard_alignment); pyobject_ownref itemsize_obj(PyLong_FromSize_t(standard_offset)); pyobject_ownref dict_obj(PyDict_New()); PyDict_SetItemString(dict_obj, "names", names_obj); PyDict_SetItemString(dict_obj, "formats", formats_obj); PyDict_SetItemString(dict_obj, "offsets", offsets_obj); PyDict_SetItemString(dict_obj, "itemsize", itemsize_obj); PyArray_Descr *result = NULL; if (!PyArray_DescrAlignConverter(dict_obj, &result)) { stringstream ss; ss << "failed to convert dynd type " << dt << " into numpy dtype via dict"; throw dynd::type_error(ss.str()); } out_numpy_dtype->reset((PyObject *)result); return; } default: { break; } } if (dt.get_base_id() == expr_kind_id) { // Convert the value type for the copy make_numpy_dtype_for_copy(out_numpy_dtype, ndim, dt.value_type(), NULL); return; } // Anything which fell through is an error stringstream ss; ss << "dynd as_numpy could not convert dynd type "; ss << dt; ss << " to a numpy dtype"; throw dynd::type_error(ss.str()); }
static void as_numpy_analysis(pyobject_ownref *out_numpy_dtype, bool *out_requires_copy, intptr_t ndim, const ndt::type &dt, const char *arrmeta) { if (dt.is_builtin()) { // DyND builtin types out_numpy_dtype->reset((PyObject *)PyArray_DescrFromType(dynd_to_numpy_id(dt.get_id()))); return; } switch (dt.get_id()) { case fixed_string_id: { const ndt::fixed_string_type *fsd = dt.extended<ndt::fixed_string_type>(); PyArray_Descr *result; switch (fsd->get_encoding()) { case string_encoding_ascii: result = PyArray_DescrNewFromType(NPY_STRING); result->elsize = (int)fsd->get_data_size(); out_numpy_dtype->reset((PyObject *)result); return; case string_encoding_utf_32: result = PyArray_DescrNewFromType(NPY_UNICODE); result->elsize = (int)fsd->get_data_size(); out_numpy_dtype->reset((PyObject *)result); return; default: out_numpy_dtype->clear(); *out_requires_copy = true; return; } break; } case string_id: { // Convert to numpy object type, requires copy out_numpy_dtype->clear(); *out_requires_copy = true; return; } case fixed_dim_id: { const ndt::base_dim_type *bdt = dt.extended<ndt::base_dim_type>(); if (ndim > 0) { // If this is one of the array dimensions, it simply // becomes one of the numpy _array dimensions as_numpy_analysis(out_numpy_dtype, out_requires_copy, ndim - 1, bdt->get_element_type(), arrmeta + sizeof(fixed_dim_type_arrmeta)); return; } else { // If this isn't one of the array dimensions, it maps into // a numpy dtype with a shape out_numpy_dtype->clear(); *out_requires_copy = true; return; } break; } /* case cfixed_dim_id: { const cfixed_dim_type *fad = dt.extended<cfixed_dim_type>(); if (ndim > 0) { // If this is one of the array dimensions, it simply // becomes one of the numpy _array dimensions as_numpy_analysis(out_numpy_dtype, out_requires_copy, ndim - 1, fad->get_element_type(), arrmeta + sizeof(cfixed_dim_type_arrmeta)); return; } else { // If this isn't one of the array dimensions, it maps into // a numpy dtype with a shape // Build up the shape of the array for NumPy pyobject_ownref shape(PyList_New(0)); ndt::type element_tp = dt; while (ndim > 0) { size_t dim_size = 0; if (dt.get_id() == cfixed_dim_id) { const cfixed_dim_type *cfd = element_tp.extended<cfixed_dim_type>(); element_tp = cfd->get_element_type(); if (cfd->get_data_size() != element_tp.get_data_size() * dim_size) { // If it's not C-order, a copy is required out_numpy_dtype->clear(); *out_requires_copy = true; return; } } else { stringstream ss; ss << "dynd as_numpy could not convert dynd type "; ss << dt; ss << " to a numpy dtype"; throw dynd::type_error(ss.str()); } --ndim; if (PyList_Append(shape.get(), PyLong_FromSize_t(dim_size)) < 0) { throw runtime_error("propagating python error"); } } // Get the numpy dtype of the element pyobject_ownref child_numpy_dtype; as_numpy_analysis(&child_numpy_dtype, out_requires_copy, 0, element_tp, arrmeta); if (*out_requires_copy) { // If the child required a copy, stop right away out_numpy_dtype->clear(); return; } // Create the result numpy dtype pyobject_ownref tuple_obj(PyTuple_New(2)); PyTuple_SET_ITEM(tuple_obj.get(), 0, child_numpy_dtype.release()); PyTuple_SET_ITEM(tuple_obj.get(), 1, shape.release()); PyArray_Descr *result = NULL; if (!PyArray_DescrConverter(tuple_obj, &result)) { throw dynd::type_error( "failed to convert dynd type into numpy subarray dtype"); } // Put the final numpy dtype reference in the output out_numpy_dtype->reset((PyObject *)result); return; } break; } */ case struct_id: { if (dt.get_id() == struct_id && arrmeta == NULL) { // If it's a struct type with no arrmeta, a copy is required out_numpy_dtype->clear(); *out_requires_copy = true; return; } const ndt::struct_type *bs = dt.extended<ndt::struct_type>(); const uintptr_t *offsets = reinterpret_cast<const uintptr_t *>(arrmeta); size_t field_count = bs->get_field_count(); pyobject_ownref names_obj(PyList_New(field_count)); for (size_t i = 0; i < field_count; ++i) { const dynd::string &fn = bs->get_field_name(i); #if PY_VERSION_HEX >= 0x03000000 pyobject_ownref name_str(PyUnicode_FromStringAndSize(fn.begin(), fn.end() - fn.begin())); #else pyobject_ownref name_str(PyString_FromStringAndSize(fn.begin(), fn.end() - fn.begin())); #endif PyList_SET_ITEM(names_obj.get(), i, name_str.release()); } pyobject_ownref formats_obj(PyList_New(field_count)); for (size_t i = 0; i < field_count; ++i) { // Get the numpy dtype of the element pyobject_ownref field_numpy_dtype; as_numpy_analysis(&field_numpy_dtype, out_requires_copy, 0, bs->get_field_type(i), arrmeta); if (*out_requires_copy) { // If the field required a copy, stop right away out_numpy_dtype->clear(); return; } PyList_SET_ITEM(formats_obj.get(), i, field_numpy_dtype.release()); } pyobject_ownref offsets_obj(PyList_New(field_count)); for (size_t i = 0; i < field_count; ++i) { PyList_SET_ITEM((PyObject *)offsets_obj, i, PyLong_FromSize_t(offsets[i])); } pyobject_ownref dict_obj(PyDict_New()); PyDict_SetItemString(dict_obj, "names", names_obj); PyDict_SetItemString(dict_obj, "formats", formats_obj); PyDict_SetItemString(dict_obj, "offsets", offsets_obj); if (dt.get_data_size() > 0) { pyobject_ownref itemsize_obj(PyLong_FromSize_t(dt.get_data_size())); PyDict_SetItemString(dict_obj, "itemsize", itemsize_obj); } PyArray_Descr *result = NULL; if (!PyArray_DescrConverter(dict_obj, &result)) { stringstream ss; ss << "failed to convert dynd type " << dt << " into numpy dtype via dict"; throw dynd::type_error(ss.str()); } out_numpy_dtype->reset((PyObject *)result); return; } default: { break; } } if (dt.get_base_id() == expr_kind_id) { // If none of the prior checks caught this expression, // a copy is required. out_numpy_dtype->clear(); *out_requires_copy = true; return; } // Anything which fell through is an error stringstream ss; ss << "dynd as_numpy could not convert dynd type "; ss << dt; ss << " to a numpy dtype"; throw dynd::type_error(ss.str()); }