static inline bool broadcast_tagged_dims_from_type(intptr_t ndim, ndt::type tp, const intptr_t *tagged_dims,
                                                   intptr_t *out_tagged_dims) {
  tp = tp.without_memory_type();
  for (intptr_t i = 0; i < ndim; ++i) {
    intptr_t tagged_dim = tagged_dims[i], dim_size;
    switch (tp.get_id()) {
    case fixed_dim_kind_id: {
      if (tagged_dim < 0) {
        out_tagged_dims[i] = -2;
      }
    } break;
    case fixed_dim_id: {
      dim_size = tp.extended<ndt::fixed_dim_type>()->get_fixed_dim_size();
      if (tagged_dim < 0 || tagged_dim == 1) {
        out_tagged_dims[i] = dim_size;
      } else if (tagged_dim != dim_size && dim_size != 1) {
        return false;
      }
    } break;
    case var_dim_id:
      // All broadcasting is done dynamically for var
      break;
    default: {
      stringstream ss;
      ss << "dim_fragment_type failed to get shape from type " << tp;
      throw type_error(ss.str());
    }
    }
    tp = tp.extended<ndt::base_dim_type>()->get_element_type();
  }
  return true;
}
static void array_getbuffer_pep3118_bytes(const ndt::type &tp, const char *arrmeta, char *data, Py_buffer *buffer,
                                          int flags)
{
  buffer->itemsize = 1;
  if (flags & PyBUF_FORMAT) {
    buffer->format = (char *)"c";
  }
  else {
    buffer->format = NULL;
  }
  buffer->ndim = 1;
#if PY_VERSION_HEX == 0x02070000
  buffer->internal = NULL;
  buffer->shape = &buffer->smalltable[0];
  buffer->strides = &buffer->smalltable[1];
#else
  buffer->internal = malloc(2 * sizeof(intptr_t));
  buffer->shape = reinterpret_cast<Py_ssize_t *>(buffer->internal);
  buffer->strides = buffer->shape + 1;
#endif
  buffer->strides[0] = 1;

  if (tp.get_id() == bytes_id) {
    // Variable-length bytes type
    buffer->buf = reinterpret_cast<bytes *>(data)->begin();
    buffer->len = reinterpret_cast<bytes *>(data)->size();
  }
  else {
    // Fixed-length bytes type
    buffer->len = tp.get_data_size();
  }
  buffer->shape[0] = buffer->len;
}
size_t pydynd::get_nonragged_dim_count(const ndt::type &tp, size_t max_count)
{
  if (tp.is_symbolic()) {
    if (tp.is_scalar()) {
      return 0;
    }
  }

  if (!tp.is_scalar()) {
    if (max_count <= 1) {
      return max_count;
    }
    else {
      return min(max_count,
                 1 + get_nonragged_dim_count(
                         static_cast<const ndt::base_dim_type *>(tp.extended())
                             ->get_element_type(),
                         max_count - 1));
    }
  }

  switch (tp.get_id()) {
  case struct_id:
  case tuple_id:
    if (max_count <= 1) {
      return max_count;
    }
    else {
      auto bsd = tp.extended<ndt::tuple_type>();
      size_t field_count = bsd->get_field_count();
      for (size_t i = 0; i != field_count; ++i) {
        size_t candidate =
            1 + get_nonragged_dim_count(bsd->get_field_type(i), max_count - 1);
        if (candidate < max_count) {
          max_count = candidate;
          if (max_count <= 1) {
            return max_count;
          }
        }
      }
      return max_count;
    }
  default:
    return 0;
  }
}
static void append_pep3118_format(intptr_t &out_itemsize, const ndt::type &tp, const char *arrmeta,
                                  std::stringstream &o)
{
  switch (tp.get_id()) {
  case bool_id:
    o << "?";
    out_itemsize = 1;
    return;
  case int8_id:
    o << "b";
    out_itemsize = 1;
    return;
  case int16_id:
    o << "h";
    out_itemsize = 2;
    return;
  case int32_id:
    o << "i";
    out_itemsize = 4;
    return;
  case int64_id:
    o << "q";
    out_itemsize = 8;
    return;
  case uint8_id:
    o << "B";
    out_itemsize = 1;
    return;
  case uint16_id:
    o << "H";
    out_itemsize = 2;
    return;
  case uint32_id:
    o << "I";
    out_itemsize = 4;
    return;
  case uint64_id:
    o << "Q";
    out_itemsize = 8;
    return;
  case float32_id:
    o << "f";
    out_itemsize = 4;
    return;
  case float64_id:
    o << "d";
    out_itemsize = 8;
    return;
  case complex_float32_id:
    o << "Zf";
    out_itemsize = 8;
    return;
  case complex_float64_id:
    o << "Zd";
    out_itemsize = 16;
    return;
  case fixed_string_id:
    switch (tp.extended<ndt::fixed_string_type>()->get_encoding()) {
    case string_encoding_ascii: {
      intptr_t element_size = tp.get_data_size();
      o << element_size << "s";
      out_itemsize = element_size;
      return;
    }
    // TODO: Couldn't find documentation for UCS-2 character code?
    case string_encoding_utf_32: {
      intptr_t element_size = tp.get_data_size();
      o << (element_size / 4) << "w";
      out_itemsize = element_size;
      return;
    }
    default:
      break;
    }
    // Pass through to error
    break;
  case fixed_dim_id: {
    ndt::type child_tp = tp;
    o << "(";
    do {
      const ndt::fixed_dim_type *tdt = child_tp.extended<ndt::fixed_dim_type>();
      intptr_t dim_size = tdt->get_fixed_dim_size();
      o << dim_size;
      if (child_tp.get_data_size() != tdt->get_element_type().get_data_size() * dim_size) {
        stringstream ss;
        ss << "Cannot convert dynd type " << tp << " into a PEP 3118 format because it is not C-order";
        throw dynd::type_error(ss.str());
      }
      o << ")";
      child_tp = tdt->get_element_type();
    } while (child_tp.get_id() == fixed_dim_id && (o << ","));
    append_pep3118_format(out_itemsize, child_tp, arrmeta, o);
    out_itemsize = tp.get_data_size();
    return;
  }
  case struct_id: {
    o << "T{";
    const ndt::struct_type *tdt = tp.extended<ndt::struct_type>();
    size_t num_fields = tdt->get_field_count();
    const uintptr_t *offsets = reinterpret_cast<const uintptr_t *>(arrmeta);
    const uintptr_t *arrmeta_offsets = tdt->get_arrmeta_offsets_raw();
    size_t format_offset = 0;
    for (size_t i = 0; i != num_fields; ++i) {
      size_t offset = offsets[i];
      // Add padding bytes
      while (offset > format_offset) {
        o << "x";
        ++format_offset;
      }
      if (offset < format_offset) {
        // DyND allows the order of fields in memory to differ from the logical
        // order, something not supported by PEP 3118
        stringstream ss;
        ss << "Cannot convert dynd type " << tp << " with out of order data layout into a PEP 3118 format string";
        throw type_error(ss.str());
      }
      // The field's type
      append_pep3118_format(out_itemsize, tdt->get_field_type(i), arrmeta ? (arrmeta + arrmeta_offsets[i]) : NULL, o);
      format_offset += out_itemsize;
      // Append the name
      o << ":" << tdt->get_field_name(i) << ":";
    }
    out_itemsize = format_offset;
    o << "}";
    return;
  }
  default:
    break;
  }
  stringstream ss;
  ss << "Cannot convert dynd type " << tp << " into a PEP 3118 format string";
  throw dynd::type_error(ss.str());
}
 void overload(const ndt::type &dst_tp, intptr_t DYND_UNUSED(nsrc), const ndt::type *DYND_UNUSED(src_tp),
               const callable &value) {
   m_dispatcher.insert({{dst_tp.get_id()}, value});
 }
      ndt::type resolve(base_callable *DYND_UNUSED(caller), char *data, call_graph &cg, const ndt::type &dst_tp,
                        size_t nsrc, const ndt::type *src_tp, size_t nkwd, const array *kwds,
                        const std::map<std::string, ndt::type> &tp_vars) {
        const ndt::callable_type *child_tp =
            reinterpret_cast<data_type *>(data)->child->get_type().template extended<ndt::callable_type>();
        bool first = reinterpret_cast<data_type *>(data)->first;
        reinterpret_cast<data_type *>(data)->first = false;
        bool state = reinterpret_cast<data_type *>(data)->state;
        bool res_ignore = reinterpret_cast<data_type *>(data)->res_ignore;

        bool dst_variadic = dst_tp.is_variadic();

        // Do a pass through the src types to classify them
        bool src_all_strided = true, src_all_strided_or_var = true;
        for (size_t i = 0; i < nsrc; ++i) {
          intptr_t src_ndim = src_tp[i].get_ndim() - child_tp->get_pos_type(i).get_ndim();
          switch (src_tp[i].get_id()) {
          case fixed_dim_id:
            break;
          case var_dim_id:
            src_all_strided = false;
            break;
          default:
            // If it's a scalar, allow it to broadcast like
            // a strided dimension
            if (src_ndim > 0) {
              src_all_strided_or_var = false;
            }
            break;
          }
        }

        bool var_broadcast = !src_all_strided;
        for (size_t i = 0; i < nsrc; ++i) {
          var_broadcast &= src_tp[i].get_id() == var_dim_id ||
                           (src_tp[i].get_id() == fixed_dim_id &&
                            src_tp[i].extended<ndt::fixed_dim_type>()->get_fixed_dim_size() == 1);
        }

        if ((dst_variadic || (dst_tp.get_id() == fixed_dim_id || res_ignore)) && src_all_strided) {
          static callable f = make_callable<elwise_callable<fixed_dim_id, fixed_dim_id, no_traits, N>>();
          static callable g = make_callable<elwise_callable<fixed_dim_id, fixed_dim_id, state_traits, N>>();
          if (!first && state) {
            return g->resolve(this, data, cg, dst_tp, nsrc, src_tp, nkwd, kwds, tp_vars);
          } else {
            return f->resolve(this, data, cg, dst_tp, nsrc, src_tp, nkwd, kwds, tp_vars);
          }
        } else if (((dst_variadic) || dst_tp.get_id() == var_dim_id) && (var_broadcast || src_all_strided)) {
          static callable f = make_callable<elwise_callable<var_dim_id, fixed_dim_id, no_traits, N>>();
          return f->resolve(this, data, cg, dst_tp, nsrc, src_tp, nkwd, kwds, tp_vars);
        } else if (src_all_strided_or_var) {
          static callable f = make_callable<elwise_callable<fixed_dim_id, var_dim_id, no_traits, N>>();
          return f->resolve(this, data, cg, dst_tp, nsrc, src_tp, nkwd, kwds, tp_vars);
        }

        std::stringstream ss;
        ss << "Cannot process lifted elwise expression from (";
        for (size_t i = 0; i < nsrc; ++i) {
          ss << src_tp[i];
          if (i != nsrc - 1) {
            ss << ", ";
          }
        }
        ss << ") to " << dst_tp;
        throw std::runtime_error(ss.str());
      }
Beispiel #7
0
static void make_numpy_dtype_for_copy(pyobject_ownref *out_numpy_dtype, intptr_t ndim, const ndt::type &dt,
                                      const char *arrmeta)
{
  // DyND builtin types
  if (dt.is_builtin()) {
    out_numpy_dtype->reset((PyObject *)PyArray_DescrFromType(dynd_to_numpy_id(dt.get_id())));
    return;
  }

  switch (dt.get_id()) {
  case fixed_string_id: {
    const ndt::fixed_string_type *fsd = dt.extended<ndt::fixed_string_type>();
    PyArray_Descr *result;
    switch (fsd->get_encoding()) {
    case string_encoding_ascii:
      result = PyArray_DescrNewFromType(NPY_STRING);
      result->elsize = (int)fsd->get_data_size();
      out_numpy_dtype->reset((PyObject *)result);
      return;
    case string_encoding_utf_32:
      result = PyArray_DescrNewFromType(NPY_UNICODE);
      result->elsize = (int)fsd->get_data_size();
      out_numpy_dtype->reset((PyObject *)result);
      return;
    default:
      // If it's not one of the encodings NumPy supports,
      // use Unicode
      result = PyArray_DescrNewFromType(NPY_UNICODE);
      result->elsize = (int)fsd->get_data_size() * 4 / string_encoding_char_size_table[fsd->get_encoding()];
      out_numpy_dtype->reset((PyObject *)result);
      return;
    }
    break;
  }
  case string_id: {
    // Convert variable-length strings into NumPy object arrays
    PyArray_Descr *dtype = PyArray_DescrNewFromType(NPY_OBJECT);
    // Add metadata to the string type being created so that
    // it can round-trip. This metadata is compatible with h5py.
    out_numpy_dtype->reset((PyObject *)dtype);
    if (dtype->metadata == NULL) {
      dtype->metadata = PyDict_New();
    }
    PyDict_SetItemString(dtype->metadata, "vlen", (PyObject *)&PyUnicode_Type);
    return;
  }
  case fixed_dim_id: {
    if (ndim > 0) {
      const ndt::base_dim_type *bdt = dt.extended<ndt::base_dim_type>();
      make_numpy_dtype_for_copy(out_numpy_dtype, ndim - 1, bdt->get_element_type(),
                                arrmeta + sizeof(fixed_dim_type_arrmeta));
      return;
    }
    else {
      // If this isn't one of the array dimensions, it maps into
      // a numpy dtype with a shape
      // Build up the shape of the array for NumPy
      pyobject_ownref shape(PyList_New(0));
      ndt::type element_tp = dt;
      while (ndim > 0) {
        const fixed_dim_type_arrmeta *am = reinterpret_cast<const fixed_dim_type_arrmeta *>(arrmeta);
        intptr_t dim_size = am->dim_size;
        element_tp = dt.extended<ndt::base_dim_type>()->get_element_type();
        arrmeta += sizeof(fixed_dim_type_arrmeta);
        --ndim;
        if (PyList_Append(shape.get(), PyLong_FromSize_t(dim_size)) < 0) {
          throw runtime_error("propagating python error");
        }
      }
      // Get the numpy dtype of the element
      pyobject_ownref child_numpy_dtype;
      make_numpy_dtype_for_copy(&child_numpy_dtype, 0, element_tp, arrmeta);
      // Create the result numpy dtype
      pyobject_ownref tuple_obj(PyTuple_New(2));
      PyTuple_SET_ITEM(tuple_obj.get(), 0, child_numpy_dtype.release());
      PyTuple_SET_ITEM(tuple_obj.get(), 1, shape.release());

      PyArray_Descr *result = NULL;
      if (!PyArray_DescrConverter(tuple_obj, &result)) {
        throw dynd::type_error("failed to convert dynd type into numpy subarray dtype");
      }
      // Put the final numpy dtype reference in the output
      out_numpy_dtype->reset((PyObject *)result);
      return;
    }
    break;
  }
  case struct_id: {
    const ndt::struct_type *bs = dt.extended<ndt::struct_type>();
    size_t field_count = bs->get_field_count();

    pyobject_ownref names_obj(PyList_New(field_count));
    for (size_t i = 0; i < field_count; ++i) {
      const dynd::string &fn = bs->get_field_name(i);
#if PY_VERSION_HEX >= 0x03000000
      pyobject_ownref name_str(PyUnicode_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#else
      pyobject_ownref name_str(PyString_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#endif
      PyList_SET_ITEM(names_obj.get(), i, name_str.release());
    }

    pyobject_ownref formats_obj(PyList_New(field_count));
    pyobject_ownref offsets_obj(PyList_New(field_count));
    size_t standard_offset = 0, standard_alignment = 1;
    for (size_t i = 0; i < field_count; ++i) {
      // Get the numpy dtype of the element
      pyobject_ownref field_numpy_dtype;
      make_numpy_dtype_for_copy(&field_numpy_dtype, 0, bs->get_field_type(i), arrmeta);
      size_t field_alignment = ((PyArray_Descr *)field_numpy_dtype.get())->alignment;
      size_t field_size = ((PyArray_Descr *)field_numpy_dtype.get())->elsize;
      standard_offset = inc_to_alignment(standard_offset, field_alignment);
      standard_alignment = max(standard_alignment, field_alignment);
      PyList_SET_ITEM(formats_obj.get(), i, field_numpy_dtype.release());
      PyList_SET_ITEM((PyObject *)offsets_obj, i, PyLong_FromSize_t(standard_offset));
      standard_offset += field_size;
    }
    // Get the full element size
    standard_offset = inc_to_alignment(standard_offset, standard_alignment);
    pyobject_ownref itemsize_obj(PyLong_FromSize_t(standard_offset));

    pyobject_ownref dict_obj(PyDict_New());
    PyDict_SetItemString(dict_obj, "names", names_obj);
    PyDict_SetItemString(dict_obj, "formats", formats_obj);
    PyDict_SetItemString(dict_obj, "offsets", offsets_obj);
    PyDict_SetItemString(dict_obj, "itemsize", itemsize_obj);

    PyArray_Descr *result = NULL;
    if (!PyArray_DescrAlignConverter(dict_obj, &result)) {
      stringstream ss;
      ss << "failed to convert dynd type " << dt << " into numpy dtype via dict";
      throw dynd::type_error(ss.str());
    }
    out_numpy_dtype->reset((PyObject *)result);
    return;
  }
  default: {
    break;
  }
  }

  if (dt.get_base_id() == expr_kind_id) {
    // Convert the value type for the copy
    make_numpy_dtype_for_copy(out_numpy_dtype, ndim, dt.value_type(), NULL);
    return;
  }

  // Anything which fell through is an error
  stringstream ss;
  ss << "dynd as_numpy could not convert dynd type ";
  ss << dt;
  ss << " to a numpy dtype";
  throw dynd::type_error(ss.str());
}
Beispiel #8
0
static void as_numpy_analysis(pyobject_ownref *out_numpy_dtype, bool *out_requires_copy, intptr_t ndim,
                              const ndt::type &dt, const char *arrmeta)
{
  if (dt.is_builtin()) {
    // DyND builtin types
    out_numpy_dtype->reset((PyObject *)PyArray_DescrFromType(dynd_to_numpy_id(dt.get_id())));
    return;
  }
  switch (dt.get_id()) {
  case fixed_string_id: {
    const ndt::fixed_string_type *fsd = dt.extended<ndt::fixed_string_type>();
    PyArray_Descr *result;
    switch (fsd->get_encoding()) {
    case string_encoding_ascii:
      result = PyArray_DescrNewFromType(NPY_STRING);
      result->elsize = (int)fsd->get_data_size();
      out_numpy_dtype->reset((PyObject *)result);
      return;
    case string_encoding_utf_32:
      result = PyArray_DescrNewFromType(NPY_UNICODE);
      result->elsize = (int)fsd->get_data_size();
      out_numpy_dtype->reset((PyObject *)result);
      return;
    default:
      out_numpy_dtype->clear();
      *out_requires_copy = true;
      return;
    }
    break;
  }
  case string_id: {
    // Convert to numpy object type, requires copy
    out_numpy_dtype->clear();
    *out_requires_copy = true;
    return;
  }
  case fixed_dim_id: {
    const ndt::base_dim_type *bdt = dt.extended<ndt::base_dim_type>();
    if (ndim > 0) {
      // If this is one of the array dimensions, it simply
      // becomes one of the numpy _array dimensions
      as_numpy_analysis(out_numpy_dtype, out_requires_copy, ndim - 1, bdt->get_element_type(),
                        arrmeta + sizeof(fixed_dim_type_arrmeta));
      return;
    }
    else {
      // If this isn't one of the array dimensions, it maps into
      // a numpy dtype with a shape
      out_numpy_dtype->clear();
      *out_requires_copy = true;
      return;
    }
    break;
  }
  /*
    case cfixed_dim_id: {
      const cfixed_dim_type *fad = dt.extended<cfixed_dim_type>();
      if (ndim > 0) {
        // If this is one of the array dimensions, it simply
        // becomes one of the numpy _array dimensions
        as_numpy_analysis(out_numpy_dtype, out_requires_copy, ndim - 1,
                          fad->get_element_type(),
                          arrmeta + sizeof(cfixed_dim_type_arrmeta));
        return;
      } else {
        // If this isn't one of the array dimensions, it maps into
        // a numpy dtype with a shape
        // Build up the shape of the array for NumPy
        pyobject_ownref shape(PyList_New(0));
        ndt::type element_tp = dt;
        while (ndim > 0) {
          size_t dim_size = 0;
          if (dt.get_id() == cfixed_dim_id) {
            const cfixed_dim_type *cfd =
    element_tp.extended<cfixed_dim_type>();
            element_tp = cfd->get_element_type();
            if (cfd->get_data_size() != element_tp.get_data_size() * dim_size)
    {
              // If it's not C-order, a copy is required
              out_numpy_dtype->clear();
              *out_requires_copy = true;
              return;
            }
          } else {
            stringstream ss;
            ss << "dynd as_numpy could not convert dynd type ";
            ss << dt;
            ss << " to a numpy dtype";
            throw dynd::type_error(ss.str());
          }
          --ndim;
          if (PyList_Append(shape.get(), PyLong_FromSize_t(dim_size)) < 0) {
            throw runtime_error("propagating python error");
          }
        }
        // Get the numpy dtype of the element
        pyobject_ownref child_numpy_dtype;
        as_numpy_analysis(&child_numpy_dtype, out_requires_copy, 0,
    element_tp,
                          arrmeta);
        if (*out_requires_copy) {
          // If the child required a copy, stop right away
          out_numpy_dtype->clear();
          return;
        }
        // Create the result numpy dtype
        pyobject_ownref tuple_obj(PyTuple_New(2));
        PyTuple_SET_ITEM(tuple_obj.get(), 0, child_numpy_dtype.release());
        PyTuple_SET_ITEM(tuple_obj.get(), 1, shape.release());

        PyArray_Descr *result = NULL;
        if (!PyArray_DescrConverter(tuple_obj, &result)) {
          throw dynd::type_error(
              "failed to convert dynd type into numpy subarray dtype");
        }
        // Put the final numpy dtype reference in the output
        out_numpy_dtype->reset((PyObject *)result);
        return;
      }
      break;
    }
  */
  case struct_id: {
    if (dt.get_id() == struct_id && arrmeta == NULL) {
      // If it's a struct type with no arrmeta, a copy is required
      out_numpy_dtype->clear();
      *out_requires_copy = true;
      return;
    }
    const ndt::struct_type *bs = dt.extended<ndt::struct_type>();
    const uintptr_t *offsets = reinterpret_cast<const uintptr_t *>(arrmeta);
    size_t field_count = bs->get_field_count();

    pyobject_ownref names_obj(PyList_New(field_count));
    for (size_t i = 0; i < field_count; ++i) {
      const dynd::string &fn = bs->get_field_name(i);
#if PY_VERSION_HEX >= 0x03000000
      pyobject_ownref name_str(PyUnicode_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#else
      pyobject_ownref name_str(PyString_FromStringAndSize(fn.begin(), fn.end() - fn.begin()));
#endif
      PyList_SET_ITEM(names_obj.get(), i, name_str.release());
    }

    pyobject_ownref formats_obj(PyList_New(field_count));
    for (size_t i = 0; i < field_count; ++i) {
      // Get the numpy dtype of the element
      pyobject_ownref field_numpy_dtype;
      as_numpy_analysis(&field_numpy_dtype, out_requires_copy, 0, bs->get_field_type(i), arrmeta);
      if (*out_requires_copy) {
        // If the field required a copy, stop right away
        out_numpy_dtype->clear();
        return;
      }
      PyList_SET_ITEM(formats_obj.get(), i, field_numpy_dtype.release());
    }

    pyobject_ownref offsets_obj(PyList_New(field_count));
    for (size_t i = 0; i < field_count; ++i) {
      PyList_SET_ITEM((PyObject *)offsets_obj, i, PyLong_FromSize_t(offsets[i]));
    }

    pyobject_ownref dict_obj(PyDict_New());
    PyDict_SetItemString(dict_obj, "names", names_obj);
    PyDict_SetItemString(dict_obj, "formats", formats_obj);
    PyDict_SetItemString(dict_obj, "offsets", offsets_obj);
    if (dt.get_data_size() > 0) {
      pyobject_ownref itemsize_obj(PyLong_FromSize_t(dt.get_data_size()));
      PyDict_SetItemString(dict_obj, "itemsize", itemsize_obj);
    }

    PyArray_Descr *result = NULL;
    if (!PyArray_DescrConverter(dict_obj, &result)) {
      stringstream ss;
      ss << "failed to convert dynd type " << dt << " into numpy dtype via dict";
      throw dynd::type_error(ss.str());
    }
    out_numpy_dtype->reset((PyObject *)result);
    return;
  }
  default: {
    break;
  }
  }

  if (dt.get_base_id() == expr_kind_id) {
    // If none of the prior checks caught this expression,
    // a copy is required.
    out_numpy_dtype->clear();
    *out_requires_copy = true;
    return;
  }

  // Anything which fell through is an error
  stringstream ss;
  ss << "dynd as_numpy could not convert dynd type ";
  ss << dt;
  ss << " to a numpy dtype";
  throw dynd::type_error(ss.str());
}
Beispiel #9
0
ndt::type ndt::detail::internal_substitute(const ndt::type &pattern, const std::map<std::string, ndt::type> &typevars,
                                           bool concrete) {
  // This function assumes that ``pattern`` is symbolic, so does not
  // have to check types that are always concrete
  switch (pattern.get_id()) {
#ifdef DYND_CUDA
  case cuda_device_id:
    return ndt::make_cuda_device(
        ndt::substitute(pattern.extended<base_memory_type>()->get_element_type(), typevars, concrete));
#endif
  case pointer_id:
    return ndt::make_type<ndt::pointer_type>(
        ndt::substitute(pattern.extended<pointer_type>()->get_target_type(), typevars, concrete));
  case fixed_dim_kind_id:
    if (!concrete) {
      return ndt::make_type<ndt::fixed_dim_kind_type>(
          ndt::substitute(pattern.extended<base_dim_type>()->get_element_type(), typevars, concrete));
    } else {
      throw invalid_argument("The dynd pattern type includes a symbolic "
                             "'fixed' dimension, which is not concrete as "
                             "requested");
    }
  case fixed_dim_id:
    return ndt::make_fixed_dim(
        pattern.extended<fixed_dim_type>()->get_fixed_dim_size(),
        ndt::substitute(pattern.extended<fixed_dim_type>()->get_element_type(), typevars, concrete));
  case var_dim_id:
    return ndt::make_type<ndt::var_dim_type>(
        ndt::substitute(pattern.extended<var_dim_type>()->get_element_type(), typevars, concrete));
  case struct_id:
    return ndt::make_type<ndt::struct_type>(
        pattern.extended<struct_type>()->get_field_names(),
        substitute_type_array(pattern.extended<struct_type>()->get_field_types(), typevars, concrete));
  case tuple_id: {
    const std::vector<ndt::type> &element_tp =
        substitute_type_array(pattern.extended<tuple_type>()->get_field_types(), typevars, concrete);
    return ndt::make_type<ndt::tuple_type>(element_tp.size(), element_tp.data());
  }
  case option_id:
    return ndt::make_type<ndt::option_type>(
        ndt::substitute(pattern.extended<option_type>()->get_value_type(), typevars, concrete));
  case callable_id:
    return ndt::make_type<ndt::callable_type>(
        substitute(pattern.extended<callable_type>()->get_return_type(), typevars, concrete),
        substitute(pattern.extended<callable_type>()->get_pos_tuple(), typevars, concrete),
        substitute(pattern.extended<callable_type>()->get_kwd_struct(), typevars, concrete));
  case typevar_constructed_id: {
    map<std::string, ndt::type>::const_iterator it =
        typevars.find(pattern.extended<typevar_constructed_type>()->get_name());
    if (it->second.get_id() == void_id) {
      return substitute(pattern.extended<typevar_constructed_type>()->get_arg(), typevars, concrete);
    }
#ifdef DYND_CUDA
    if (it->second.get_id() == cuda_device_id) {
      return ndt::make_cuda_device(
          substitute(pattern.extended<typevar_constructed_type>()->get_arg(), typevars, concrete));
    }
#endif
  }
  case typevar_id: {
    map<std::string, ndt::type>::const_iterator it = typevars.find(pattern.extended<typevar_type>()->get_name());
    if (it != typevars.end()) {
      if (it->second.get_ndim() != 0) {
        stringstream ss;
        ss << "The substitution for dynd typevar " << pattern << ", " << it->second
           << ", is a dimension, expected a dtype";
        throw invalid_argument(ss.str());
      }
      if (!concrete || !it->second.is_symbolic()) {
        return it->second;
      } else {
        stringstream ss;
        ss << "The substitution for dynd typevar " << pattern << ", " << it->second << ", is not concrete as required";
        throw invalid_argument(ss.str());
      }
    } else {
      if (concrete) {
        stringstream ss;
        ss << "No substitution type for dynd type var " << pattern << " was available";
        throw invalid_argument(ss.str());
      } else {
        return pattern;
      }
    }
  }
  case typevar_dim_id: {
    map<std::string, ndt::type>::const_iterator it = typevars.find(pattern.extended<typevar_dim_type>()->get_name());
    if (it != typevars.end()) {
      if (it->second.get_ndim() == 0) {
        stringstream ss;
        ss << "The substitution for dynd typevar " << pattern << ", " << it->second
           << ", is a dtype, expected a dimension";
        throw invalid_argument(ss.str());
      }
      if (!concrete || !it->second.is_symbolic()) {
        switch (it->second.get_id()) {
        case fixed_dim_kind_id:
          return ndt::make_type<ndt::fixed_dim_kind_type>(
              ndt::substitute(pattern.extended<typevar_dim_type>()->get_element_type(), typevars, concrete));
        case fixed_dim_id:
          return ndt::make_fixed_dim(
              it->second.extended<fixed_dim_type>()->get_fixed_dim_size(),
              ndt::substitute(pattern.extended<typevar_dim_type>()->get_element_type(), typevars, concrete));
        case var_dim_id:
          return ndt::make_type<ndt::var_dim_type>(
              ndt::substitute(pattern.extended<typevar_dim_type>()->get_element_type(), typevars, concrete));
        default: {
          stringstream ss;
          ss << "The substitution for dynd typevar " << pattern << ", " << it->second
             << ", is not a substitutable dimension type";
          throw invalid_argument(ss.str());
        }
        }
      } else {
        stringstream ss;
        ss << "The substitution for dynd typevar " << pattern << ", " << it->second << ", is not concrete as required";
        throw invalid_argument(ss.str());
      }
    } else {
      if (concrete) {
        stringstream ss;
        ss << "No substitution type for dynd typevar " << pattern << " was available";
        throw invalid_argument(ss.str());
      } else {
        return ndt::make_type<ndt::typevar_dim_type>(
            pattern.extended<typevar_dim_type>()->get_name(),
            ndt::substitute(pattern.extended<typevar_dim_type>()->get_element_type(), typevars, concrete));
      }
    }
  }
  case pow_dimsym_id: {
    // Look up to the exponent typevar
    std::string exponent_name = pattern.extended<pow_dimsym_type>()->get_exponent();
    map<std::string, ndt::type>::const_iterator tv_type = typevars.find(exponent_name);
    intptr_t exponent = -1;
    if (tv_type != typevars.end()) {
      if (tv_type->second.get_id() == fixed_dim_id) {
        exponent = tv_type->second.extended<fixed_dim_type>()->get_fixed_dim_size();
      } else if (tv_type->second.get_id() == typevar_dim_id) {
        // If it's a typevar, substitute the new name in
        exponent_name = tv_type->second.extended<typevar_dim_type>()->get_name();
        if (concrete) {
          stringstream ss;
          ss << "The substitution for dynd typevar " << exponent_name << ", " << tv_type->second
             << ", is not concrete as required";
          throw invalid_argument(ss.str());
        }
      } else {
        stringstream ss;
        ss << "The substitution for dynd typevar " << exponent_name << ", " << tv_type->second
           << ", is not a fixed_dim integer as required";
        throw invalid_argument(ss.str());
      }
    }
    // If the exponent is zero, just substitute the rest of the type
    if (exponent == 0) {
      return ndt::substitute(pattern.extended<pow_dimsym_type>()->get_element_type(), typevars, concrete);
    }
    // Get the base type
    ndt::type base_tp = pattern.extended<pow_dimsym_type>()->get_base_type();
    if (base_tp.get_id() == typevar_dim_id) {
      map<std::string, ndt::type>::const_iterator btv_type =
          typevars.find(base_tp.extended<typevar_dim_type>()->get_name());
      if (btv_type == typevars.end()) {
        // We haven't seen this typevar yet, check if concrete
        // is required
        if (concrete) {
          stringstream ss;
          ss << "No substitution type for dynd typevar " << base_tp << " was available";
          throw invalid_argument(ss.str());
        }
      } else if (btv_type->second.get_ndim() > 0 && btv_type->second.get_id() != dim_fragment_id) {
        // Swap in for the base type
        base_tp = btv_type->second;
      } else {
        stringstream ss;
        ss << "The substitution for dynd typevar " << base_tp << ", " << btv_type->second
           << ", is not a substitutable dimension type";
        throw invalid_argument(ss.str());
      }
    }
    // Substitute the element type, then apply the exponent
    ndt::type result = ndt::substitute(pattern.extended<pow_dimsym_type>()->get_element_type(), typevars, concrete);
    if (exponent == 0) {
      return result;
    } else if (exponent < 0) {
      return ndt::make_type<ndt::pow_dimsym_type>(base_tp, exponent_name, result);
    } else {
      switch (base_tp.get_id()) {
      case fixed_dim_kind_id: {
        if (concrete) {
          stringstream ss;
          ss << "The base for a dimensional power type, 'Fixed ** " << exponent << "', is not concrete as required";
          throw invalid_argument(ss.str());
        }
        for (intptr_t i = 0; i < exponent; ++i) {
          result = ndt::make_type<ndt::fixed_dim_kind_type>(result);
        }
        return result;
      }
      case fixed_dim_id: {
        intptr_t dim_size = base_tp.extended<fixed_dim_type>()->get_fixed_dim_size();
        for (intptr_t i = 0; i < exponent; ++i) {
          result = ndt::make_fixed_dim(dim_size, result);
        }
        return result;
      }
      case var_dim_id:
        for (intptr_t i = 0; i < exponent; ++i) {
          result = ndt::make_type<ndt::var_dim_type>(result);
        }
        return result;
      case typevar_dim_id: {
        const std::string &tvname = base_tp.extended<typevar_dim_type>()->get_name();
        for (intptr_t i = 0; i < exponent; ++i) {
          result = ndt::make_type<ndt::typevar_dim_type>(tvname, result);
        }
        return result;
      }
      default: {
        stringstream ss;
        ss << "Cannot substitute " << base_tp << " as the base of a dynd dimensional power type";
        throw invalid_argument(ss.str());
      }
      }
    }
  }
  case ellipsis_dim_id: {
    const std::string &name = pattern.extended<ellipsis_dim_type>()->get_name();
    if (!name.empty()) {
      map<std::string, ndt::type>::const_iterator it = typevars.find(pattern.extended<typevar_dim_type>()->get_name());
      if (it != typevars.end()) {
        if (it->second.get_id() == dim_fragment_id) {
          return it->second.extended<dim_fragment_type>()->apply_to_dtype(
              ndt::substitute(pattern.extended<ellipsis_dim_type>()->get_element_type(), typevars, concrete));
        } else {
          stringstream ss;
          ss << "The substitution for dynd typevar " << pattern << ", " << it->second
             << ", is not a dim fragment as required";
          throw invalid_argument(ss.str());
        }
      } else {
        if (concrete) {
          stringstream ss;
          ss << "No substitution type for dynd typevar " << pattern << " was available";
          throw invalid_argument(ss.str());
        } else {
          return ndt::make_type<ellipsis_dim_type>(
              pattern.extended<ellipsis_dim_type>()->get_name(),
              ndt::substitute(pattern.extended<ellipsis_dim_type>()->get_element_type(), typevars, concrete));
        }
      }
    } else {
      throw invalid_argument("Cannot substitute into an unnamed ellipsis typevar");
    }
  }
  case any_kind_id: {
    if (concrete) {
      stringstream ss;
      ss << "The dynd type " << pattern << " is not concrete as required";
      throw invalid_argument(ss.str());
    } else {
      return pattern;
    }
  }
  case scalar_kind_id: {
    if (concrete) {
      stringstream ss;
      ss << "The dynd type " << pattern << " is not concrete as required";
      throw invalid_argument(ss.str());
    } else {
      return pattern;
    }
  }
  default:
    break;
  }

  stringstream ss;
  ss << "Unsupported dynd type \"" << pattern << "\" encountered for substituting typevars";
  throw invalid_argument(ss.str());
}
void pydynd::deduce_pyseq_shape_using_dtype(PyObject *obj, const ndt::type &tp,
                                            std::vector<intptr_t> &shape,
                                            bool initial_pass,
                                            size_t current_axis)
{
  bool is_sequence = (PySequence_Check(obj) != 0 && !PyUnicode_Check(obj) &&
                      !PyDict_Check(obj));
#if PY_VERSION_HEX < 0x03000000
  is_sequence = is_sequence && !PyString_Check(obj);
#endif
  Py_ssize_t size = 0;
  if (is_sequence) {
    size = PySequence_Size(obj);
    if (size == -1 && PyErr_Occurred()) {
      PyErr_Clear();
      is_sequence = false;
    }
  }

  if (is_sequence) {
    if (shape.size() == current_axis) {
      if (initial_pass) {
        shape.push_back(size);
      }
      else if (tp.get_id() == struct_id || tp.get_id() == tuple_id) {
        // Signal that this is a dimension which is sometimes scalar, to allow
        // for
        // raggedness in the struct type's fields
        shape.push_back(pydynd_shape_deduction_ragged);
      }
      else {
        throw runtime_error(
            "dynd array doesn't support dimensions"
            " which are sometimes scalars and sometimes arrays");
      }
    }
    else {
      if (shape[current_axis] != size && shape[current_axis] >= 0) {
        // A variable-sized dimension
        shape[current_axis] = pydynd_shape_deduction_var;
      }
    }

    for (Py_ssize_t i = 0; i < size; ++i) {
      pyobject_ownref item(PySequence_GetItem(obj, i));
      deduce_pyseq_shape_using_dtype(item.get(), tp, shape,
                                     i == 0 && initial_pass, current_axis + 1);
    }
  }
  else {
    if (PyDict_Check(obj) && tp.get_id() == struct_id) {
      if (shape.size() == current_axis) {
        shape.push_back(pydynd_shape_deduction_dict);
      }
      else if (shape[current_axis] != pydynd_shape_deduction_ragged) {
        shape[current_axis] = pydynd_shape_deduction_dict;
      }
    }
    else if (shape.size() != current_axis) {
      if (tp.get_id() == struct_id || tp.get_id() == tuple_id) {
        shape[current_axis] = pydynd_shape_deduction_ragged;
      }
      else {
        throw runtime_error(
            "dynd array doesn't support dimensions"
            " which are sometimes scalars and sometimes arrays");
      }
    }
  }
}