groupby_type::groupby_type(const ndt::type& data_values_tp,
const ndt::type& by_values_tp)
: base_expr_type(groupby_type_id, expr_kind,
sizeof(groupby_type_data), sizeof(void *), type_flag_none,
0, 1 + data_values_tp.get_ndim())
{
m_groups_type = by_values_tp.at_single(0).value_type();
if (m_groups_type.get_type_id() != categorical_type_id) {
stringstream ss;
ss << "to construct a groupby type, the by type, " << by_values_tp.at_single(0);
ss << ", must have a categorical value type";
throw runtime_error(ss.str());
}
if (data_values_tp.get_ndim() < 1) {
throw runtime_error("to construct a groupby type, its values type must have at least one array dimension");
}
if (by_values_tp.get_ndim() < 1) {
throw runtime_error("to construct a groupby type, its values type must have at least one array dimension");
}
m_operand_type = ndt::make_cstruct(ndt::make_pointer(data_values_tp), "data",
ndt::make_pointer(by_values_tp), "by");
m_members.arrmeta_size = m_operand_type.get_arrmeta_size();
const categorical_type *cd = m_groups_type.extended<categorical_type>();
m_value_type = ndt::make_cfixed_dim(cd->get_category_count(),
ndt::make_var_dim(data_values_tp.at_single(0)));
m_members.flags = inherited_flags(m_value_type.get_flags(), m_operand_type.get_flags());
}
size_t cfixed_dim_type::make_assignment_kernel(
ckernel_builder *ckb, intptr_t ckb_offset, const ndt::type &dst_tp,
const char *dst_arrmeta, const ndt::type &src_tp, const char *src_arrmeta,
kernel_request_t kernreq, const eval::eval_context *ectx) const
{
if (this == dst_tp.extended()) {
intptr_t src_size, src_stride;
ndt::type src_el_tp;
const char *src_el_arrmeta;
if (src_tp.get_ndim() < dst_tp.get_ndim()) {
kernels::strided_assign_ck *self =
kernels::strided_assign_ck::create(ckb, kernreq, ckb_offset);
self->m_size = get_fixed_dim_size();
self->m_dst_stride = get_fixed_stride();
// If the src has fewer dimensions, broadcast it across this one
self->m_src_stride = 0;
return ::make_assignment_kernel(
ckb, ckb_offset, m_element_tp,
dst_arrmeta + sizeof(cfixed_dim_type_arrmeta), src_tp, src_arrmeta,
kernel_request_strided, ectx);
} else if (src_tp.get_as_strided(src_arrmeta, &src_size, &src_stride,
&src_el_tp, &src_el_arrmeta)) {
kernels::strided_assign_ck *self =
kernels::strided_assign_ck::create(ckb, kernreq, ckb_offset);
self->m_size = get_fixed_dim_size();
self->m_dst_stride = get_fixed_stride();
self->m_src_stride = src_stride;
// Check for a broadcasting error
if (src_size != 1 && get_fixed_dim_size() != src_size) {
throw broadcast_error(dst_tp, dst_arrmeta, src_tp, src_arrmeta);
}
return ::make_assignment_kernel(
ckb, ckb_offset, m_element_tp,
dst_arrmeta + sizeof(cfixed_dim_type_arrmeta), src_el_tp,
src_el_arrmeta, kernel_request_strided, ectx);
} else if (!src_tp.is_builtin()) {
// Give the src type a chance to make a kernel
return src_tp.extended()->make_assignment_kernel(
ckb, ckb_offset, dst_tp, dst_arrmeta, src_tp, src_arrmeta,
kernreq, ectx);
} else {
stringstream ss;
ss << "Cannot assign from " << src_tp << " to " << dst_tp;
throw dynd::type_error(ss.str());
}
} else if (dst_tp.get_kind() == string_kind) {
return make_any_to_string_assignment_kernel(ckb, ckb_offset, dst_tp,
dst_arrmeta, src_tp,
src_arrmeta, kernreq, ectx);
} else if (dst_tp.get_ndim() < src_tp.get_ndim()) {
throw broadcast_error(dst_tp, dst_arrmeta, src_tp, src_arrmeta);
} else {
stringstream ss;
ss << "Cannot assign from " << src_tp << " to " << dst_tp;
throw dynd::type_error(ss.str());
}
}
static intptr_t instantiate_lifted_expr_arrfunc_data(
const arrfunc_type_data *self, const arrfunc_type *DYND_UNUSED(af_tp),
void *ckb, intptr_t ckb_offset, const ndt::type &dst_tp,
const char *dst_arrmeta, const ndt::type *src_tp,
const char *const *src_arrmeta, kernel_request_t kernreq,
const eval::eval_context *ectx, const nd::array &DYND_UNUSED(args),
const nd::array &DYND_UNUSED(kwds))
{
const array_preamble *data = *self->get_data_as<const array_preamble *>();
const arrfunc_type_data *child_af =
reinterpret_cast<const arrfunc_type_data *>(data->m_data_pointer);
const arrfunc_type *child_af_tp =
reinterpret_cast<const arrfunc_type *>(data->m_type);
intptr_t src_count = child_af_tp->get_npos();
dimvector src_ndim(src_count);
for (int i = 0; i < src_count; ++i) {
src_ndim[i] =
src_tp[i].get_ndim() - child_af_tp->get_arg_type(i).get_ndim();
}
return make_lifted_expr_ckernel(
child_af, child_af_tp, ckb, ckb_offset,
dst_tp.get_ndim() - child_af_tp->get_return_type().get_ndim(), dst_tp,
dst_arrmeta, src_ndim.get(), src_tp, src_arrmeta,
static_cast<dynd::kernel_request_t>(kernreq), ectx);
}
nd::array nd::view(const nd::array &arr, const ndt::type &tp)
{
if (arr.get_type() == tp) {
// If the types match exactly, simply return 'arr'
return arr;
} else if (tp.get_type_id() == bytes_type_id) {
// If it's a request to view the data as raw bytes
nd::array result = view_as_bytes(arr, tp);
if (!result.is_null()) {
return result;
}
} else if (arr.get_type().get_type_id() == bytes_type_id) {
// If it's a request to view raw bytes as something else
nd::array result = view_from_bytes(arr, tp);
if (!result.is_null()) {
return result;
}
} else if (arr.get_ndim() == tp.get_ndim()) {
// If the type is symbolic, e.g. has a "Fixed" symbolic dimension,
// first substitute in the shape from the array
if (tp.is_symbolic()) {
dimvector shape(arr.get_ndim());
arr.get_shape(shape.get());
return view_concrete(arr, substitute_shape(tp, arr.get_ndim(), shape.get()));
} else {
return view_concrete(arr, tp);
}
}
stringstream ss;
ss << "Unable to view nd::array of type " << arr.get_type();
ss << " as type " << tp;
throw type_error(ss.str());
}
nd::array nd::view(const nd::array& arr, const ndt::type& tp)
{
// If the types match exactly, simply return 'arr'
if (arr.get_type() == tp) {
return arr;
} else if (arr.get_ndim() == tp.get_ndim()) {
// Allocate a result array to attempt the view in it
array result(make_array_memory_block(tp.get_metadata_size()));
// Copy the fields
result.get_ndo()->m_data_pointer = arr.get_ndo()->m_data_pointer;
if (arr.get_ndo()->m_data_reference == NULL) {
// Embedded data, need reference to the array
result.get_ndo()->m_data_reference = arr.get_memblock().release();
} else {
// Use the same data reference, avoid producing a chain
result.get_ndo()->m_data_reference = arr.get_data_memblock().release();
}
result.get_ndo()->m_type = ndt::type(tp).release();
result.get_ndo()->m_flags = arr.get_ndo()->m_flags;
// Now try to copy the metadata as a view
if (try_view(arr.get_type(), arr.get_ndo_meta(), tp,
result.get_ndo_meta(), arr.get_memblock().get())) {
// If it succeeded, return it
return result;
}
// Otherwise fall through, let it get destructed, and raise an error
}
stringstream ss;
ss << "Unable to view nd::array of type " << arr.get_type();
ss << "as type " << tp;
throw type_error(ss.str());
}
expr_type::expr_type(const ndt::type& value_type, const ndt::type& operand_type,
const expr_kernel_generator *kgen)
: base_expression_type(expr_type_id, expression_kind,
operand_type.get_data_size(), operand_type.get_data_alignment(),
inherited_flags(value_type.get_flags(), operand_type.get_flags()),
operand_type.get_metadata_size(), value_type.get_ndim()),
m_value_type(value_type), m_operand_type(operand_type),
m_kgen(kgen)
{
if (operand_type.get_type_id() != cstruct_type_id) {
stringstream ss;
ss << "expr_type can only be constructed with a cstruct as its operand, given ";
ss << operand_type;
throw runtime_error(ss.str());
}
const cstruct_type *fsd = static_cast<const cstruct_type *>(operand_type.extended());
size_t field_count = fsd->get_field_count();
if (field_count == 1) {
throw runtime_error("expr_type is for 2 or more operands, use unary_expr_type for 1 operand");
}
const ndt::type *field_types = fsd->get_field_types();
for (size_t i = 0; i != field_count; ++i) {
if (field_types[i].get_type_id() != pointer_type_id) {
stringstream ss;
ss << "each field of the expr_type's operand must be a pointer, field " << i;
ss << " is " << field_types[i];
throw runtime_error(ss.str());
}
}
}
ndt::adapt_type::adapt_type(const ndt::type &value_tp, const ndt::type &storage_tp, const nd::callable &forward,
const nd::callable &inverse)
: base_expr_type(adapt_id, storage_tp.get_data_size(), storage_tp.get_data_alignment(), type_flag_none,
storage_tp.get_arrmeta_size(), storage_tp.get_ndim()),
m_value_tp(value_tp), m_storage_tp(storage_tp), m_forward(forward), m_inverse(inverse)
{
}
inline void init(const ndt::type& tp0, const char *metadata0, char *data0)
{
m_array_tp = tp0;
m_iter_ndim = m_array_tp.get_ndim();
m_itersize = 1;
if (m_iter_ndim != 0) {
m_iterindex.init(m_iter_ndim);
memset(m_iterindex.get(), 0, sizeof(intptr_t) * m_iter_ndim);
m_itershape.init(m_iter_ndim);
m_array_tp.extended()->get_shape(m_iter_ndim, 0, m_itershape.get(), metadata0);
size_t iterdata_size = m_array_tp.extended()->get_iterdata_size(m_iter_ndim);
m_iterdata = reinterpret_cast<iterdata_common *>(malloc(iterdata_size));
if (!m_iterdata) {
throw std::bad_alloc();
}
m_metadata = metadata0;
m_array_tp.iterdata_construct(m_iterdata,
&m_metadata, m_iter_ndim, m_itershape.get(), m_uniform_tp);
m_data = m_iterdata->reset(m_iterdata, data0, m_iter_ndim);
for (size_t i = 0, i_end = m_iter_ndim; i != i_end; ++i) {
m_itersize *= m_itershape[i];
}
} else {
m_iterdata = NULL;
m_uniform_tp = m_array_tp;
m_data = data0;
m_metadata = metadata0;
}
}
inline base_uniform_dim_type(type_id_t type_id, const ndt::type& element_tp, size_t data_size,
size_t alignment, size_t element_metadata_offset,
flags_type flags)
: base_type(type_id, uniform_dim_kind, data_size,
alignment, flags, element_metadata_offset + element_tp.get_metadata_size(),
1 + element_tp.get_ndim()),
m_element_tp(element_tp), m_element_metadata_offset(element_metadata_offset)
{
}
unary_expr_type::unary_expr_type(const ndt::type& value_type, const ndt::type& operand_type,
const expr_kernel_generator *kgen)
: base_expression_type(unary_expr_type_id, expression_kind,
operand_type.get_data_size(), operand_type.get_data_alignment(),
inherited_flags(value_type.get_flags(), operand_type.get_flags()),
operand_type.get_metadata_size(), value_type.get_ndim()),
m_value_type(value_type), m_operand_type(operand_type),
m_kgen(kgen)
{
}
static size_t make_elwise_strided_or_var_to_var_dimension_expr_kernel_for_N(
void *ckb, intptr_t ckb_offset, const ndt::type &dst_tp, const char *dst_arrmeta, size_t DYND_UNUSED(src_count),
const ndt::type *src_tp, const char *const *src_arrmeta, kernel_request_t kernreq, const eval::eval_context *ectx,
const expr_kernel_generator *elwise_handler)
{
intptr_t undim = dst_tp.get_ndim();
const char *dst_child_arrmeta;
const char *src_child_arrmeta[N];
ndt::type dst_child_dt;
ndt::type src_child_dt[N];
strided_or_var_to_var_expr_kernel_extra<N> *e =
strided_or_var_to_var_expr_kernel_extra<N>::make(ckb, kernreq, ckb_offset);
// The dst var parameters
const ndt::var_dim_type *dst_vdd = dst_tp.extended<ndt::var_dim_type>();
const var_dim_type_arrmeta *dst_md = reinterpret_cast<const var_dim_type_arrmeta *>(dst_arrmeta);
e->dst_memblock = dst_md->blockref.get();
e->dst_stride = dst_md->stride;
e->dst_offset = dst_md->offset;
e->dst_target_alignment = dst_vdd->get_target_alignment();
dst_child_arrmeta = dst_arrmeta + sizeof(var_dim_type_arrmeta);
dst_child_dt = dst_vdd->get_element_type();
for (int i = 0; i < N; ++i) {
intptr_t src_size;
// The src[i] strided parameters
if (src_tp[i].get_ndim() < undim) {
// This src value is getting broadcasted
e->src_stride[i] = 0;
e->src_offset[i] = 0;
e->is_src_var[i] = false;
src_child_arrmeta[i] = src_arrmeta[i];
src_child_dt[i] = src_tp[i];
} else if (src_tp[i].get_as_strided(src_arrmeta[i], &src_size, &e->src_stride[i], &src_child_dt[i],
&src_child_arrmeta[i])) {
// Check for a broadcasting error (the strided dimension size must be 1,
// otherwise the destination should be strided, not var)
if (src_size != 1) {
throw broadcast_error(dst_tp, dst_arrmeta, src_tp[i], src_arrmeta[i]);
}
e->src_offset[i] = 0;
e->is_src_var[i] = false;
} else {
const ndt::var_dim_type *vdd = static_cast<const ndt::var_dim_type *>(src_tp[i].extended());
const var_dim_type_arrmeta *src_md = reinterpret_cast<const var_dim_type_arrmeta *>(src_arrmeta[i]);
e->src_stride[i] = src_md->stride;
e->src_offset[i] = src_md->offset;
e->is_src_var[i] = true;
src_child_arrmeta[i] = src_arrmeta[i] + sizeof(var_dim_type_arrmeta);
src_child_dt[i] = vdd->get_element_type();
}
}
return elwise_handler->make_expr_kernel(ckb, ckb_offset, dst_child_dt, dst_child_arrmeta, N, src_child_dt,
src_child_arrmeta, kernel_request_strided, ectx);
}
static size_t make_elwise_strided_or_var_to_strided_dimension_expr_kernel_for_N(
void *ckb, intptr_t ckb_offset, const ndt::type &dst_tp, const char *dst_arrmeta, size_t DYND_UNUSED(src_count),
const ndt::type *src_tp, const char *const *src_arrmeta, kernel_request_t kernreq, const eval::eval_context *ectx,
const expr_kernel_generator *elwise_handler)
{
intptr_t undim = dst_tp.get_ndim();
const char *dst_child_arrmeta;
const char *src_child_arrmeta[N];
ndt::type dst_child_dt;
ndt::type src_child_dt[N];
strided_or_var_to_strided_expr_kernel_extra<N> *e = reinterpret_cast<ckernel_builder<kernel_request_host> *>(
ckb)->alloc_ck<strided_or_var_to_strided_expr_kernel_extra<N>>(ckb_offset);
strided_or_var_to_strided_expr_kernel_extra<N>::make(ckb, kernreq, ckb_offset);
// The dst strided parameters
if (!dst_tp.get_as_strided(dst_arrmeta, &e->size, &e->dst_stride, &dst_child_dt, &dst_child_arrmeta)) {
throw type_error("make_elwise_strided_dimension_expr_kernel: dst was not "
"strided as expected");
}
for (int i = 0; i < N; ++i) {
intptr_t src_size;
// The src[i] strided parameters
if (src_tp[i].get_ndim() < undim) {
// This src value is getting broadcasted
e->src_stride[i] = 0;
e->src_offset[i] = 0;
e->is_src_var[i] = false;
src_child_arrmeta[i] = src_arrmeta[i];
src_child_dt[i] = src_tp[i];
} else if (src_tp[i].get_as_strided(src_arrmeta[i], &src_size, &e->src_stride[i], &src_child_dt[i],
&src_child_arrmeta[i])) {
// Check for a broadcasting error
if (src_size != 1 && e->size != src_size) {
throw broadcast_error(dst_tp, dst_arrmeta, src_tp[i], src_arrmeta[i]);
}
e->src_offset[i] = 0;
e->is_src_var[i] = false;
} else {
const ndt::var_dim_type *vdd = static_cast<const ndt::var_dim_type *>(src_tp[i].extended());
const var_dim_type_arrmeta *src_md = reinterpret_cast<const var_dim_type_arrmeta *>(src_arrmeta[i]);
e->src_stride[i] = src_md->stride;
e->src_offset[i] = src_md->offset;
e->is_src_var[i] = true;
src_child_arrmeta[i] = src_arrmeta[i] + sizeof(var_dim_type_arrmeta);
src_child_dt[i] = vdd->get_element_type();
}
}
return elwise_handler->make_expr_kernel(ckb, ckb_offset, dst_child_dt, dst_child_arrmeta, N, src_child_dt,
src_child_arrmeta, kernel_request_strided, ectx);
}
ndt::type ndt::substitute_shape(const ndt::type &pattern, intptr_t ndim,
const intptr_t *shape)
{
substitute_shape_data ssd;
ssd.ndim = ndim;
ssd.i = 0;
ssd.shape = shape;
ssd.fulltype = &pattern;
ndt::type transformed_tp;
bool was_transformed = false;
if (ndim > pattern.get_ndim()) {
ssd.throw_error();
}
substitute_shape_visitor(pattern, 0, &ssd, transformed_tp, was_transformed);
return transformed_tp;
}
base_memory_type(type_id_t type_id, const ndt::type &element_tp,
size_t data_size, size_t alignment,
size_t storage_arrmeta_offset, flags_type flags)
: base_type(type_id, memory_kind, data_size, alignment, flags,
storage_arrmeta_offset + element_tp.get_arrmeta_size(),
element_tp.get_ndim(), 0),
m_element_tp(element_tp),
m_storage_arrmeta_offset(storage_arrmeta_offset)
{
if (element_tp.get_kind() == memory_kind ||
element_tp.get_kind() == symbolic_kind) {
stringstream ss;
ss << "a memory space cannot be specified for type " << element_tp;
throw runtime_error(ss.str());
}
}
convert_type::convert_type(const ndt::type &value_type,
const ndt::type &operand_type)
: base_expr_type(
convert_type_id, expr_kind, operand_type.get_data_size(),
operand_type.get_data_alignment(),
inherited_flags(value_type.get_flags(), operand_type.get_flags()),
operand_type.get_arrmeta_size(), value_type.get_ndim()),
m_value_type(value_type), m_operand_type(operand_type)
{
// An alternative to this error would be to use value_type.value_type(), cutting
// away the expression part of the given value_type.
if (m_value_type.get_kind() == expr_kind) {
std::stringstream ss;
ss << "convert_type: The destination type " << m_value_type;
ss << " should not be an expr_kind";
throw dynd::type_error(ss.str());
}
}
pointer_type::pointer_type(const ndt::type& target_tp)
: base_expr_type(pointer_type_id, expr_kind, sizeof(void *),
sizeof(void *),
inherited_flags(target_tp.get_flags(), type_flag_zeroinit|type_flag_blockref),
sizeof(pointer_type_arrmeta) + target_tp.get_arrmeta_size(),
target_tp.get_ndim()),
m_target_tp(target_tp)
{
// I'm not 100% sure how blockref pointer types should interact with
// the computational subsystem, the details will have to shake out
// when we want to actually do something with them.
if (target_tp.get_kind() == expr_kind && target_tp.get_type_id() != pointer_type_id) {
stringstream ss;
ss << "A dynd pointer type's target cannot be the expression type ";
ss << target_tp;
throw dynd::type_error(ss.str());
}
}
static size_t make_elwise_strided_dimension_expr_kernel_for_N(void *ckb, intptr_t ckb_offset, const ndt::type &dst_tp,
const char *dst_arrmeta, size_t DYND_UNUSED(src_count),
const ndt::type *src_tp, const char *const *src_arrmeta,
kernel_request_t kernreq, const eval::eval_context *ectx,
const expr_kernel_generator *elwise_handler)
{
intptr_t undim = dst_tp.get_ndim();
const char *dst_child_arrmeta;
const char *src_child_arrmeta[N];
ndt::type dst_child_dt;
ndt::type src_child_dt[N];
strided_expr_kernel_extra<N> *e = strided_expr_kernel_extra<N>::make(ckb, kernreq, ckb_offset);
// The dst strided parameters
if (!dst_tp.get_as_strided(dst_arrmeta, &e->size, &e->dst_stride, &dst_child_dt, &dst_child_arrmeta)) {
throw type_error("make_elwise_strided_dimension_expr_kernel: dst was not "
"strided as expected");
}
for (int i = 0; i < N; ++i) {
intptr_t src_size;
// The src[i] strided parameters
if (src_tp[i].get_ndim() < undim) {
// This src value is getting broadcasted
e->src_stride[i] = 0;
src_child_arrmeta[i] = src_arrmeta[i];
src_child_dt[i] = src_tp[i];
} else if (src_tp[i].get_as_strided(src_arrmeta[i], &src_size, &e->src_stride[i], &src_child_dt[i],
&src_child_arrmeta[i])) {
// Check for a broadcasting error
if (src_size != 1 && e->size != src_size) {
throw broadcast_error(dst_tp, dst_arrmeta, src_tp[i], src_arrmeta[i]);
}
} else {
throw type_error("make_elwise_strided_dimension_expr_kernel: src was "
"not strided as expected");
}
}
return elwise_handler->make_expr_kernel(ckb, ckb_offset, dst_child_dt, dst_child_arrmeta, N, src_child_dt,
src_child_arrmeta, kernel_request_strided, ectx);
}
void strided_dim_type::reorder_default_constructed_strides(char *dst_arrmeta,
const ndt::type& src_tp, const char *src_arrmeta) const
{
if (m_element_tp.get_type_id() != strided_dim_type_id) {
// Nothing to do if there's just one reorderable dimension
return;
}
if (get_ndim() > src_tp.get_ndim()) {
// If the destination has more dimensions than the source,
// do the reordering starting from where they match, to
// follow the broadcasting rules.
if (m_element_tp.get_type_id() == strided_dim_type_id) {
const strided_dim_type *sdd = m_element_tp.tcast<strided_dim_type>();
sdd->reorder_default_constructed_strides(
dst_arrmeta + sizeof(strided_dim_type_arrmeta),
src_tp, src_arrmeta);
}
return;
}
// Find the total number of dimensions we might be reordering, then process
// them all at once. This code handles a whole chain of strided_dim_type
// instances at once.
size_t ndim = 1;
ndt::type last_dt = m_element_tp;
do {
++ndim;
last_dt = last_dt.tcast<strided_dim_type>()->get_element_type();
} while (last_dt.get_type_id() == strided_dim_type_id);
dimvector strides(ndim);
ndt::type last_src_tp = src_tp;
intptr_t previous_stride = 0;
size_t ndim_partial = 0;
// Get representative strides from all the strided source dimensions
bool c_order = true;
for (size_t i = 0; i < ndim; ++i) {
intptr_t stride;
switch (last_src_tp.get_type_id()) {
case cfixed_dim_type_id: {
const cfixed_dim_type *fdd = last_src_tp.tcast<cfixed_dim_type>();
stride = fdd->get_fixed_stride();
last_src_tp = fdd->get_element_type();
src_arrmeta += sizeof(cfixed_dim_type_arrmeta);
break;
}
case strided_dim_type_id: {
const strided_dim_type *sdd = last_src_tp.tcast<strided_dim_type>();
const strided_dim_type_arrmeta *md = reinterpret_cast<const strided_dim_type_arrmeta *>(src_arrmeta);
stride = md->stride;
last_src_tp = sdd->get_element_type();
src_arrmeta += sizeof(strided_dim_type_arrmeta);
break;
}
default:
stride = numeric_limits<intptr_t>::max();
break;
}
ndim_partial = i + 1;
// To check for C-order, we skip over any 0-strides, and
// check if a stride ever gets bigger instead of always
// getting smaller.
if (stride != 0) {
if (stride == numeric_limits<intptr_t>::max()) {
break;
}
if (previous_stride != 0 && previous_stride < stride) {
c_order = false;
}
previous_stride = stride;
}
strides[i] = stride;
}
// If it wasn't all C-order, reorder the axes
if (!c_order) {
shortvector<int> axis_perm(ndim_partial);
strides_to_axis_perm(ndim_partial, strides.get(), axis_perm.get());
strided_dim_type_arrmeta *md =
reinterpret_cast<strided_dim_type_arrmeta *>(dst_arrmeta);
intptr_t stride = md[ndim_partial-1].stride;
if (stride == 0) {
// Because of the rule that size one dimensions have
// zero stride, may have to look further
intptr_t i = ndim_partial-2;
do {
stride = md[i].stride;
} while (stride == 0 && i >= 0);
}
for (size_t i = 0; i < ndim_partial; ++i) {
int i_perm = axis_perm[i];
strided_dim_type_arrmeta& i_md = md[i_perm];
intptr_t dim_size = i_md.dim_size;
i_md.stride = dim_size > 1 ? stride : 0;
stride *= dim_size;
}
}
// If that didn't cover all the dimensions, then get the
// axis order classification to handle the rest
if (ndim_partial < ndim && !last_src_tp.is_builtin()) {
axis_order_classification_t aoc = last_src_tp.extended()->classify_axis_order(src_arrmeta);
// TODO: Allow user control by adding a "default axis order" to the evaluation context
if (aoc == axis_order_f) {
// If it's F-order, reverse the ordering of the strides
strided_dim_type_arrmeta *md =
reinterpret_cast<strided_dim_type_arrmeta *>(dst_arrmeta);
intptr_t stride = md[ndim-1].stride;
if (stride == 0) {
// Because of the rule that size one dimensions have
// zero stride, may have to look further
intptr_t i = ndim-2;
do {
stride = md[i].stride;
} while (stride == 0 && i >= (intptr_t)ndim_partial);
}
for (size_t i = ndim_partial; i != ndim; ++i) {
intptr_t dim_size = md[i].dim_size;
md[i].stride = dim_size > 1 ? stride : 0;
stride *= dim_size;
}
}
}
}
size_t dynd::make_lifted_reduction_ckernel(
const callable_type_data *elwise_reduction_const,
const ndt::callable_type *elwise_reduction_tp,
const callable_type_data *dst_initialization_const,
const ndt::callable_type *dst_initialization_tp, void *ckb,
intptr_t ckb_offset, const ndt::type &dst_tp, const char *dst_arrmeta,
const ndt::type &src_tp, const char *src_arrmeta, intptr_t reduction_ndim,
const bool *reduction_dimflags, bool associative, bool commutative,
bool right_associative, const nd::array &reduction_identity,
dynd::kernel_request_t kernreq, const eval::eval_context *ectx)
{
callable_type_data *elwise_reduction =
const_cast<callable_type_data *>(elwise_reduction_const);
callable_type_data *dst_initialization =
const_cast<callable_type_data *>(dst_initialization_const);
// Count the number of dimensions being reduced
intptr_t reducedim_count = 0;
for (intptr_t i = 0; i < reduction_ndim; ++i) {
reducedim_count += reduction_dimflags[i];
}
if (reducedim_count == 0) {
if (reduction_ndim == 0) {
// If there are no dimensions to reduce, it's
// just a dst_initialization operation, so create
// that ckernel directly
if (dst_initialization != NULL) {
return dst_initialization->instantiate(
dst_initialization->static_data, 0, NULL, ckb, ckb_offset, dst_tp,
dst_arrmeta, elwise_reduction_tp->get_npos(), &src_tp, &src_arrmeta,
kernreq, ectx, nd::array(), std::map<nd::string, ndt::type>());
} else if (reduction_identity.is_null()) {
return make_assignment_kernel(ckb, ckb_offset, dst_tp, dst_arrmeta,
src_tp, src_arrmeta, kernreq, ectx);
} else {
// Create the kernel which copies the identity and then
// does one reduction
return make_strided_inner_reduction_dimension_kernel(
elwise_reduction, elwise_reduction_tp, dst_initialization,
dst_initialization_tp, ckb, ckb_offset, 0, 1, dst_tp, dst_arrmeta,
src_tp, src_arrmeta, right_associative, reduction_identity, kernreq,
ectx);
}
}
throw runtime_error("make_lifted_reduction_ckernel: no dimensions were "
"flagged for reduction");
}
if (!(reducedim_count == 1 || (associative && commutative))) {
throw runtime_error(
"make_lifted_reduction_ckernel: for reducing along multiple dimensions,"
" the reduction function must be both associative and commutative");
}
if (right_associative) {
throw runtime_error("make_lifted_reduction_ckernel: right_associative is "
"not yet supported");
}
ndt::type dst_el_tp = elwise_reduction_tp->get_return_type();
ndt::type src_el_tp = elwise_reduction_tp->get_pos_type(0);
// This is the number of dimensions being processed by the reduction
if (reduction_ndim != src_tp.get_ndim() - src_el_tp.get_ndim()) {
stringstream ss;
ss << "make_lifted_reduction_ckernel: wrong number of reduction "
"dimensions, ";
ss << "requested " << reduction_ndim << ", but types have ";
ss << (src_tp.get_ndim() - src_el_tp.get_ndim());
ss << " lifting from " << src_el_tp << " to " << src_tp;
throw runtime_error(ss.str());
}
// Determine whether reduced dimensions are being kept or not
bool keep_dims;
if (reduction_ndim == dst_tp.get_ndim() - dst_el_tp.get_ndim()) {
keep_dims = true;
} else if (reduction_ndim - reducedim_count ==
dst_tp.get_ndim() - dst_el_tp.get_ndim()) {
keep_dims = false;
} else {
stringstream ss;
ss << "make_lifted_reduction_ckernel: The number of dimensions flagged for "
"reduction, ";
ss << reducedim_count << ", is not consistent with the destination type ";
ss << "reducing " << dst_tp << " with element " << dst_el_tp;
throw runtime_error(ss.str());
}
ndt::type dst_i_tp = dst_tp, src_i_tp = src_tp;
for (intptr_t i = 0; i < reduction_ndim; ++i) {
intptr_t dst_stride, dst_size, src_stride, src_size;
// Get the striding parameters for the source dimension
if (!src_i_tp.get_as_strided(src_arrmeta, &src_size, &src_stride, &src_i_tp,
&src_arrmeta)) {
stringstream ss;
ss << "make_lifted_reduction_ckernel: type " << src_i_tp
<< " not supported as source";
throw type_error(ss.str());
}
if (reduction_dimflags[i]) {
// This dimension is being reduced
if (src_size == 0 && reduction_identity.is_null()) {
// If the size of the src is 0, a reduction identity is required to get
// a value
stringstream ss;
ss << "cannot reduce a zero-sized dimension (axis ";
ss << i << " of " << src_i_tp << ") because the operation";
ss << " has no identity";
throw invalid_argument(ss.str());
}
if (keep_dims) {
// If the dimensions are being kept, the output should be a
// a strided dimension of size one
if (dst_i_tp.get_as_strided(dst_arrmeta, &dst_size, &dst_stride,
&dst_i_tp, &dst_arrmeta)) {
if (dst_size != 1 || dst_stride != 0) {
stringstream ss;
ss << "make_lifted_reduction_ckernel: destination of a reduction "
"dimension ";
ss << "must have size 1, not size" << dst_size << "/stride "
<< dst_stride;
ss << " in type " << dst_i_tp;
throw type_error(ss.str());
}
} else {
stringstream ss;
ss << "make_lifted_reduction_ckernel: type " << dst_i_tp;
ss << " not supported the destination of a dimension being reduced";
throw type_error(ss.str());
}
}
if (i < reduction_ndim - 1) {
// An initial dimension being reduced
ckb_offset = make_strided_initial_reduction_dimension_kernel(
ckb, ckb_offset, src_stride, src_size, kernreq);
// The next request should be single, as that's the kind of
// ckernel the 'first_call' should be in this case
kernreq = kernel_request_single;
} else {
// The innermost dimension being reduced
return make_strided_inner_reduction_dimension_kernel(
elwise_reduction, elwise_reduction_tp, dst_initialization,
dst_initialization_tp, ckb, ckb_offset, src_stride, src_size,
dst_i_tp, dst_arrmeta, src_i_tp, src_arrmeta, right_associative,
reduction_identity, kernreq, ectx);
}
} else {
// This dimension is being broadcast, not reduced
if (!dst_i_tp.get_as_strided(dst_arrmeta, &dst_size, &dst_stride,
&dst_i_tp, &dst_arrmeta)) {
stringstream ss;
ss << "make_lifted_reduction_ckernel: type " << dst_i_tp
<< " not supported as destination";
throw type_error(ss.str());
}
if (dst_size != src_size) {
stringstream ss;
ss << "make_lifted_reduction_ckernel: the dst dimension size "
<< dst_size;
ss << " must equal the src dimension size " << src_size
<< " for broadcast dimensions";
throw runtime_error(ss.str());
}
if (i < reduction_ndim - 1) {
// An initial dimension being broadcast
ckb_offset = make_strided_initial_broadcast_dimension_kernel(
ckb, ckb_offset, dst_stride, src_stride, src_size, kernreq);
// The next request should be strided, as that's the kind of
// ckernel the 'first_call' should be in this case
kernreq = kernel_request_strided;
} else {
// The innermost dimension being broadcast
return make_strided_inner_broadcast_dimension_kernel(
elwise_reduction, elwise_reduction_tp, dst_initialization,
dst_initialization_tp, ckb, ckb_offset, dst_stride, src_stride,
src_size, dst_i_tp, dst_arrmeta, src_i_tp, src_arrmeta,
right_associative, reduction_identity, kernreq, ectx);
}
}
}
throw runtime_error("make_lifted_reduction_ckernel: internal error, "
"should have returned in the loop");
}
axis_order_classification_t dynd::classify_strided_axis_order(intptr_t current_stride,
const ndt::type& element_tp, const char *element_arrmeta)
{
switch (element_tp.get_type_id()) {
case cfixed_dim_type_id: {
const cfixed_dim_type *edt = element_tp.extended<cfixed_dim_type>();
intptr_t estride = intptr_abs(edt->get_fixed_stride());
if (estride != 0) {
axis_order_classification_t aoc;
// Get the classification from the next dimension onward
if (edt->get_ndim() > 1) {
aoc = classify_strided_axis_order(current_stride,
edt->get_element_type(),
element_arrmeta);
} else {
aoc = axis_order_none;
}
if (current_stride > estride) {
// C order
return (aoc == axis_order_none || aoc == axis_order_c)
? axis_order_c : axis_order_neither;
} else {
// F order
return (aoc == axis_order_none || aoc == axis_order_f)
? axis_order_f : axis_order_neither;
}
} else if (element_tp.get_ndim() > 1) {
// Skip the zero-stride dimensions (DyND requires that the stride
// be zero when the dimension size is one)
return classify_strided_axis_order(current_stride,
edt->get_element_type(),
element_arrmeta);
} else {
// There was only one dimension with a nonzero stride
return axis_order_none;
}
}
case fixed_dim_type_id: {
const fixed_dim_type *edt = element_tp.extended<fixed_dim_type>();
const fixed_dim_type_arrmeta *emd =
reinterpret_cast<const fixed_dim_type_arrmeta *>(
element_arrmeta);
intptr_t estride = intptr_abs(emd->stride);
if (estride != 0) {
axis_order_classification_t aoc;
// Get the classification from the next dimension onward
if (edt->get_ndim() > 1) {
aoc = classify_strided_axis_order(current_stride,
edt->get_element_type(),
element_arrmeta + sizeof(fixed_dim_type_arrmeta));
} else {
aoc = axis_order_none;
}
if (current_stride > estride) {
// C order
return (aoc == axis_order_none || aoc == axis_order_c)
? axis_order_c : axis_order_neither;
} else {
// F order
return (aoc == axis_order_none || aoc == axis_order_f)
? axis_order_f : axis_order_neither;
}
} else if (element_tp.get_ndim() > 1) {
// Skip the zero-stride dimensions (DyND requires that the stride
// be zero when the dimension size is one)
return classify_strided_axis_order(current_stride,
edt->get_element_type(),
element_arrmeta + sizeof(fixed_dim_type_arrmeta));
} else {
// There was only one dimension with a nonzero stride
return axis_order_none;
}
}
case pointer_type_id:
case var_dim_type_id: {
// A pointer or a var type is treated like C-order
axis_order_classification_t aoc =
element_tp.extended()->classify_axis_order(element_arrmeta);
return (aoc == axis_order_none || aoc == axis_order_c)
? axis_order_c : axis_order_neither;
}
default: {
stringstream ss;
ss << "classify_strided_axis_order not implemented for dynd type ";
ss << element_tp;
throw runtime_error(ss.str());
}
}
}
size_t fixed_dim_type::make_assignment_kernel(
ckernel_builder *out, size_t offset_out,
const ndt::type& dst_tp, const char *dst_metadata,
const ndt::type& src_tp, const char *src_metadata,
kernel_request_t kernreq, assign_error_mode errmode,
const eval::eval_context *ectx) const
{
if (this == dst_tp.extended()) {
out->ensure_capacity(offset_out + sizeof(strided_assign_kernel_extra));
strided_assign_kernel_extra *e = out->get_at<strided_assign_kernel_extra>(offset_out);
switch (kernreq) {
case kernel_request_single:
e->base.set_function<unary_single_operation_t>(&strided_assign_kernel_extra::single);
break;
case kernel_request_strided:
e->base.set_function<unary_strided_operation_t>(&strided_assign_kernel_extra::strided);
break;
default: {
stringstream ss;
ss << "strided_dim_type::make_assignment_kernel: unrecognized request " << (int)kernreq;
throw runtime_error(ss.str());
}
}
e->base.destructor = strided_assign_kernel_extra::destruct;
if (src_tp.get_ndim() < dst_tp.get_ndim()) {
// If the src has fewer dimensions, broadcast it across this one
e->size = get_fixed_dim_size();
e->dst_stride = get_fixed_stride();
e->src_stride = 0;
return ::make_assignment_kernel(out, offset_out + sizeof(strided_assign_kernel_extra),
m_element_tp, dst_metadata,
src_tp, src_metadata,
kernel_request_strided, errmode, ectx);
} else if (src_tp.get_type_id() == fixed_dim_type_id) {
// fixed_array -> strided_dim
const fixed_dim_type *src_fad = static_cast<const fixed_dim_type *>(src_tp.extended());
intptr_t src_size = src_fad->get_fixed_dim_size();
intptr_t dst_size = get_fixed_dim_size();
// Check for a broadcasting error
if (src_size != 1 && dst_size != src_size) {
throw broadcast_error(dst_tp, dst_metadata, src_tp, src_metadata);
}
e->size = dst_size;
e->dst_stride = get_fixed_stride();
// In DyND, the src stride is required to be zero for size-one dimensions,
// so we don't have to check the size here.
e->src_stride = src_fad->get_fixed_stride();
return ::make_assignment_kernel(out, offset_out + sizeof(strided_assign_kernel_extra),
m_element_tp, dst_metadata,
src_fad->get_element_type(), src_metadata,
kernel_request_strided, errmode, ectx);
} else if (src_tp.get_type_id() == strided_dim_type_id) {
// strided_dim -> strided_dim
const strided_dim_type *src_sad = static_cast<const strided_dim_type *>(src_tp.extended());
const strided_dim_type_metadata *src_md =
reinterpret_cast<const strided_dim_type_metadata *>(src_metadata);
intptr_t dst_size = get_fixed_dim_size();
// Check for a broadcasting error
if (src_md->size != 1 && dst_size != src_md->size) {
throw broadcast_error(dst_tp, dst_metadata, src_tp, src_metadata);
}
e->size = dst_size;
e->dst_stride = get_fixed_stride();
// In DyND, the src stride is required to be zero for size-one dimensions,
// so we don't have to check the size here.
e->src_stride = src_md->stride;
return ::make_assignment_kernel(out, offset_out + sizeof(strided_assign_kernel_extra),
m_element_tp, dst_metadata,
src_sad->get_element_type(), src_metadata + sizeof(strided_dim_type_metadata),
kernel_request_strided, errmode, ectx);
} else if (!src_tp.is_builtin()) {
// Give the src type a chance to make a kernel
return src_tp.extended()->make_assignment_kernel(out, offset_out,
dst_tp, dst_metadata,
src_tp, src_metadata,
kernreq, errmode, ectx);
} else {
stringstream ss;
ss << "Cannot assign from " << src_tp << " to " << dst_tp;
throw runtime_error(ss.str());
}
} else if (dst_tp.get_ndim() < src_tp.get_ndim()) {
throw broadcast_error(dst_tp, dst_metadata, src_tp, src_metadata);
} else {
stringstream ss;
ss << "Cannot assign from " << src_tp << " to " << dst_tp;
throw runtime_error(ss.str());
}
}
static void refine_bytes_view(memory_block_ptr &data_ref, char *&data_ptr, ndt::type &data_tp, const char *&data_meta,
intptr_t &data_dim_size, intptr_t &data_stride)
{
// Handle sequence of strided dims
intptr_t dim_size, stride;
ndt::type el_tp;
const char *el_meta;
if (data_tp.get_as_strided(data_meta, &dim_size, &stride, &el_tp, &el_meta)) {
dimvector shape(data_tp.get_ndim());
dimvector strides(data_tp.get_ndim());
intptr_t ndim = 1;
shape[0] = dim_size;
strides[0] = stride;
bool csorted = true;
// Get all the strided dimensions we can in a row
while (el_tp.get_as_strided(el_meta, &dim_size, &stride, &el_tp, &el_meta)) {
shape[ndim] = dim_size;
strides[ndim] = stride;
if (stride > strides[ndim - 1]) {
csorted = false;
}
++ndim;
}
if (!csorted) {
// If the strides weren't sorted in C order, sort them
shortvector<int> axis_perm(ndim);
strides_to_axis_perm(ndim, strides.get(), axis_perm.get());
dimvector shape_sorted(ndim);
dimvector strides_sorted(ndim);
for (intptr_t i = 0; i < ndim; ++i) {
int i_perm = axis_perm[i];
shape_sorted[ndim - i - 1] = shape[i_perm];
strides_sorted[ndim - i - 1] = strides[i_perm];
}
shape.swap(shape_sorted);
strides.swap(strides_sorted);
}
// Try to collapse the shape/strides into a single strided array
intptr_t i = 0;
while (data_dim_size == -1 && i < ndim) {
// If there's not already a dim_size/stride, start one
if (shape[i] != 1) {
data_dim_size = shape[i];
data_stride = strides[i];
}
++i;
}
for (; i < ndim; ++i) {
if (shape[i] != 1) {
if (shape[i] * strides[i] != data_stride) {
// Indicate we couldn't view this as bytes
data_tp = ndt::type();
data_dim_size = -1;
return;
}
data_dim_size *= shape[i];
data_stride = strides[i];
}
}
data_tp = el_tp;
data_meta = el_meta;
return;
}
switch (data_tp.get_type_id()) {
case var_dim_type_id: {
// We can only allow leading var_dim
if (data_dim_size != -1) {
data_tp = ndt::type();
data_dim_size = -1;
return;
}
const var_dim_type_arrmeta *meta = reinterpret_cast<const var_dim_type_arrmeta *>(data_meta);
if (meta->blockref != NULL) {
data_ref = meta->blockref;
}
var_dim_type_data *d = reinterpret_cast<var_dim_type_data *>(data_ptr);
data_ptr = d->begin + meta->offset;
if (d->size != 1) {
data_dim_size = d->size;
data_stride = meta->stride;
}
data_tp = data_tp.extended<ndt::var_dim_type>()->get_element_type();
data_meta += sizeof(var_dim_type_arrmeta);
return;
}
case pointer_type_id: {
// We can only strip away leading pointers
if (data_dim_size != -1) {
data_tp = ndt::type();
data_dim_size = -1;
return;
}
const pointer_type_arrmeta *meta = reinterpret_cast<const pointer_type_arrmeta *>(data_meta);
if (meta->blockref != NULL) {
data_ref = meta->blockref;
}
data_ptr = *reinterpret_cast<char **>(data_ptr) + meta->offset;
data_tp = data_tp.extended<ndt::pointer_type>()->get_target_type();
data_meta += sizeof(pointer_type_arrmeta);
return;
}
case string_type_id: {
// We can only view leading strings
if (data_dim_size != -1) {
data_tp = ndt::type();
data_dim_size = -1;
return;
}
// Look at the actual string data, not the pointer to it
const string_type_arrmeta *meta = reinterpret_cast<const string_type_arrmeta *>(data_meta);
if (meta->blockref != NULL) {
data_ref = meta->blockref;
}
const dynd::string *str_ptr = reinterpret_cast<const dynd::string *>(data_ptr);
data_ptr = str_ptr->begin;
data_tp = ndt::type();
data_dim_size = str_ptr->end - str_ptr->begin;
data_stride = 1;
return;
}
default:
break;
}
// If the data type has a fixed size, check if it fits the strides
size_t data_tp_size = data_tp.get_data_size();
if (data_tp_size > 0) {
if (data_dim_size == -1) {
// Indicate success (one item)
data_tp = ndt::type();
data_dim_size = data_tp_size;
data_stride = 1;
return;
} else if ((intptr_t)data_tp_size == data_stride) {
data_tp = ndt::type();
data_dim_size *= data_tp_size;
data_stride = 1;
return;
}
}
// Indicate we couldn't view this as bytes
data_tp = ndt::type();
data_dim_size = -1;
}