inline static bool run(nd::array &a)
{
const ndt::type &tp = a.get_type();
if (a.is_immutable() && tp.get_type_id() == fixed_dim_type_id) {
// It's immutable and "N * <something>"
const ndt::type &et = tp.extended<fixed_dim_type>()->get_element_type();
const fixed_dim_type_arrmeta *md =
reinterpret_cast<const fixed_dim_type_arrmeta *>(a.get_arrmeta());
if (et.get_type_id() == type_type_id &&
md->stride == sizeof(ndt::type)) {
// It also has the right type and is contiguous,
// so no modification necessary.
return true;
}
}
// We have to make a copy, check that it's a 1D array, and that
// it has the same array kind as the requested type.
if (tp.get_ndim() == 1) {
// It's a 1D array
const ndt::type &et = tp.get_type_at_dimension(NULL, 1).value_type();
if (et.get_type_id() == type_type_id) {
// It also has the same array type as requested
nd::array tmp = nd::empty(a.get_dim_size(), ndt::make_type());
tmp.vals() = a;
tmp.flag_as_immutable();
a.swap(tmp);
return true;
}
}
// It's not compatible, so return false
return false;
}
/**
* Substitutes the field types for contiguous array of types
*/
static std::vector<ndt::type> substitute_type_array(const nd::array &type_array,
const std::map<std::string, ndt::type> &typevars, bool concrete) {
intptr_t field_count = type_array.get_dim_size();
const ndt::type *field_types = reinterpret_cast<const ndt::type *>(type_array.cdata());
std::vector<ndt::type> tmp_field_types(field_count);
for (intptr_t i = 0; i < field_count; ++i) {
tmp_field_types[i] = ndt::substitute(field_types[i], typevars, concrete);
}
return tmp_field_types;
}
/**
* Substitutes the field types for contiguous array of types
*/
static nd::array substitute_type_array(const nd::array &type_array, const std::map<std::string, ndt::type> &typevars,
bool concrete)
{
intptr_t field_count = type_array.get_dim_size();
const ndt::type *field_types = reinterpret_cast<const ndt::type *>(type_array.cdata());
nd::array tmp_field_types(nd::empty(field_count, ndt::make_type()));
ndt::type *ftraw = reinterpret_cast<ndt::type *>(tmp_field_types.data());
for (intptr_t i = 0; i < field_count; ++i) {
ftraw[i] = ndt::substitute(field_types[i], typevars, concrete);
}
return tmp_field_types;
}
base_tuple_type::base_tuple_type(type_id_t type_id,
const nd::array &field_types, flags_type flags,
bool variable_layout)
: base_type(type_id, tuple_kind, 0, 1, flags, 0, 0, 0),
m_field_count(field_types.get_dim_size()), m_field_types(field_types),
m_arrmeta_offsets(nd::empty(m_field_count, ndt::make_type<uintptr_t>()))
{
if (!nd::ensure_immutable_contig<ndt::type>(m_field_types)) {
stringstream ss;
ss << "dynd tuple type requires an array of types, got an array with "
"type " << m_field_types.get_type();
throw invalid_argument(ss.str());
}
// Calculate the needed element alignment and arrmeta offsets
size_t arrmeta_offset = 0;
if (variable_layout) {
arrmeta_offset = get_field_count() * sizeof(size_t);
}
uintptr_t *arrmeta_offsets = reinterpret_cast<uintptr_t *>(
m_arrmeta_offsets.get_readwrite_originptr());
m_members.data_alignment = 1;
for (intptr_t i = 0, i_end = get_field_count(); i != i_end; ++i) {
const ndt::type &ft = get_field_type(i);
size_t field_alignment = ft.get_data_alignment();
// Accumulate the biggest field alignment as the type alignment
if (field_alignment > m_members.data_alignment) {
m_members.data_alignment = (uint8_t)field_alignment;
}
// Inherit any operand flags from the fields
m_members.flags |= (ft.get_flags()&type_flags_operand_inherited);
// Calculate the arrmeta offsets
arrmeta_offsets[i] = arrmeta_offset;
arrmeta_offset += ft.get_arrmeta_size();
}
m_members.arrmeta_size = arrmeta_offset;
m_arrmeta_offsets.flag_as_immutable();
}
categorical_type::categorical_type(const nd::array& categories, bool presorted)
: base_type(categorical_type_id, custom_kind, 4, 4, type_flag_scalar, 0, 0, 0)
{
intptr_t category_count;
if (presorted) {
// This is construction shortcut, for the case when the categories are already
// sorted. No validation of this is done, the caller should have ensured it
// was correct already, typically by construction.
m_categories = categories.eval_immutable();
m_category_tp = m_categories.get_type().at(0);
category_count = categories.get_dim_size();
m_value_to_category_index.resize(category_count);
m_category_index_to_value.resize(category_count);
for (size_t i = 0; i != (size_t)category_count; ++i) {
m_value_to_category_index[i] = i;
m_category_index_to_value[i] = i;
}
} else {
// Process the categories array to make sure it's valid
const ndt::type& cdt = categories.get_type();
if (cdt.get_type_id() != strided_dim_type_id) {
throw dynd::type_error("categorical_type only supports construction from a strided array of categories");
}
m_category_tp = categories.get_type().at(0);
if (!m_category_tp.is_scalar()) {
throw dynd::type_error("categorical_type only supports construction from a 1-dimensional strided array of categories");
}
category_count = categories.get_dim_size();
intptr_t categories_stride = reinterpret_cast<const strided_dim_type_arrmeta *>(categories.get_arrmeta())->stride;
const char *categories_element_arrmeta = categories.get_arrmeta() + sizeof(strided_dim_type_arrmeta);
comparison_ckernel_builder k;
::make_comparison_kernel(&k, 0,
m_category_tp, categories_element_arrmeta,
m_category_tp, categories_element_arrmeta,
comparison_type_sorting_less, &eval::default_eval_context);
cmp less(k.get_function(), k.get());
set<const char *, cmp> uniques(less);
m_value_to_category_index.resize(category_count);
m_category_index_to_value.resize(category_count);
// create the mapping from indices of (to be lexicographically sorted) categories to values
for (size_t i = 0; i != (size_t)category_count; ++i) {
m_category_index_to_value[i] = i;
const char *category_value = categories.get_readonly_originptr() +
i * categories_stride;
if (uniques.find(category_value) == uniques.end()) {
uniques.insert(category_value);
} else {
stringstream ss;
ss << "categories must be unique: category value ";
m_category_tp.print_data(ss, categories_element_arrmeta, category_value);
ss << " appears more than once";
throw std::runtime_error(ss.str());
}
}
// TODO: Putting everything in a set already caused a sort operation to occur,
// there's no reason we should need a second sort.
std::sort(m_category_index_to_value.begin(), m_category_index_to_value.end(),
sorter(categories.get_readonly_originptr(), categories_stride,
k.get_function(), k.get()));
// invert the m_category_index_to_value permutation
for (uint32_t i = 0; i < m_category_index_to_value.size(); ++i) {
m_value_to_category_index[m_category_index_to_value[i]] = i;
}
m_categories = make_sorted_categories(uniques, m_category_tp,
categories_element_arrmeta);
}
// Use the number of categories to set which underlying integer storage to use
if (category_count <= 256) {
m_storage_type = ndt::make_type<uint8_t>();
} else if (category_count <= 65536) {
m_storage_type = ndt::make_type<uint16_t>();
} else {
m_storage_type = ndt::make_type<uint32_t>();
}
m_members.data_size = m_storage_type.get_data_size();
m_members.data_alignment = (uint8_t)m_storage_type.get_data_alignment();
}
ndt::categorical_type::categorical_type(const nd::array &categories, bool presorted)
: base_type(categorical_id, 4, 4, type_flag_none, 0, 0, 0)
{
intptr_t category_count;
if (presorted) {
// This is construction shortcut, for the case when the categories are
// already
// sorted. No validation of this is done, the caller should have ensured it
// was correct already, typically by construction.
m_categories = categories.eval_immutable();
m_category_tp = m_categories.get_type().at(0);
category_count = categories.get_dim_size();
m_value_to_category_index = nd::range(category_count);
m_value_to_category_index.flag_as_immutable();
m_category_index_to_value = m_value_to_category_index;
}
else {
// Process the categories array to make sure it's valid
const type &cdt = categories.get_type();
if (cdt.get_id() != fixed_dim_id) {
throw dynd::type_error("categorical_type only supports construction from "
"a fixed-dim array of categories");
}
m_category_tp = categories.get_type().at(0);
if (!m_category_tp.is_scalar()) {
throw dynd::type_error("categorical_type only supports construction from "
"a 1-dimensional strided array of categories");
}
category_count = categories.get_dim_size();
intptr_t categories_stride = reinterpret_cast<const fixed_dim_type_arrmeta *>(categories.get()->metadata())->stride;
const char *categories_element_arrmeta = categories.get()->metadata() + sizeof(fixed_dim_type_arrmeta);
nd::kernel_builder k;
kernel_single_t fn = k.get()->get_function<kernel_single_t>();
cmp less(fn, k.get());
set<const char *, cmp> uniques(less);
m_value_to_category_index = nd::empty(category_count, make_type<intptr_t>());
m_category_index_to_value = nd::empty(category_count, make_type<intptr_t>());
// create the mapping from indices of (to be lexicographically sorted)
// categories to values
for (size_t i = 0; i != (size_t)category_count; ++i) {
unchecked_fixed_dim_get_rw<intptr_t>(m_category_index_to_value, i) = i;
const char *category_value = categories.cdata() + i * categories_stride;
if (uniques.find(category_value) == uniques.end()) {
uniques.insert(category_value);
}
else {
stringstream ss;
ss << "categories must be unique: category value ";
m_category_tp.print_data(ss, categories_element_arrmeta, category_value);
ss << " appears more than once";
throw std::runtime_error(ss.str());
}
}
// TODO: Putting everything in a set already caused a sort operation to
// occur,
// there's no reason we should need a second sort.
std::sort(&unchecked_fixed_dim_get_rw<intptr_t>(m_category_index_to_value, 0),
&unchecked_fixed_dim_get_rw<intptr_t>(m_category_index_to_value, category_count),
sorter(categories.cdata(), categories_stride, fn, k.get()));
// invert the m_category_index_to_value permutation
for (intptr_t i = 0; i < category_count; ++i) {
unchecked_fixed_dim_get_rw<intptr_t>(m_value_to_category_index,
unchecked_fixed_dim_get<intptr_t>(m_category_index_to_value, i)) = i;
}
m_categories = make_sorted_categories(uniques, m_category_tp, categories_element_arrmeta);
}
// Use the number of categories to set which underlying integer storage to use
if (category_count <= 256) {
m_storage_type = make_type<uint8_t>();
}
else if (category_count <= 65536) {
m_storage_type = make_type<uint16_t>();
}
else {
m_storage_type = make_type<uint32_t>();
}
this->data_size = m_storage_type.get_data_size();
this->data_alignment = (uint8_t)m_storage_type.get_data_alignment();
}