static nd::array function_type_construct(const ndt::type& DYND_UNUSED(dt),
const nd::array& year, const nd::array& month, const nd::array& day)
{
// TODO proper buffering
nd::array year_as_int = year.ucast(ndt::make_type<int32_t>()).eval();
nd::array month_as_int = month.ucast(ndt::make_type<int32_t>()).eval();
nd::array day_as_int = day.ucast(ndt::make_type<int32_t>()).eval();
nd::array result;
array_iter<1,3> iter(ndt::make_date(), result, year_as_int, month_as_int, day_as_int);
if (!iter.empty()) {
datetime::date_ymd ymd;
do {
ymd.year = *reinterpret_cast<const int32_t *>(iter.data<1>());
ymd.month = *reinterpret_cast<const int32_t *>(iter.data<2>());
ymd.day = *reinterpret_cast<const int32_t *>(iter.data<3>());
if (!datetime::is_valid_ymd(ymd)) {
stringstream ss;
ss << "invalid year/month/day " << ymd.year << "/" << ymd.month << "/" << ymd.day;
throw runtime_error(ss.str());
}
*reinterpret_cast<int32_t *>(iter.data<0>()) = datetime::ymd_to_days(ymd);
} while (iter.next());
}
return result;
}
inline static bool run(nd::array &a)
{
const ndt::type &tp = a.get_type();
if (a.is_immutable() && tp.get_type_id() == fixed_dim_type_id) {
// It's immutable and "N * <something>"
const ndt::type &et = tp.extended<fixed_dim_type>()->get_element_type();
const fixed_dim_type_arrmeta *md =
reinterpret_cast<const fixed_dim_type_arrmeta *>(a.get_arrmeta());
if (et.get_type_id() == type_type_id &&
md->stride == sizeof(ndt::type)) {
// It also has the right type and is contiguous,
// so no modification necessary.
return true;
}
}
// We have to make a copy, check that it's a 1D array, and that
// it has the same array kind as the requested type.
if (tp.get_ndim() == 1) {
// It's a 1D array
const ndt::type &et = tp.get_type_at_dimension(NULL, 1).value_type();
if (et.get_type_id() == type_type_id) {
// It also has the same array type as requested
nd::array tmp = nd::empty(a.get_dim_size(), ndt::make_type());
tmp.vals() = a;
tmp.flag_as_immutable();
a.swap(tmp);
return true;
}
}
// It's not compatible, so return false
return false;
}
nd::array dynd::format_json(const nd::array& n)
{
// Create a UTF-8 string
nd::array result = nd::empty(ndt::make_string());
// Initialize the output with some memory
output_data out;
out.blockref = reinterpret_cast<const string_type_metadata *>(result.get_ndo_meta())->blockref;
out.api = get_memory_block_pod_allocator_api(out.blockref);
out.api->allocate(out.blockref, 1024, 1, &out.out_begin, &out.out_capacity_end);
out.out_end = out.out_begin;
if (!n.get_type().is_expression()) {
::format_json(out, n.get_type(), n.get_ndo_meta(), n.get_readonly_originptr());
} else {
nd::array tmp = n.eval();
::format_json(out, tmp.get_type(), tmp.get_ndo_meta(), tmp.get_readonly_originptr());
}
// Shrink the memory to fit, and set the pointers in the output
string_type_data *d = reinterpret_cast<string_type_data *>(result.get_readwrite_originptr());
d->begin = out.out_begin;
d->end = out.out_capacity_end;
out.api->resize(out.blockref, out.out_end - out.out_begin, &d->begin, &d->end);
// Finalize processing and mark the result as immutable
result.get_type().extended()->metadata_finalize_buffers(result.get_ndo_meta());
result.flag_as_immutable();
return result;
}
/**
* This function promotes the requested `axis` from
* a strided dim to a var dim. It modifies `shape`, `coord`,
* `elem`, and `arr` to point to a new array, and
* copies the data over.
*/
static void promote_nd_arr_dim(std::vector<intptr_t> &shape,
std::vector<afpd_coordentry> &coord,
afpd_dtype &elem, nd::array &arr, intptr_t axis,
bool copy_final_coord)
{
vector<afpd_coordentry> newcoord;
afpd_dtype newelem;
newelem.dtp = elem.dtp;
// Convert the axis into a var dim
shape[axis] = -1;
// Create the new array
nd::array newarr = allocate_nd_arr(shape, newcoord, newelem, axis);
// Copy the data up to, but not including, the current `coord`
// from the old `arr` to the new one. The recursion stops
// at `axis`, where all subsequent dimensions are handled by the
// created kernel.
ckernel_builder<kernel_request_host> k;
if (elem.dtp.get_type_id() != uninitialized_type_id) {
make_assignment_kernel(&k, 0, newcoord[axis].tp, newcoord[axis].arrmeta_ptr,
coord[axis].tp, coord[axis].arrmeta_ptr,
kernel_request_strided, &eval::default_eval_context);
}
copy_to_promoted_nd_arr(shape, newarr.get_readwrite_originptr(), newcoord,
newelem, arr.get_readonly_originptr(), coord, elem, k,
0, axis, copy_final_coord, true);
arr.swap(newarr);
coord.swap(newcoord);
elem.swap(newelem);
}
void dynd::parse_json(nd::array &out, const char *json_begin,
const char *json_end, const eval::eval_context *ectx)
{
try {
const char *begin = json_begin, *end = json_end;
ndt::type tp = out.get_type();
::parse_json(tp, out.get_ndo_meta(), out.get_readwrite_originptr(), begin, end, ectx);
begin = skip_whitespace(begin, end);
if (begin != end) {
throw json_parse_error(begin, "unexpected trailing JSON text", tp);
}
} catch (const json_parse_error& e) {
stringstream ss;
string line_prev, line_cur;
int line, column;
get_error_line_column(json_begin, json_end, e.get_position(),
line_prev, line_cur, line, column);
ss << "Error parsing JSON at line " << line << ", column " << column << "\n";
if (e.get_type().get_type_id() != uninitialized_type_id) {
ss << "DType: " << e.get_type() << "\n";
}
ss << "Message: " << e.get_message() << "\n";
print_json_parse_error_marker(ss, line_prev, line_cur, line, column);
throw runtime_error(ss.str());
}
}
nd::array dynd::format_json(const nd::array &n, bool struct_as_list)
{
// Create a UTF-8 string
nd::array result = nd::empty(ndt::string_type::make());
// Initialize the output with some memory
output_data out;
out.out_string.resize(1024);
out.out_begin = out.out_string.begin();
out.out_capacity_end = out.out_string.end();
out.out_end = out.out_begin;
out.struct_as_list = struct_as_list;
if (!n.get_type().is_expression()) {
::format_json(out, n.get_type(), n.get_arrmeta(), n.get_readonly_originptr());
} else {
nd::array tmp = n.eval();
::format_json(out, tmp.get_type(), tmp.get_arrmeta(), tmp.get_readonly_originptr());
}
// Shrink the memory to fit, and set the pointers in the output
string *d = reinterpret_cast<string *>(result.get_readwrite_originptr());
d->assign(out.out_string.data(), out.out_end - out.out_begin);
// Finalize processing and mark the result as immutable
result.get_type().extended()->arrmeta_finalize_buffers(result.get_arrmeta());
result.flag_as_immutable();
return result;
}
static void instantiate(const callable_type_data *DYND_UNUSED(self), const callable_type *DYND_UNUSED(self_tp),
kernel_builder *ckb, intptr_t ckb_offset, const ndt::type &DYND_UNUSED(dst_tp),
const char *dst_arrmeta, intptr_t DYND_UNUSED(nsrc), const ndt::type *src_tp,
const char *const *src_arrmeta, kernel_request_t kernreq,
const eval::eval_context *DYND_UNUSED(ectx), const nd::array &kwds,
const std::map<std::string, ndt::type> &DYND_UNUSED(tp_vars))
{
const size_stride_t *dst_size_stride = reinterpret_cast<const size_stride_t *>(dst_arrmeta);
const size_stride_t *src_size_stride = reinterpret_cast<const size_stride_t *>(src_arrmeta[0]);
array axes = kwds.p("axes");
array shape = kwds.p("shape");
int ndim = static_cast<int>(src_tp[0].get_ndim());
int rank = static_cast<int>(axes.is_missing() ? ndim : (ndim - 1));
int istride = static_cast<int>(src_size_stride[ndim - 1].stride / sizeof(src_type));
int idist = static_cast<int>(src_size_stride[0].stride / sizeof(src_type));
int ostride = static_cast<int>(dst_size_stride[ndim - 1].stride / sizeof(dst_type));
int odist = static_cast<int>(dst_size_stride[0].stride / sizeof(dst_type));
std::vector<int> n(rank), inembed(rank), onembed(rank);
for (int i = 0, j = axes.is_missing() ? 0 : 1; j < ndim; ++i, ++j) {
n[i] = static_cast<int>(src_size_stride[j].dim_size);
inembed[i] = static_cast<int>(src_size_stride[j].dim_size);
onembed[i] = static_cast<int>(dst_size_stride[j].dim_size);
}
int batch = static_cast<int>(axes.is_missing() ? 1 : src_size_stride[0].dim_size);
self_type *self = self_type::create(ckb, kernreq, ckb_offset);
cufftPlanMany(&self->plan, rank, n.data(), inembed.data(), istride, idist, onembed.data(), ostride, odist,
CUFFT_Z2Z, batch);
}
static void set(const ndt::type& paramtype, char *metadata, char *data, const nd::array& value) {
if (paramtype.get_type_id() == void_pointer_type_id) {
// TODO: switch to a better mechanism for passing nd::array references
*reinterpret_cast<const array_preamble **>(data) = value.get_ndo();
} else {
typed_data_assign(paramtype, metadata, data, value.get_type(), value.get_ndo_meta(), value.get_ndo()->m_data_pointer);
}
}
static nd::array function_ndo_strftime(const nd::array& n, const std::string& format) {
// TODO: Allow 'format' itself to be an array, with broadcasting, etc.
if (format.empty()) {
throw runtime_error("format string for strftime should not be empty");
}
return n.replace_dtype(ndt::make_unary_expr(ndt::make_string(), n.get_dtype(),
make_strftime_kernelgen(format)));
}
void dynd::typed_data_assign(const ndt::type &dst_tp, const char *dst_arrmeta,
char *dst_data, const nd::array &src_arr,
const eval::eval_context *ectx)
{
typed_data_assign(dst_tp, dst_arrmeta, dst_data, src_arr.get_type(),
src_arr.get_arrmeta(), src_arr.get_readonly_originptr(),
ectx);
}
const char *get_category_data_from_value(size_t value) const {
if (value >= get_category_count()) {
throw std::runtime_error("category value is out of bounds");
}
return m_categories.get_readonly_originptr() +
m_value_to_category_index[value] *
reinterpret_cast<const strided_dim_type_arrmeta *>(
m_categories.get_arrmeta())->stride;
}
static void linspace_specialization(float start, float stop, intptr_t count, nd::array& result)
{
intptr_t stride = result.get_strides()[0];
char *dst = result.get_readwrite_originptr();
for (intptr_t i = 0; i < count; ++i, dst += stride) {
double val = ((count - i - 1) * double(start) + i * double(stop)) / double(count - 1);
*reinterpret_cast<float *>(dst) = static_cast<float>(val);
}
}
static nd::array function_ndo_replace(const nd::array& n, int32_t year, int32_t month, int32_t day) {
// TODO: Allow 'year', 'month', and 'day' to be arrays, with broadcasting, etc.
if (year == numeric_limits<int32_t>::max() && month == numeric_limits<int32_t>::max() &&
day == numeric_limits<int32_t>::max()) {
throw std::runtime_error("no parameters provided to date.replace, should provide at least one");
}
return n.replace_dtype(ndt::make_unary_expr(ndt::make_date(), n.get_dtype(),
make_replace_kernelgen(year, month, day)));
}
nd::array dynd::nd::linspace(const nd::array& start, const nd::array& stop, intptr_t count)
{
ndt::type dt = promote_types_arithmetic(start.get_dtype(), stop.get_dtype());
// Make sure it's at least floating point
if (dt.get_kind() == bool_kind || dt.get_kind() == int_kind || dt.get_kind() == uint_kind) {
dt = ndt::make_type<double>();
}
return linspace(start, stop, count, dt);
}
static void linspace_specialization(double start, double stop, intptr_t count, nd::array& result)
{
intptr_t stride = result.get_strides()[0];
char *dst = result.get_readwrite_originptr();
for (intptr_t i = 0; i < count; ++i, dst += stride) {
double val = ((count - i - 1) * start + i * stop) / double(count - 1);
*reinterpret_cast<double *>(dst) = val;
}
}
static void linspace_specialization(dynd_complex<double> start, dynd_complex<double> stop, intptr_t count, nd::array& result)
{
intptr_t stride = result.get_strides()[0];
char *dst = result.get_readwrite_originptr();
for (intptr_t i = 0; i < count; ++i, dst += stride) {
dynd_complex<double> val = (double(count - i - 1) * dynd_complex<double>(start) +
double(i) * dynd_complex<double>(stop)) / double(count - 1);
*reinterpret_cast<dynd_complex<double> *>(dst) = val;
}
}
/**
* Substitutes the field types for contiguous array of types
*/
static std::vector<ndt::type> substitute_type_array(const nd::array &type_array,
const std::map<std::string, ndt::type> &typevars, bool concrete) {
intptr_t field_count = type_array.get_dim_size();
const ndt::type *field_types = reinterpret_cast<const ndt::type *>(type_array.cdata());
std::vector<ndt::type> tmp_field_types(field_count);
for (intptr_t i = 0; i < field_count; ++i) {
tmp_field_types[i] = ndt::substitute(field_types[i], typevars, concrete);
}
return tmp_field_types;
}
nd::array dynd::nd::linspace(const nd::array& start, const nd::array& stop, intptr_t count, const ndt::type& dt)
{
nd::array start_cleaned = start.ucast(dt).eval();
nd::array stop_cleaned = stop.ucast(dt).eval();
if (start_cleaned.is_scalar() && stop_cleaned.is_scalar()) {
return linspace(dt, start_cleaned.get_readonly_originptr(), stop_cleaned.get_readonly_originptr(), count);
} else {
throw runtime_error("dynd::linspace presently only supports scalar parameters");
}
}
const char *get_category_data_from_value(uint32_t value) const
{
if (value >= get_category_count()) {
throw std::runtime_error("category value is out of bounds");
}
return m_categories.get_readonly_originptr() +
unchecked_fixed_dim_get<intptr_t>(m_value_to_category_index,
value) *
reinterpret_cast<const fixed_dim_type_arrmeta *>(
m_categories.get_arrmeta())->stride;
}
uint32_t categorical_type::get_value_from_category(const nd::array& category) const
{
if (category.get_type() == m_category_tp) {
// If the type is right, get the category value directly
return get_value_from_category(category.get_arrmeta(), category.get_readonly_originptr());
} else {
// Otherwise convert to the correct type, then get the category value
nd::array c = nd::empty(m_category_tp);
c.val_assign(category);
return get_value_from_category(c.get_arrmeta(), c.get_readonly_originptr());
}
}
/**
* Substitutes the field types for contiguous array of types
*/
static nd::array substitute_type_array(const nd::array &type_array, const std::map<std::string, ndt::type> &typevars,
bool concrete)
{
intptr_t field_count = type_array.get_dim_size();
const ndt::type *field_types = reinterpret_cast<const ndt::type *>(type_array.cdata());
nd::array tmp_field_types(nd::empty(field_count, ndt::make_type()));
ndt::type *ftraw = reinterpret_cast<ndt::type *>(tmp_field_types.data());
for (intptr_t i = 0; i < field_count; ++i) {
ftraw[i] = ndt::substitute(field_types[i], typevars, concrete);
}
return tmp_field_types;
}
inline string broadcast_error_message(const nd::array& dst, const nd::array& src)
{
vector<intptr_t> dst_shape = dst.get_shape(), src_shape = src.get_shape();
stringstream ss;
ss << "cannot broadcast dynd array with type ";
ss << src.get_type() << " and shape ";
print_shape(ss, src_shape);
ss << " to type " << dst.get_type() << " and shape ";
print_shape(ss, dst_shape);
return ss.str();
}
dynd::busdate_type::busdate_type(busdate_roll_t roll, const bool *weekmask, const nd::array& holidays)
: base_type(busdate_type_id, datetime_kind, 4, 4, type_flag_scalar, 0, 0), m_roll(roll)
{
memcpy(m_workweek, weekmask, sizeof(m_workweek));
m_busdays_in_weekmask = 0;
for (int i = 0; i < 7; ++i) {
m_busdays_in_weekmask += weekmask[i] ? 1 : 0;
}
if (!holidays.is_empty()) {
nd::array hol = holidays.ucast(ndt::make_date()).eval_immutable();
// TODO: Make sure hol is contiguous and one-dimensional
m_holidays = hol;
}
}
const arrfunc_type *get_is_avail_arrfunc_type() const
{
return m_nafunc.get_type()
.extended<base_tuple_type>()
->get_field_type(0)
.extended<arrfunc_type>();
}
const arrfunc_type *get_assign_na_arrfunc_type() const
{
return m_nafunc.get_type()
.extended<base_tuple_type>()
->get_field_type(1)
.extended<arrfunc_type>();
}
static inline nd::array To(const nd::array &a) {
#ifdef DYND_CUDA
return a.to_host();
#else
return a;
#endif // DYND_CUDA
}
ndt::type dynd::ndt::factor_categorical(const nd::array& values)
{
// Do the factor operation on a concrete version of the values
// TODO: Some cases where we don't want to do this?
nd::array values_eval = values.eval();
array_iter<0, 1> iter(values_eval);
comparison_ckernel_builder k;
::make_comparison_kernel(&k, 0,
iter.get_uniform_dtype(), iter.arrmeta(),
iter.get_uniform_dtype(), iter.arrmeta(),
comparison_type_sorting_less, &eval::default_eval_context);
cmp less(k.get_function(), k.get());
set<const char *, cmp> uniques(less);
if (!iter.empty()) {
do {
if (uniques.find(iter.data()) == uniques.end()) {
uniques.insert(iter.data());
}
} while (iter.next());
}
// Copy the values (now sorted and unique) into a new nd::array
nd::array categories = make_sorted_categories(uniques,
iter.get_uniform_dtype(), iter.arrmeta());
return ndt::type(new categorical_type(categories, true), false);
}
uint32_t ndt::categorical_type::get_value_from_category(const nd::array &category) const
{
nd::array c;
if (category.get_type() == m_category_tp) {
// If the type is right, get the category value directly
c = category;
}
else {
// Otherwise convert to the correct type, then get the category value
c = nd::empty(m_category_tp);
c.assign(category);
}
intptr_t i = nd::binary_search(m_categories, c).as<intptr_t>();
if (i < 0) {
stringstream ss;
ss << "Unrecognized category value ";
m_category_tp.print_data(ss, c.get()->metadata(), c.data());
ss << " assigning to dynd type " << type(this, true);
throw std::runtime_error(ss.str());
}
else {
return (uint32_t)unchecked_fixed_dim_get<intptr_t>(m_category_index_to_value, i);
}
}
ndt::type ndt::factor_categorical(const nd::array &values)
{
// Do the factor operation on a concrete version of the values
// TODO: Some cases where we don't want to do this?
nd::array values_eval = values.eval();
intptr_t dim_size, stride;
type el_tp;
const char *el_arrmeta;
values_eval.get_type().get_as_strided(values_eval.get_arrmeta(), &dim_size,
&stride, &el_tp, &el_arrmeta);
ckernel_builder<kernel_request_host> k;
::make_comparison_kernel(&k, 0, el_tp, el_arrmeta, el_tp, el_arrmeta,
comparison_type_sorting_less,
&eval::default_eval_context);
expr_single_t fn = k.get()->get_function<expr_single_t>();
cmp less(fn, k.get());
set<const char *, cmp> uniques(less);
for (intptr_t i = 0; i < dim_size; ++i) {
const char *data = values_eval.get_readonly_originptr() + i * stride;
if (uniques.find(data) == uniques.end()) {
uniques.insert(data);
}
}
// Copy the values (now sorted and unique) into a new nd::array
nd::array categories = make_sorted_categories(uniques, el_tp, el_arrmeta);
return type(new categorical_type(categories, true), false);
}
ndt::type ndt::factor_categorical(const nd::array &values)
{
// Do the factor operation on a concrete version of the values
// TODO: Some cases where we don't want to do this?
nd::array values_eval = values.eval();
intptr_t dim_size, stride;
type el_tp;
const char *el_arrmeta;
values_eval.get_type().get_as_strided(values_eval.get()->metadata(), &dim_size, &stride, &el_tp, &el_arrmeta);
nd::kernel_builder k;
kernel_single_t fn = k.get()->get_function<kernel_single_t>();
cmp less(fn, k.get());
set<const char *, cmp> uniques(less);
for (intptr_t i = 0; i < dim_size; ++i) {
const char *data = values_eval.cdata() + i * stride;
if (uniques.find(data) == uniques.end()) {
uniques.insert(data);
}
}
// Copy the values (now sorted and unique) into a new nd::array
nd::array categories = make_sorted_categories(uniques, el_tp, el_arrmeta);
return type(new categorical_type(categories, true), false);
}