static nd::array function_type_construct(const ndt::type& DYND_UNUSED(dt), const nd::array& year, const nd::array& month, const nd::array& day) { // TODO proper buffering nd::array year_as_int = year.ucast(ndt::make_type<int32_t>()).eval(); nd::array month_as_int = month.ucast(ndt::make_type<int32_t>()).eval(); nd::array day_as_int = day.ucast(ndt::make_type<int32_t>()).eval(); nd::array result; array_iter<1,3> iter(ndt::make_date(), result, year_as_int, month_as_int, day_as_int); if (!iter.empty()) { datetime::date_ymd ymd; do { ymd.year = *reinterpret_cast<const int32_t *>(iter.data<1>()); ymd.month = *reinterpret_cast<const int32_t *>(iter.data<2>()); ymd.day = *reinterpret_cast<const int32_t *>(iter.data<3>()); if (!datetime::is_valid_ymd(ymd)) { stringstream ss; ss << "invalid year/month/day " << ymd.year << "/" << ymd.month << "/" << ymd.day; throw runtime_error(ss.str()); } *reinterpret_cast<int32_t *>(iter.data<0>()) = datetime::ymd_to_days(ymd); } while (iter.next()); } return result; }
inline static bool run(nd::array &a) { const ndt::type &tp = a.get_type(); if (a.is_immutable() && tp.get_type_id() == fixed_dim_type_id) { // It's immutable and "N * <something>" const ndt::type &et = tp.extended<fixed_dim_type>()->get_element_type(); const fixed_dim_type_arrmeta *md = reinterpret_cast<const fixed_dim_type_arrmeta *>(a.get_arrmeta()); if (et.get_type_id() == type_type_id && md->stride == sizeof(ndt::type)) { // It also has the right type and is contiguous, // so no modification necessary. return true; } } // We have to make a copy, check that it's a 1D array, and that // it has the same array kind as the requested type. if (tp.get_ndim() == 1) { // It's a 1D array const ndt::type &et = tp.get_type_at_dimension(NULL, 1).value_type(); if (et.get_type_id() == type_type_id) { // It also has the same array type as requested nd::array tmp = nd::empty(a.get_dim_size(), ndt::make_type()); tmp.vals() = a; tmp.flag_as_immutable(); a.swap(tmp); return true; } } // It's not compatible, so return false return false; }
nd::array dynd::format_json(const nd::array& n) { // Create a UTF-8 string nd::array result = nd::empty(ndt::make_string()); // Initialize the output with some memory output_data out; out.blockref = reinterpret_cast<const string_type_metadata *>(result.get_ndo_meta())->blockref; out.api = get_memory_block_pod_allocator_api(out.blockref); out.api->allocate(out.blockref, 1024, 1, &out.out_begin, &out.out_capacity_end); out.out_end = out.out_begin; if (!n.get_type().is_expression()) { ::format_json(out, n.get_type(), n.get_ndo_meta(), n.get_readonly_originptr()); } else { nd::array tmp = n.eval(); ::format_json(out, tmp.get_type(), tmp.get_ndo_meta(), tmp.get_readonly_originptr()); } // Shrink the memory to fit, and set the pointers in the output string_type_data *d = reinterpret_cast<string_type_data *>(result.get_readwrite_originptr()); d->begin = out.out_begin; d->end = out.out_capacity_end; out.api->resize(out.blockref, out.out_end - out.out_begin, &d->begin, &d->end); // Finalize processing and mark the result as immutable result.get_type().extended()->metadata_finalize_buffers(result.get_ndo_meta()); result.flag_as_immutable(); return result; }
/** * This function promotes the requested `axis` from * a strided dim to a var dim. It modifies `shape`, `coord`, * `elem`, and `arr` to point to a new array, and * copies the data over. */ static void promote_nd_arr_dim(std::vector<intptr_t> &shape, std::vector<afpd_coordentry> &coord, afpd_dtype &elem, nd::array &arr, intptr_t axis, bool copy_final_coord) { vector<afpd_coordentry> newcoord; afpd_dtype newelem; newelem.dtp = elem.dtp; // Convert the axis into a var dim shape[axis] = -1; // Create the new array nd::array newarr = allocate_nd_arr(shape, newcoord, newelem, axis); // Copy the data up to, but not including, the current `coord` // from the old `arr` to the new one. The recursion stops // at `axis`, where all subsequent dimensions are handled by the // created kernel. ckernel_builder<kernel_request_host> k; if (elem.dtp.get_type_id() != uninitialized_type_id) { make_assignment_kernel(&k, 0, newcoord[axis].tp, newcoord[axis].arrmeta_ptr, coord[axis].tp, coord[axis].arrmeta_ptr, kernel_request_strided, &eval::default_eval_context); } copy_to_promoted_nd_arr(shape, newarr.get_readwrite_originptr(), newcoord, newelem, arr.get_readonly_originptr(), coord, elem, k, 0, axis, copy_final_coord, true); arr.swap(newarr); coord.swap(newcoord); elem.swap(newelem); }
void dynd::parse_json(nd::array &out, const char *json_begin, const char *json_end, const eval::eval_context *ectx) { try { const char *begin = json_begin, *end = json_end; ndt::type tp = out.get_type(); ::parse_json(tp, out.get_ndo_meta(), out.get_readwrite_originptr(), begin, end, ectx); begin = skip_whitespace(begin, end); if (begin != end) { throw json_parse_error(begin, "unexpected trailing JSON text", tp); } } catch (const json_parse_error& e) { stringstream ss; string line_prev, line_cur; int line, column; get_error_line_column(json_begin, json_end, e.get_position(), line_prev, line_cur, line, column); ss << "Error parsing JSON at line " << line << ", column " << column << "\n"; if (e.get_type().get_type_id() != uninitialized_type_id) { ss << "DType: " << e.get_type() << "\n"; } ss << "Message: " << e.get_message() << "\n"; print_json_parse_error_marker(ss, line_prev, line_cur, line, column); throw runtime_error(ss.str()); } }
nd::array dynd::format_json(const nd::array &n, bool struct_as_list) { // Create a UTF-8 string nd::array result = nd::empty(ndt::string_type::make()); // Initialize the output with some memory output_data out; out.out_string.resize(1024); out.out_begin = out.out_string.begin(); out.out_capacity_end = out.out_string.end(); out.out_end = out.out_begin; out.struct_as_list = struct_as_list; if (!n.get_type().is_expression()) { ::format_json(out, n.get_type(), n.get_arrmeta(), n.get_readonly_originptr()); } else { nd::array tmp = n.eval(); ::format_json(out, tmp.get_type(), tmp.get_arrmeta(), tmp.get_readonly_originptr()); } // Shrink the memory to fit, and set the pointers in the output string *d = reinterpret_cast<string *>(result.get_readwrite_originptr()); d->assign(out.out_string.data(), out.out_end - out.out_begin); // Finalize processing and mark the result as immutable result.get_type().extended()->arrmeta_finalize_buffers(result.get_arrmeta()); result.flag_as_immutable(); return result; }
static void instantiate(const callable_type_data *DYND_UNUSED(self), const callable_type *DYND_UNUSED(self_tp), kernel_builder *ckb, intptr_t ckb_offset, const ndt::type &DYND_UNUSED(dst_tp), const char *dst_arrmeta, intptr_t DYND_UNUSED(nsrc), const ndt::type *src_tp, const char *const *src_arrmeta, kernel_request_t kernreq, const eval::eval_context *DYND_UNUSED(ectx), const nd::array &kwds, const std::map<std::string, ndt::type> &DYND_UNUSED(tp_vars)) { const size_stride_t *dst_size_stride = reinterpret_cast<const size_stride_t *>(dst_arrmeta); const size_stride_t *src_size_stride = reinterpret_cast<const size_stride_t *>(src_arrmeta[0]); array axes = kwds.p("axes"); array shape = kwds.p("shape"); int ndim = static_cast<int>(src_tp[0].get_ndim()); int rank = static_cast<int>(axes.is_missing() ? ndim : (ndim - 1)); int istride = static_cast<int>(src_size_stride[ndim - 1].stride / sizeof(src_type)); int idist = static_cast<int>(src_size_stride[0].stride / sizeof(src_type)); int ostride = static_cast<int>(dst_size_stride[ndim - 1].stride / sizeof(dst_type)); int odist = static_cast<int>(dst_size_stride[0].stride / sizeof(dst_type)); std::vector<int> n(rank), inembed(rank), onembed(rank); for (int i = 0, j = axes.is_missing() ? 0 : 1; j < ndim; ++i, ++j) { n[i] = static_cast<int>(src_size_stride[j].dim_size); inembed[i] = static_cast<int>(src_size_stride[j].dim_size); onembed[i] = static_cast<int>(dst_size_stride[j].dim_size); } int batch = static_cast<int>(axes.is_missing() ? 1 : src_size_stride[0].dim_size); self_type *self = self_type::create(ckb, kernreq, ckb_offset); cufftPlanMany(&self->plan, rank, n.data(), inembed.data(), istride, idist, onembed.data(), ostride, odist, CUFFT_Z2Z, batch); }
static void set(const ndt::type& paramtype, char *metadata, char *data, const nd::array& value) { if (paramtype.get_type_id() == void_pointer_type_id) { // TODO: switch to a better mechanism for passing nd::array references *reinterpret_cast<const array_preamble **>(data) = value.get_ndo(); } else { typed_data_assign(paramtype, metadata, data, value.get_type(), value.get_ndo_meta(), value.get_ndo()->m_data_pointer); } }
static nd::array function_ndo_strftime(const nd::array& n, const std::string& format) { // TODO: Allow 'format' itself to be an array, with broadcasting, etc. if (format.empty()) { throw runtime_error("format string for strftime should not be empty"); } return n.replace_dtype(ndt::make_unary_expr(ndt::make_string(), n.get_dtype(), make_strftime_kernelgen(format))); }
void dynd::typed_data_assign(const ndt::type &dst_tp, const char *dst_arrmeta, char *dst_data, const nd::array &src_arr, const eval::eval_context *ectx) { typed_data_assign(dst_tp, dst_arrmeta, dst_data, src_arr.get_type(), src_arr.get_arrmeta(), src_arr.get_readonly_originptr(), ectx); }
const char *get_category_data_from_value(size_t value) const { if (value >= get_category_count()) { throw std::runtime_error("category value is out of bounds"); } return m_categories.get_readonly_originptr() + m_value_to_category_index[value] * reinterpret_cast<const strided_dim_type_arrmeta *>( m_categories.get_arrmeta())->stride; }
static void linspace_specialization(float start, float stop, intptr_t count, nd::array& result) { intptr_t stride = result.get_strides()[0]; char *dst = result.get_readwrite_originptr(); for (intptr_t i = 0; i < count; ++i, dst += stride) { double val = ((count - i - 1) * double(start) + i * double(stop)) / double(count - 1); *reinterpret_cast<float *>(dst) = static_cast<float>(val); } }
static nd::array function_ndo_replace(const nd::array& n, int32_t year, int32_t month, int32_t day) { // TODO: Allow 'year', 'month', and 'day' to be arrays, with broadcasting, etc. if (year == numeric_limits<int32_t>::max() && month == numeric_limits<int32_t>::max() && day == numeric_limits<int32_t>::max()) { throw std::runtime_error("no parameters provided to date.replace, should provide at least one"); } return n.replace_dtype(ndt::make_unary_expr(ndt::make_date(), n.get_dtype(), make_replace_kernelgen(year, month, day))); }
nd::array dynd::nd::linspace(const nd::array& start, const nd::array& stop, intptr_t count) { ndt::type dt = promote_types_arithmetic(start.get_dtype(), stop.get_dtype()); // Make sure it's at least floating point if (dt.get_kind() == bool_kind || dt.get_kind() == int_kind || dt.get_kind() == uint_kind) { dt = ndt::make_type<double>(); } return linspace(start, stop, count, dt); }
static void linspace_specialization(double start, double stop, intptr_t count, nd::array& result) { intptr_t stride = result.get_strides()[0]; char *dst = result.get_readwrite_originptr(); for (intptr_t i = 0; i < count; ++i, dst += stride) { double val = ((count - i - 1) * start + i * stop) / double(count - 1); *reinterpret_cast<double *>(dst) = val; } }
static void linspace_specialization(dynd_complex<double> start, dynd_complex<double> stop, intptr_t count, nd::array& result) { intptr_t stride = result.get_strides()[0]; char *dst = result.get_readwrite_originptr(); for (intptr_t i = 0; i < count; ++i, dst += stride) { dynd_complex<double> val = (double(count - i - 1) * dynd_complex<double>(start) + double(i) * dynd_complex<double>(stop)) / double(count - 1); *reinterpret_cast<dynd_complex<double> *>(dst) = val; } }
/** * Substitutes the field types for contiguous array of types */ static std::vector<ndt::type> substitute_type_array(const nd::array &type_array, const std::map<std::string, ndt::type> &typevars, bool concrete) { intptr_t field_count = type_array.get_dim_size(); const ndt::type *field_types = reinterpret_cast<const ndt::type *>(type_array.cdata()); std::vector<ndt::type> tmp_field_types(field_count); for (intptr_t i = 0; i < field_count; ++i) { tmp_field_types[i] = ndt::substitute(field_types[i], typevars, concrete); } return tmp_field_types; }
nd::array dynd::nd::linspace(const nd::array& start, const nd::array& stop, intptr_t count, const ndt::type& dt) { nd::array start_cleaned = start.ucast(dt).eval(); nd::array stop_cleaned = stop.ucast(dt).eval(); if (start_cleaned.is_scalar() && stop_cleaned.is_scalar()) { return linspace(dt, start_cleaned.get_readonly_originptr(), stop_cleaned.get_readonly_originptr(), count); } else { throw runtime_error("dynd::linspace presently only supports scalar parameters"); } }
const char *get_category_data_from_value(uint32_t value) const { if (value >= get_category_count()) { throw std::runtime_error("category value is out of bounds"); } return m_categories.get_readonly_originptr() + unchecked_fixed_dim_get<intptr_t>(m_value_to_category_index, value) * reinterpret_cast<const fixed_dim_type_arrmeta *>( m_categories.get_arrmeta())->stride; }
uint32_t categorical_type::get_value_from_category(const nd::array& category) const { if (category.get_type() == m_category_tp) { // If the type is right, get the category value directly return get_value_from_category(category.get_arrmeta(), category.get_readonly_originptr()); } else { // Otherwise convert to the correct type, then get the category value nd::array c = nd::empty(m_category_tp); c.val_assign(category); return get_value_from_category(c.get_arrmeta(), c.get_readonly_originptr()); } }
/** * Substitutes the field types for contiguous array of types */ static nd::array substitute_type_array(const nd::array &type_array, const std::map<std::string, ndt::type> &typevars, bool concrete) { intptr_t field_count = type_array.get_dim_size(); const ndt::type *field_types = reinterpret_cast<const ndt::type *>(type_array.cdata()); nd::array tmp_field_types(nd::empty(field_count, ndt::make_type())); ndt::type *ftraw = reinterpret_cast<ndt::type *>(tmp_field_types.data()); for (intptr_t i = 0; i < field_count; ++i) { ftraw[i] = ndt::substitute(field_types[i], typevars, concrete); } return tmp_field_types; }
inline string broadcast_error_message(const nd::array& dst, const nd::array& src) { vector<intptr_t> dst_shape = dst.get_shape(), src_shape = src.get_shape(); stringstream ss; ss << "cannot broadcast dynd array with type "; ss << src.get_type() << " and shape "; print_shape(ss, src_shape); ss << " to type " << dst.get_type() << " and shape "; print_shape(ss, dst_shape); return ss.str(); }
dynd::busdate_type::busdate_type(busdate_roll_t roll, const bool *weekmask, const nd::array& holidays) : base_type(busdate_type_id, datetime_kind, 4, 4, type_flag_scalar, 0, 0), m_roll(roll) { memcpy(m_workweek, weekmask, sizeof(m_workweek)); m_busdays_in_weekmask = 0; for (int i = 0; i < 7; ++i) { m_busdays_in_weekmask += weekmask[i] ? 1 : 0; } if (!holidays.is_empty()) { nd::array hol = holidays.ucast(ndt::make_date()).eval_immutable(); // TODO: Make sure hol is contiguous and one-dimensional m_holidays = hol; } }
const arrfunc_type *get_is_avail_arrfunc_type() const { return m_nafunc.get_type() .extended<base_tuple_type>() ->get_field_type(0) .extended<arrfunc_type>(); }
const arrfunc_type *get_assign_na_arrfunc_type() const { return m_nafunc.get_type() .extended<base_tuple_type>() ->get_field_type(1) .extended<arrfunc_type>(); }
static inline nd::array To(const nd::array &a) { #ifdef DYND_CUDA return a.to_host(); #else return a; #endif // DYND_CUDA }
ndt::type dynd::ndt::factor_categorical(const nd::array& values) { // Do the factor operation on a concrete version of the values // TODO: Some cases where we don't want to do this? nd::array values_eval = values.eval(); array_iter<0, 1> iter(values_eval); comparison_ckernel_builder k; ::make_comparison_kernel(&k, 0, iter.get_uniform_dtype(), iter.arrmeta(), iter.get_uniform_dtype(), iter.arrmeta(), comparison_type_sorting_less, &eval::default_eval_context); cmp less(k.get_function(), k.get()); set<const char *, cmp> uniques(less); if (!iter.empty()) { do { if (uniques.find(iter.data()) == uniques.end()) { uniques.insert(iter.data()); } } while (iter.next()); } // Copy the values (now sorted and unique) into a new nd::array nd::array categories = make_sorted_categories(uniques, iter.get_uniform_dtype(), iter.arrmeta()); return ndt::type(new categorical_type(categories, true), false); }
uint32_t ndt::categorical_type::get_value_from_category(const nd::array &category) const { nd::array c; if (category.get_type() == m_category_tp) { // If the type is right, get the category value directly c = category; } else { // Otherwise convert to the correct type, then get the category value c = nd::empty(m_category_tp); c.assign(category); } intptr_t i = nd::binary_search(m_categories, c).as<intptr_t>(); if (i < 0) { stringstream ss; ss << "Unrecognized category value "; m_category_tp.print_data(ss, c.get()->metadata(), c.data()); ss << " assigning to dynd type " << type(this, true); throw std::runtime_error(ss.str()); } else { return (uint32_t)unchecked_fixed_dim_get<intptr_t>(m_category_index_to_value, i); } }
ndt::type ndt::factor_categorical(const nd::array &values) { // Do the factor operation on a concrete version of the values // TODO: Some cases where we don't want to do this? nd::array values_eval = values.eval(); intptr_t dim_size, stride; type el_tp; const char *el_arrmeta; values_eval.get_type().get_as_strided(values_eval.get_arrmeta(), &dim_size, &stride, &el_tp, &el_arrmeta); ckernel_builder<kernel_request_host> k; ::make_comparison_kernel(&k, 0, el_tp, el_arrmeta, el_tp, el_arrmeta, comparison_type_sorting_less, &eval::default_eval_context); expr_single_t fn = k.get()->get_function<expr_single_t>(); cmp less(fn, k.get()); set<const char *, cmp> uniques(less); for (intptr_t i = 0; i < dim_size; ++i) { const char *data = values_eval.get_readonly_originptr() + i * stride; if (uniques.find(data) == uniques.end()) { uniques.insert(data); } } // Copy the values (now sorted and unique) into a new nd::array nd::array categories = make_sorted_categories(uniques, el_tp, el_arrmeta); return type(new categorical_type(categories, true), false); }
ndt::type ndt::factor_categorical(const nd::array &values) { // Do the factor operation on a concrete version of the values // TODO: Some cases where we don't want to do this? nd::array values_eval = values.eval(); intptr_t dim_size, stride; type el_tp; const char *el_arrmeta; values_eval.get_type().get_as_strided(values_eval.get()->metadata(), &dim_size, &stride, &el_tp, &el_arrmeta); nd::kernel_builder k; kernel_single_t fn = k.get()->get_function<kernel_single_t>(); cmp less(fn, k.get()); set<const char *, cmp> uniques(less); for (intptr_t i = 0; i < dim_size; ++i) { const char *data = values_eval.cdata() + i * stride; if (uniques.find(data) == uniques.end()) { uniques.insert(data); } } // Copy the values (now sorted and unique) into a new nd::array nd::array categories = make_sorted_categories(uniques, el_tp, el_arrmeta); return type(new categorical_type(categories, true), false); }