inline ssize_t offset(int x, int y) const
{
const Py_ssize_t* m = strides();
assert((Py_ssize_t)x<shape()[0]);
assert((Py_ssize_t)y<shape()[1]);
return NUMPY_HPP_OFFSET2;
}
PyObject *
convertToNumPyArray(const Domi::MDArrayView< T > & mdArrayView)
{
// Get the number of dimensions and initialize the dimensions and
// strides arrays
int numDims = mdArrayView.numDims();
Teuchos::Array< npy_intp > dims(numDims);
Teuchos::Array< npy_intp > strides(numDims);
int typecode = NumPy_TypeCode< T >();
// Set the dimensions and strides
for (int axis = 0; axis < numDims; ++axis)
{
dims[ axis] = mdArrayView.dimension(axis);
strides[axis] = mdArrayView.strides()[axis];
}
// Get the data pointer and flags, based on data layout
void * data = (void*) mdArrayView.getRawPtr();
int flags = (mdArrayView.layout() == Domi::C_ORDER) ? NPY_ARRAY_CARRAY :
NPY_ARRAY_FARRAY;
// Return the result
return PyArray_New(&PyArray_Type,
numDims,
dims.getRawPtr(),
typecode,
strides.getRawPtr(),
data,
0,
flags,
NULL);
}
static blitz::Array<T,1> np_to_blitz_1d(
PyObject *ovec,
std::string const &vname,
std::array<int,1> dims)
{
// Check that it's type PyArrayObject
if (!PyArray_Check(ovec)) {
(*icebin_error)(-1,
"check_dimensions: Object %s is not a Numpy array", vname.c_str());
}
PyArrayObject *vec = (PyArrayObject *)ovec;
// Set up shape and strides
int const T_size = sizeof(T);
size_t len = 1;
auto min_stride = PyArray_STRIDE(vec, 0);
for (int i=0;i<PyArray_NDIM(vec); ++i) {
len *= PyArray_DIM(vec, i);
// Python/Numpy strides are in bytes, Blitz++ in sizeof(T) units.
min_stride = std::min(min_stride, PyArray_STRIDE(vec, i) / T_size);
}
assert(T_size == PyArray_ITEMSIZE(vec));
blitz::TinyVector<int,1> shape(0);
shape[0] = len;
blitz::TinyVector<int,1> strides(0);
strides[0] = min_stride;
return blitz::Array<T,1>((T*) PyArray_DATA(vec),shape,strides,
blitz::neverDeleteData);
}
std::vector<unsigned int> VertexBufferGroup::CreateStrides(const VertexBufferVec& buffers)
{
std::vector<unsigned int> strides(buffers.size());
for(const auto& vertexBuffer : buffers)
strides.push_back(vertexBuffer->GetBufferStride());
return strides;
}
inline ssize_t offset(int x,int y, int z, int w) const
{
const Py_ssize_t* m = strides();
assert((Py_ssize_t)x<shape()[0]);
assert((Py_ssize_t)y<shape()[1]);
assert((Py_ssize_t)z<shape()[2]);
assert((Py_ssize_t)w<shape()[3]);
return NUMPY_HPP_OFFSET4;
}
inline bool array_is_c_contiguous(const dynd::nd::array &n)
{
intptr_t ndim = n.get_ndim();
dynd::dimvector shape(ndim), strides(ndim);
n.get_shape(shape.get());
n.get_strides(strides.get());
return dynd::strides_are_c_contiguous(ndim, n.get_dtype().get_data_size(),
shape.get(), strides.get());
}
static std::vector<int64_t> defaultStrides(IntList sizes) {
std::vector<int64_t> strides(sizes.size());
int64_t stride = 1;
for(size_t i = sizes.size(); i > 0; --i) {
strides[i-1] = stride;
stride *= sizes[i-1];
}
return strides;
}
Node_ptr Array<T>::getNode() const {
if (node->isBuffer()) {
BufferNode<T> *bufNode = reinterpret_cast<BufferNode<T> *>(node.get());
unsigned bytes = this->getDataDims().elements() * sizeof(T);
bufNode->setData(data, bytes, getOffset(), dims().get(),
strides().get(), isLinear());
}
return node;
}
operator Param<T>()
{
Param<T> out;
out.ptr = this->get();
for (int i = 0; i < 4; i++) {
out.dims[i] = dims()[i];
out.strides[i] = strides()[i];
}
return out;
}
blitz::Array<T,1> vector_to_blitz(std::vector<T> &vec)
{
blitz::TinyVector<int,1> shape(0);
blitz::TinyVector<int,1> strides(0);
shape[0] = vec.size();
strides[0] = 1; // Blitz++ strides in sizeof(T) units
return blitz::Array<T,1>(&vec[0], shape, strides,
blitz::neverDeleteData);
}
IndexMapType indexmap() const {
// extract strides and lengths
const size_t dim =arr_obj->nd;
std::vector<size_t> lengths (dim);
std::vector<std::ptrdiff_t> strides(dim);
for (size_t i=0; i< dim ; ++i) {
lengths[i] = size_t(arr_obj-> dimensions[i]);
strides[i] = std::ptrdiff_t(arr_obj-> strides[i])/sizeof(ValueType);
}
return IndexMapType (mini_vector<size_t,IndexMapType::rank>(lengths), mini_vector<std::ptrdiff_t,IndexMapType::rank>(strides),0);
}
inline ssize_t offset(const int *c) const /// _c_ must point to an array of at least rank() items.
{
const Py_ssize_t* m = strides();
int r = rank();
Py_ssize_t q = 0;
for(int i=0; i<r; ++i)
{
assert((Py_ssize_t)(c[i])<shape()[i]);
q += c[i]*m[i];
}
return q;
}
blitz::Array<T,1> const vector_to_blitz(std::vector<T> const &vec)
{
blitz::TinyVector<int,1> shape(0);
blitz::TinyVector<int,1> strides(0);
shape[0] = vec.size();
strides[0] = 1; // Blitz++ strides in sizeof(T) units
// const_cast because Blitz++ can't construct a const Array
T *vecp = const_cast<T *>(&vec[0]);
return blitz::Array<T,1>(vecp, shape, strides,
blitz::neverDeleteData);
}
//Conversion part 2 (C++ -> Python)
static py::handle cast(const rd::Array2D<T>& src, py::return_value_policy policy, py::handle parent)
{
std::vector<size_t> shape (2);
std::vector<size_t> strides(2);
shape[0] = src.height();
shape[1] = src.width();
strides[0] = src.width();
strides[1] = strides[0]*sizeof(T);
py::array a(std::move(shape), std::move(strides), src.data() );
return a.release();
}
Node_ptr Array<T>::getNode() const
{
if (!node) {
unsigned bytes = this->getDataDims().elements() * sizeof(T);
BufferNode<T> *buf_node = new BufferNode<T>(data,
bytes,
offset,
dims().get(),
strides().get());
const_cast<Array<T> *>(this)->node = Node_ptr(reinterpret_cast<Node *>(buf_node));
}
return node;
}
static Tensor new_from_sequence(ScalarType scalarType, PyObject* data) {
if (!PySequence_Check(data)) {
throw TypeError("new(): data must be a sequence (got %s)", Py_TYPE(data)->tp_name);
}
if (THPUtils_checkString(data)) {
throw TypeError("new(): invalid data type '%s'", Py_TYPE(data)->tp_name);
}
#ifdef WITH_NUMPY
if (PyArray_Check(data)) {
return autograd::make_variable(tensor_from_numpy(data), false);
}
#endif
auto sizes = compute_sizes(data);
auto tensor = autograd::make_variable(CPU(scalarType).tensor(sizes), false);
recursive_store(
(char*)tensor.data_ptr(), tensor.sizes(), tensor.strides(), 0,
scalarType, tensor.type().elementSizeInBytes(), data);
return tensor;
}
Teuchos::Array< SIZE_TYPE >
computeStrides(const Teuchos::Array< DIM_TYPE > & dimensions,
const Layout layout)
{
int n = dimensions.size();
Teuchos::Array< SIZE_TYPE > strides(n);
if (n == 0) return strides;
if (layout == FIRST_INDEX_FASTEST)
{
strides[0] = 1;
for (int axis = 1; axis < n; ++axis)
strides[axis] = strides[axis-1] * dimensions[axis-1];
}
else
{
strides[n-1] = 1;
for (int axis = n-2; axis >= 0; --axis)
strides[axis] = strides[axis+1] * dimensions[axis+1];
}
return strides;
}
void Array<T>::eval()
{
if (isReady()) return;
this->setId(getActiveDeviceId());
data = std::shared_ptr<T>(memAlloc<T>(elements()), memFree<T>);
T *ptr = data.get();
dim4 ostrs = strides();
dim4 odims = dims();
for (int w = 0; w < (int)odims[3]; w++) {
dim_t offw = w * ostrs[3];
for (int z = 0; z < (int)odims[2]; z++) {
dim_t offz = z * ostrs[2] + offw;
for (int y = 0; y < (int)odims[1]; y++) {
dim_t offy = y * ostrs[1] + offz;
for (int x = 0; x < (int)odims[0]; x++) {
dim_t id = x + offy;
ptr[id] = *(T *)node->calc(x, y, z, w);
}
}
}
}
ready = true;
Node_ptr prev = node;
prev->reset();
// FIXME: Replace the current node in any JIT possible trees with the new BufferNode
node.reset();
}
unique_ptr<BaseSignal> ndarray_to_matrix(bpyn::array a){
int ndim = bpy::extract<int>(a.attr("ndim"));
if (ndim == 1){
int size = bpy::extract<int>(a.attr("size"));
auto ret = unique_ptr<BaseSignal>(new BaseSignal(size, 1));
for(unsigned i = 0; i < size; i++){
(*ret)(i, 0) = bpy::extract<dtype>(a[i]);
}
return ret;
}else{
int size = bpy::extract<int>(a.attr("size"));
vector<int> shape(ndim);
vector<int> strides(ndim);
bpy::object python_shape = a.attr("shape");
bpy::object python_strides = a.attr("strides");
for(unsigned i = 0; i < ndim; i++){
shape[i] = bpy::extract<int>(python_shape[i]);
strides[i] = bpy::extract<int>(python_strides[i]);
}
auto ret = unique_ptr<BaseSignal>(new BaseSignal(shape[0], shape[1]));
for(unsigned i = 0; i < shape[0]; i++){
for(unsigned j = 0; j < shape[1]; j++){
(*ret)(i, j) = bpy::extract<dtype>(a[i][j]);
}
}
return ret;
}
}
Domi::MDArrayRCP< T >
convertToMDArrayRCP(PyArrayObject * pyArray)
{
// Get the number of dimensions and initialize the dimensions and
// strides arrays
int numDims = PyArray_NDIM(pyArray);
Teuchos::Array< Domi::dim_type > dims( numDims);
Teuchos::Array< Domi::size_type > strides(numDims);
// Set the dimensions and strides
for (int axis = 0; axis < numDims; ++axis)
{
dims[ axis] = (Domi::dim_type ) PyArray_DIM( pyArray, axis);
strides[axis] = (Domi::size_type) PyArray_STRIDE(pyArray, axis);
}
// Get the data pointer and layout
T * data = (T*) PyArray_DATA(pyArray);
Domi::Layout layout = PyArray_IS_C_CONTIGUOUS(pyArray) ? Domi::C_ORDER :
Domi::FORTRAN_ORDER;
// Return the result
return Domi::MDArrayRCP< T >(dims, strides, data, layout);
}
void* VisusAxisAlignedExtractor::extractionLoop(void *)
{
VisusDataDescription dataset; // The name of the data set we are
// working on
VisusFieldIndex field_index; // The index of the current field we
// are extracting
VisusGlobalTime current_time; // The time of field we are extracting
VisusDataRequest request; // The current request
VisusDataSource *old_source;
int stride_index;
std::vector<int> strides(3);
std::vector<double> extent(3);
VisusBoundingBox bbox; // The domain bounding box of the data
VisusUnit unit; // The physical unit of the domain
VisusBoundingBox query_region;
// Make sure the constructor of the thread is finished
while (mThread == NULL)
sleepmillisec(sUpdateInterval);
// Indicate that we have started processing
mThread->status(WORKING);
// Until we get an outside signal to stop we continue
while (!mThread->terminated()) {
////////////////////////////////////////////////////////////
////// Stage 1: Request Synchronization //////////
////////////////////////////////////////////////////////////
// Access the dataset, data request, and field index currently
// stored in our parameter list
getValue(mDataDescription);
getValue(mRequest);
getValue(mFieldIndex);
getValue(mCurrentTime);
// If the dataset has changed
if (mDataDescription != dataset) {
dataset = mDataDescription;
// store the out-dated data source
old_source = mDataSource;
// Indicate that we have not worked on this data at all
stride_index = -1;
// Create a new appropriate data source
mDataSource = VisusDataSourceFactory::make(dataset);
//sleepmillisec(100);
//fprintf(stderr,"Continue exraction\n");
// Store the data source specific domain attribute
unit = mDataSource->unit();
// Get the new domain box
mDataSource->domainBoundingBox(bbox);
// And make sure that our bouding box is up to date
this->setValue(bbox);
// Get the new time parameters
mCurrentTime = mDataSource->timeInfo();
// And update the shared parameter accordingly
setValue(mCurrentTime);
// Delete the now obsolete data source
delete old_source;
}
// If the current data source for some reason is not valid
if (!mDataSource->isValid()) {
// Indicate that we cannot work on anything
mStatus = VISUS_EXTRACTION_INVALID;
// Make clear that we have just verified our status
mSynchronizationFlag = false;
vmessage("Data source not valid extraction will hold.\nDataDescriptor:\n%s\n",dataset.c_str());
// Wait till hopefully new valid values come it
sleepmillisec(sUpdateInterval);
// and start over
continue;
}
// If we have a valid data source we want to make sure that the
// request reflects the current bounding box.
mRequest.domainBBox(bbox);
// Copy the data source dependent information into mData. Note
// that even though this information only changes when a new data
// source is opened the copy here is necessary. Currently, some
// data is passed on by swapping in which case the current unit
// and domain bounding box might be invalid for the new data we
// want to extract. Maybe for performance reasons this should be
// changed later.
mData.unit(unit);
// If the scalar function has changed
if (mFieldIndex != field_index) {
field_index = mFieldIndex;
stride_index = -1;
}
// If the field index is out of range
if ((field_index < 0) || (field_index >= mDataSource->numberOfFields())) {
// Indicate that we cannot work on anything
mStatus = VISUS_EXTRACTION_INVALID;
// Make clear that we have just verified our status
mSynchronizationFlag = false;
vmessage("Field index %d out of range for this data set, cannot extract data",(int)field_index);
// Wait till hopefully new valid values come it
sleepmillisec(sUpdateInterval);
// and start over
continue;
}
// If the current time requested has changed
if (mCurrentTime != current_time) {
current_time = mCurrentTime;
vverbose("VisusAxisAlignedExtractor - Time has changed!\n", VISUS_TIME_VERBOSE);
// Indicate that we have not worked on this data at all
stride_index = -1;
}
/*
// In the new implementation of VisusGlobalTime and the data
// source the data source will automatically clamp any illegal
// value to the closest valid one. Thus this test is no longer
// necessary. However, this might trigger a "busy" wait if the
// extractor continuously feeds incorrect data to the data
// source. If this ever becomes an issue (it shouldn't since the
// global shared time should reflect the data souce time) this
// code needs to be re-activated. (ptb 03/05/09)
// If the field index is out of range
if ((current_time < mDataSource->timeBegin()) ||
(current_time > mDataSource->timeEnd())) {
vwarning("Current time %s out of range for this data set, cannot extract data",current_time.time().toString().c_str());
// Indicate that we cannot work on anything
mStatus = VISUS_EXTRACTION_INVALID;
// Make clear that we have just verified our status
mSynchronizationFlag = false;
// Wait till hopefully new valid values come it
sleepmillisec(sUpdateInterval);
// and start over
continue;
}
*/
// If we have gotten a new request
if (mRequest != request) {
request = mRequest;
setValue(mRequest);
stride_index = -1;
}
// If the current request is not valid
if (!request.valid()) {
mStatus = VISUS_EXTRACTION_INVALID;
// Make clear that we have just verified our status
mSynchronizationFlag = false;
vwarning("DataRequest invalid cannot extract data");
sleepmillisec(sUpdateInterval);
continue;
}
////////////////////////////////////////////////////////////
////// Stage 2: Resolution Refinement //////////
////////////////////////////////////////////////////////////
// If we have already worked on the finest resolution requested
if (stride_index == request.numberOfResolutions()-1) {
// Indicate that we are done with the last request
mStatus = VISUS_EXTRACTION_FINISHED;
// Make clear that we have just verified our status
mSynchronizationFlag = false;
sleepmillisec(sUpdateInterval);
continue;
}
// Increase our target resolution
stride_index++;
// Set our status depending on whether we are starting or refining
// this request
if (stride_index == 0)
mStatus = VISUS_EXTRACTION_STARTED;
else
mStatus = VISUS_EXTRACTION_REFINING;
// Make clear that we just verified our status
mSynchronizationFlag = false;
strides = request.strides(stride_index);
////////////////////////////////////////////////////////////
////// Stage 4: Extracting Data //////////
////////////////////////////////////////////////////////////
// First we determined the actual bounding box of the query region
// Since this is an axis-aligned extractor we assume that the
// transformed extent is still axis aligned. Otherwise, we will
// inflate the bbox.
query_region = request.queryRegion();
/*
fprintf(stderr,"VisusAxisAlignedExtractor::extractionLoop accessing Data\n");
fprintf(stderr,"[%f,%f,%f] x [%f,%f,%f] at strides <%d,%d,%d>\n\n",
query_region.leftLower()[0],query_region.leftLower()[1],query_region.leftLower()[2],
query_region.rightUpper()[0],query_region.rightUpper()[1],query_region.rightUpper()[2],
strides[0],strides[1],strides[2]);
*/
// Access the data
if (mDataSource->accessAlignedData(query_region.leftLower(),query_region.rightUpper(),
request.startStrides(),request.endStrides(),strides,
field_index, current_time.time(),
mData, this->mProductLock) != 0) {
// If the data access was successful then we have extracted a
// valid piece of data and the mProductLock is *locked*
// If the data is supposed to be lower dimensional we
// potentially have to compact the dimensions and adjust the
// matrices
if (request.requestType() == VISUS_1D_REQUEST)
mData.compactDimensions(1);
else if (request.requestType() == VISUS_2D_REQUEST)
mData.compactDimensions(2);
if (this->mProductLock.unlock() == 0)
vwarning("Could not unlock data after loading new data.");
this->markAsDirty();
}
}
mThread->status(FINISHED);
return NULL;
}
void sortByKeyBatched(Array<Tk> okey, Array<Tv> oval, const int dim, bool isAscending)
{
af::dim4 inDims = okey.dims();
af::dim4 tileDims(1);
af::dim4 seqDims = inDims;
tileDims[dim] = inDims[dim];
seqDims[dim] = 1;
uint* key = memAlloc<uint>(inDims.elements());
// IOTA
{
af::dim4 dims = inDims;
uint* out = key;
af::dim4 strides(1);
for(int i = 1; i < 4; i++)
strides[i] = strides[i-1] * dims[i-1];
for(dim_t w = 0; w < dims[3]; w++) {
dim_t offW = w * strides[3];
uint okeyW = (w % seqDims[3]) * seqDims[0] * seqDims[1] * seqDims[2];
for(dim_t z = 0; z < dims[2]; z++) {
dim_t offWZ = offW + z * strides[2];
uint okeyZ = okeyW + (z % seqDims[2]) * seqDims[0] * seqDims[1];
for(dim_t y = 0; y < dims[1]; y++) {
dim_t offWZY = offWZ + y * strides[1];
uint okeyY = okeyZ + (y % seqDims[1]) * seqDims[0];
for(dim_t x = 0; x < dims[0]; x++) {
dim_t id = offWZY + x;
out[id] = okeyY + (x % seqDims[0]);
}
}
}
}
}
// initialize original index locations
Tk *okey_ptr = okey.get();
Tv *oval_ptr = oval.get();
typedef KeyIndexPair<Tk, Tv> CurrentTuple;
size_t size = okey.elements();
size_t bytes = okey.elements() * sizeof(CurrentTuple);
CurrentTuple *tupleKeyValIdx = (CurrentTuple *)memAlloc<char>(bytes);
for(unsigned i = 0; i < size; i++) {
tupleKeyValIdx[i] = std::make_tuple(okey_ptr[i], oval_ptr[i], key[i]);
}
memFree(key); // key is no longer required
if(isAscending) {
std::stable_sort(tupleKeyValIdx, tupleKeyValIdx + size, KIPCompareV<Tk, Tv, true>());
}
else {
std::stable_sort(tupleKeyValIdx, tupleKeyValIdx + size, KIPCompareV<Tk, Tv, false>());
}
std::stable_sort(tupleKeyValIdx, tupleKeyValIdx + size, KIPCompareK<Tk, Tv, true>());
for(unsigned x = 0; x < okey.elements(); x++) {
okey_ptr[x] = std::get<0>(tupleKeyValIdx[x]);
oval_ptr[x] = std::get<1>(tupleKeyValIdx[x]);
}
memFree((char *)tupleKeyValIdx);
return;
}
/** Returns the position within memory corresponding to the given integer coordinates. No range checking is
* performed in release builds. However access is protected by use of the assert macro.
* @sa ssize_t offset(int x, int y) const
* @sa ssize_t offset(int x, int y, int z) const
* @sa ssize_t offset(int x,int y, int z, int w) const
* @sa ssize_t offset(const int *c) const
*/
inline ssize_t offset(int x) const
{
const Py_ssize_t* m = strides();
assert((unsigned int)x<shape()[0]);
return NUMPY_HPP_OFFSET1;
}
