static void f2py_report_on_array_copy(PyArrayObject* arr) {
const long arr_size = PyArray_Size((PyObject *)arr);
if (arr_size>F2PY_REPORT_ON_ARRAY_COPY) {
fprintf(stderr,"copied an array: size=%ld, elsize=%d\n",
arr_size, PyArray_ITEMSIZE(arr));
}
}
NRT_adapt_ndarray_from_python(PyObject *obj, arystruct_t* arystruct) {
PyArrayObject *ndary;
int i, ndim;
npy_intp *p;
void *data;
if (!PyArray_Check(obj)) {
return -1;
}
ndary = (PyArrayObject*)obj;
ndim = PyArray_NDIM(ndary);
data = PyArray_DATA(ndary);
arystruct->meminfo = meminfo_new_from_pyobject((void*)data, obj);
arystruct->data = data;
arystruct->nitems = PyArray_SIZE(ndary);
arystruct->itemsize = PyArray_ITEMSIZE(ndary);
arystruct->parent = obj;
p = arystruct->shape_and_strides;
for (i = 0; i < ndim; i++, p++) {
*p = PyArray_DIM(ndary, i);
}
for (i = 0; i < ndim; i++, p++) {
*p = PyArray_STRIDE(ndary, i);
}
NRT_Debug(nrt_debug_print("NRT_adapt_ndarray_from_python %p\n",
arystruct->meminfo));
return 0;
}
static blitz::Array<T,1> np_to_blitz_1d(
PyObject *ovec,
std::string const &vname,
std::array<int,1> dims)
{
// Check that it's type PyArrayObject
if (!PyArray_Check(ovec)) {
(*icebin_error)(-1,
"check_dimensions: Object %s is not a Numpy array", vname.c_str());
}
PyArrayObject *vec = (PyArrayObject *)ovec;
// Set up shape and strides
int const T_size = sizeof(T);
size_t len = 1;
auto min_stride = PyArray_STRIDE(vec, 0);
for (int i=0;i<PyArray_NDIM(vec); ++i) {
len *= PyArray_DIM(vec, i);
// Python/Numpy strides are in bytes, Blitz++ in sizeof(T) units.
min_stride = std::min(min_stride, PyArray_STRIDE(vec, i) / T_size);
}
assert(T_size == PyArray_ITEMSIZE(vec));
blitz::TinyVector<int,1> shape(0);
shape[0] = len;
blitz::TinyVector<int,1> strides(0);
strides[0] = min_stride;
return blitz::Array<T,1>((T*) PyArray_DATA(vec),shape,strides,
blitz::neverDeleteData);
}
static PyObject * permute(PyObject * self,
PyObject * args, PyObject * kwds) {
static char * kwlist[] = {
"array", "index", NULL
};
PyArrayObject * array, * index, *out;
if(!PyArg_ParseTupleAndKeywords(args, kwds,
"O!O!", kwlist,
&PyArray_Type, &array,
&PyArray_Type, &index)) return NULL;
size_t n = PyArray_SIZE(index);
size_t * p = PyArray_DATA(index);
char * data = PyArray_DATA(array);
int itemsize = PyArray_ITEMSIZE(array);
size_t i, k, pk;
for (i = 0; i < n; i++)
{
k = p[i];
while (k > i)
k = p[k];
if (k < i)
continue ;
/* Now have k == i, i.e the least in its cycle */
pk = p[k];
if (pk == i)
continue ;
/* shuffle the elements of the cycle */
{
unsigned int a;
char t[itemsize];
memmove(t, & data[i * itemsize], itemsize);
while (pk != i)
{
memmove(&data[k * itemsize], & data[pk * itemsize], itemsize);
k = pk;
pk = p[k];
};
memmove(&data[k * itemsize], t, itemsize);
}
}
Py_INCREF(array);
return (PyObject*) array;
}
/* Gets a half-open range [start, end) which contains the array data */
static void
get_array_memory_extents(PyArrayObject *arr,
npy_uintp *out_start, npy_uintp *out_end,
npy_uintp *num_bytes)
{
npy_intp low, upper;
int j;
offset_bounds_from_strides(PyArray_ITEMSIZE(arr), PyArray_NDIM(arr),
PyArray_DIMS(arr), PyArray_STRIDES(arr),
&low, &upper);
*out_start = (npy_uintp)PyArray_DATA(arr) + (npy_uintp)low;
*out_end = (npy_uintp)PyArray_DATA(arr) + (npy_uintp)upper;
*num_bytes = PyArray_ITEMSIZE(arr);
for (j = 0; j < PyArray_NDIM(arr); ++j) {
*num_bytes *= PyArray_DIM(arr, j);
}
}
std::valarray<T> convert2valarray(boost::python::object arr)
{
import(); // ### WARNING: forgetting this will end up in segmentation fault!
std::size_t vec_size = PyArray_Size(arr.ptr());
T * data = (T *) PyArray_DATA(arr.ptr());
std::valarray<T> vec(vec_size);
memcpy(&vec[0],data, PyArray_ITEMSIZE((PyArrayObject*) arr.ptr()) * vec_size);
return vec;
}
boost::python::numeric::array convert2numpy(std::valarray<T> vec)
{
import(); // ### WARNING: forgetting this will end up in segmentation fault!
npy_intp arr_size= vec.size(); // ### NOTE: npy_intp is nothing but just signed size_t
boost::python::object obj(boost::python::handle<>(PyArray_SimpleNew(1, &arr_size, getEnum<T>()))); // ### NOTE: PyArray_SimpleNew is the new version of PyArray_FromDims
void *arr_data= PyArray_DATA((PyArrayObject*) obj.ptr());
memcpy(arr_data, &vec[0], PyArray_ITEMSIZE((PyArrayObject*) obj.ptr()) * arr_size);
return boost::python::extract<boost::python::numeric::array>(obj);
}
static Bool
_default_nonzero(void *ip, void *arr)
{
int elsize = PyArray_ITEMSIZE(arr);
char *ptr = ip;
while (elsize--) {
if (*ptr++ != 0) {
return TRUE;
}
}
return FALSE;
}
/* Fetch vector data from an iterator (view or copy) */
void _fff_vector_sync_with_PyArrayIter(fff_vector* y, const PyArrayIterObject* it, npy_intp axis)
{
if (y->owner) {
PyArrayObject* ao = (PyArrayObject*) it->ao;
COPY_BUFFER(y, PyArray_ITER_DATA(it), PyArray_STRIDE(ao, axis),
PyArray_TYPE(ao), PyArray_ITEMSIZE(ao));
}
else
y->data = (double*) PyArray_ITER_DATA(it);
return;
}
void local_histogram(double* H,
unsigned int clamp,
PyArrayIterObject* iter,
const unsigned int* size)
{
PyArrayObject *block, *im = iter->ao;
PyArrayIterObject* block_iter;
unsigned int i, left, right, center, halfsize, dim, offset=0;
npy_intp block_dims[3];
UPDATE_ITERATOR_COORDS(iter);
/* Compute block corners */
for (i=0; i<3; i++) {
center = iter->coordinates[i];
halfsize = size[i]/2;
dim = PyArray_DIM(im, i);
/* Left handside corner */
if (center<halfsize)
left = 0;
else
left = center-halfsize;
/* Right handside corner (plus one)*/
right = center+halfsize+1;
if (right>dim)
right = dim;
/* Block properties */
offset += left*PyArray_STRIDE(im, i);
block_dims[i] = right-left;
}
/* Create the block as a vew and the block iterator */
block = (PyArrayObject*)PyArray_New(&PyArray_Type, 3, block_dims,
PyArray_TYPE(im), PyArray_STRIDES(im),
(void*)(PyArray_DATA(im)+offset),
PyArray_ITEMSIZE(im),
NPY_BEHAVED, NULL);
block_iter = (PyArrayIterObject*)PyArray_IterNew((PyObject*)block);
/* Compute block histogram */
histogram(H, clamp, block_iter);
/* Free memory */
Py_XDECREF(block_iter);
Py_XDECREF(block);
return;
}
/* Create an fff_vector from a PyArrayIter object */
fff_vector* _fff_vector_new_from_PyArrayIter(const PyArrayIterObject* it, npy_intp axis)
{
fff_vector* y;
char* data = PyArray_ITER_DATA(it);
PyArrayObject* ao = (PyArrayObject*) it->ao;
npy_intp dim = PyArray_DIM(ao, axis);
npy_intp stride = PyArray_STRIDE(ao, axis);
int type = PyArray_TYPE(ao);
int itemsize = PyArray_ITEMSIZE(ao);
y = _fff_vector_new_from_buffer(data, dim, stride, type, itemsize);
return y;
}
static int
map_inc(PyArrayMapIterObject *mit, PyObject *op)
{
PyObject *arr = NULL;
PyArrayIterObject *it;
int index;
PyArray_Descr *descr;
/* Unbound Map Iterator */
if (mit->ait == NULL) {
return -1;
}
descr = mit->ait->ao->descr;
Py_INCREF(descr);
arr = PyArray_FromAny(op, descr, 0, 0, NPY_FORCECAST, NULL);
if (arr == NULL) {
return -1;
}
if ((mit->subspace != NULL) && (mit->consec)) {
if (mit->iteraxes[0] > 0) { /* then we need to swap */
_swap_axes(mit, (PyArrayObject **)&arr, 0);
if (arr == NULL) {
return -1;
}
}
}
/* Be sure values array is "broadcastable"
to shape of mit->dimensions, mit->nd */
if ((it = (PyArrayIterObject *)\
PyArray_BroadcastToShape(arr, mit->dimensions, mit->nd))==NULL) {
Py_DECREF(arr);
return -1;
}
index = mit->size;
PyArray_MapIterReset(mit);
while(index--) {
memmove(mit->dataptr, it->dataptr, PyArray_ITEMSIZE(arr));
*(double*)(mit->dataptr) = *(double*)(mit->dataptr) + *(double*)(it->dataptr);
PyArray_MapIterNext(mit);
PyArray_ITER_NEXT(it);
}
Py_DECREF(arr);
Py_DECREF(it);
return 0;
}
PyObject *cloneToNDArray(const ContainerType &cdata) {
npy_intp dims[1] = {cdata.size()};
int datatype = NDArrayTypeIndex<typename ContainerType::value_type>::typenum;
PyObject *nparray =
PyArray_NewFromDescr(&PyArray_Type, PyArray_DescrFromType(datatype),
1, // rank 1
dims, // Length in each dimension
NULL, NULL, 0, NULL);
void *arrayData = PyArray_DATA(nparray);
const void *data = cdata.data();
std::memcpy(arrayData, data, PyArray_ITEMSIZE(nparray) * dims[0]);
return (PyObject *)nparray;
}
/* This also makes sure that the data segment is aligned with
an itemsize address as well by returning one if not true.
*/
static int
_bad_strides(PyArrayObject *ap)
{
register int itemsize = PyArray_ITEMSIZE(ap);
register int i, N=PyArray_NDIM(ap);
register intp *strides = PyArray_STRIDES(ap);
if (((intp)(ap->data) % itemsize) != 0)
return 1;
for (i=0; i<N; i++) {
if ((strides[i] < 0) || (strides[i] % itemsize) != 0)
return 1;
}
return 0;
}
fff_vector* fff_vector_fromPyArray(const PyArrayObject* x)
{
fff_vector* y;
int ok;
npy_intp axis = _PyArray_main_axis(x, &ok);
if (!ok) {
FFF_ERROR("Input array is not a vector", EINVAL);
return NULL;
}
y = _fff_vector_new_from_buffer(PyArray_DATA(x),
PyArray_DIM(x, axis),
PyArray_STRIDE(x, axis),
PyArray_TYPE(x),
PyArray_ITEMSIZE(x));
return y;
}
NPY_NO_EXPORT int
_IsAligned(PyArrayObject *ap)
{
unsigned int i;
npy_uintp aligned;
npy_uintp alignment = PyArray_DESCR(ap)->alignment;
/* alignment 1 types should have a efficient alignment for copy loops */
if (PyArray_ISFLEXIBLE(ap) || PyArray_ISSTRING(ap)) {
npy_intp itemsize = PyArray_ITEMSIZE(ap);
/* power of two sizes may be loaded in larger moves */
if (((itemsize & (itemsize - 1)) == 0)) {
alignment = itemsize > NPY_MAX_COPY_ALIGNMENT ?
NPY_MAX_COPY_ALIGNMENT : itemsize;
}
else {
/* if not power of two it will be accessed bytewise */
alignment = 1;
}
}
if (alignment == 1) {
return 1;
}
aligned = (npy_uintp)PyArray_DATA(ap);
for (i = 0; i < PyArray_NDIM(ap); i++) {
#if NPY_RELAXED_STRIDES_CHECKING
/* skip dim == 1 as it is not required to have stride 0 */
if (PyArray_DIM(ap, i) > 1) {
/* if shape[i] == 1, the stride is never used */
aligned |= (npy_uintp)PyArray_STRIDES(ap)[i];
}
else if (PyArray_DIM(ap, i) == 0) {
/* an array with zero elements is always aligned */
return 1;
}
#else /* not NPY_RELAXED_STRIDES_CHECKING */
aligned |= (npy_uintp)PyArray_STRIDES(ap)[i];
#endif /* not NPY_RELAXED_STRIDES_CHECKING */
}
return npy_is_aligned((void *)aligned, alignment);
}
static int copy_object(PyArrayIterObject *itx, PyArrayNeighborhoodIterObject *niterx,
npy_intp *bounds,
PyObject **out)
{
npy_intp i, j;
npy_intp odims[NPY_MAXDIMS];
PyArrayObject *aout;
PyArray_CopySwapFunc *copyswap = PyArray_DESCR(itx->ao)->f->copyswap;
npy_int itemsize = PyArray_ITEMSIZE(itx->ao);
/*
* For each point in itx, copy the current neighborhood into an array which
* is appended at the output list
*/
for (i = 0; i < itx->size; ++i) {
PyArrayNeighborhoodIter_Reset(niterx);
for (j = 0; j < PyArray_NDIM(itx->ao); ++j) {
odims[j] = bounds[2 * j + 1] - bounds[2 * j] + 1;
}
aout = (PyArrayObject*)PyArray_SimpleNew(PyArray_NDIM(itx->ao), odims, NPY_OBJECT);
if (aout == NULL) {
return -1;
}
for (j = 0; j < niterx->size; ++j) {
copyswap(PyArray_BYTES(aout) + j * itemsize, niterx->dataptr, 0, NULL);
PyArrayNeighborhoodIter_Next(niterx);
}
PyList_Append(*out, (PyObject*)aout);
Py_DECREF(aout);
PyArray_ITER_NEXT(itx);
}
return 0;
}
static PyObject* initCentersKMpp(PyObject *self, PyObject *args) {
int k, centers_found, first_center_index, i, j, n_trials;
int some_not_done;
float d;
float dist_sum;
float sum;
Py_ssize_t dim, n_frames;
PyObject *ret_init_centers;
PyArrayObject *np_data;
PyObject *py_callback_result;
char *metric;
npy_intp dims[2];
int *taken_points;
int best_candidate = -1;
float best_potential = FLT_MAX;
int *next_center_candidates;
float *next_center_candidates_rand;
float *next_center_candidates_potential;
float *data, *init_centers;
float *buffer_a, *buffer_b;
void *arr_data;
float *squared_distances;
float (*distance)(float*, float*, size_t, float*, float*);
ret_init_centers = Py_BuildValue("");
py_callback_result = NULL;
np_data = NULL; metric = NULL; data = NULL;
init_centers = NULL; taken_points = NULL;
centers_found = 0; squared_distances = NULL;
buffer_a = NULL; buffer_b = NULL;
next_center_candidates = NULL;
next_center_candidates_rand = NULL;
next_center_candidates_potential = NULL;
dist_sum = 0.0;
#ifndef _KMEANS_INIT_RANDOM_SEED
#define _KMEANS_INIT_RANDOM_SEED
/* set random seed */
srand(time(NULL));
#endif
/* parse python input (np_data, metric, k) */
if (!PyArg_ParseTuple(args, "O!si", &PyArray_Type, &np_data, &metric, &k)) {
goto error;
}
n_frames = np_data->dimensions[0];
dim = np_data->dimensions[1];
data = PyArray_DATA(np_data);
/* number of trials before choosing the data point with the best potential */
n_trials = 2 + (int) log(k);
/* allocate space for the index giving away which point has already been used as a cluster center */
if(!(taken_points = (int*) calloc(n_frames, sizeof(int)))) { PyErr_NoMemory(); goto error; }
/* allocate space for the array holding the cluster centers to be returned */
if(!(init_centers = (float*) calloc(k * dim, sizeof(float)))) { PyErr_NoMemory(); goto error; }
/* allocate space for the array holding the squared distances to the assigned cluster centers */
if(!(squared_distances = (float*) calloc(n_frames, sizeof(float)))) { PyErr_NoMemory(); goto error; }
/* candidates allocations */
if(!(next_center_candidates = (int*) malloc(n_trials * sizeof(int)))) { PyErr_NoMemory(); goto error; }
if(!(next_center_candidates_rand = (float*) malloc(n_trials * sizeof(float)))) { PyErr_NoMemory(); goto error; }
if(!(next_center_candidates_potential = (float*) malloc(n_trials * sizeof(float)))) { PyErr_NoMemory(); goto error; }
/* parse and initialize metric */
if(strcmp(metric,"euclidean")==0) {
distance = euclidean_distance;
} /*else if(strcmp(metric,"minRMSD")==0) {
distance = minRMSD_distance;
buffer_a = malloc(dim*sizeof(float));
buffer_b = malloc(dim*sizeof(float));
if(!buffer_a || !buffer_b) { PyErr_NoMemory(); goto error; }
} */
else {
PyErr_SetString(PyExc_ValueError, "metric must be one of \"euclidean\" or \"minRMSD\".");
goto error;
}
/* pick first center randomly */
first_center_index = rand() % n_frames;
/* and mark it as assigned */
taken_points[first_center_index] = 1;
/* write its coordinates into the init_centers array */
for(j = 0; j < dim; j++) {
(*(init_centers + centers_found*dim + j)) = data[first_center_index*dim + j];
}
/* increase number of found centers */
centers_found++;
/* perform callback */
if(set_callback) {
py_callback_result = PyObject_CallObject(set_callback, NULL);
if(py_callback_result) Py_DECREF(py_callback_result);
}
/* iterate over all data points j, measuring the squared distance between j and the initial center i: */
/* squared_distances[i] = distance(x_j, x_i)*distance(x_j, x_i) */
for(i = 0; i < n_frames; i++) {
if(i != first_center_index) {
d = pow(distance(&data[i*dim], &data[first_center_index*dim], dim, buffer_a, buffer_b), 2);
squared_distances[i] = d;
/* build up dist_sum which keeps the sum of all squared distances */
dist_sum += d;
}
}
/* keep picking centers while we do not have enough of them... */
while(centers_found < k) {
/* initialize the trials random values by the D^2-weighted distribution */
for(j = 0; j < n_trials; j++) {
next_center_candidates[j] = -1;
next_center_candidates_rand[j] = dist_sum * ((float)rand()/(float)RAND_MAX);
next_center_candidates_potential[j] = 0.0;
}
/* pick candidate data points corresponding to their random value */
sum = 0.0;
for(i = 0; i < n_frames; i++) {
if (!taken_points[i]) {
sum += squared_distances[i];
some_not_done = 0;
for(j = 0; j < n_trials; j++) {
if(next_center_candidates[j] == -1) {
if (sum >= next_center_candidates_rand[j]) {
next_center_candidates[j] = i;
} else {
some_not_done = 1;
}
}
}
if(!some_not_done) break;
}
}
/* now find the maximum squared distance for each trial... */
for(i = 0; i < n_frames; i++) {
if (!taken_points[i]) {
for(j = 0; j < n_trials; j++) {
if(next_center_candidates[j] == -1) break;
if(next_center_candidates[j] != i) {
d = pow(distance(&data[i*dim], &data[next_center_candidates[j]*dim], dim, buffer_a, buffer_b), 2);
if(d < squared_distances[i]) {
next_center_candidates_potential[j] += d;
} else {
next_center_candidates_potential[j] += squared_distances[i];
}
}
}
}
}
/* ... and select the best candidate by the minimum value of the maximum squared distances */
best_candidate = -1;
best_potential = FLT_MAX;
for(j = 0; j < n_trials; j++) {
if(next_center_candidates[j] != -1 && next_center_candidates_potential[j] < best_potential) {
best_potential = next_center_candidates_potential[j];
best_candidate = next_center_candidates[j];
}
}
/* if for some reason we did not find a best candidate, just take the next available point */
if(best_candidate == -1) {
for(i = 0; i < n_frames; i++) {
if(!taken_points[i]) {
best_candidate = i;
break;
}
}
}
/* check if best_candidate was set, otherwise break to avoid an infinite loop should things go wrong */
if(best_candidate >= 0) {
/* write the best_candidate's components into the init_centers array */
for(j = 0; j < dim; j++) {
(*(init_centers + centers_found*dim + j)) = (*(data + best_candidate*dim + j));
}
/* increase centers_found */
centers_found++;
/* perform the callback */
if(set_callback) {
py_callback_result = PyObject_CallObject(set_callback, NULL);
if(py_callback_result) Py_DECREF(py_callback_result);
}
/* mark the data point as assigned center */
taken_points[best_candidate] = 1;
/* update the sum of squared distances by removing the assigned center */
dist_sum -= squared_distances[best_candidate];
/* if we still have centers to assign, the squared distances array has to be updated */
if(centers_found < k) {
/* Check for each data point if its squared distance to the freshly added center is smaller than */
/* the squared distance to the previously picked centers. If so, update the squared_distances */
/* array by the new value and also update the dist_sum value by removing the old value and adding */
/* the new one. */
for(i = 0; i < n_frames; i++) {
if(!taken_points[i]) {
d = pow(distance(&data[i*dim], &data[best_candidate*dim], dim, buffer_a, buffer_b), 2);
if(d < squared_distances[i]) {
dist_sum += d - squared_distances[i];
squared_distances[i] = d;
}
}
}
}
} else {
break;
}
}
/* create the output objects */
dims[0] = k;
dims[1] = dim;
ret_init_centers = PyArray_SimpleNew(2, dims, NPY_FLOAT32);
if (ret_init_centers == NULL){
PyErr_SetString(PyExc_MemoryError, "Error occurs when creating a new PyArray");
goto error;
}
arr_data = PyArray_DATA((PyArrayObject*)ret_init_centers);
/* Need to copy the data of the malloced buffer to the PyObject
since the malloced buffer will disappear after the C extension is called. */
memcpy(arr_data, init_centers, PyArray_ITEMSIZE((PyArrayObject*) ret_init_centers) * k * dim);
Py_INCREF(ret_init_centers); /* The returned list should still exist after calling the C extension */
error:
free(buffer_a);
free(buffer_b);
free(taken_points);
free(init_centers);
free(squared_distances);
free(next_center_candidates);
free(next_center_candidates_rand);
free(next_center_candidates_potential);
return ret_init_centers;
}
/**
* Determine whether two arrays share some memory.
*
* Returns: 0 (no shared memory), 1 (shared memory), or < 0 (failed to solve).
*
* Note that failures to solve can occur due to integer overflows, or effort
* required solving the problem exceeding max_work. The general problem is
* NP-hard and worst case runtime is exponential in the number of dimensions.
* max_work controls the amount of work done, either exact (max_work == -1), only
* a simple memory extent check (max_work == 0), or set an upper bound
* max_work > 0 for the number of solution candidates considered.
*/
NPY_VISIBILITY_HIDDEN mem_overlap_t
solve_may_share_memory(PyArrayObject *a, PyArrayObject *b,
Py_ssize_t max_work)
{
npy_int64 rhs;
diophantine_term_t terms[2*NPY_MAXDIMS+2];
npy_uintp start1 = 0, start2 = 0, end1 = 0, end2 = 0, size1 = 0, size2 = 0;
npy_int64 x[2*NPY_MAXDIMS+2];
unsigned int nterms;
get_array_memory_extents(a, &start1, &end1, &size1);
get_array_memory_extents(b, &start2, &end2, &size2);
if (!(start1 < end2 && start2 < end1 && start1 < end1 && start2 < end2)) {
/* Memory extents don't overlap */
return MEM_OVERLAP_NO;
}
if (max_work == 0) {
/* Too much work required, give up */
return MEM_OVERLAP_TOO_HARD;
}
/* Convert problem to Diophantine equation form with positive coefficients.
The bounds computed by offset_bounds_from_strides correspond to
all-positive strides.
start1 + sum(abs(stride1)*x1)
== start2 + sum(abs(stride2)*x2)
== end1 - 1 - sum(abs(stride1)*x1')
== end2 - 1 - sum(abs(stride2)*x2')
<=>
sum(abs(stride1)*x1) + sum(abs(stride2)*x2')
== end2 - 1 - start1
OR
sum(abs(stride1)*x1') + sum(abs(stride2)*x2)
== end1 - 1 - start2
We pick the problem with the smaller RHS (they are non-negative due to
the extent check above.)
*/
rhs = MIN(end2 - 1 - start1, end1 - 1 - start2);
if (rhs != (npy_uintp)rhs) {
/* Integer overflow */
return MEM_OVERLAP_OVERFLOW;
}
nterms = 0;
if (strides_to_terms(a, terms, &nterms, 1)) {
return MEM_OVERLAP_OVERFLOW;
}
if (strides_to_terms(b, terms, &nterms, 1)) {
return MEM_OVERLAP_OVERFLOW;
}
if (PyArray_ITEMSIZE(a) > 1) {
terms[nterms].a = 1;
terms[nterms].ub = PyArray_ITEMSIZE(a) - 1;
++nterms;
}
if (PyArray_ITEMSIZE(b) > 1) {
terms[nterms].a = 1;
terms[nterms].ub = PyArray_ITEMSIZE(b) - 1;
++nterms;
}
/* Simplify, if possible */
if (diophantine_simplify(&nterms, terms, rhs)) {
/* Integer overflow */
return MEM_OVERLAP_OVERFLOW;
}
/* Solve */
return solve_diophantine(nterms, terms, rhs, max_work, 0, x);
}
static int
fortran_setattr(PyFortranObject *fp, char *name, PyObject *v) {
int i,j,flag;
PyArrayObject *arr = NULL;
for (i=0,j=1;i<fp->len && (j=strcmp(name,fp->defs[i].name));i++);
if (j==0) {
if (fp->defs[i].rank==-1) {
PyErr_SetString(PyExc_AttributeError,"over-writing fortran routine");
return -1;
}
if (fp->defs[i].func!=NULL) { /* is allocatable array */
npy_intp dims[F2PY_MAX_DIMS];
int k;
save_def = &fp->defs[i];
if (v!=Py_None) { /* set new value (reallocate if needed --
see f2py generated code for more
details ) */
for(k=0;k<fp->defs[i].rank;k++) dims[k]=-1;
if ((arr = array_from_pyobj(fp->defs[i].type,dims,fp->defs[i].rank,F2PY_INTENT_IN,v))==NULL)
return -1;
(*(fp->defs[i].func))(&fp->defs[i].rank,arr->dimensions,set_data,&flag);
} else { /* deallocate */
for(k=0;k<fp->defs[i].rank;k++) dims[k]=0;
(*(fp->defs[i].func))(&fp->defs[i].rank,dims,set_data,&flag);
for(k=0;k<fp->defs[i].rank;k++) dims[k]=-1;
}
memcpy(fp->defs[i].dims.d,dims,fp->defs[i].rank*sizeof(npy_intp));
} else { /* not allocatable array */
if ((arr = array_from_pyobj(fp->defs[i].type,fp->defs[i].dims.d,fp->defs[i].rank,F2PY_INTENT_IN,v))==NULL)
return -1;
}
if (fp->defs[i].data!=NULL) { /* copy Python object to Fortran array */
npy_intp s = PyArray_MultiplyList(fp->defs[i].dims.d,arr->nd);
if (s==-1)
s = PyArray_MultiplyList(arr->dimensions,arr->nd);
if (s<0 ||
(memcpy(fp->defs[i].data,arr->data,s*PyArray_ITEMSIZE(arr)))==NULL) {
if ((PyObject*)arr!=v) {
Py_DECREF(arr);
}
return -1;
}
if ((PyObject*)arr!=v) {
Py_DECREF(arr);
}
} else return (fp->defs[i].func==NULL?-1:0);
return 0; /* succesful */
}
if (fp->dict == NULL) {
fp->dict = PyDict_New();
if (fp->dict == NULL)
return -1;
}
if (v == NULL) {
int rv = PyDict_DelItemString(fp->dict, name);
if (rv < 0)
PyErr_SetString(PyExc_AttributeError,"delete non-existing fortran attribute");
return rv;
}
else
return PyDict_SetItemString(fp->dict, name, v);
}
/**
* Determine whether an array has internal overlap.
*
* Returns: 0 (no overlap), 1 (overlap), or < 0 (failed to solve).
*
* max_work and reasons for solver failures are as in solve_may_share_memory.
*/
NPY_VISIBILITY_HIDDEN mem_overlap_t
solve_may_have_internal_overlap(PyArrayObject *a, Py_ssize_t max_work)
{
diophantine_term_t terms[NPY_MAXDIMS+1];
npy_int64 x[NPY_MAXDIMS+1];
unsigned int nterms;
int i, j;
if (PyArray_ISCONTIGUOUS(a)) {
/* Quick case */
return MEM_OVERLAP_NO;
}
/* The internal memory overlap problem is looking for two different
solutions to
sum(a*x) = b, 0 <= x[i] <= ub[i]
for any b. Equivalently,
sum(a*x0) - sum(a*x1) = 0
Mapping the coefficients on the left by x0'[i] = x0[i] if a[i] > 0
else ub[i]-x0[i] and opposite for x1, we have
sum(abs(a)*(x0' + x1')) = sum(abs(a)*ub)
Now, x0!=x1 if for some i we have x0'[i] + x1'[i] != ub[i].
We can now change variables to z[i] = x0'[i] + x1'[i] so the problem
becomes
sum(abs(a)*z) = sum(abs(a)*ub), 0 <= z[i] <= 2*ub[i], z != ub
This can be solved with solve_diophantine.
*/
nterms = 0;
if (strides_to_terms(a, terms, &nterms, 0)) {
return MEM_OVERLAP_OVERFLOW;
}
if (PyArray_ITEMSIZE(a) > 1) {
terms[nterms].a = 1;
terms[nterms].ub = PyArray_ITEMSIZE(a) - 1;
++nterms;
}
/* Get rid of zero coefficients and empty terms */
i = 0;
for (j = 0; j < nterms; ++j) {
if (terms[j].ub == 0) {
continue;
}
else if (terms[j].ub < 0) {
return MEM_OVERLAP_NO;
}
else if (terms[j].a == 0) {
return MEM_OVERLAP_YES;
}
if (i != j) {
terms[i] = terms[j];
}
++i;
}
nterms = i;
/* Double bounds to get the internal overlap problem */
for (j = 0; j < nterms; ++j) {
terms[j].ub *= 2;
}
/* Sort vs. coefficients; cannot call diophantine_simplify because it may
change the decision problem inequality part */
qsort(terms, nterms, sizeof(diophantine_term_t), diophantine_sort_A);
/* Solve */
return solve_diophantine(nterms, terms, -1, max_work, 1, x);
}
/* Calculate the offsets to the filter points, for all border regions and
the interior of the array: */
int init_filter_offsets(PyArrayObject *array, bool *footprint,
const npy_intp * const fshape, npy_intp* origins,
const ExtendMode mode, npy_intp **offsets, npy_intp *border_flag_value,
npy_intp **coordinate_offsets)
{
npy_intp coordinates[NPY_MAXDIMS], position[NPY_MAXDIMS];
npy_intp forigins[NPY_MAXDIMS];
const int rank = array->nd;
const npy_intp* const ashape = array->dimensions;
const npy_intp* const astrides = array->strides;
const npy_intp sizeof_element = PyArray_ITEMSIZE(array);
/* calculate how many sets of offsets must be stored: */
npy_intp offsets_size = 1;
for(int ii = 0; ii < rank; ii++)
offsets_size *= (ashape[ii] < fshape[ii] ? ashape[ii] : fshape[ii]);
/* the size of the footprint array: */
npy_intp filter_size = 1;
for(int i = 0; i < rank; ++i) filter_size *= fshape[i];
/* calculate the number of non-zero elements in the footprint: */
npy_intp footprint_size = 0;
if (footprint) {
for(int i = 0; i < filter_size; ++i) footprint_size += footprint[i];
} else {
footprint_size = filter_size;
}
if (int(mode) < 0 || int(mode) > EXTEND_LAST) {
PyErr_SetString(PyExc_RuntimeError, "boundary mode not supported");
return 0;
}
try {
*offsets = 0;
if (coordinate_offsets) *coordinate_offsets = 0;
*offsets = new npy_intp[offsets_size * footprint_size];
if (coordinate_offsets) {
*coordinate_offsets = new npy_intp[offsets_size * rank * footprint_size];
}
} catch (std::bad_alloc&) {
if (*offsets) delete [] offsets;
PyErr_NoMemory();
return -1;
}
// from here on, we cannot fail anymore:
for(int ii = 0; ii < rank; ii++) {
forigins[ii] = fshape[ii]/2 + (origins ? *origins++ : 0);
}
npy_intp max_size = 0; // maximum ashape[i]
npy_intp max_stride = 0; // maximum abs( astrides[i] )
for(int ii = 0; ii < rank; ii++) {
const npy_intp stride = astrides[ii] < 0 ? -astrides[ii] : astrides[ii];
if (stride > max_stride)
max_stride = stride;
if (ashape[ii] > max_size)
max_size = ashape[ii];
/* coordinates for iterating over the kernel elements: */
coordinates[ii] = 0;
/* keep track of the kernel position: */
position[ii] = 0;
}
/* the flag to indicate that we are outside the border must have a
value that is larger than any possible offset: */
*border_flag_value = max_size * max_stride + 1;
/* calculate all possible offsets to elements in the filter kernel,
for all regions in the array (interior and border regions): */
npy_intp* po = *offsets;
npy_intp* pc = coordinate_offsets ? *coordinate_offsets : 0;
/* iterate over all regions: */
for(int ll = 0; ll < offsets_size; ll++) {
/* iterate over the elements in the footprint array: */
for(int kk = 0; kk < filter_size; kk++) {
npy_intp offset = 0;
/* only calculate an offset if the footprint is 1: */
if (!footprint || footprint[kk]) {
/* find offsets along all axes: */
for(int ii = 0; ii < rank; ii++) {
const npy_intp orgn = forigins[ii];
npy_intp cc = coordinates[ii] - orgn + position[ii];
cc = fix_offset(mode, cc, ashape[ii], *border_flag_value);
/* calculate offset along current axis: */
if (cc == *border_flag_value) {
/* just flag that we are outside the border */
offset = *border_flag_value;
if (coordinate_offsets)
pc[ii] = 0;
break;
} else {
/* use an offset that is possibly mapped from outside the border: */
cc -= position[ii];
offset += astrides[ii] * cc;
if (coordinate_offsets)
pc[ii] = cc;
}
}
if (offset != *border_flag_value) offset /= sizeof_element;
/* store the offset */
*po++ = offset;
if (coordinate_offsets)
pc += rank;
}
/* next point in the filter: */
for(int ii = rank - 1; ii >= 0; ii--) {
if (coordinates[ii] < fshape[ii] - 1) {
coordinates[ii]++;
break;
} else {
coordinates[ii] = 0;
}
}
}
/* move to the next array region: */
for(int ii = rank - 1; ii >= 0; ii--) {
const int orgn = forigins[ii];
if (position[ii] == orgn) {
position[ii] += ashape[ii] - fshape[ii] + 1;
if (position[ii] <= orgn)
position[ii] = orgn + 1;
} else {
position[ii]++;
}
if (position[ii] < ashape[ii]) {
break;
} else {
position[ii] = 0;
}
}
}
return footprint_size;
}
/*
* Retrieving buffers for ndarray
*/
static int
array_getbuffer(PyObject *obj, Py_buffer *view, int flags)
{
PyArrayObject *self;
_buffer_info_t *info = NULL;
self = (PyArrayObject*)obj;
/* Check whether we can provide the wanted properties */
if ((flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS &&
!PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)) {
PyErr_SetString(PyExc_ValueError, "ndarray is not C-contiguous");
goto fail;
}
if ((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS &&
!PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)) {
PyErr_SetString(PyExc_ValueError, "ndarray is not Fortran contiguous");
goto fail;
}
if ((flags & PyBUF_ANY_CONTIGUOUS) == PyBUF_ANY_CONTIGUOUS
&& !PyArray_ISONESEGMENT(self)) {
PyErr_SetString(PyExc_ValueError, "ndarray is not contiguous");
goto fail;
}
if ((flags & PyBUF_STRIDES) != PyBUF_STRIDES &&
!PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)) {
/* Non-strided N-dim buffers must be C-contiguous */
PyErr_SetString(PyExc_ValueError, "ndarray is not C-contiguous");
goto fail;
}
if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) {
if (PyArray_FailUnlessWriteable(self, "buffer source array") < 0) {
goto fail;
}
}
/*
* If a read-only buffer is requested on a read-write array, we return a
* read-write buffer, which is dubious behavior. But that's why this call
* is guarded by PyArray_ISWRITEABLE rather than (flags &
* PyBUF_WRITEABLE).
*/
if (PyArray_ISWRITEABLE(self)) {
if (array_might_be_written(self) < 0) {
goto fail;
}
}
if (view == NULL) {
PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
goto fail;
}
/* Fill in information */
info = _buffer_get_info(obj);
if (info == NULL) {
goto fail;
}
view->buf = PyArray_DATA(self);
view->suboffsets = NULL;
view->itemsize = PyArray_ITEMSIZE(self);
view->readonly = !PyArray_ISWRITEABLE(self);
view->internal = NULL;
view->len = PyArray_NBYTES(self);
if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) {
view->format = info->format;
} else {
view->format = NULL;
}
if ((flags & PyBUF_ND) == PyBUF_ND) {
view->ndim = info->ndim;
view->shape = info->shape;
}
else {
view->ndim = 0;
view->shape = NULL;
}
if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) {
view->strides = info->strides;
#ifdef NPY_RELAXED_STRIDES_CHECKING
/*
* If NPY_RELAXED_STRIDES_CHECKING is on, the array may be
* contiguous, but it won't look that way to Python when it
* tries to determine contiguity by looking at the strides
* (since one of the elements may be -1). In that case, just
* regenerate strides from shape.
*/
if (PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS) &&
!((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS)) {
Py_ssize_t sd = view->itemsize;
int i;
for (i = view->ndim-1; i >= 0; --i) {
view->strides[i] = sd;
sd *= view->shape[i];
}
}
else if (PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)) {
Py_ssize_t sd = view->itemsize;
int i;
for (i = 0; i < view->ndim; ++i) {
view->strides[i] = sd;
sd *= view->shape[i];
}
}
#endif
}
else {
view->strides = NULL;
}
view->obj = (PyObject*)self;
Py_INCREF(self);
return 0;
fail:
return -1;
}
/*
* Check whether the given array is stored contiguously
* in memory. And update the passed in ap flags appropriately.
*
* The traditional rule is that for an array to be flagged as C contiguous,
* the following must hold:
*
* strides[-1] == itemsize
* strides[i] == shape[i+1] * strides[i + 1]
*
* And for an array to be flagged as F contiguous, the obvious reversal:
*
* strides[0] == itemsize
* strides[i] == shape[i - 1] * strides[i - 1]
*
* According to these rules, a 0- or 1-dimensional array is either both
* C- and F-contiguous, or neither; and an array with 2+ dimensions
* can be C- or F- contiguous, or neither, but not both. Though there
* there are exceptions for arrays with zero or one item, in the first
* case the check is relaxed up to and including the first dimension
* with shape[i] == 0. In the second case `strides == itemsize` will
* can be true for all dimensions and both flags are set.
*
* When NPY_RELAXED_STRIDES_CHECKING is set, we use a more accurate
* definition of C- and F-contiguity, in which all 0-sized arrays are
* contiguous (regardless of dimensionality), and if shape[i] == 1
* then we ignore strides[i] (since it has no affect on memory layout).
* With these new rules, it is possible for e.g. a 10x1 array to be both
* C- and F-contiguous -- but, they break downstream code which assumes
* that for contiguous arrays strides[-1] (resp. strides[0]) always
* contains the itemsize.
*/
static void
_UpdateContiguousFlags(PyArrayObject *ap)
{
npy_intp sd;
npy_intp dim;
int i;
npy_bool is_c_contig = 1;
sd = PyArray_ITEMSIZE(ap);
for (i = PyArray_NDIM(ap) - 1; i >= 0; --i) {
dim = PyArray_DIMS(ap)[i];
#if NPY_RELAXED_STRIDES_CHECKING
/* contiguous by definition */
if (dim == 0) {
PyArray_ENABLEFLAGS(ap, NPY_ARRAY_C_CONTIGUOUS);
PyArray_ENABLEFLAGS(ap, NPY_ARRAY_F_CONTIGUOUS);
return;
}
if (dim != 1) {
if (PyArray_STRIDES(ap)[i] != sd) {
is_c_contig = 0;
}
sd *= dim;
}
#else /* not NPY_RELAXED_STRIDES_CHECKING */
if (PyArray_STRIDES(ap)[i] != sd) {
is_c_contig = 0;
break;
}
/* contiguous, if it got this far */
if (dim == 0) {
break;
}
sd *= dim;
#endif /* not NPY_RELAXED_STRIDES_CHECKING */
}
if (is_c_contig) {
PyArray_ENABLEFLAGS(ap, NPY_ARRAY_C_CONTIGUOUS);
}
else {
PyArray_CLEARFLAGS(ap, NPY_ARRAY_C_CONTIGUOUS);
}
/* check if fortran contiguous */
sd = PyArray_ITEMSIZE(ap);
for (i = 0; i < PyArray_NDIM(ap); ++i) {
dim = PyArray_DIMS(ap)[i];
#if NPY_RELAXED_STRIDES_CHECKING
if (dim != 1) {
if (PyArray_STRIDES(ap)[i] != sd) {
PyArray_CLEARFLAGS(ap, NPY_ARRAY_F_CONTIGUOUS);
return;
}
sd *= dim;
}
#else /* not NPY_RELAXED_STRIDES_CHECKING */
if (PyArray_STRIDES(ap)[i] != sd) {
PyArray_CLEARFLAGS(ap, NPY_ARRAY_F_CONTIGUOUS);
return;
}
if (dim == 0) {
break;
}
sd *= dim;
#endif /* not NPY_RELAXED_STRIDES_CHECKING */
}
PyArray_ENABLEFLAGS(ap, NPY_ARRAY_F_CONTIGUOUS);
return;
}
//python main function
static PyObject *
gadget(PyObject *self, PyObject *args)
{
const char *file;
int i,j,k,pos,nfiles,tempint,dummy,ntot_withmasses;
int NumPart,Ngas,t,n,off,pc,pc_new,pc_sph;
npy_intp dim[1] = {1};
npy_intp ndim = 1;
PyObject *array;
float *data;
float *temp,tempfloat;
struct io_header header;
FILE *fd;
float *dataout;
char buf[500];
double x0,y0,z0,xmin,xmax,ymin,ymax,cellsize,rho,boxsize,radius,length;
double x_axis[3],y_axis[3],z_axis[3],rs[0],x,y,double_buf;
int tot_x, tot_y, x_block,y_block;
double **count;
long countp,countk;
size_t size,il;
if (!PyArg_ParseTuple(args, "siddddddddddddddddddii",
&file, &nfiles,
&x0,&y0,&z0,
&(x_axis[0]),&(x_axis[1]),&(x_axis[2]),
&(y_axis[0]),&(y_axis[1]),&(y_axis[2]),
&(z_axis[0]),&(z_axis[1]),&(z_axis[2]),
&xmin,&xmax,
&ymin,&ymax,
&boxsize,
&cellsize,
&(tot_x),&(tot_y)
))
return NULL;
//read from file(s) in a loop
//***keep track of global particle index through pc and local file index through pc_new***
pc = 0;
pc_new = 0;
countp = 0;
for(i=0;i<nfiles;i++)
{
if(nfiles > 1) //multiple file names are different
sprintf(buf,"%s.%d",file,i);
else
sprintf(buf,"%s",file);
if(!(fd=fopen(buf,"r"))) //error check and complain
{
printf("gadgetIO module can't open file: '%s'\n",buf);
fflush(stdout);
return NULL;
}
printf("reading: '%s'\n",buf);
fflush(stdout);
fread(&dummy, sizeof(dummy), 1, fd);
fread(&header, sizeof(header), 1, fd);
fread(&dummy, sizeof(dummy), 1, fd);
//allocate data array and get total number of particles to start reading
if(i == 0)
{
if(nfiles == 1)
{
for(k=0,NumPart=0;k<5;k++)
NumPart += header.npart[k];
Ngas = header.npart[0];
}
else
{
for(k=0,NumPart=0;k<5;k++)
NumPart += header.npartTotal[k];
Ngas = header.npartTotal[0];
}
//printf("allocate array N = %d\n",NumPart);
data = (float *) malloc(NumPart*3*sizeof(float));
}
for(k=0,ntot_withmasses=0;k<5;k++)
if(header.mass[k] == 0)
ntot_withmasses += header.npart[k];
fread(&dummy, sizeof(dummy), 1, fd);
for(k=0;k<6;k++)
{
//printf("Read particle %d : N = %d\n",k,header.npart[k]);
//printf("Read byte %d\n",fread(&(data[3*countp]),3*sizeof(float),header.npart[k],fd));
fread(&(data[3*countp]),3*sizeof(float),header.npart[k],fd);
countp += header.npart[k];
}
fread(&dummy, sizeof(dummy), 1, fd);
fclose(fd);
}
printf("Finish reading particles\n");
radius = (xmax-xmin)/2.;
//count = calloc(tot_x,sizeof(double *));
countk = 0;
for(i=0;i<tot_x;i++)
{
//count[i] = calloc(tot_y,sizeof(double));
}
dataout = malloc(2*(countk+1)*sizeof(float));
for(i=0;i<NumPart;i++)
{
// Mpc => kpc
// printf("%g\t%g\t%g\n",data[i*3],data[i*3+1],data[i*3+2]);
rs[0] = data[i*3]*1000. - x0;
rs[1] = data[i*3+1]*1000. - y0;
rs[2] = data[i*3+2]*1000. -z0;
for(j=0;j<3;j++)
{
if(rs[j] > boxsize/2.)
rs[j] -= boxsize;
if(rs[j] < -1.*boxsize/2.)
rs[j] += boxsize;
}
length = rs[0]*z_axis[0]+rs[1]*z_axis[1]+rs[2]*z_axis[2];
//printf("Length = %f\n",length);
if(fabs(length) < radius)
{
//printf("Length = %f\n",length);
for(j=0;j<3;j++)
{
rs[j] -= z_axis[j]*length;
}
x = rs[0]*x_axis[0]+rs[1]*x_axis[1]+rs[2]*x_axis[2];
y = rs[0]*y_axis[0]+rs[1]*y_axis[1]+rs[2]*y_axis[2];
if(x < xmax && x > xmin && y < ymax && y > ymin)
{
//printf("Length = %f\n",length);
//printf("%f %f\n",x,y);
x_block = (int) ((x-xmin)/cellsize);
y_block = (int) ((y-ymin)/cellsize);
//printf("block %d %d\n",x_block, y_block);
//printf("size of float = %d\n",sizeof(float));
//printf("Going to allocate %ld elements = %ld bytes\n",2*(countk+1),2*(countk+1)*sizeof(float));
dataout = realloc(dataout,2*(countk+1)*4);
dataout[2*countk] = x;
dataout[2*countk+1] = y;
//printf("data = %g\t%g\n",dataout[2*(countk)],dataout[2*(countk)+1]);
//count[x_block][y_block]+= 1.;
countk++;
}
}
}
free(data);
printf("finish transforming\n");
//dim[0] = tot_x*tot_y;
dim[0] = countk*2;
rho = (float) NumPart/pow(boxsize,3)*(2.*radius*cellsize*cellsize);
k=0;
//dataout = calloc(tot_x*tot_y,sizeof(double));
//smooth
for(i=1;i<tot_x-1;i++)
{
for(j=1;j<tot_y-1;j++)
{
//count[i][j] = count[i][j] + count[i+1][j] + count[i+1][j+1] + count[i+1][j-1] + count[i][j+1] + count[i][j-1] + count[i-1][j-1] +count[i-1][j] +count[i-1][j+1];
//count[i][j] /= 9.;
}
}
k=0;
for(i=0;i<tot_x;i++)
{
for(j=0;j<tot_y;j++)
{
//dataout[k] = (float) (count[i][j] / rho);
k++;
}
}
for(i=0;i<sizeof(dataout)/sizeof(dataout[0]); i+=2)
{
//printf("%f\t%f\n",dataout[i],dataout[i+1]);
}
printf("\n\n");
printf("Allocate python object and copy data inz\n");
//array = (PyArrayObject *) PyArray_FromDimsAndData(ndim, dim, NPY_DOUBLE,dataout);
printf("total element %d %d\n",ndim,dim[0]/2);
array = (PyArrayObject *)PyArray_SimpleNew(ndim, &dim,PyArray_FLOAT);
//array = (PyArrayObject *) PyArray_Return(array);
void *arr_data = PyArray_DATA((PyArrayObject*)array);
memcpy(arr_data, dataout, PyArray_ITEMSIZE((PyArrayObject*) array) * dim[0]);
return (PyObject *)array;
}
/* Calculate the offsets to the filter points, for all border regions and
the interior of the array: */
int init_filter_offsets(PyArrayObject *array, bool *footprint,
const npy_intp * const fshape, npy_intp* origins,
const ExtendMode mode, std::vector<npy_intp>& offsets,
std::vector<npy_intp>* coordinate_offsets)
{
npy_intp coordinates[NPY_MAXDIMS], position[NPY_MAXDIMS];
npy_intp forigins[NPY_MAXDIMS];
const int rank = array->nd;
const npy_intp* const ashape = array->dimensions;
npy_intp astrides[NPY_MAXDIMS];
for (int d = 0; d != rank; ++d) astrides[d] = array->strides[d]/PyArray_ITEMSIZE(array);
/* calculate how many sets of offsets must be stored: */
npy_intp offsets_size = 1;
for(int ii = 0; ii < rank; ii++)
offsets_size *= (ashape[ii] < fshape[ii] ? ashape[ii] : fshape[ii]);
/* the size of the footprint array: */
npy_intp filter_size = 1;
for(int i = 0; i < rank; ++i) filter_size *= fshape[i];
/* calculate the number of non-zero elements in the footprint: */
npy_intp footprint_size = 0;
if (footprint) {
for(int i = 0; i < filter_size; ++i) footprint_size += footprint[i];
} else {
footprint_size = filter_size;
}
if (int(mode) < 0 || int(mode) > EXTEND_LAST) {
throw PythonException(PyExc_RuntimeError, "boundary mode not supported");
}
offsets.resize(offsets_size * footprint_size);
if (coordinate_offsets) coordinate_offsets->resize(offsets_size * footprint_size);
// from here on, we cannot fail anymore:
for(int ii = 0; ii < rank; ii++) {
forigins[ii] = fshape[ii]/2 + (origins ? *origins++ : 0);
}
std::fill(coordinates, coordinates + rank, 0);
std::fill(position, position + rank, 0);
/* calculate all possible offsets to elements in the filter kernel,
for all regions in the array (interior and border regions): */
unsigned poi = 0;
npy_intp* pc = coordinate_offsets ? &(*coordinate_offsets)[0] : 0;
/* iterate over all regions: */
for(int ll = 0; ll < offsets_size; ll++) {
/* iterate over the elements in the footprint array: */
for(int kk = 0; kk < filter_size; kk++) {
npy_intp offset = 0;
/* only calculate an offset if the footprint is 1: */
if (!footprint || footprint[kk]) {
/* find offsets along all axes: */
for(int ii = 0; ii < rank; ii++) {
const npy_intp orgn = forigins[ii];
npy_intp cc = coordinates[ii] - orgn + position[ii];
cc = fix_offset(mode, cc, ashape[ii]);
/* calculate offset along current axis: */
if (cc == border_flag_value) {
/* just flag that we are outside the border */
offset = border_flag_value;
if (coordinate_offsets)
pc[ii] = 0;
break;
} else {
/* use an offset that is possibly mapped from outside the border: */
cc -= position[ii];
offset += astrides[ii] * cc;
if (coordinate_offsets)
pc[ii] = cc;
}
}
/* store the offset */
offsets[poi++] = offset;
if (coordinate_offsets)
pc += rank;
}
/* next point in the filter: */
for(int ii = rank - 1; ii >= 0; ii--) {
if (coordinates[ii] < fshape[ii] - 1) {
coordinates[ii]++;
break;
} else {
coordinates[ii] = 0;
}
}
}
/* move to the next array region: */
for(int ii = rank - 1; ii >= 0; ii--) {
const int orgn = forigins[ii];
if (position[ii] == orgn) {
position[ii] += ashape[ii] - fshape[ii] + 1;
if (position[ii] <= orgn)
position[ii] = orgn + 1;
} else {
position[ii]++;
}
if (position[ii] < ashape[ii]) {
break;
} else {
position[ii] = 0;
}
}
}
assert(poi <= offsets.size());
return footprint_size;
}
static PyObject *cluster(PyObject *self, PyObject *args) {
int debug;
PyObject *py_centers, *py_item, *py_res;
PyArrayObject *np_chunk, *np_item;
Py_ssize_t N_centers, N_frames, dim;
float *chunk;
float **centers;
char *metric;
float mindist;
float d;
float *buffer_a, *buffer_b;
int l;
int *centers_counter;
void *arr_data;
float *new_centers;
int i, j;
int closest_center_index;
npy_intp dims[2];
float (*distance)(float*, float*, size_t, float*, float*);
PyObject* return_new_centers;
debug = 0;
if(debug) printf("KMEANS: \n----------- cluster called ----------\n");
if(debug) printf("KMEANS: declaring variables...");
if(debug) printf("done.\n");
if(debug) printf("KMEANS: initializing some of them...");
py_centers = NULL; py_item = NULL; py_res = NULL;
np_chunk = NULL; np_item = NULL;
centers = NULL; metric=""; chunk = NULL;
centers_counter = NULL; new_centers = NULL;
buffer_a = NULL; buffer_b = NULL;
return_new_centers = Py_BuildValue("");
if(debug) printf("done\n");
if(debug) printf("KMEANS: attempting to parse args...");
if (!PyArg_ParseTuple(args, "O!O!s", &PyArray_Type, &np_chunk, &PyList_Type, &py_centers, &metric)) {
goto error;
}
if(debug) printf("done\n");
/* import chunk */
if(debug) printf("KMEANS: importing chunk...");
if(PyArray_TYPE(np_chunk)!=NPY_FLOAT32) { PyErr_SetString(PyExc_ValueError, "dtype of \"chunk\" isn\'t float (32)."); goto error; };
if(!PyArray_ISCARRAY_RO(np_chunk) ) { PyErr_SetString(PyExc_ValueError, "\"chunk\" isn\'t C-style contiguous or isn\'t behaved."); goto error; };
if(PyArray_NDIM(np_chunk)!=2) { PyErr_SetString(PyExc_ValueError, "Number of dimensions of \"chunk\" isn\'t 2."); goto error; };
N_frames = np_chunk->dimensions[0];
dim = np_chunk->dimensions[1];
if(dim==0) {
PyErr_SetString(PyExc_ValueError, "chunk dimension must be larger than zero.");
goto error;
}
chunk = PyArray_DATA(np_chunk);
if(debug) printf("done with N_frames=%zd, dim=%zd\n", N_frames, dim);
if(debug) printf("KMEANS: creating metric function...");
if(strcmp(metric,"euclidean")==0) {
distance = euclidean_distance;
} /*else if(strcmp(metric,"minRMSD")==0) {
distance = minRMSD_distance;
buffer_a = malloc(dim*sizeof(float));
buffer_b = malloc(dim*sizeof(float));
if(!buffer_a || !buffer_b) { PyErr_NoMemory(); goto error; }
} */
else {
PyErr_SetString(PyExc_ValueError, "metric must be one of \"euclidean\" or \"minRMSD\".");
goto error;
}
if(debug) printf("done\n");
/* import list of cluster centers */
if(debug) printf("KMEANS: importing list of cluster centers...");
N_centers = PyList_Size(py_centers);
if(!(centers = malloc(N_centers*sizeof(float*)))) {
PyErr_NoMemory(); goto error;
}
for(i = 0; i < N_centers; ++i) {
l = 0;
if(debug) printf("%d", l++); /* 0 */
py_item = PyList_GetItem(py_centers,i); /* ref:borr. */
if(debug) printf("%d", l++); /* 1 */
if(!py_item) goto error;
if(debug) printf("%d", l++); /* 2 */
if(!PyArray_Check(py_item)) { PyErr_SetString(PyExc_ValueError, "Elements of centers must be numpy arrays."); goto error; }
if(debug) printf("%d", l++); /* 3 */
np_item = (PyArrayObject*)py_item;
if(debug) printf("%d", l++); /* 4 */
if(PyArray_TYPE(np_item)!=NPY_FLOAT32) { PyErr_SetString(PyExc_ValueError, "dtype of cluster center isn\'t float (32)."); goto error; };
if(debug) printf("%d", l++); /* 5 */
if(!PyArray_ISBEHAVED_RO(np_item) ) { PyErr_SetString(PyExc_ValueError, "cluster center isn\'t behaved."); goto error; };
if(debug) printf("%d", l++); /* 6 */
if(PyArray_NDIM(np_item)!=1) { PyErr_SetString(PyExc_ValueError, "Number of dimensions of cluster centers must be 1."); goto error; };
if(debug) printf("%d", l++); /* 7 */
if(np_item->dimensions[0]!=dim) {
PyErr_SetString(PyExc_ValueError, "Dimension of cluster centers doesn\'t match dimension of frames.");
goto error;
}
if(debug) printf("%d", l++); /* 8 */
centers[i] = (float*)PyArray_DATA(np_item);
if(debug) printf("%d", l++); /* 9 */
}
if(debug) printf("done, k=%zd\n", N_centers);
/* initialize centers_counter and new_centers with zeros */
if(debug) printf("KMEANS: initializing calloc centers counter and new_centers stuff...");
centers_counter = (int*) calloc(N_centers, sizeof(int));
new_centers = (float*) calloc(N_centers * dim, sizeof(float));
if(!centers_counter || !new_centers) { PyErr_NoMemory(); goto error; }
if(debug) printf("done\n");
/* do the clustering */
if(debug) printf("KMEANS: performing the clustering...");
for (i = 0; i < N_frames; i++) {
mindist = FLT_MAX;
for(j = 0; j < N_centers; ++j) {
d = distance(&chunk[i*dim], centers[j], dim, buffer_a, buffer_b);
if(d<mindist) {
mindist = d;
closest_center_index = j;
}
}
(*(centers_counter + closest_center_index))++;
for (j = 0; j < dim; j++) {
new_centers[closest_center_index*dim+j] += chunk[i*dim+j];
}
}
for (i = 0; i < N_centers; i++) {
if (*(centers_counter + i) == 0) {
for (j = 0; j < dim; j++) {
(*(new_centers + i * dim + j)) = centers[i][j];
}
} else {
for (j=0; j < dim; j++) {
(*(new_centers + i * dim + j)) /= (*(centers_counter + i));
}
}
}
if(debug) printf("done\n");
if(debug) printf("KMEANS: creating return_new_centers...");
dims[0] = N_centers; dims[1] = dim;
return_new_centers = PyArray_SimpleNew(2, dims, NPY_FLOAT32);
if (return_new_centers == NULL){
PyErr_SetString(PyExc_MemoryError, "Error occurs when creating a new PyArray");
goto error;
}
arr_data = PyArray_DATA((PyArrayObject*)return_new_centers);
if(debug) printf("done\n");
/* Need to copy the data of the malloced buffer to the PyObject
since the malloced buffer will disappear after the C extension is called. */
if(debug) printf("KMEANS: attempting memcopy...");
memcpy(arr_data, new_centers, PyArray_ITEMSIZE((PyArrayObject*) return_new_centers) * N_centers * dim);
if(debug) printf("done\n");
if(debug) printf("KMEANS: increasting ref to return_new_centers thingy...");
Py_INCREF(return_new_centers); /* The returned list should still exist after calling the C extension */
if(debug) printf("done\n");
/* fall through */
error:
free(centers_counter);
free(new_centers);
free(centers);
free(buffer_a);
free(buffer_b);
return return_new_centers;
}
static int
array_getbuffer(PyObject *obj, Py_buffer *view, int flags)
{
PyArrayObject *self;
_buffer_info_t *info = NULL;
self = (PyArrayObject*)obj;
/* Check whether we can provide the wanted properties */
if ((flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS &&
!PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)) {
PyErr_SetString(PyExc_ValueError, "ndarray is not C-contiguous");
goto fail;
}
if ((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS &&
!PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)) {
PyErr_SetString(PyExc_ValueError, "ndarray is not Fortran contiguous");
goto fail;
}
if ((flags & PyBUF_ANY_CONTIGUOUS) == PyBUF_ANY_CONTIGUOUS
&& !PyArray_ISONESEGMENT(self)) {
PyErr_SetString(PyExc_ValueError, "ndarray is not contiguous");
goto fail;
}
if ((flags & PyBUF_STRIDES) != PyBUF_STRIDES &&
!PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)) {
/* Non-strided N-dim buffers must be C-contiguous */
PyErr_SetString(PyExc_ValueError, "ndarray is not C-contiguous");
goto fail;
}
if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) {
if (PyArray_FailUnlessWriteable(self, "buffer source array") < 0) {
goto fail;
}
}
/*
* If a read-only buffer is requested on a read-write array, we return a
* read-write buffer, which is dubious behavior. But that's why this call
* is guarded by PyArray_ISWRITEABLE rather than (flags &
* PyBUF_WRITEABLE).
*/
if (PyArray_ISWRITEABLE(self)) {
if (array_might_be_written(self) < 0) {
goto fail;
}
}
if (view == NULL) {
PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
goto fail;
}
/* Fill in information */
info = _buffer_get_info(obj);
if (info == NULL) {
goto fail;
}
view->buf = PyArray_DATA(self);
view->suboffsets = NULL;
view->itemsize = PyArray_ITEMSIZE(self);
view->readonly = !PyArray_ISWRITEABLE(self);
view->internal = NULL;
view->len = PyArray_NBYTES(self);
if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) {
view->format = info->format;
} else {
view->format = NULL;
}
if ((flags & PyBUF_ND) == PyBUF_ND) {
view->ndim = info->ndim;
view->shape = info->shape;
}
else {
view->ndim = 0;
view->shape = NULL;
}
if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) {
view->strides = info->strides;
}
else {
view->strides = NULL;
}
view->obj = (PyObject*)self;
Py_INCREF(self);
return 0;
fail:
return -1;
}
static int
array_getbuffer(PyObject *obj, Py_buffer *view, int flags)
{
PyArrayObject *self;
_buffer_info_t *info = NULL;
self = (PyArrayObject*)obj;
/* Check whether we can provide the wanted properties */
if ((flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS &&
!PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)) {
PyErr_SetString(PyExc_ValueError, "ndarray is not C-contiguous");
goto fail;
}
if ((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS &&
!PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)) {
PyErr_SetString(PyExc_ValueError, "ndarray is not Fortran contiguous");
goto fail;
}
if ((flags & PyBUF_ANY_CONTIGUOUS) == PyBUF_ANY_CONTIGUOUS
&& !PyArray_ISONESEGMENT(self)) {
PyErr_SetString(PyExc_ValueError, "ndarray is not contiguous");
goto fail;
}
if ((flags & PyBUF_STRIDES) != PyBUF_STRIDES &&
(flags & PyBUF_ND) == PyBUF_ND &&
!PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)) {
/* Non-strided N-dim buffers must be C-contiguous */
PyErr_SetString(PyExc_ValueError, "ndarray is not C-contiguous");
goto fail;
}
if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE &&
!PyArray_ISWRITEABLE(self)) {
PyErr_SetString(PyExc_ValueError, "ndarray is not writeable");
goto fail;
}
if (view == NULL) {
PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
goto fail;
}
/* Fill in information */
info = _buffer_get_info(obj);
if (info == NULL) {
goto fail;
}
view->buf = PyArray_DATA(self);
view->suboffsets = NULL;
view->itemsize = PyArray_ITEMSIZE(self);
view->readonly = !PyArray_ISWRITEABLE(self);
view->internal = NULL;
view->len = PyArray_NBYTES(self);
if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) {
view->format = info->format;
} else {
view->format = NULL;
}
if ((flags & PyBUF_ND) == PyBUF_ND) {
view->ndim = info->ndim;
view->shape = info->shape;
}
else {
view->ndim = 0;
view->shape = NULL;
}
if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) {
view->strides = info->strides;
}
else {
view->strides = NULL;
}
view->obj = (PyObject*)self;
Py_INCREF(self);
return 0;
fail:
return -1;
}
extern
PyArrayObject* array_from_pyobj(const int type_num,
npy_intp *dims,
const int rank,
const int intent,
PyObject *obj) {
/* Note about reference counting
-----------------------------
If the caller returns the array to Python, it must be done with
Py_BuildValue("N",arr).
Otherwise, if obj!=arr then the caller must call Py_DECREF(arr).
Note on intent(cache,out,..)
---------------------
Don't expect correct data when returning intent(cache) array.
*/
char mess[200];
PyArrayObject *arr = NULL;
PyArray_Descr *descr;
char typechar;
int elsize;
if ((intent & F2PY_INTENT_HIDE)
|| ((intent & F2PY_INTENT_CACHE) && (obj==Py_None))
|| ((intent & F2PY_OPTIONAL) && (obj==Py_None))
) {
/* intent(cache), optional, intent(hide) */
if (count_nonpos(rank,dims)) {
int i;
strcpy(mess, "failed to create intent(cache|hide)|optional array"
"-- must have defined dimensions but got (");
for(i=0;i<rank;++i)
sprintf(mess+strlen(mess),"%" NPY_INTP_FMT ",",dims[i]);
strcat(mess, ")");
PyErr_SetString(PyExc_ValueError,mess);
return NULL;
}
arr = (PyArrayObject *)
PyArray_New(&PyArray_Type, rank, dims, type_num,
NULL,NULL,0,
!(intent&F2PY_INTENT_C),
NULL);
if (arr==NULL) return NULL;
if (!(intent & F2PY_INTENT_CACHE))
PyArray_FILLWBYTE(arr, 0);
return arr;
}
descr = PyArray_DescrFromType(type_num);
elsize = descr->elsize;
typechar = descr->type;
Py_DECREF(descr);
if (PyArray_Check(obj)) {
arr = (PyArrayObject *)obj;
if (intent & F2PY_INTENT_CACHE) {
/* intent(cache) */
if (PyArray_ISONESEGMENT(arr)
&& PyArray_ITEMSIZE(arr)>=elsize) {
if (check_and_fix_dimensions(arr,rank,dims)) {
return NULL; /*XXX: set exception */
}
if (intent & F2PY_INTENT_OUT)
Py_INCREF(arr);
return arr;
}
strcpy(mess, "failed to initialize intent(cache) array");
if (!PyArray_ISONESEGMENT(arr))
strcat(mess, " -- input must be in one segment");
if (PyArray_ITEMSIZE(arr)<elsize)
sprintf(mess+strlen(mess),
" -- expected at least elsize=%d but got %" NPY_INTP_FMT,
elsize,
(npy_intp)PyArray_ITEMSIZE(arr)
);
PyErr_SetString(PyExc_ValueError,mess);
return NULL;
}
/* here we have always intent(in) or intent(inout) or intent(inplace) */
if (check_and_fix_dimensions(arr,rank,dims)) {
return NULL; /*XXX: set exception */
}
/*
printf("intent alignement=%d\n", F2PY_GET_ALIGNMENT(intent));
printf("alignement check=%d\n", F2PY_CHECK_ALIGNMENT(arr, intent));
int i;
for (i=1;i<=16;i++)
printf("i=%d isaligned=%d\n", i, ARRAY_ISALIGNED(arr, i));
*/
if ((! (intent & F2PY_INTENT_COPY))
&& PyArray_ITEMSIZE(arr)==elsize
&& ARRAY_ISCOMPATIBLE(arr,type_num)
&& F2PY_CHECK_ALIGNMENT(arr, intent)
) {
if ((intent & F2PY_INTENT_C)?PyArray_ISCARRAY(arr):PyArray_ISFARRAY(arr)) {
if ((intent & F2PY_INTENT_OUT)) {
Py_INCREF(arr);
}
/* Returning input array */
return arr;
}
}
if (intent & F2PY_INTENT_INOUT) {
strcpy(mess, "failed to initialize intent(inout) array");
if ((intent & F2PY_INTENT_C) && !PyArray_ISCARRAY(arr))
strcat(mess, " -- input not contiguous");
if (!(intent & F2PY_INTENT_C) && !PyArray_ISFARRAY(arr))
strcat(mess, " -- input not fortran contiguous");
if (PyArray_ITEMSIZE(arr)!=elsize)
sprintf(mess+strlen(mess),
" -- expected elsize=%d but got %" NPY_INTP_FMT,
elsize,
(npy_intp)PyArray_ITEMSIZE(arr)
);
if (!(ARRAY_ISCOMPATIBLE(arr,type_num)))
sprintf(mess+strlen(mess)," -- input '%c' not compatible to '%c'",
PyArray_DESCR(arr)->type,typechar);
if (!(F2PY_CHECK_ALIGNMENT(arr, intent)))
sprintf(mess+strlen(mess)," -- input not %d-aligned", F2PY_GET_ALIGNMENT(intent));
PyErr_SetString(PyExc_ValueError,mess);
return NULL;
}
/* here we have always intent(in) or intent(inplace) */
{
PyArrayObject *retarr = (PyArrayObject *) \
PyArray_New(&PyArray_Type, PyArray_NDIM(arr), PyArray_DIMS(arr), type_num,
NULL,NULL,0,
!(intent&F2PY_INTENT_C),
NULL);
if (retarr==NULL)
return NULL;
F2PY_REPORT_ON_ARRAY_COPY_FROMARR;
if (PyArray_CopyInto(retarr, arr)) {
Py_DECREF(retarr);
return NULL;
}
if (intent & F2PY_INTENT_INPLACE) {
if (swap_arrays(arr,retarr))
return NULL; /* XXX: set exception */
Py_XDECREF(retarr);
if (intent & F2PY_INTENT_OUT)
Py_INCREF(arr);
} else {
arr = retarr;
}
}
return arr;
}
if ((intent & F2PY_INTENT_INOUT) ||
(intent & F2PY_INTENT_INPLACE) ||
(intent & F2PY_INTENT_CACHE)) {
PyErr_SetString(PyExc_TypeError,
"failed to initialize intent(inout|inplace|cache) "
"array, input not an array");
return NULL;
}
{
F2PY_REPORT_ON_ARRAY_COPY_FROMANY;
arr = (PyArrayObject *) \
PyArray_FromAny(obj,PyArray_DescrFromType(type_num), 0,0,
((intent & F2PY_INTENT_C)?NPY_ARRAY_CARRAY:NPY_ARRAY_FARRAY) \
| NPY_ARRAY_FORCECAST, NULL);
if (arr==NULL)
return NULL;
if (check_and_fix_dimensions(arr,rank,dims))
return NULL; /*XXX: set exception */
return arr;
}
}
