/*
* dimension/stride cache, uses a different allocator and is always a multiple
* of npy_intp
*/
NPY_NO_EXPORT void *
npy_alloc_cache_dim(npy_uintp sz)
{
/* dims + strides */
if (NPY_UNLIKELY(sz < 2)) {
sz = 2;
}
return _npy_alloc_cache(sz, sizeof(npy_intp), NBUCKETS_DIM, dimcache,
&PyArray_malloc);
}
NPY_NO_EXPORT void
npy_free_cache_dim(void * p, npy_uintp sz)
{
/* dims + strides */
if (NPY_UNLIKELY(sz < 2)) {
sz = 2;
}
_npy_free_cache(p, sz, NBUCKETS_DIM, dimcache,
&PyArray_free);
}
static int
check_callers(int * cannot)
{
/*
* get base addresses of multiarray and python, check if
* backtrace is in these libraries only calling dladdr if a new max address
* is found.
* When after the initial multiarray stack everything is inside python we
* can elide as no C-API user could have messed up the reference counts.
* Only check until the python frame evaluation function is found
* approx 10us overhead for stack size of 10
*
* TODO some calls go over scalarmath in umath but we cannot get the base
* address of it from multiarraymodule as it is not linked against it
*/
static int init = 0;
/*
* measured DSO object memory start and end, if an address is located
* inside these bounds it is part of that library so we don't need to call
* dladdr on it (assuming linear memory)
*/
static void * pos_python_start;
static void * pos_python_end;
static void * pos_ma_start;
static void * pos_ma_end;
/* known address storage to save dladdr calls */
static void * py_addr[64];
static void * pyeval_addr[64];
static npy_intp n_py_addr = 0;
static npy_intp n_pyeval = 0;
void *buffer[NPY_MAX_STACKSIZE];
int i, nptrs;
int ok = 0;
/* cannot determine callers */
if (init == -1) {
*cannot = 1;
return 0;
}
nptrs = backtrace(buffer, NPY_MAX_STACKSIZE);
if (nptrs == 0) {
/* complete failure, disable elision */
init = -1;
*cannot = 1;
return 0;
}
/* setup DSO base addresses, ends updated later */
if (NPY_UNLIKELY(init == 0)) {
Dl_info info;
/* get python base address */
if (dladdr(&PyNumber_Or, &info)) {
pos_python_start = info.dli_fbase;
pos_python_end = info.dli_fbase;
}
else {
init = -1;
return 0;
}
/* get multiarray base address */
if (dladdr(&PyArray_SetNumericOps, &info)) {
pos_ma_start = info.dli_fbase;
pos_ma_end = info.dli_fbase;
}
else {
init = -1;
return 0;
}
init = 1;
}
/* loop over callstack addresses to check if they leave numpy or cpython */
for (i = 0; i < nptrs; i++) {
Dl_info info;
int in_python = 0;
int in_multiarray = 0;
#if NPY_ELIDE_DEBUG >= 2
dladdr(buffer[i], &info);
printf("%s(%p) %s(%p)\n", info.dli_fname, info.dli_fbase,
info.dli_sname, info.dli_saddr);
#endif
/* check stored DSO boundaries first */
if (buffer[i] >= pos_python_start && buffer[i] <= pos_python_end) {
in_python = 1;
}
else if (buffer[i] >= pos_ma_start && buffer[i] <= pos_ma_end) {
in_multiarray = 1;
}
/* update DSO boundaries via dladdr if necessary */
if (!in_python && !in_multiarray) {
if (dladdr(buffer[i], &info) == 0) {
init = -1;
ok = 0;
break;
}
/* update DSO end */
if (info.dli_fbase == pos_python_start) {
pos_python_end = NPY_NUMBER_MAX(buffer[i], pos_python_end);
in_python = 1;
}
else if (info.dli_fbase == pos_ma_start) {
pos_ma_end = NPY_NUMBER_MAX(buffer[i], pos_ma_end);
in_multiarray = 1;
}
}
/* no longer in ok libraries and not reached PyEval -> no elide */
if (!in_python && !in_multiarray) {
ok = 0;
break;
}
/* in python check if the frame eval function was reached */
if (in_python) {
/* if reached eval we are done */
if (find_addr(pyeval_addr, n_pyeval, buffer[i])) {
ok = 1;
break;
}
/*
* check if its some other function, use pointer lookup table to
* save expensive dladdr calls
*/
if (find_addr(py_addr, n_py_addr, buffer[i])) {
continue;
}
/* new python address, check for PyEvalFrame */
if (dladdr(buffer[i], &info) == 0) {
init = -1;
ok = 0;
break;
}
if (info.dli_sname &&
strcmp(info.dli_sname, PYFRAMEEVAL_FUNC) == 0) {
if (n_pyeval < sizeof(pyeval_addr) / sizeof(pyeval_addr[0])) {
/* store address to not have to dladdr it again */
pyeval_addr[n_pyeval++] = buffer[i];
}
ok = 1;
break;
}
else if (n_py_addr < sizeof(py_addr) / sizeof(py_addr[0])) {
/* store other py function to not have to dladdr it again */
py_addr[n_py_addr++] = buffer[i];
}
}
}
/* all stacks after numpy are from python, we can elide */
if (ok) {
*cannot = 0;
return 1;
}
else {
#if NPY_ELIDE_DEBUG != 0
puts("cannot elide due to c-api usage");
#endif
*cannot = 1;
return 0;
}
}
int NI_ZoomShift(PyArrayObject *input, PyArrayObject* zoom_ar,
PyArrayObject* shift_ar, PyArrayObject *output,
int order, int mode, double cval)
{
char *po, *pi;
npy_intp **zeros = NULL, **offsets = NULL, ***edge_offsets = NULL;
npy_intp ftmp[NPY_MAXDIMS], *fcoordinates = NULL, *foffsets = NULL;
npy_intp jj, hh, kk, filter_size, odimensions[NPY_MAXDIMS];
npy_intp idimensions[NPY_MAXDIMS], istrides[NPY_MAXDIMS];
npy_intp size;
double ***splvals = NULL;
NI_Iterator io;
npy_double *zooms = zoom_ar ? (npy_double*)PyArray_DATA(zoom_ar) : NULL;
npy_double *shifts = shift_ar ? (npy_double*)PyArray_DATA(shift_ar) : NULL;
int rank = 0;
NPY_BEGIN_THREADS_DEF;
NPY_BEGIN_THREADS;
for (kk = 0; kk < PyArray_NDIM(input); kk++) {
idimensions[kk] = PyArray_DIM(input, kk);
istrides[kk] = PyArray_STRIDE(input, kk);
odimensions[kk] = PyArray_DIM(output, kk);
}
rank = PyArray_NDIM(input);
/* if the mode is 'constant' we need some temps later: */
if (mode == NI_EXTEND_CONSTANT) {
zeros = malloc(rank * sizeof(npy_intp*));
if (NPY_UNLIKELY(!zeros)) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
for(jj = 0; jj < rank; jj++)
zeros[jj] = NULL;
for(jj = 0; jj < rank; jj++) {
zeros[jj] = malloc(odimensions[jj] * sizeof(npy_intp));
if (NPY_UNLIKELY(!zeros[jj])) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
}
}
/* store offsets, along each axis: */
offsets = malloc(rank * sizeof(npy_intp*));
/* store spline coefficients, along each axis: */
splvals = malloc(rank * sizeof(double**));
/* store offsets at all edges: */
edge_offsets = malloc(rank * sizeof(npy_intp**));
if (NPY_UNLIKELY(!offsets || !splvals || !edge_offsets)) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
for(jj = 0; jj < rank; jj++) {
offsets[jj] = NULL;
splvals[jj] = NULL;
edge_offsets[jj] = NULL;
}
for(jj = 0; jj < rank; jj++) {
offsets[jj] = malloc(odimensions[jj] * sizeof(npy_intp));
splvals[jj] = malloc(odimensions[jj] * sizeof(double*));
edge_offsets[jj] = malloc(odimensions[jj] * sizeof(npy_intp*));
if (NPY_UNLIKELY(!offsets[jj] || !splvals[jj] || !edge_offsets[jj])) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
for(hh = 0; hh < odimensions[jj]; hh++) {
splvals[jj][hh] = NULL;
edge_offsets[jj][hh] = NULL;
}
}
/* precalculate offsets, and offsets at the edge: */
for(jj = 0; jj < rank; jj++) {
double shift = 0.0, zoom = 0.0;
if (shifts)
shift = shifts[jj];
if (zooms)
zoom = zooms[jj];
for(kk = 0; kk < odimensions[jj]; kk++) {
double cc = (double)kk;
if (shifts)
cc += shift;
if (zooms)
cc *= zoom;
cc = map_coordinate(cc, idimensions[jj], mode);
if (cc > -1.0) {
npy_intp start;
if (zeros && zeros[jj])
zeros[jj][kk] = 0;
if (order & 1) {
start = (npy_intp)floor(cc) - order / 2;
} else {
start = (npy_intp)floor(cc + 0.5) - order / 2;
}
offsets[jj][kk] = istrides[jj] * start;
if (start < 0 || start + order >= idimensions[jj]) {
edge_offsets[jj][kk] = malloc((order + 1) * sizeof(npy_intp));
if (NPY_UNLIKELY(!edge_offsets[jj][kk])) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
for(hh = 0; hh <= order; hh++) {
npy_intp idx = start + hh;
npy_intp len = idimensions[jj];
if (len <= 1) {
idx = 0;
} else {
npy_intp s2 = 2 * len - 2;
if (idx < 0) {
idx = s2 * (npy_intp)(-idx / s2) + idx;
idx = idx <= 1 - len ? idx + s2 : -idx;
} else if (idx >= len) {
idx -= s2 * (npy_intp)(idx / s2);
if (idx >= len)
idx = s2 - idx;
}
}
edge_offsets[jj][kk][hh] = istrides[jj] * (idx - start);
}
}
if (order > 0) {
splvals[jj][kk] = malloc((order + 1) * sizeof(double));
if (NPY_UNLIKELY(!splvals[jj][kk])) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
spline_coefficients(cc, order, splvals[jj][kk]);
}
} else {
zeros[jj][kk] = 1;
}
}
}
filter_size = 1;
for(jj = 0; jj < rank; jj++)
filter_size *= order + 1;
if (!NI_InitPointIterator(output, &io))
goto exit;
pi = (void *)PyArray_DATA(input);
po = (void *)PyArray_DATA(output);
/* store all coordinates and offsets with filter: */
fcoordinates = malloc(rank * filter_size * sizeof(npy_intp));
foffsets = malloc(filter_size * sizeof(npy_intp));
if (NPY_UNLIKELY(!fcoordinates || !foffsets)) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
for(jj = 0; jj < rank; jj++)
ftmp[jj] = 0;
kk = 0;
for(hh = 0; hh < filter_size; hh++) {
for(jj = 0; jj < rank; jj++)
fcoordinates[jj + hh * rank] = ftmp[jj];
foffsets[hh] = kk;
for(jj = rank - 1; jj >= 0; jj--) {
if (ftmp[jj] < order) {
ftmp[jj]++;
kk += istrides[jj];
break;
} else {
ftmp[jj] = 0;
kk -= istrides[jj] * order;
}
}
}
size = PyArray_SIZE(output);
for(kk = 0; kk < size; kk++) {
double t = 0.0;
npy_intp edge = 0, oo = 0, zero = 0;
for(hh = 0; hh < rank; hh++) {
if (zeros && zeros[hh][io.coordinates[hh]]) {
/* we use constant border condition */
zero = 1;
break;
}
oo += offsets[hh][io.coordinates[hh]];
if (edge_offsets[hh][io.coordinates[hh]])
edge = 1;
}
if (!zero) {
npy_intp *ff = fcoordinates;
const int type_num = PyArray_TYPE(input);
t = 0.0;
for(hh = 0; hh < filter_size; hh++) {
npy_intp idx = 0;
double coeff = 0.0;
if (NPY_UNLIKELY(edge)) {
/* use precalculated edge offsets: */
for(jj = 0; jj < rank; jj++) {
if (edge_offsets[jj][io.coordinates[jj]])
idx += edge_offsets[jj][io.coordinates[jj]][ff[jj]];
else
idx += ff[jj] * istrides[jj];
}
idx += oo;
} else {
/* use normal offsets: */
idx += oo + foffsets[hh];
}
switch (type_num) {
CASE_INTERP_COEFF(NPY_BOOL, npy_bool,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_UBYTE, npy_ubyte,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_USHORT, npy_ushort,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_UINT, npy_uint,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_ULONG, npy_ulong,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_ULONGLONG, npy_ulonglong,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_BYTE, npy_byte,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_SHORT, npy_short,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_INT, npy_int,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_LONG, npy_long,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_LONGLONG, npy_longlong,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_FLOAT, npy_float,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_DOUBLE, npy_double,
coeff, pi, idx);
default:
NPY_END_THREADS;
PyErr_SetString(PyExc_RuntimeError,
"data type not supported");
goto exit;
}
/* calculate interpolated value: */
for(jj = 0; jj < rank; jj++)
if (order > 0)
coeff *= splvals[jj][io.coordinates[jj]][ff[jj]];
t += coeff;
ff += rank;
}
} else {
t = cval;
}
/* store output: */
switch (PyArray_TYPE(output)) {
CASE_INTERP_OUT(NPY_BOOL, npy_bool, po, t);
CASE_INTERP_OUT_UINT(UBYTE, npy_ubyte, po, t);
CASE_INTERP_OUT_UINT(USHORT, npy_ushort, po, t);
CASE_INTERP_OUT_UINT(UINT, npy_uint, po, t);
CASE_INTERP_OUT_UINT(ULONG, npy_ulong, po, t);
CASE_INTERP_OUT_UINT(ULONGLONG, npy_ulonglong, po, t);
CASE_INTERP_OUT_INT(BYTE, npy_byte, po, t);
CASE_INTERP_OUT_INT(SHORT, npy_short, po, t);
CASE_INTERP_OUT_INT(INT, npy_int, po, t);
CASE_INTERP_OUT_INT(LONG, npy_long, po, t);
CASE_INTERP_OUT_INT(LONGLONG, npy_longlong, po, t);
CASE_INTERP_OUT(NPY_FLOAT, npy_float, po, t);
CASE_INTERP_OUT(NPY_DOUBLE, npy_double, po, t);
default:
NPY_END_THREADS;
PyErr_SetString(PyExc_RuntimeError, "data type not supported");
goto exit;
}
NI_ITERATOR_NEXT(io, po);
}
exit:
NPY_END_THREADS;
if (zeros) {
for(jj = 0; jj < rank; jj++)
free(zeros[jj]);
free(zeros);
}
if (offsets) {
for(jj = 0; jj < rank; jj++)
free(offsets[jj]);
free(offsets);
}
if (splvals) {
for(jj = 0; jj < rank; jj++) {
if (splvals[jj]) {
for(hh = 0; hh < odimensions[jj]; hh++)
free(splvals[jj][hh]);
free(splvals[jj]);
}
}
free(splvals);
}
if (edge_offsets) {
for(jj = 0; jj < rank; jj++) {
if (edge_offsets[jj]) {
for(hh = 0; hh < odimensions[jj]; hh++)
free(edge_offsets[jj][hh]);
free(edge_offsets[jj]);
}
}
free(edge_offsets);
}
free(foffsets);
free(fcoordinates);
return PyErr_Occurred() ? 0 : 1;
}
int
NI_GeometricTransform(PyArrayObject *input, int (*map)(npy_intp*, double*,
int, int, void*), void* map_data, PyArrayObject* matrix_ar,
PyArrayObject* shift_ar, PyArrayObject *coordinates,
PyArrayObject *output, int order, int mode, double cval)
{
char *po, *pi, *pc = NULL;
npy_intp **edge_offsets = NULL, **data_offsets = NULL, filter_size;
npy_intp ftmp[NPY_MAXDIMS], *fcoordinates = NULL, *foffsets = NULL;
npy_intp cstride = 0, kk, hh, ll, jj;
npy_intp size;
double **splvals = NULL, icoor[NPY_MAXDIMS];
npy_intp idimensions[NPY_MAXDIMS], istrides[NPY_MAXDIMS];
NI_Iterator io, ic;
npy_double *matrix = matrix_ar ? (npy_double*)PyArray_DATA(matrix_ar) : NULL;
npy_double *shift = shift_ar ? (npy_double*)PyArray_DATA(shift_ar) : NULL;
int irank = 0, orank;
NPY_BEGIN_THREADS_DEF;
NPY_BEGIN_THREADS;
for(kk = 0; kk < PyArray_NDIM(input); kk++) {
idimensions[kk] = PyArray_DIM(input, kk);
istrides[kk] = PyArray_STRIDE(input, kk);
}
irank = PyArray_NDIM(input);
orank = PyArray_NDIM(output);
/* if the mapping is from array coordinates: */
if (coordinates) {
/* initialize a line iterator along the first axis: */
if (!NI_InitPointIterator(coordinates, &ic))
goto exit;
cstride = ic.strides[0];
if (!NI_LineIterator(&ic, 0))
goto exit;
pc = (void *)(PyArray_DATA(coordinates));
}
/* offsets used at the borders: */
edge_offsets = malloc(irank * sizeof(npy_intp*));
data_offsets = malloc(irank * sizeof(npy_intp*));
if (NPY_UNLIKELY(!edge_offsets || !data_offsets)) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
for(jj = 0; jj < irank; jj++)
data_offsets[jj] = NULL;
for(jj = 0; jj < irank; jj++) {
data_offsets[jj] = malloc((order + 1) * sizeof(npy_intp));
if (NPY_UNLIKELY(!data_offsets[jj])) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
}
/* will hold the spline coefficients: */
splvals = malloc(irank * sizeof(double*));
if (NPY_UNLIKELY(!splvals)) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
for(jj = 0; jj < irank; jj++)
splvals[jj] = NULL;
for(jj = 0; jj < irank; jj++) {
splvals[jj] = malloc((order + 1) * sizeof(double));
if (NPY_UNLIKELY(!splvals[jj])) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
}
filter_size = 1;
for(jj = 0; jj < irank; jj++)
filter_size *= order + 1;
/* initialize output iterator: */
if (!NI_InitPointIterator(output, &io))
goto exit;
/* get data pointers: */
pi = (void *)PyArray_DATA(input);
po = (void *)PyArray_DATA(output);
/* make a table of all possible coordinates within the spline filter: */
fcoordinates = malloc(irank * filter_size * sizeof(npy_intp));
/* make a table of all offsets within the spline filter: */
foffsets = malloc(filter_size * sizeof(npy_intp));
if (NPY_UNLIKELY(!fcoordinates || !foffsets)) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
for(jj = 0; jj < irank; jj++)
ftmp[jj] = 0;
kk = 0;
for(hh = 0; hh < filter_size; hh++) {
for(jj = 0; jj < irank; jj++)
fcoordinates[jj + hh * irank] = ftmp[jj];
foffsets[hh] = kk;
for(jj = irank - 1; jj >= 0; jj--) {
if (ftmp[jj] < order) {
ftmp[jj]++;
kk += istrides[jj];
break;
} else {
ftmp[jj] = 0;
kk -= istrides[jj] * order;
}
}
}
size = PyArray_SIZE(output);
for(kk = 0; kk < size; kk++) {
double t = 0.0;
int constant = 0, edge = 0;
npy_intp offset = 0;
if (map) {
NPY_END_THREADS;
/* call mappint functions: */
if (!map(io.coordinates, icoor, orank, irank, map_data)) {
if (!PyErr_Occurred())
PyErr_SetString(PyExc_RuntimeError,
"unknown error in mapping function");
goto exit;
}
NPY_BEGIN_THREADS;
} else if (matrix) {
/* do an affine transformation: */
npy_double *p = matrix;
for(hh = 0; hh < irank; hh++) {
icoor[hh] = 0.0;
for(ll = 0; ll < orank; ll++)
icoor[hh] += io.coordinates[ll] * *p++;
icoor[hh] += shift[hh];
}
} else if (coordinates) {
/* mapping is from an coordinates array: */
char *p = pc;
switch (PyArray_TYPE(coordinates)) {
CASE_MAP_COORDINATES(NPY_BOOL, npy_bool,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_UBYTE, npy_ubyte,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_USHORT, npy_ushort,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_UINT, npy_uint,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_ULONG, npy_ulong,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_ULONGLONG, npy_ulonglong,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_BYTE, npy_byte,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_SHORT, npy_short,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_INT, npy_int,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_LONG, npy_long,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_LONGLONG, npy_longlong,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_FLOAT, npy_float,
p, icoor, irank, cstride);
CASE_MAP_COORDINATES(NPY_DOUBLE, npy_double,
p, icoor, irank, cstride);
default:
NPY_END_THREADS;
PyErr_SetString(PyExc_RuntimeError,
"coordinate array data type not supported");
goto exit;
}
}
/* iterate over axes: */
for(hh = 0; hh < irank; hh++) {
/* if the input coordinate is outside the borders, map it: */
double cc = map_coordinate(icoor[hh], idimensions[hh], mode);
if (cc > -1.0) {
/* find the filter location along this axis: */
npy_intp start;
if (order & 1) {
start = (npy_intp)floor(cc) - order / 2;
} else {
start = (npy_intp)floor(cc + 0.5) - order / 2;
}
/* get the offset to the start of the filter: */
offset += istrides[hh] * start;
if (start < 0 || start + order >= idimensions[hh]) {
/* implement border mapping, if outside border: */
edge = 1;
edge_offsets[hh] = data_offsets[hh];
for(ll = 0; ll <= order; ll++) {
npy_intp idx = start + ll;
npy_intp len = idimensions[hh];
if (len <= 1) {
idx = 0;
} else {
npy_intp s2 = 2 * len - 2;
if (idx < 0) {
idx = s2 * (int)(-idx / s2) + idx;
idx = idx <= 1 - len ? idx + s2 : -idx;
} else if (idx >= len) {
idx -= s2 * (int)(idx / s2);
if (idx >= len)
idx = s2 - idx;
}
}
/* calculate and store the offests at this edge: */
edge_offsets[hh][ll] = istrides[hh] * (idx - start);
}
} else {
/* we are not at the border, use precalculated offsets: */
edge_offsets[hh] = NULL;
}
spline_coefficients(cc, order, splvals[hh]);
} else {
/* we use the constant border condition: */
constant = 1;
break;
}
}
if (!constant) {
npy_intp *ff = fcoordinates;
const int type_num = PyArray_TYPE(input);
t = 0.0;
for(hh = 0; hh < filter_size; hh++) {
double coeff = 0.0;
npy_intp idx = 0;
if (NPY_UNLIKELY(edge)) {
for(ll = 0; ll < irank; ll++) {
if (edge_offsets[ll])
idx += edge_offsets[ll][ff[ll]];
else
idx += ff[ll] * istrides[ll];
}
} else {
idx = foffsets[hh];
}
idx += offset;
switch (type_num) {
CASE_INTERP_COEFF(NPY_BOOL, npy_bool,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_UBYTE, npy_ubyte,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_USHORT, npy_ushort,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_UINT, npy_uint,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_ULONG, npy_ulong,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_ULONGLONG, npy_ulonglong,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_BYTE, npy_byte,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_SHORT, npy_short,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_INT, npy_int,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_LONG, npy_long,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_LONGLONG, npy_longlong,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_FLOAT, npy_float,
coeff, pi, idx);
CASE_INTERP_COEFF(NPY_DOUBLE, npy_double,
coeff, pi, idx);
default:
NPY_END_THREADS;
PyErr_SetString(PyExc_RuntimeError,
"data type not supported");
goto exit;
}
/* calculate the interpolated value: */
for(ll = 0; ll < irank; ll++)
if (order > 0)
coeff *= splvals[ll][ff[ll]];
t += coeff;
ff += irank;
}
} else {
t = cval;
}
/* store output value: */
switch (PyArray_TYPE(output)) {
CASE_INTERP_OUT(NPY_BOOL, npy_bool, po, t);
CASE_INTERP_OUT_UINT(UBYTE, npy_ubyte, po, t);
CASE_INTERP_OUT_UINT(USHORT, npy_ushort, po, t);
CASE_INTERP_OUT_UINT(UINT, npy_uint, po, t);
CASE_INTERP_OUT_UINT(ULONG, npy_ulong, po, t);
CASE_INTERP_OUT_UINT(ULONGLONG, npy_ulonglong, po, t);
CASE_INTERP_OUT_INT(BYTE, npy_byte, po, t);
CASE_INTERP_OUT_INT(SHORT, npy_short, po, t);
CASE_INTERP_OUT_INT(INT, npy_int, po, t);
CASE_INTERP_OUT_INT(LONG, npy_long, po, t);
CASE_INTERP_OUT_INT(LONGLONG, npy_longlong, po, t);
CASE_INTERP_OUT(NPY_FLOAT, npy_float, po, t);
CASE_INTERP_OUT(NPY_DOUBLE, npy_double, po, t);
default:
NPY_END_THREADS;
PyErr_SetString(PyExc_RuntimeError, "data type not supported");
goto exit;
}
if (coordinates) {
NI_ITERATOR_NEXT2(io, ic, po, pc);
} else {
NI_ITERATOR_NEXT(io, po);
}
}
exit:
NPY_END_THREADS;
free(edge_offsets);
if (data_offsets) {
for(jj = 0; jj < irank; jj++)
free(data_offsets[jj]);
free(data_offsets);
}
if (splvals) {
for(jj = 0; jj < irank; jj++)
free(splvals[jj]);
free(splvals);
}
free(foffsets);
free(fcoordinates);
return PyErr_Occurred() ? 0 : 1;
}
int NI_WatershedIFT(PyArrayObject* input, PyArrayObject* markers,
PyArrayObject* strct, PyArrayObject* output)
{
char *pl, *pm, *pi;
int ll;
npy_intp size, jj, hh, kk, maxval;
npy_intp strides[WS_MAXDIM], coordinates[WS_MAXDIM];
npy_intp *nstrides = NULL, nneigh, ssize;
int i_contiguous, o_contiguous;
NI_WatershedElement *temp = NULL, **first = NULL, **last = NULL;
npy_bool *ps = NULL;
NI_Iterator mi, ii, li;
NPY_BEGIN_THREADS_DEF;
i_contiguous = PyArray_ISCONTIGUOUS(input);
o_contiguous = PyArray_ISCONTIGUOUS(output);
ssize = PyArray_SIZE(strct);
if (PyArray_NDIM(input) > WS_MAXDIM) {
PyErr_SetString(PyExc_RuntimeError, "too many dimensions");
goto exit;
}
size = PyArray_SIZE(input);
/* Storage for the temporary queue data. */
temp = malloc(size * sizeof(NI_WatershedElement));
if (!temp) {
PyErr_NoMemory();
goto exit;
}
NPY_BEGIN_THREADS;
pi = (void *)PyArray_DATA(input);
if (!NI_InitPointIterator(input, &ii))
goto exit;
/* Initialization and find the maximum of the input. */
maxval = 0;
for(jj = 0; jj < size; jj++) {
npy_intp ival = 0;
switch (PyArray_TYPE(input)) {
CASE_GET_INPUT(NPY_UINT8, npy_uint8, ival, pi);
CASE_GET_INPUT(NPY_UINT16, npy_uint16, ival, pi);
default:
NPY_END_THREADS;
PyErr_SetString(PyExc_RuntimeError, "data type not supported");
goto exit;
}
temp[jj].index = jj;
temp[jj].done = 0;
if (ival > maxval)
maxval = ival;
NI_ITERATOR_NEXT(ii, pi);
}
pi = (void *)PyArray_DATA(input);
/* Allocate and initialize the storage for the queue. */
first = malloc((maxval + 1) * sizeof(NI_WatershedElement*));
last = malloc((maxval + 1) * sizeof(NI_WatershedElement*));
if (NPY_UNLIKELY(!first || !last)) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
for(hh = 0; hh <= maxval; hh++) {
first[hh] = NULL;
last[hh] = NULL;
}
if (!NI_InitPointIterator(markers, &mi))
goto exit;
if (!NI_InitPointIterator(output, &li))
goto exit;
pm = (void *)PyArray_DATA(markers);
pl = (void *)PyArray_DATA(output);
/* initialize all nodes */
for (ll = 0; ll < PyArray_NDIM(input); ll++) {
coordinates[ll] = 0;
}
for(jj = 0; jj < size; jj++) {
/* get marker */
int label = 0;
switch (PyArray_TYPE(markers)) {
CASE_GET_LABEL(NPY_UBYTE, npy_ubyte, label, pm);
CASE_GET_LABEL(NPY_USHORT, npy_ushort, label, pm);
CASE_GET_LABEL(NPY_UINT, npy_uint, label, pm);
CASE_GET_LABEL(NPY_ULONG, npy_ulong, label, pm);
CASE_GET_LABEL(NPY_ULONGLONG, npy_ulonglong, label, pm);
CASE_GET_LABEL(NPY_BYTE, npy_byte, label, pm);
CASE_GET_LABEL(NPY_SHORT, npy_short, label, pm);
CASE_GET_LABEL(NPY_INT, npy_int, label, pm);
CASE_GET_LABEL(NPY_LONG, npy_long, label, pm);
CASE_GET_LABEL(NPY_LONGLONG, npy_longlong, label, pm);
default:
NPY_END_THREADS;
PyErr_SetString(PyExc_RuntimeError, "data type not supported");
goto exit;
}
switch (PyArray_TYPE(output)) {
CASE_PUT_LABEL(NPY_UBYTE, npy_ubyte, label, pl);
CASE_PUT_LABEL(NPY_USHORT, npy_ushort, label, pl);
CASE_PUT_LABEL(NPY_UINT, npy_uint, label, pl);
CASE_PUT_LABEL(NPY_ULONG, npy_ulong, label, pl);
CASE_PUT_LABEL(NPY_ULONGLONG, npy_ulonglong, label, pl);
CASE_PUT_LABEL(NPY_BYTE, npy_byte, label, pl);
CASE_PUT_LABEL(NPY_SHORT, npy_short, label, pl);
CASE_PUT_LABEL(NPY_INT, npy_int, label, pl);
CASE_PUT_LABEL(NPY_LONG, npy_long, label, pl);
CASE_PUT_LABEL(NPY_LONGLONG, npy_longlong, label, pl);
default:
NPY_END_THREADS;
PyErr_SetString(PyExc_RuntimeError, "data type not supported");
goto exit;
}
NI_ITERATOR_NEXT2(mi, li, pm, pl);
if (label != 0) {
/* This node is a marker */
temp[jj].cost = 0;
if (!first[0]) {
first[0] = &(temp[jj]);
first[0]->next = NULL;
first[0]->prev = NULL;
last[0] = first[0];
} else {
if (label > 0) {
/* object markers are enqueued at the beginning, so they
are processed first. */
temp[jj].next = first[0];
temp[jj].prev = NULL;
first[0]->prev = &(temp[jj]);
first[0] = &(temp[jj]);
} else {
/* background markers are enqueued at the end, so they are
processed after the object markers. */
temp[jj].next = NULL;
temp[jj].prev = last[0];
last[0]->next = &(temp[jj]);
last[0] = &(temp[jj]);
}
}
} else {
/* This node is not a marker */
temp[jj].cost = maxval + 1;
temp[jj].next = NULL;
temp[jj].prev = NULL;
}
for (ll = PyArray_NDIM(input) - 1; ll >= 0; ll--) {
if (coordinates[ll] < PyArray_DIMS(input)[ll] - 1) {
coordinates[ll]++;
break;
} else {
coordinates[ll] = 0;
}
}
}
pl = (void *)PyArray_DATA(output);
ps = (npy_bool*)PyArray_DATA(strct);
nneigh = 0;
for (kk = 0; kk < ssize; kk++)
if (ps[kk] && kk != (ssize / 2))
++nneigh;
nstrides = malloc(nneigh * sizeof(npy_intp));
if (NPY_UNLIKELY(!nstrides)) {
NPY_END_THREADS;
PyErr_NoMemory();
goto exit;
}
strides[PyArray_NDIM(input) - 1] = 1;
for (ll = PyArray_NDIM(input) - 2; ll >= 0; ll--) {
strides[ll] = PyArray_DIM(input, ll + 1) * strides[ll + 1];
}
for (ll = 0; ll < PyArray_NDIM(input); ll++) {
coordinates[ll] = -1;
}
for(kk = 0; kk < nneigh; kk++)
nstrides[kk] = 0;
jj = 0;
for(kk = 0; kk < ssize; kk++) {
if (ps[kk]) {
int offset = 0;
for (ll = 0; ll < PyArray_NDIM(input); ll++) {
offset += coordinates[ll] * strides[ll];
}
if (offset != 0)
nstrides[jj++] += offset;
}
for (ll = PyArray_NDIM(input) - 1; ll >= 0; ll--) {
if (coordinates[ll] < 1) {
coordinates[ll]++;
break;
} else {
coordinates[ll] = -1;
}
}
}
/* Propagation phase: */
for(jj = 0; jj <= maxval; jj++) {
while (first[jj]) {
/* dequeue first element: */
NI_WatershedElement *v = first[jj];
first[jj] = first[jj]->next;
if (first[jj])
first[jj]->prev = NULL;
v->prev = NULL;
v->next = NULL;
/* Mark element as done: */
v->done = 1;
/* Iterate over the neighbors of the element: */
for(hh = 0; hh < nneigh; hh++) {
npy_intp v_index = v->index, p_index = v->index, idx, cc;
int qq, outside = 0;
p_index += nstrides[hh];
/* check if the neighbor is within the extent of the array: */
idx = p_index;
for (qq = 0; qq < PyArray_NDIM(input); qq++) {
cc = idx / strides[qq];
if (cc < 0 || cc >= PyArray_DIM(input, qq)) {
outside = 1;
break;
}
idx -= cc * strides[qq];
}
if (!outside) {
NI_WatershedElement *p = &(temp[p_index]);
if (!(p->done)) {
/* If the neighbor was not processed yet: */
int max, pval, vval, wvp, pcost, label, p_idx, v_idx;
switch (PyArray_TYPE(input)) {
CASE_WINDEX1(NPY_UBYTE, npy_ubyte,
v_index, p_index, strides,
PyArray_STRIDES(input), PyArray_NDIM(input),
i_contiguous, p_idx, v_idx, pi, vval, pval);
CASE_WINDEX1(NPY_USHORT, npy_ushort,
v_index, p_index, strides,
PyArray_STRIDES(input), PyArray_NDIM(input),
i_contiguous, p_idx, v_idx, pi, vval, pval);
default:
NPY_END_THREADS;
PyErr_SetString(PyExc_RuntimeError,
"data type not supported");
goto exit;
}
/* Calculate cost: */
wvp = pval - vval;
if (wvp < 0)
wvp = -wvp;
/* Find the maximum of this cost and the current
element cost: */
pcost = p->cost;
max = v->cost > wvp ? v->cost : wvp;
if (max < pcost) {
/* If this maximum is less than the neighbors cost,
adapt the cost and the label of the neighbor: */
int idx;
p->cost = max;
switch (PyArray_TYPE(output)) {
CASE_WINDEX2(NPY_UBYTE, npy_ubyte,
v_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX2(NPY_USHORT, npy_ushort,
v_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX2(NPY_UINT, npy_uint,
v_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX2(NPY_ULONG, npy_ulong,
v_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX2(NPY_ULONGLONG, npy_ulonglong,
v_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX2(NPY_BYTE, npy_byte,
v_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX2(NPY_SHORT, npy_short,
v_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX2(NPY_INT, npy_int,
v_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX2(NPY_LONG, npy_long,
v_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX2(NPY_LONGLONG, npy_longlong,
v_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
default:
NPY_END_THREADS;
PyErr_SetString(PyExc_RuntimeError,
"data type not supported");
goto exit;
}
switch (PyArray_TYPE(output)) {
CASE_WINDEX3(NPY_UBYTE, npy_ubyte,
p_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX3(NPY_USHORT, npy_ushort,
p_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX3(NPY_UINT, npy_uint,
p_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX3(NPY_ULONG, npy_ulong,
p_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX3(NPY_ULONGLONG, npy_ulonglong,
p_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX3(NPY_BYTE, npy_byte,
p_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX3(NPY_SHORT, npy_short,
p_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX3(NPY_INT, npy_int,
p_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX3(NPY_LONG, npy_long,
p_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
CASE_WINDEX3(NPY_LONGLONG, npy_longlong,
p_index, strides,
PyArray_STRIDES(output),
PyArray_NDIM(input),
idx, o_contiguous, label, pl);
default:
NPY_END_THREADS;
PyErr_SetString(PyExc_RuntimeError,
"data type not supported");
goto exit;
}
/* If the neighbor is in a queue, remove it: */
if (p->next || p->prev) {
NI_WatershedElement *prev = p->prev, *next = p->next;
if (first[pcost] == p)
first[pcost] = next;
if (last[pcost] == p)
last[pcost] = prev;
if (prev)
prev->next = next;
if (next)
next->prev = prev;
}
/* Insert the neighbor in the appropriate queue: */
if (label < 0) {
p->prev = last[max];
p->next = NULL;
if (last[max])
last[max]->next = p;
last[max] = p;
if (!first[max])
first[max] = p;
} else {
p->next = first[max];
p->prev = NULL;
if (first[max])
first[max]->prev = p;
first[max] = p;
if (!last[max])
last[max] = p;
}
}
}
}
}
}
}
exit:
NPY_END_THREADS;
free(temp);
free(first);
free(last);
free(nstrides);
return PyErr_Occurred() ? 0 : 1;
}
