struct training load_pattern(char *dir)
{
char path[2048];
char filename[2048];
t_img_desc *img;
strcpy(path, dir);
for (dir = path; *dir; ++dir);
if (*(--dir) != '/')
strcat(path, "/");
const size_t n_font = 7;
const size_t n_letter = 52;
const size_t n_set = n_letter * n_font;
const size_t size_in = 400;
const size_t size_out = 52;
struct training t = { NULL, NULL, n_set, size_in, size_out };
t.in = malloc(sizeof(double) * t.n_in * t.n_set);
assert(t.in);
t.out = malloc(sizeof(double) * t.n_out * t.n_set);
assert(t.out);
for (size_t f = 0; f < n_font; ++f) {
/* Lower */
for (size_t l = 0; l < n_letter / 2; ++l) {
sprintf(filename, "%s%s/%c/%lu.png", path, "lower",
(char)l+'A', f);
img = load_image(filename, 1);
printf("[INFO] Load %s (%ix%i -- %i)\n", filename,
img->x, img->y, img->comp);
process_pattern(img, 1);
size_t line = l + f * n_letter;
gen_input(img, get_in(&t, line));
gen_output(get_out(&t, line), size_out, (char)l+'a');
free_image(img);
}
/* Upper */
for (size_t l = 0; l < n_letter / 2; ++l) {
sprintf(filename, "%s%s/%c/%lu.png", path, "upper",
(char)l+'A', f);
img = load_image(filename, 1);
printf("[INFO] Load %s (%ix%i -- %i)\n", filename,
img->x, img->y, img->comp);
process_pattern(img, 1);
size_t line = (l + 26) + f * n_letter;
gen_input(img, get_in(&t, line));
gen_output(get_out(&t, line), size_out, (char)l+'A');
free_image(img);
}
}
return t;
}
void* gen_xop_set(BoundBuffer* boundbuffer)
{
uint32_t random = 0;
void* pages = VirtualAlloc(
NULL,
BLOCK_SIZE,
MEM_COMMIT | MEM_RESERVE,
PAGE_EXECUTE_READWRITE);
if (pages == NULL)
return NULL;
uint8_t* nextfree = (uint8_t*)pages;
nextfree += gen_header((void*)nextfree);
while (nextfree < (uint8_t*)pages + BLOCK_SIZE - MAX_GEN_SIZE - 1)
{
boundbuffer->bounds[boundbuffer->free] = (void*)nextfree;
rand_s(&random);
random &= 0x0f;
switch (random)
{
case 0x0f:
nextfree += gen_input((void*)nextfree);
break;
case 0x00:
nextfree += gen_output((void*)nextfree);
break;
case 0x01:
nextfree += gen_xop_vprot((void*)nextfree);
break;
case 0x02:
nextfree += gen_xop_vpcom((void*)nextfree);
nextfree += gen_xop_vpcmov((void*)nextfree);
break;
case 0x03:
if (boundbuffer->count > MIN_LOOP_SIZE * 4)
{
nextfree += gen_sse41_ptest((void*)nextfree);
nextfree += gen_jcc((void*)nextfree, boundbuffer);
break;
}
default:
nextfree += gen_xop_vpperm((void*)nextfree);
break;
}
if (boundbuffer->count < BOUND_BUFFER_SIZE)
boundbuffer->count++;
boundbuffer->free = ++boundbuffer->free & BOUND_BUFFER_MASK;
}
// return pointer to end of output ...
// mov rax, rdx
// ret
*((uint32_t*)nextfree++) = 0xc3c28b48u;
nextfree--;
while (!*++nextfree
&& nextfree < (uint8_t*)pages + BLOCK_SIZE - 1)
{
*nextfree = 0xcc;
}
return pages;
}
PyObject *odr(PyObject * self, PyObject * args, PyObject * kwds)
{
PyObject *fcn, *initbeta, *py, *px, *pwe = NULL, *pwd = NULL, *fjacb = NULL;
PyObject *fjacd = NULL, *pifixb = NULL, *pifixx = NULL;
PyObject *pstpb = NULL, *pstpd = NULL, *psclb = NULL, *pscld = NULL;
PyObject *pwork = NULL, *piwork = NULL, *extra_args = NULL;
int job = 0, ndigit = 0, maxit = -1, iprint = 0;
int full_output = 0;
double taufac = 0.0, sstol = -1.0, partol = -1.0;
char *errfile = NULL, *rptfile = NULL;
int lerrfile = 0, lrptfile = 0;
PyArrayObject *beta = NULL, *y = NULL, *x = NULL, *we = NULL, *wd = NULL;
PyArrayObject *ifixb = NULL, *ifixx = NULL;
PyArrayObject *stpb = NULL, *stpd = NULL, *sclb = NULL, *scld = NULL;
PyArrayObject *work = NULL, *iwork = NULL;
int n, m, np, nq, ldy, ldx, ldwe, ld2we, ldwd, ld2wd, ldifx;
int lunerr = -1, lunrpt = -1, ldstpd, ldscld, lwork, liwork, info = 0;
static char *kw_list[] = { "fcn", "initbeta", "y", "x", "we", "wd", "fjacb",
"fjacd", "extra_args", "ifixb", "ifixx", "job", "iprint", "errfile",
"rptfile", "ndigit", "taufac", "sstol", "partol",
"maxit", "stpb", "stpd", "sclb", "scld", "work",
"iwork", "full_output", NULL
};
int isodr = 1;
PyObject *result;
npy_intp dim1[1], dim2[2], dim3[3];
int implicit; /* flag for implicit model */
if (kwds == NULL)
{
if (!PyArg_ParseTuple(args, "OOOO|OOOOOOOiiz#z#idddiOOOOOOi:odr",
&fcn, &initbeta, &py, &px, &pwe, &pwd,
&fjacb, &fjacd, &extra_args, &pifixb, &pifixx,
&job, &iprint, &errfile, &lerrfile, &rptfile,
&lrptfile, &ndigit, &taufac, &sstol, &partol,
&maxit, &pstpb, &pstpd, &psclb, &pscld, &pwork,
&piwork, &full_output))
{
return NULL;
}
}
else
{
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"OOOO|OOOOOOOiiz#z#idddiOOOOOOi:odr",
kw_list, &fcn, &initbeta, &py, &px,
&pwe, &pwd, &fjacb, &fjacd,
&extra_args, &pifixb, &pifixx, &job,
&iprint, &errfile, &lerrfile, &rptfile,
&lrptfile, &ndigit, &taufac, &sstol,
&partol, &maxit, &pstpb, &pstpd,
&psclb, &pscld, &pwork, &piwork,
&full_output))
{
return NULL;
}
}
/* Check the validity of all arguments */
if (!PyCallable_Check(fcn))
{
PYERR(PyExc_TypeError, "fcn must be callable");
}
if (!PySequence_Check(initbeta))
{
PYERR(PyExc_TypeError, "initbeta must be a sequence");
}
if (!PySequence_Check(py))
{
/* Checking whether py is an int
*
* XXX: PyInt_Check for np.int32 instances does not work on python 2.6 -
* we should fix this in numpy, workaround by trying to cast to an int
* for now */
long val;
PyErr_Clear();
val = PyInt_AsLong(py);
if (val == -1 && PyErr_Occurred()) {
PYERR(PyExc_TypeError,
"y must be a sequence or integer (if model is implicit)");
}
}
if (!PySequence_Check(px))
{
PYERR(PyExc_TypeError, "x must be a sequence");
}
if (pwe != NULL && !PySequence_Check(pwe) && !PyNumber_Check(pwe))
{
PYERR(PyExc_TypeError, "we must be a sequence or a number");
}
if (pwd != NULL && !PySequence_Check(pwd) && !PyNumber_Check(pwd))
{
PYERR(PyExc_TypeError, "wd must be a sequence or a number");
}
if (fjacb != NULL && !PyCallable_Check(fjacb))
{
PYERR(PyExc_TypeError, "fjacb must be callable");
}
if (fjacd != NULL && !PyCallable_Check(fjacd))
{
PYERR(PyExc_TypeError, "fjacd must be callable");
}
if (extra_args != NULL && !PySequence_Check(extra_args))
{
PYERR(PyExc_TypeError, "extra_args must be a sequence");
}
if (pifixx != NULL && !PySequence_Check(pifixx))
{
PYERR(PyExc_TypeError, "ifixx must be a sequence");
}
if (pifixb != NULL && !PySequence_Check(pifixb))
{
PYERR(PyExc_TypeError, "ifixb must be a sequence");
}
if (pstpb != NULL && !PySequence_Check(pstpb))
{
PYERR(PyExc_TypeError, "stpb must be a sequence");
}
if (pstpd != NULL && !PySequence_Check(pstpd))
{
PYERR(PyExc_TypeError, "stpd must be a sequence");
}
if (psclb != NULL && !PySequence_Check(psclb))
{
PYERR(PyExc_TypeError, "sclb must be a sequence");
}
if (pscld != NULL && !PySequence_Check(pscld))
{
PYERR(PyExc_TypeError, "scld must be a sequence");
}
if (pwork != NULL && !PyArray_Check(pwork))
{
PYERR(PyExc_TypeError, "work must be an array");
}
if (piwork != NULL && !PyArray_Check(piwork))
{
PYERR(PyExc_TypeError, "iwork must be an array");
}
/* start processing the arguments and check for errors on the way */
/* check for implicit model */
implicit = (job % 10 == 1);
if (!implicit)
{
if ((y =
(PyArrayObject *) PyArray_CopyFromObject(py, NPY_DOUBLE, 1,
2)) == NULL)
{
PYERR(PyExc_ValueError,
"y could not be made into a suitable array");
}
n = y->dimensions[y->nd - 1]; /* pick the last dimension */
if ((x =
(PyArrayObject *) PyArray_CopyFromObject(px, NPY_DOUBLE, 1,
2)) == NULL)
{
PYERR(PyExc_ValueError,
"x could not be made into a suitable array");
}
if (n != x->dimensions[x->nd - 1])
{
PYERR(PyExc_ValueError,
"x and y don't have matching numbers of observations");
}
if (y->nd == 1)
{
nq = 1;
}
else
{
nq = y->dimensions[0];
}
ldx = ldy = n;
}
else
{ /* we *do* have an implicit model */
ldy = 1;
nq = (int)PyInt_AsLong(py);
dim1[0] = 1;
/* initialize y to a dummy array; never referenced */
y = (PyArrayObject *) PyArray_SimpleNew(1, dim1, NPY_DOUBLE);
if ((x =
(PyArrayObject *) PyArray_CopyFromObject(px, NPY_DOUBLE, 1,
2)) == NULL)
{
PYERR(PyExc_ValueError,
"x could not be made into a suitable array");
}
n = x->dimensions[x->nd - 1];
ldx = n;
}
if (x->nd == 1)
{
m = 1;
}
else
{
m = x->dimensions[0];
} /* x, y */
if ((beta =
(PyArrayObject *) PyArray_CopyFromObject(initbeta, NPY_DOUBLE, 1,
1)) == NULL)
{
PYERR(PyExc_ValueError,
"initbeta could not be made into a suitable array");
}
np = beta->dimensions[0];
if (pwe == NULL)
{
ldwe = ld2we = 1;
dim1[0] = n;
we = (PyArrayObject *) PyArray_SimpleNew(1, dim1, NPY_DOUBLE);
((double *)(we->data))[0] = -1.0;
}
else if (PyNumber_Check(pwe) && !PyArray_Check(pwe))
{
/* we is a single weight, set the first value of we to -pwe */
PyObject *tmp;
double val;
tmp = PyNumber_Float(pwe);
if (tmp == NULL)
PYERR(PyExc_ValueError, "could not convert we to a suitable array");
val = PyFloat_AsDouble(tmp);
Py_DECREF(tmp);
dim3[0] = nq;
dim3[1] = 1;
dim3[2] = 1;
we = (PyArrayObject *) PyArray_SimpleNew(3, dim3, NPY_DOUBLE);
if (implicit)
{
((double *)(we->data))[0] = val;
}
else
{
((double *)(we->data))[0] = -val;
}
ldwe = ld2we = 1;
}
else if (PySequence_Check(pwe))
{
/* we needs to be turned into an array */
if ((we =
(PyArrayObject *) PyArray_CopyFromObject(pwe, NPY_DOUBLE, 1,
3)) == NULL)
{
PYERR(PyExc_ValueError, "could not convert we to a suitable array");
}
if (we->nd == 1 && nq == 1)
{
ldwe = n;
ld2we = 1;
}
else if (we->nd == 1 && we->dimensions[0] == nq)
{
/* we is a rank-1 array with diagonal weightings to be broadcast
* to all observations */
ldwe = 1;
ld2we = 1;
}
else if (we->nd == 3 && we->dimensions[0] == nq
&& we->dimensions[1] == nq && we->dimensions[2] == 1)
{
/* we is a rank-3 array with the covariant weightings
to be broadcast to all observations */
ldwe = 1;
ld2we = nq;
}
else if (we->nd == 2 && we->dimensions[0] == nq
&& we->dimensions[1] == nq)
{
/* we is a rank-2 array with the full covariant weightings
to be broadcast to all observations */
ldwe = 1;
ld2we = nq;
}
else if (we->nd == 2 && we->dimensions[0] == nq
&& we->dimensions[1] == n)
{
/* we is a rank-2 array with the diagonal elements of the
covariant weightings for each observation */
ldwe = n;
ld2we = 1;
}
else if (we->nd == 3 && we->dimensions[0] == nq
&& we->dimensions[1] == nq && we->dimensions[2] == n)
{
/* we is the full specification of the covariant weights
for each observation */
ldwe = n;
ld2we = nq;
}
else
{
PYERR(PyExc_ValueError, "could not convert we to a suitable array");
}
} /* we */
if (pwd == NULL)
{
ldwd = ld2wd = 1;
dim1[0] = m;
wd = (PyArrayObject *) PyArray_SimpleNew(1, dim1, NPY_DOUBLE);
((double *)(wd->data))[0] = -1.0;
}
else if (PyNumber_Check(pwd) && !PyArray_Check(pwd))
{
/* wd is a single weight, set the first value of wd to -pwd */
PyObject *tmp;
double val;
tmp = PyNumber_Float(pwd);
if (tmp == NULL)
PYERR(PyExc_ValueError, "could not convert wd to a suitable array");
val = PyFloat_AsDouble(tmp);
Py_DECREF(tmp);
dim3[0] = 1;
dim3[1] = 1;
dim3[2] = m;
wd = (PyArrayObject *) PyArray_SimpleNew(3, dim3, NPY_DOUBLE);
((double *)(wd->data))[0] = -val;
ldwd = ld2wd = 1;
}
else if (PySequence_Check(pwd))
{
/* wd needs to be turned into an array */
if ((wd =
(PyArrayObject *) PyArray_CopyFromObject(pwd, NPY_DOUBLE, 1,
3)) == NULL)
{
PYERR(PyExc_ValueError, "could not convert wd to a suitable array");
}
if (wd->nd == 1 && m == 1)
{
ldwd = n;
ld2wd = 1;
}
else if (wd->nd == 1 && wd->dimensions[0] == m)
{
/* wd is a rank-1 array with diagonal weightings to be broadcast
* to all observations */
ldwd = 1;
ld2wd = 1;
}
else if (wd->nd == 3 && wd->dimensions[0] == m
&& wd->dimensions[1] == m && wd->dimensions[2] == 1)
{
/* wd is a rank-3 array with the covariant wdightings
to be broadcast to all observations */
ldwd = 1;
ld2wd = m;
}
else if (wd->nd == 2 && wd->dimensions[0] == m
&& wd->dimensions[1] == m)
{
/* wd is a rank-2 array with the full covariant weightings
to be broadcast to all observations */
ldwd = 1;
ld2wd = m;
}
else if (wd->nd == 2 && wd->dimensions[0] == m
&& wd->dimensions[1] == n)
{
/* wd is a rank-2 array with the diagonal elements of the
covariant weightings for each observation */
ldwd = n;
ld2wd = 1;
}
else if (wd->nd == 3 && wd->dimensions[0] == m
&& wd->dimensions[1] == m && wd->dimensions[2] == n)
{
/* wd is the full specification of the covariant weights
for each observation */
ldwd = n;
ld2wd = m;
}
else
{
PYERR(PyExc_ValueError, "could not convert wd to a suitable array");
}
} /* wd */
if (pifixb == NULL)
{
dim1[0] = np;
ifixb = (PyArrayObject *) PyArray_SimpleNew(1, dim1, NPY_INT);
*(int *)(ifixb->data) = -1; /* set first element negative */
}
else
{
/* pifixb is a sequence as checked before */
if ((ifixb =
(PyArrayObject *) PyArray_CopyFromObject(pifixb, NPY_INT, 1,
1)) == NULL)
{
PYERR(PyExc_ValueError,
"could not convert ifixb to a suitable array");
}
if (ifixb->dimensions[0] != np)
{
PYERR(PyExc_ValueError,
"could not convert ifixb to a suitable array");
}
} /* ifixb */
if (pifixx == NULL)
{
dim2[0] = m;
dim2[1] = 1;
ifixx = (PyArrayObject *) PyArray_SimpleNew(2, dim2, NPY_INT);
*(int *)(ifixx->data) = -1; /* set first element negative */
ldifx = 1;
}
else
{
/* pifixx is a sequence as checked before */
if ((ifixx =
(PyArrayObject *) PyArray_CopyFromObject(pifixx, NPY_INT, 1,
2)) == NULL)
{
PYERR(PyExc_ValueError,
"could not convert ifixx to a suitable array");
}
if (ifixx->nd == 1 && ifixx->dimensions[0] == m)
{
ldifx = 1;
}
else if (ifixx->nd == 1 && ifixx->dimensions[0] == n && m == 1)
{
ldifx = n;
}
else if (ifixx->nd == 2 && ifixx->dimensions[0] == m
&& ifixx->dimensions[1] == n)
{
ldifx = n;
}
else
{
PYERR(PyExc_ValueError,
"could not convert ifixx to a suitable array");
}
} /* ifixx */
if (errfile != NULL)
{
/* call FORTRAN's OPEN to open the file with a logical unit of 18 */
lunerr = 18;
F_FUNC(dluno,DLUNO)(&lunerr, errfile, lerrfile);
}
if (rptfile != NULL)
{
/* call FORTRAN's OPEN to open the file with a logical unit of 19 */
lunrpt = 19;
F_FUNC(dluno,DLUNO)(&lunrpt, rptfile, lrptfile);
}
if (pstpb == NULL)
{
dim1[0] = np;
stpb = (PyArrayObject *) PyArray_SimpleNew(1, dim1, NPY_DOUBLE);
*(double *)(stpb->data) = 0.0;
}
else /* pstpb is a sequence */
{
if ((stpb =
(PyArrayObject *) PyArray_CopyFromObject(pstpb, NPY_DOUBLE, 1,
1)) == NULL
|| stpb->dimensions[0] != np)
{
PYERR(PyExc_ValueError,
"could not convert stpb to a suitable array");
}
} /* stpb */
if (pstpd == NULL)
{
dim2[0] = 1;
dim2[1] = m;
stpd = (PyArrayObject *) PyArray_SimpleNew(2, dim2, NPY_DOUBLE);
*(double *)(stpd->data) = 0.0;
ldstpd = 1;
}
else
{
if ((stpd =
(PyArrayObject *) PyArray_CopyFromObject(pstpd, NPY_DOUBLE, 1,
2)) == NULL)
{
PYERR(PyExc_ValueError,
"could not convert stpb to a suitable array");
}
if (stpd->nd == 1 && stpd->dimensions[0] == m)
{
ldstpd = 1;
}
else if (stpd->nd == 1 && stpd->dimensions[0] == n && m == 1)
{
ldstpd = n;
}
else if (stpd->nd == 2 && stpd->dimensions[0] == n &&
stpd->dimensions[1] == m)
{
ldstpd = n;
}
} /* stpd */
if (psclb == NULL)
{
dim1[0] = np;
sclb = (PyArrayObject *) PyArray_SimpleNew(1, dim1, NPY_DOUBLE);
*(double *)(sclb->data) = 0.0;
}
else /* psclb is a sequence */
{
if ((sclb =
(PyArrayObject *) PyArray_CopyFromObject(psclb, NPY_DOUBLE, 1,
1)) == NULL
|| sclb->dimensions[0] != np)
{
PYERR(PyExc_ValueError,
"could not convert sclb to a suitable array");
}
} /* sclb */
if (pscld == NULL)
{
dim2[0] = 1;
dim2[1] = n;
scld = (PyArrayObject *) PyArray_SimpleNew(2, dim2, NPY_DOUBLE);
*(double *)(scld->data) = 0.0;
ldscld = 1;
}
else
{
if ((scld =
(PyArrayObject *) PyArray_CopyFromObject(pscld, NPY_DOUBLE, 1,
2)) == NULL)
{
PYERR(PyExc_ValueError,
"could not convert stpb to a suitable array");
}
if (scld->nd == 1 && scld->dimensions[0] == m)
{
ldscld = 1;
}
else if (scld->nd == 1 && scld->dimensions[0] == n && m == 1)
{
ldscld = n;
}
else if (scld->nd == 2 && scld->dimensions[0] == n &&
scld->dimensions[1] == m)
{
ldscld = n;
}
} /* scld */
if (job % 10 < 2)
{
/* ODR, not OLS */
lwork =
18 + 11 * np + np * np + m + m * m + 4 * n * nq + 6 * n * m +
2 * n * nq * np + 2 * n * nq * m + nq * nq + 5 * nq + nq * (np + m) +
ldwe * ld2we * nq;
isodr = 1;
}
else
{
/* OLS, not ODR */
lwork =
18 + 11 * np + np * np + m + m * m + 4 * n * nq + 2 * n * m +
2 * n * nq * np + 5 * nq + nq * (np + m) + ldwe * ld2we * nq;
isodr = 0;
}
liwork = 20 + np + nq * (np + m);
if ((job / 10000) % 10 >= 1)
{
/* fit is a restart, make sure work and iwork are input */
if (pwork == NULL || piwork == NULL)
{
PYERR(PyExc_ValueError,
"need to input work and iwork arrays to restart");
}
}
if ((job / 1000) % 10 >= 1)
{
/* delta should be supplied, make sure the user does */
if (pwork == NULL)
{
PYERR(PyExc_ValueError,
"need to input work array for delta initialization");
}
}
if (pwork != NULL)
{
if ((work =
(PyArrayObject *) PyArray_CopyFromObject(pwork, NPY_DOUBLE, 1,
1)) == NULL)
{
PYERR(PyExc_ValueError,
"could not convert work to a suitable array");
}
if (work->dimensions[0] < lwork)
{
printf("%d %d\n", work->dimensions[0], lwork);
PYERR(PyExc_ValueError, "work is too small");
}
}
else
{
/* initialize our own work array */
dim1[0] = lwork;
work = (PyArrayObject *) PyArray_SimpleNew(1, dim1, NPY_DOUBLE);
} /* work */
if (piwork != NULL)
{
if ((iwork =
(PyArrayObject *) PyArray_CopyFromObject(piwork, NPY_INT, 1,
1)) == NULL)
{
PYERR(PyExc_ValueError,
"could not convert iwork to a suitable array");
}
if (iwork->dimensions[0] < liwork)
{
PYERR(PyExc_ValueError, "iwork is too small");
}
}
else
{
/* initialize our own iwork array */
dim1[0] = liwork;
iwork = (PyArrayObject *) PyArray_SimpleNew(1, dim1, NPY_INT);
} /* iwork */
/* check if what JOB requests can be done with what the user has
input into the function */
if ((job / 10) % 10 >= 2)
{
/* derivatives are supposed to be supplied */
if (fjacb == NULL || fjacd == NULL)
{
PYERR(PyExc_ValueError,
"need fjacb and fjacd to calculate derivatives");
}
}
/* setup the global data for the callback */
odr_global.fcn = fcn;
Py_INCREF(fcn);
odr_global.fjacb = fjacb;
Py_XINCREF(fjacb);
odr_global.fjacd = fjacd;
Py_XINCREF(fjacd);
odr_global.pyBeta = (PyObject *) beta;
Py_INCREF(beta);
odr_global.extra_args = extra_args;
Py_XINCREF(extra_args);
/* now call DODRC */
F_FUNC(dodrc,DODRC)(fcn_callback, &n, &m, &np, &nq, (double *)(beta->data),
(double *)(y->data), &ldy, (double *)(x->data), &ldx,
(double *)(we->data), &ldwe, &ld2we,
(double *)(wd->data), &ldwd, &ld2wd,
(int *)(ifixb->data), (int *)(ifixx->data), &ldifx,
&job, &ndigit, &taufac, &sstol, &partol, &maxit,
&iprint, &lunerr, &lunrpt,
(double *)(stpb->data), (double *)(stpd->data), &ldstpd,
(double *)(sclb->data), (double *)(scld->data), &ldscld,
(double *)(work->data), &lwork, (int *)(iwork->data), &liwork,
&info);
result = gen_output(n, m, np, nq, ldwe, ld2we,
beta, work, iwork, isodr, info, full_output);
if (result == NULL)
PYERR(PyExc_RuntimeError, "could not generate output");
if (lunerr != -1)
{
F_FUNC(dlunc,DLUNC)(&lunerr);
}
if (lunrpt != -1)
{
F_FUNC(dlunc,DLUNC)(&lunrpt);
}
Py_DECREF(odr_global.fcn);
Py_XDECREF(odr_global.fjacb);
Py_XDECREF(odr_global.fjacd);
Py_DECREF(odr_global.pyBeta);
Py_XDECREF(odr_global.extra_args);
odr_global.fcn = odr_global.fjacb = odr_global.fjacd = odr_global.pyBeta =
odr_global.extra_args = NULL;
Py_DECREF(beta);
Py_DECREF(y);
Py_DECREF(x);
Py_DECREF(we);
Py_DECREF(wd);
Py_DECREF(ifixb);
Py_DECREF(ifixx);
Py_DECREF(stpb);
Py_DECREF(stpd);
Py_DECREF(sclb);
Py_DECREF(scld);
Py_DECREF(work);
Py_DECREF(iwork);
return result;
fail:
if (lunerr != -1)
{
F_FUNC(dlunc,DLUNC)(&lunerr);
}
if (lunrpt != -1)
{
F_FUNC(dlunc,DLUNC)(&lunrpt);
}
Py_XDECREF(beta);
Py_XDECREF(y);
Py_XDECREF(x);
Py_XDECREF(we);
Py_XDECREF(wd);
Py_XDECREF(ifixb);
Py_XDECREF(ifixx);
Py_XDECREF(stpb);
Py_XDECREF(stpd);
Py_XDECREF(sclb);
Py_XDECREF(scld);
Py_XDECREF(work);
Py_XDECREF(iwork);
return NULL;
}
int main (int argc, char *argv[]){
//directory or disk device
int devicetype=-1;
//path to the device
char devicepath[100];
//path output log
int outputfile=0;
char outputpath[100];
//path of databases folder
int dbfolder=0;
char dbfolderpath[100];
while ((argc > 1) && (argv[1][0] == '-'))
{
switch (argv[1][1])
{
case 'f':
//Test if -d is not being used also
if(devicetype!=DEVICE)
devicetype=FOLDER;
else{
printf("Cannot use both -f and -d\n");
usage();
}
break;
case 'd':
//test if -f is not being used also
if(devicetype!=FOLDER)
devicetype=DEVICE;
else{
printf("Cannot use both -f and -d\n\n");
usage();
}
break;
case 'p':
strcpy(devicepath,&argv[1][2]);
break;
case 'o':
outputfile=1;
strcpy(outputpath,&argv[1][2]);
break;
case 'b':
str_split(&argv[1][2]);
break;
case 'z':
dbfolder=1;
strcpy(dbfolderpath,&argv[1][2]);
break;
case 'h':
help();
break;
default:
printf("Wrong Argument: %s\n", argv[1]);
usage();
exit(0);
break;
}
++argv;
--argc;
}
//test if iotype is defined
if(devicetype!=FOLDER && devicetype!=DEVICE){
printf("missing -f or -d\n\n");
usage();
exit(0);
}
//test if testype is defined
if(strlen(devicepath)==0){
printf("missing -p<value>\n\n");
usage();
exit(0);
}
//test if blocksize >0
if(nr_sizes_proc==0){
nr_sizes_proc=1;
sizes_proc=malloc(sizeof(int));
sizes_proc[0]=4096;
}
//Initialize variables
total_blocks=malloc(sizeof(uint64_t)*nr_sizes_proc);
eq=malloc(sizeof(uint64_t)*nr_sizes_proc);
dif=malloc(sizeof(uint64_t)*nr_sizes_proc);
distinctdup=malloc(sizeof(uint64_t)*nr_sizes_proc);
//zeroed_blocks=malloc(sizeof(uint64_t)*nr_sizes_proc);
space=malloc(sizeof(uint64_t)*nr_sizes_proc);
dbporiginal=malloc(sizeof(DB**)*nr_sizes_proc);
envporiginal=malloc(sizeof(DB_ENV**)*nr_sizes_proc);
dbprinter=malloc(sizeof(DB**)*nr_sizes_proc);
envprinter=malloc(sizeof(DB_ENV**)*nr_sizes_proc);
int aux=0;
for(aux=0;aux<nr_sizes_proc;aux++){
dbporiginal[aux]=malloc(sizeof(DB *));
envporiginal[aux]=malloc(sizeof(DB_ENV *));
dbprinter[aux]=malloc(sizeof(DB *));
envprinter[aux]=malloc(sizeof(DB_ENV *));
char printdbpath[100];
char duplicatedbpath[100];
char sizeid[5];
sprintf(sizeid,"%d",aux);
//if a folder were specified for databases
if(dbfolder==1){
strcpy(printdbpath,PRINTDB);
strcat(printdbpath,sizeid);
strcpy(duplicatedbpath,dbfolderpath);
strcat(duplicatedbpath,sizeid);
}
else{
strcpy(printdbpath,PRINTDB);
strcat(printdbpath,sizeid);
strcpy(duplicatedbpath,DUPLICATEDB);
strcat(duplicatedbpath,sizeid);
}
char mkcmd[200];
sprintf(mkcmd, "mkdir -p %s", printdbpath);
int ress = system(mkcmd);
sprintf(mkcmd, "mkdir -p %s", duplicatedbpath);
ress=system(mkcmd);
if(ress<0)
perror("Error creating folders for databases\n");
printf("Removing old databases\n");
//remove databases if exist
remove_db(duplicatedbpath,dbporiginal[aux],envporiginal[aux]);
remove_db(printdbpath,dbprinter[aux],envprinter[aux]);
printf("Initing new database\n");
init_db(duplicatedbpath,dbporiginal[aux],envporiginal[aux]);
if(outputfile==1){
init_db(printdbpath,dbprinter[aux],envprinter[aux]);
}
}
//initialize analysis variables
bzero(total_blocks,nr_sizes_proc*(sizeof(uint64_t)));
//identical chunks (that could be eliminated)
bzero(eq,nr_sizes_proc*(sizeof(uint64_t)));
//distinct chunks
bzero(dif,nr_sizes_proc*(sizeof(uint64_t)));
//distinct chunks with duplicates
bzero(distinctdup,nr_sizes_proc*(sizeof(uint64_t)));
//chunks that were appended with zeros due to their size
//bzero(zeroed_blocks,nr_sizes_proc*(sizeof(uint64_t)));
//duplicated disk space
bzero(space,nr_sizes_proc*(sizeof(uint64_t)));
//check if it is a folder or device and start processing
if(devicetype==FOLDER){
printf("start processing folder %s\n",devicepath);
search_dir(devicepath);
}
else{
printf("start processing device %s\n",devicepath);
extract_blocks(devicepath);
}
for(aux=0;aux<nr_sizes_proc;aux++){
fprintf(stderr,"\n\n\nResults for %d\n",sizes_proc[aux]);
fprintf(stderr,"files scanned %llu\n",(unsigned long long int)nfiles);
fprintf(stderr,"total blocks scanned %llu\n",(unsigned long long int)total_blocks[aux]);
//fprintf(stderr,"total blocks with zeros appended %llu\n",(unsigned long long int)zeroed_blocks[aux]);
//blocks without any duplicate are the distinct block minus the distinct blocks with duplicates
uint64_t zerodups=dif[aux]-distinctdup[aux];
fprintf(stderr,"blocks without duplicates %llu\n",(unsigned long long int)zerodups);
fprintf(stderr,"distinct blocks with duplicates %llu\n",(unsigned long long int)distinctdup[aux]);
fprintf(stderr,"duplicate blocks %llu\n",(unsigned long long int)eq[aux]);
fprintf(stderr,"space saved %llu Bytes\n",(unsigned long long int)space[aux]);
//if outputdist was chosen and specified generate it
if(outputfile==1){
printf("before generating output dist\n");
gen_output(dbporiginal[aux],envporiginal[aux],dbprinter[aux],envprinter[aux]);
printf("before printing output dist\n");
char outputfilename[100];
char sizeid[10];
sprintf(sizeid,"%d",sizes_proc[aux]);
strcpy(outputfilename,outputpath);
strcat(outputfilename,sizeid);
FILE* fpp=fopen(outputfilename,"w");
print_elements_print(dbprinter[aux], envprinter[aux],fpp);
fclose(fpp);
close_db(dbprinter[aux],envprinter[aux]);
//TODO this is not removed now but in the future it can be...
//remove_db(printdbpath,dbprinter[aux],envprinter[aux]);
}
close_db(dbporiginal[aux],envporiginal[aux]);
//TODO this is not removed to keep the database for dedisgen-utils
//remove_db(duplicatedbpath,dbporiginal,envporiginal);
}
//free memory
free(total_blocks);
free(eq);
free(dif);
free(distinctdup);
//free(zeroed_blocks);
free(space);
free(dbporiginal);
free(envporiginal);
free(dbprinter);
free(envprinter);
return 0;
}
