int main(int argc, char **argv)
{
if (argc < 2) {
fprintf(stderr, "Usage:\n");
fprintf(stderr, "create_discodb discodb.out input.txt\n");
fprintf(stderr, "where input.txt contain a key-value pair on each line, devided by space.\n");
exit(1);
}
FILE *in;
FILE *out;
uint64_t size;
char *data;
struct ddb_cons *db = ddb_cons_new();
uint64_t flags = 0;
flags |= getenv("DONT_COMPRESS") ? DDB_OPT_DISABLE_COMPRESSION: 0;
flags |= getenv("UNIQUE_ITEMS") ? DDB_OPT_UNIQUE_ITEMS: 0;
if (!db) {
fprintf(stderr, "DB init failed\n");
exit(1);
}
if (!(in = fopen(argv[2], "r"))) {
fprintf(stderr, "Couldn't open %s\n", argv[2]);
exit(1);
}
if (getenv("KEYS_ONLY"))
read_keys(in, db);
else
read_pairs(in, db);
fprintf(stderr, "Packing the index..\n");
if (!(data = ddb_finalize(db, &size, flags))) {
fprintf(stderr, "Packing the index failed\n");
exit(1);
}
ddb_cons_free(db);
if (!(out = fopen(argv[1], "w"))) {
fprintf(stderr, "Opening file %s failed\n", argv[1]);
exit(1);
}
if (!fwrite(data, size, 1, out)) {
fprintf(stderr, "Writing file %s failed\n", argv[1]);
exit(1);
}
fclose(out);
free(data);
fprintf(stderr, "Ok! Index written to %s\n", argv[1]);
return 0;
}
int main(int argc, char **argv)
{
if (argc < 2){
fprintf(stderr, "Usage:\n");
fprintf(stderr, "create_discodb [discodb.out] [file_1] ... [file_N]\n");
fprintf(stderr, "where input files contain key on the first line and values on the next lines.\n");
exit(1);
}
struct ddb_cons *db = ddb_cons_new();
if (!db){
fprintf(stderr, "DB init failed\n");
exit(1);
}
int i = 0;
for (i = 2; i < argc; i++){
uint32_t n;
struct ddb_entry key = {NULL, 0};
struct ddb_entry *values = read_file(argv[i], &key, &n);
if (ddb_add(db, &key, values, n)){
fprintf(stderr, "Adding entries from %s failed: out of memory\n",
argv[i]);
exit(1);
}
while(n--)
free((char*)values[n].data);
free(values);
free((char*)key.data);
}
fprintf(stderr, "%u files read. Packing the index..\n", argc - 2);
uint64_t size;
char *data;
if (!(data = ddb_finalize(db, &size))){
fprintf(stderr, "Packing the index failed: duplicate keys or out of memory\n");
exit(1);
}
FILE *out;
if (!(out = fopen(argv[1], "w"))){
fprintf(stderr, "Opening file %s failed\n", argv[1]);
exit(1);
}
fwrite(data, size, 1, out);
fclose(out);
free(data);
fprintf(stderr, "Ok! Index written to %s\n", argv[1]);
return 0;
}
static PyObject *
DiscoDB_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
DiscoDB *self = (DiscoDB *)type->tp_alloc(type, 0);
PyObject
*arg = NULL,
*item = NULL,
*items = NULL,
*iteritems = NULL,
*itervalues = NULL,
*vpack = NULL,
*value = NULL,
*values = NULL,
*valueseq = NULL;
struct ddb_cons *ddb_cons = NULL;
struct ddb_entry
*kentry = NULL,
*ventry = NULL;
uint64_t n,
flags = 0,
disable_compression = 0,
unique_items = 0;
static char *kwlist[] = {"arg",
"disable_compression",
"unique_items",
NULL};
if (self != NULL) {
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OII", kwlist,
&arg,
&disable_compression,
&unique_items))
goto Done;
if (disable_compression)
flags |= DDB_OPT_DISABLE_COMPRESSION;
if (unique_items)
flags |= DDB_OPT_UNIQUE_ITEMS;
if (arg == NULL) /* null constructor */
items = PyTuple_New(0);
else if (PyMapping_Check(arg)) /* copy constructor */
items = PyMapping_Items(arg);
else /* iter constructor */
Py_INCREF(items = arg);
iteritems = PyObject_GetIter(items);
if (iteritems == NULL)
goto Done;
ddb_cons = ddb_cons_alloc();
if (ddb_cons == NULL)
goto Done;
while ((item = PyIter_Next(iteritems))) {
kentry = ddb_entry_alloc(1);
if (kentry == NULL)
goto Done;
if (!PyArg_ParseTuple(item, "s#O", &kentry->data, &kentry->length, &values))
goto Done;
Py_XINCREF(values);
if (values == NULL)
values = PyTuple_New(0);
if (PyString_Check(values))
valueseq = Py_BuildValue("(O)", values);
else
Py_XINCREF(valueseq = values);
if (valueseq == NULL)
goto Done;
itervalues = PyObject_GetIter(valueseq);
if (itervalues == NULL)
goto Done;
for (n = 0; (value = PyIter_Next(itervalues)); n++) {
ventry = ddb_entry_alloc(1);
if (ventry == NULL)
goto Done;
vpack = Py_BuildValue("(O)", value);
if (vpack == NULL)
goto Done;
if (!PyArg_ParseTuple(vpack, "s#", &ventry->data, &ventry->length))
goto Done;
if (ddb_add(ddb_cons, kentry, ventry)) {
PyErr_SetString(DiscoDBError, "Construction failed");
goto Done;
}
Py_CLEAR(vpack);
Py_CLEAR(value);
DiscoDB_CLEAR(ventry);
}
if (n == 0)
if (ddb_add(ddb_cons, kentry, NULL)) {
PyErr_SetString(DiscoDBError, "Construction failed");
goto Done;
}
Py_CLEAR(itervalues);
Py_CLEAR(item);
Py_CLEAR(values);
Py_CLEAR(valueseq);
DiscoDB_CLEAR(kentry);
}
}
self->obuffer = NULL;
self->cbuffer = ddb_finalize(ddb_cons, &n, flags);
if (self->cbuffer == NULL) {
PyErr_SetString(DiscoDBError, "Construction finalization failed");
goto Done;
}
self->discodb = ddb_alloc();
if (self->discodb == NULL)
goto Done;
if (ddb_loads(self->discodb, self->cbuffer, n))
if (ddb_has_error(self->discodb))
goto Done;
Done:
ddb_cons_dealloc(ddb_cons);
Py_CLEAR(item);
Py_CLEAR(items);
Py_CLEAR(iteritems);
Py_CLEAR(itervalues);
Py_CLEAR(vpack);
Py_CLEAR(value);
Py_CLEAR(values);
Py_CLEAR(valueseq);
DiscoDB_CLEAR(kentry);
DiscoDB_CLEAR(ventry);
if (PyErr_Occurred()) {
Py_CLEAR(self);
return NULL;
}
return (PyObject *)self;
}
