int main(int argc, char **argv) { if (argc < 2){ fprintf(stderr, "Usage:\n"); fprintf(stderr, "create_discodb [discodb.out] [file_1] ... [file_N]\n"); fprintf(stderr, "where input files contain key on the first line and values on the next lines.\n"); exit(1); } struct ddb_cons *db = ddb_cons_new(); if (!db){ fprintf(stderr, "DB init failed\n"); exit(1); } int i = 0; for (i = 2; i < argc; i++){ uint32_t n; struct ddb_entry key = {NULL, 0}; struct ddb_entry *values = read_file(argv[i], &key, &n); if (ddb_add(db, &key, values, n)){ fprintf(stderr, "Adding entries from %s failed: out of memory\n", argv[i]); exit(1); } while(n--) free((char*)values[n].data); free(values); free((char*)key.data); } fprintf(stderr, "%u files read. Packing the index..\n", argc - 2); uint64_t size; char *data; if (!(data = ddb_finalize(db, &size))){ fprintf(stderr, "Packing the index failed: duplicate keys or out of memory\n"); exit(1); } FILE *out; if (!(out = fopen(argv[1], "w"))){ fprintf(stderr, "Opening file %s failed\n", argv[1]); exit(1); } fwrite(data, size, 1, out); fclose(out); free(data); fprintf(stderr, "Ok! Index written to %s\n", argv[1]); return 0; }
static void read_keys(FILE *in, struct ddb_cons *db) { char *key; uint32_t lc = 0; while(fscanf(in, "%as\n", &key) == 1) { struct ddb_entry key_e = {.data = key, .length = strlen(key)}; if (ddb_add(db, &key_e, NULL)) { fprintf(stderr, "Adding '%s' failed\n", key); exit(1); } free(key); ++lc; } fclose(in); fprintf(stderr, "%u keys read.\n", lc); }
static void read_pairs(FILE *in, struct ddb_cons *db) { char *key; char *value; uint32_t lc = 0; while(fscanf(in, "%as %as\n", &key, &value) == 2) { struct ddb_entry key_e = {.data = key, .length = strlen(key)}; struct ddb_entry val_e = {.data = value, .length = strlen(value)}; if (ddb_add(db, &key_e, &val_e)) { fprintf(stderr, "Adding '%s':'%s' failed\n", key, value); exit(1); } free(key); free(value); ++lc; } fclose(in); fprintf(stderr, "%u key-value pairs read.\n", lc); }
static PyObject * DiscoDB_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { DiscoDB *self = (DiscoDB *)type->tp_alloc(type, 0); PyObject *arg = NULL, *item = NULL, *items = NULL, *iteritems = NULL, *itervalues = NULL, *vpack = NULL, *value = NULL, *values = NULL, *valueseq = NULL; struct ddb_cons *ddb_cons = NULL; struct ddb_entry *kentry = NULL, *ventry = NULL; uint64_t n, flags = 0, disable_compression = 0, unique_items = 0; static char *kwlist[] = {"arg", "disable_compression", "unique_items", NULL}; if (self != NULL) { if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OII", kwlist, &arg, &disable_compression, &unique_items)) goto Done; if (disable_compression) flags |= DDB_OPT_DISABLE_COMPRESSION; if (unique_items) flags |= DDB_OPT_UNIQUE_ITEMS; if (arg == NULL) /* null constructor */ items = PyTuple_New(0); else if (PyMapping_Check(arg)) /* copy constructor */ items = PyMapping_Items(arg); else /* iter constructor */ Py_INCREF(items = arg); iteritems = PyObject_GetIter(items); if (iteritems == NULL) goto Done; ddb_cons = ddb_cons_alloc(); if (ddb_cons == NULL) goto Done; while ((item = PyIter_Next(iteritems))) { kentry = ddb_entry_alloc(1); if (kentry == NULL) goto Done; if (!PyArg_ParseTuple(item, "s#O", &kentry->data, &kentry->length, &values)) goto Done; Py_XINCREF(values); if (values == NULL) values = PyTuple_New(0); if (PyString_Check(values)) valueseq = Py_BuildValue("(O)", values); else Py_XINCREF(valueseq = values); if (valueseq == NULL) goto Done; itervalues = PyObject_GetIter(valueseq); if (itervalues == NULL) goto Done; for (n = 0; (value = PyIter_Next(itervalues)); n++) { ventry = ddb_entry_alloc(1); if (ventry == NULL) goto Done; vpack = Py_BuildValue("(O)", value); if (vpack == NULL) goto Done; if (!PyArg_ParseTuple(vpack, "s#", &ventry->data, &ventry->length)) goto Done; if (ddb_add(ddb_cons, kentry, ventry)) { PyErr_SetString(DiscoDBError, "Construction failed"); goto Done; } Py_CLEAR(vpack); Py_CLEAR(value); DiscoDB_CLEAR(ventry); } if (n == 0) if (ddb_add(ddb_cons, kentry, NULL)) { PyErr_SetString(DiscoDBError, "Construction failed"); goto Done; } Py_CLEAR(itervalues); Py_CLEAR(item); Py_CLEAR(values); Py_CLEAR(valueseq); DiscoDB_CLEAR(kentry); } } self->obuffer = NULL; self->cbuffer = ddb_finalize(ddb_cons, &n, flags); if (self->cbuffer == NULL) { PyErr_SetString(DiscoDBError, "Construction finalization failed"); goto Done; } self->discodb = ddb_alloc(); if (self->discodb == NULL) goto Done; if (ddb_loads(self->discodb, self->cbuffer, n)) if (ddb_has_error(self->discodb)) goto Done; Done: ddb_cons_dealloc(ddb_cons); Py_CLEAR(item); Py_CLEAR(items); Py_CLEAR(iteritems); Py_CLEAR(itervalues); Py_CLEAR(vpack); Py_CLEAR(value); Py_CLEAR(values); Py_CLEAR(valueseq); DiscoDB_CLEAR(kentry); DiscoDB_CLEAR(ventry); if (PyErr_Occurred()) { Py_CLEAR(self); return NULL; } return (PyObject *)self; }