int leveldb_client_stop(leveldb *db)
{
int retval = 0;
#if DELETE_DB_FROM_DISK
char *err = NULL;
#endif
kp_debug("stopping db [%s]\n", db->name);
leveldb_close(db->db);
#if DELETE_DB_FROM_DISK
leveldb_destroy_db(db->options, db->name, &err); //destroy db on disk
if (err) {
kp_error("leveldb_destroy_db() returned error: %s\n", err);
retval = -1;
}
free_err(&err);
#endif
leveldb_options_destroy(db->options);
leveldb_readoptions_destroy(db->roptions);
leveldb_writeoptions_destroy(db->woptions);
leveldb_cache_destroy(db->cache);
leveldb_comparator_destroy(db->cmp);
leveldb_env_destroy(db->env);
free(db->name);
free(db);
kp_debug("freed the leveldb, returning %d\n", retval);
return retval;
}
int leveldb_client_get(leveldb *db, const char *key, char **value,
bool add_null_zero)
{
char *err = NULL;
size_t keylen, vallen;
char *re_value;
if (!db || !key) {
kp_error("got a NULL argument\n");
return -1;
}
keylen = strlen(key);
if (keylen < 1) {
kp_error("got empty string for key (%zu)\n", keylen);
}
kp_debug("calling leveldb_get() with key=%s, keylen=%zu\n", key, keylen);
*value = leveldb_get(db->db, db->roptions, key, keylen, &vallen, &err);
if (err) {
kp_error("leveldb_get() returned error: %s\n", err);
free_err(&err);
return -1;
}
/* leveldb_get() will return NULL if the key is not found. */
if (*value == NULL) {
kp_debug("did not find key=%s in db; returning 1\n", key);
return 1;
}
/* If we got a value back from the db, then it is not null-zero-
* terminated. If the caller asks us to, we can allocate another
* string that is null-terminated.
*/
kp_debug("get(%s) returned vallen=%zu, not-null-terminated-value=[%s]\n",
key, vallen, *value);
if (add_null_zero) {
re_value = (char *)malloc(vallen + 1);
strncpy(re_value, *value, vallen);
re_value[vallen] = '\0';
free(*value);
*value = re_value;
kp_debug("created null-terminated value=%s; vallen=%zu, strlen=%zu\n",
*value, vallen, strlen(*value));
} else {
kp_debug("add_null_zero is false, so value returned will not end "
"in a null-zero\n");
}
kp_debug("got a value, returning 0\n");
return 0;
}
int leveldb_client_put(leveldb *db, const char *key, const char *value)
{
char *err = NULL;
size_t keylen, vallen;
if (!db || !key || !value) {
kp_error("got a NULL argument\n");
return -1;
}
/* The lengths that we pass into leveldb_put() should NOT include
* the null-zero!
*/
keylen = strlen(key);
vallen = strlen(value);
if (keylen < 1 || vallen < 1) {
kp_error("got empty string for key (%zu) or val (%zu)\n",
keylen, vallen);
}
kp_debug("putting [%s:%s] into db [%s]; keylen=%zu, vallen=%zu\n",
key, value, db->name, keylen, vallen);
/* leveldb_put() goes to internal leveldb code that will copy the key
* and the value; we do not have to do this here in our leveldb client.
* After our client function returns, the caller will be able to free
* the key and the value. The leveldb code will call Slice() to slice
* off the null-zero from the key and value strings we are passing it,
* and just the character bytes will be stored in the db.
*/
leveldb_put(db->db, db->woptions, key, keylen, value, vallen, &err);
if (err) {
kp_error("leveldb_put() returned error: %s\n", err);
free_err(&err);
return -1;
}
kp_debug("leveldb_put() succeeded\n");
return 0;
}
int leveldb_client_delete(leveldb *db, const char *key)
{
char *err = NULL;
size_t keylen;
if (!db || !key){
kp_error("got a NULL arguement\n");
return -1;
}
keylen = strlen(key);
if (keylen < 1 )
kp_error("got empty string for key (%zu)\n", keylen);
kp_debug("deleting{%s} from db [%s]; keylen=%zu\n", key, db->name, keylen);
leveldb_delete(db->db, db->woptions, key, keylen, &err);
if (err){
kp_error("leveldb_delete() returned error: %s\n", err);
free_err(&err);
return -1;
}
kp_debug("leveledb_delete() succeeded\n");
return 0;
}
//TODO: add "bool create_if_missing" to this API!
int leveldb_client_start(leveldb **db, const char *name,
size_t write_buffer_size, bool use_compression, bool sync_writes)
{
int namelen;
char *db_name;
char *err = NULL;
leveldb_t* db_t;
leveldb_comparator_t *cmp;
leveldb_cache_t *cache;
leveldb_env_t *env;
leveldb_options_t *options;
leveldb_readoptions_t *roptions;
leveldb_writeoptions_t *woptions;
*db = (leveldb *)malloc(sizeof(leveldb));
if (*db == NULL) {
kp_error("malloc(leveldb) failed!\n");
return -1;
}
/* Check arguments: */
if (strlen(name) < 1) {
kp_error("name has no length!\n");
return -1;
}
if (write_buffer_size == 0) {
write_buffer_size = WRITE_BUFFER_DEFAULT;
}
if (write_buffer_size < WRITE_BUFFER_MIN) {
kp_error("write_buffer_size=%zu too small; min=%u\n",
write_buffer_size, WRITE_BUFFER_MIN);
return -1;
}
kp_debug("got args: name=%s, write_buffer_size=%zu, use_compression=%s, "
"sync_writes=%s\n",
name, write_buffer_size, use_compression ? "true" : "false",
sync_writes ? "true" : "false");
/* Format of db name: LEVELDB_DIR + / + name */
namelen = strlen(name);
db_name = malloc(namelen + 1);
if (!db_name) {
kp_error("malloc(db_name) failed!\n");
return -1;
}
strncpy(db_name, name, namelen+1); //copy null-zero
(*db)->name = db_name;
kp_debug("constructed db->name=[%s]\n", (*db)->name);
/* Set up the database's comparator, environment, cache, etc. According
* to leveldb/include/leveldb/c.h, all functions that can raise an error
* must be passed a "char **errptr" (set to NULL!) as the last argument;
* I guess functions that don't take this pointer can't return errors.
*/
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_compare,
cmp_name); //first arg is "state"
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(CACHE_CAPACITY);
// cache = NULL;
(*db)->cmp = cmp;
(*db)->env = env;
(*db)->cache = cache;
/* Set up the database's various options. Many of these will affect the
* database's performance! (see leveldb/include/leveldb/options.h).
*/
options = leveldb_options_create();
leveldb_options_set_create_if_missing(options, 1);
leveldb_options_set_comparator(options, cmp);
leveldb_options_set_error_if_exists(options, 1); //raise error if db already exists
leveldb_options_set_cache(options, cache);
// leveldb_options_set_cache(options, NULL); //disable cache??
leveldb_options_set_env(options, env);
leveldb_options_set_info_log(options, NULL); //NULL: write info to file in db's dir
leveldb_options_set_write_buffer_size(options, write_buffer_size);
/* Amount of data to build up in memory (backed by an unsorted log
* on disk) before converting to a sorted on-disk file.
* Larger values increase performance, especially during bulk loads.
* Up to two write buffers may be held in memory at the same time,
* so you may wish to adjust this parameter to control memory usage.
* Also, a larger write buffer will result in a longer recovery time
* the next time the database is opened.
* Default: 4MB (test file uses 100000 bytes)
*/
leveldb_options_set_paranoid_checks(options, 0); //default false; test file uses true
leveldb_options_set_max_open_files(options, MAX_OPEN_FILES);
leveldb_options_set_block_size(options, BLOCK_SIZE);
leveldb_options_set_block_restart_interval(options, BLOCK_RESTART_INTERVAL);
leveldb_options_set_compression(options,
use_compression ? leveldb_snappy_compression : leveldb_no_compression);
(*db)->options = options;
roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(roptions, 0);
/* If true, all data read from underlying storage will be
* verified against corresponding checksums.
* Default false; test file uses true.
*/
leveldb_readoptions_set_fill_cache(roptions, true);
/* Should the data read for this iteration be cached in memory?
* Callers may wish to set this field to false for bulk scans.
* Default true; test file uses false.
*/
(*db)->roptions = roptions;
woptions = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(woptions, sync_writes ? 1 : 0);
/* If true, the write will be flushed from the operating system
* buffer cache (by calling WritableFile::Sync()) before the write
* is considered complete. If this flag is true, writes will be
* slower.
* If this flag is false, and the machine crashes, some recent
* writes may be lost. Note that if it is just the process that
* crashes (i.e., the machine does not reboot), no writes will be
* lost even if sync==false.
* In other words, a DB write with sync==false has similar
* crash semantics as the "write()" system call. A DB write
* with sync==true has similar crash semantics to a "write()"
* system call followed by "fsync()".
*/
(*db)->woptions = woptions;
kp_debug("destroying previous copy of database, if it exists\n");
leveldb_destroy_db((*db)->options, (*db)->name, &err);
free_err(&err);
kp_debug("opening/creating database [%s]\n", (*db)->name);
db_t = leveldb_open((*db)->options, (*db)->name, &err);
if (err) {
kp_error("opening db returned error: %s\n", err);
return -1;
}
free_err(&err);
(*db)->db = db_t;
kp_debug("successfully started leveldb [%s]\n", (*db)->name);
return 0;
}
int main (int argc, const char ** argv)
{
void * p = NIL; /* WGetOpt object */
void * a = NIL; /* an ARC file object */
void * w = NIL; /* a WARC file object */
void * u = NIL; /* a UUID object */
char * aname = NIL;
afile_comp_t amode = ARC_FILE_DETECT_COMPRESSION;
warc_bool_t b = WARC_FALSE;
warc_i32_t c = 0;
warc_u8_t * flags = uS ("ca:f:t:");
char * fname = NIL;
char * wdir = ".";
wfile_comp_t cmode = WARC_FILE_UNCOMPRESSED;
if (argc < 5 || argc > 9)
{
fprintf (stderr, "ARC to WARC convertor\n");
fprintf (stderr, "Usage: %s -a <file.arc> -f <file.warc> [-c] [-t <working_dir>]\n",
argv [0]);
fprintf (stderr, "\t-a : valid ARC file name\n");
fprintf (stderr, "\t-f : valid WARC file name\n");
fprintf (stderr, "\t[-c] : WARC file will be GZIP compressed (default no)\n");
fprintf (stderr, "\t[-t] : temporary working directory (default \".\")\n");
return (2);
}
p = bless (WGetOpt, makeS (flags) );
assert (p);
/* parse command line parameters */
while ( (c = WGetOpt_parse (p, argc, argv) ) != -1)
{
switch (c)
{
case 'f' :
if (w_index (flags, c) [1] == ':')
fname = WGetOpt_argument (p);
break;
case 'c' :
cmode = WARC_FILE_COMPRESSED_GZIP;
break;
case 'a' :
if (w_index (flags, c) [1] == ':')
aname = WGetOpt_argument (p);
break;
case 't' :
if (w_index (flags, c) [1] == ':')
wdir = WGetOpt_argument (p);
break;
case '?' : /* illegal option or missing argument */
destroy (p);
return (1);
}
}
unless (aname)
{
fprintf (stderr, "missing ARC file name. Use -a option\n");
destroy (p);
return (1);
}
unless (fname)
{
fprintf (stderr, "missing WARC file name. Use -f option\n");
destroy (p);
return (1);
}
/* open an existing ARC file */
a = bless (AFile, aname, amode, wdir);
unless (a)
{
fprintf (stderr, "unable to create the Arc object\n");
free_p;
return (2);
}
/* open or create a WARC file */
w = bless (WFile, fname, WARC_MAX_SIZE,
WARC_FILE_WRITER, cmode, wdir);
unless (w)
{
fprintf (stderr, "unable to create the Warc object\n");
free_p;
free_a;
return (3);
}
/* create a UUID object */
u = bless (WUUID);
unless (u)
{
fprintf (stderr, "unable to create a UUID object\n");
free_p;
free_w;
free_a;
return (4);
}
/* loop over all ARC records */
while (AFile_hasMoreRecords (a) )
{
void * ar = AFile_nextRecord (a);
void * wr = NIL;
/* check the next ARC record */
unless (ar)
{
fprintf (stderr, "corrupted ARC\n");
free_err_out ( 5);
}
/* create an empty WARC record */
wr = bless (WRecord);
unless (wr)
{
fprintf (stderr, "unable to create the WARC record object\n");
free_ar;
free_err_out (6);
}
/* set the subject URI */
b = WRecord_setTargetUri (wr, makeS (ARecord_getUrl (ar) ) );
if (b)
free_err (7);
/* set the record tyep */
b = WRecord_setRecordType (wr, WARC_RESPONSE_RECORD);
if (b)
free_err (8);
/* set the creation date */
b = WRecord_setDateFromArc (wr, makeS (ARecord_getCreationDate (ar) ) );
if (b)
free_err (9);
/* set the content type */
b = WRecord_setContentType (wr, makeS (ARecord_getMimeType (ar) ) );
if (b)
free_err (10);
/* Create a UUID (Universal Unique IDentifier) based on URL + Timestamp */
WUUID_hash (u, makeU (ARecord_getUrl (ar) ) );
WUUID_hash (u, makeU (ARecord_getCreationDate (ar) ) );
b = WRecord_setRecordId (wr, makeS (WUUID_text (u) ) );
if (b)
free_err (11);
WUUID_reinit (u); /* re-initialize the UUID object */
/* add the ARC IP as an Anvl */
b = WRecord_setIpAddress (wr, makeS (ARecord_getIpAddress (ar) ) );
if (b)
free_err (12);
/* move the ARC record payload to the WARC record */
b = ARecord_transferContent (ar, wr, a);
if (b)
free_err (13);
/* save the WARC record into the WARC file */
b = WFile_storeRecord (w, wr, NIL);
if (b)
free_err (14);
/* free the ARC and the WARC records */
free_in;
} /* end of while */
/* free the ARC and the WARC files */
free_out;
return (0);
}
