gpointer fslog_thread(gpointer d)
{
while (TRUE) {
struct item *item = get_item();
int fd = openat(dir_fd, item->key, O_CREAT|O_WRONLY|O_TRUNC, 0666);
g_assert(fd >= 0);
writebuf(fd, item->data, item->len);
finish_item(item);
fsync(fd);
fsync(dir_fd);
close(fd);
}
return NULL;
}
/****************************** Single-File Log ******************************/
gpointer flatlog_thread(gpointer d)
{
int fd = open("logfile", O_CREAT|O_WRONLY|O_TRUNC, 0666);
g_assert(fd >= 0);
int count = 0;
while (TRUE) {
struct item *item = get_item();
writebuf(fd, item->key, strlen(item->key) + 1);
writebuf(fd, (char *)&item->len, sizeof(item->len));
writebuf(fd, item->data, item->len);
count++;
if (count % opt_batchsize == 0)
fdatasync(fd);
finish_item(item);
}
return NULL;
}
int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
_cleanup_fclose_ FILE *f = NULL;
_cleanup_free_ char *payload = NULL;
unsigned n = 0;
sd_id128_t id;
char language[32];
bool got_id = false, empty_line = true;
int r;
assert(h);
assert(sb);
assert(path);
f = fopen(path, "re");
if (!f) {
log_error("Failed to open file %s: %m", path);
return -errno;
}
for (;;) {
char line[LINE_MAX];
size_t a, b, c;
char *t;
if (!fgets(line, sizeof(line), f)) {
if (feof(f))
break;
log_error("Failed to read file %s: %m", path);
return -errno;
}
n++;
truncate_nl(line);
if (line[0] == 0) {
empty_line = true;
continue;
}
if (strchr(COMMENTS "\n", line[0]))
continue;
if (empty_line &&
strlen(line) >= 2+1+32 &&
line[0] == '-' &&
line[1] == '-' &&
line[2] == ' ' &&
(line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
bool with_language;
sd_id128_t jd;
/* New entry */
with_language = line[2+1+32] != '\0';
line[2+1+32] = '\0';
if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
if (got_id) {
r = finish_item(h, sb, id, language, payload);
if (r < 0)
return r;
}
if (with_language) {
t = strstrip(line + 2 + 1 + 32 + 1);
c = strlen(t);
if (c <= 0) {
log_error("[%s:%u] Language too short.", path, n);
return -EINVAL;
}
if (c > sizeof(language) - 1) {
log_error("[%s:%u] language too long.", path, n);
return -EINVAL;
}
strscpy(language, sizeof(language), t);
} else
language[0] = '\0';
got_id = true;
empty_line = false;
id = jd;
if (payload)
payload[0] = '\0';
continue;
}
}
/* Payload */
if (!got_id) {
log_error("[%s:%u] Got payload before ID.", path, n);
return -EINVAL;
}
a = payload ? strlen(payload) : 0;
b = strlen(line);
c = a + (empty_line ? 1 : 0) + b + 1 + 1;
t = realloc(payload, c);
if (!t)
return log_oom();
if (empty_line) {
t[a] = '\n';
memcpy(t + a + 1, line, b);
t[a+b+1] = '\n';
t[a+b+2] = 0;
} else {
memcpy(t + a, line, b);
t[a+b] = '\n';
t[a+b+1] = 0;
}
payload = t;
empty_line = false;
}
if (got_id) {
r = finish_item(h, sb, id, language, payload);
if (r < 0)
return r;
}
return 0;
}
/************************* Transactional Berkeley DB *************************/
gpointer bdb_thread(gpointer d)
{
int res;
DB_ENV *env;
DB *db;
DB_TXN *txn = NULL;
int count = 0;
res = db_env_create(&env, 0);
g_assert(res == 0);
res = env->open(env, ".",
DB_CREATE | DB_RECOVER | DB_INIT_LOCK | DB_INIT_LOG
| DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD, 0644);
g_assert(res == 0);
if (opt_bdb_async) {
res = env->set_flags(env, DB_TXN_WRITE_NOSYNC, 1);
g_assert(res == 0);
}
res = db_create(&db, env, 0);
g_assert(res == 0);
res = db->open(db, NULL, "log.db", "log", DB_BTREE,
DB_CREATE | DB_THREAD | DB_AUTO_COMMIT, 0644);
g_assert(res == 0);
while (TRUE) {
if (txn == NULL && !opt_bdb_async) {
res = env->txn_begin(env, NULL, &txn, 0);
g_assert(res == 0);
}
struct item *item = get_item();
DBT key, value;
memset(&key, 0, sizeof(key));
memset(&value, 0, sizeof(value));
key.data = item->key;
key.size = strlen(item->key);
value.data = item->data;
value.size = item->len;
res = db->put(db, opt_bdb_async ? NULL : txn, &key, &value, 0);
g_assert(res == 0);
count++;
if (count % opt_batchsize == 0) {
if (opt_bdb_async) {
env->txn_checkpoint(env, 0, 0, 0);
} else {
txn->commit(txn, 0);
txn = NULL;
}
}
finish_item(item);
}
return NULL;
}