static void test_empty(void) {
JournalFile *f1, *f2, *f3, *f4;
char t[] = "/tmp/journal-XXXXXX";
test_setup_logging(LOG_DEBUG);
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f1) == 0);
assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f2) == 0);
assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f3) == 0);
assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f4) == 0);
journal_file_print_header(f1);
puts("");
journal_file_print_header(f2);
puts("");
journal_file_print_header(f3);
puts("");
journal_file_print_header(f4);
puts("");
log_info("Done...");
if (arg_keep)
log_info("Not removing %s", t);
else {
journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
}
(void) journal_file_close(f1);
(void) journal_file_close(f2);
(void) journal_file_close(f3);
(void) journal_file_close(f4);
}
static int raw_verify(const char *fn, const char *verification_key) {
JournalFile *f;
int r;
r = journal_file_open(fn, O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, &f);
if (r < 0)
return r;
r = journal_file_verify(f, verification_key, NULL, NULL, NULL, false);
journal_file_close(f);
return r;
}
int main(int argc, char *argv[]) {
_cleanup_free_ char *fn = NULL;
char dn[] = "/var/tmp/test-journal-flush.XXXXXX";
JournalFile *new_journal = NULL;
sd_journal *j = NULL;
unsigned n = 0;
int r;
assert_se(mkdtemp(dn));
fn = strappend(dn, "/test.journal");
r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, false, NULL, NULL, NULL, NULL, &new_journal);
assert_se(r >= 0);
r = sd_journal_open(&j, 0);
assert_se(r >= 0);
sd_journal_set_data_threshold(j, 0);
SD_JOURNAL_FOREACH(j) {
Object *o;
JournalFile *f;
f = j->current_file;
assert_se(f && f->current_offset > 0);
r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
assert_se(r >= 0);
r = journal_file_copy_entry(f, new_journal, o, f->current_offset, NULL, NULL, NULL);
assert_se(r >= 0);
n++;
if (n > 10000)
break;
}
sd_journal_close(j);
(void) journal_file_close(new_journal);
unlink(fn);
assert_se(rmdir(dn) == 0);
return 0;
}
int main(int argc, char *argv[]) {
JournalFile *one, *two, *three;
char t[] = "/tmp/journal-stream-XXXXXX";
unsigned i;
_cleanup_journal_close_ sd_journal *j = NULL;
char *z;
const void *data;
size_t l;
log_set_max_level(LOG_DEBUG);
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0666, true, NULL, NULL, NULL, &one) == 0);
assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0666, true, NULL, NULL, NULL, &two) == 0);
assert_se(journal_file_open("three.journal", O_RDWR|O_CREAT, 0666, true, NULL, NULL, NULL, &three) == 0);
for (i = 0; i < N_ENTRIES; i++) {
char *p, *q;
dual_timestamp ts;
struct iovec iovec[2];
dual_timestamp_get(&ts);
assert_se(asprintf(&p, "NUMBER=%u", i) >= 0);
iovec[0].iov_base = p;
iovec[0].iov_len = strlen(p);
assert_se(asprintf(&q, "MAGIC=%s", i % 5 == 0 ? "quux" : "waldo") >= 0);
iovec[1].iov_base = q;
iovec[1].iov_len = strlen(q);
if (i % 10 == 0)
assert_se(journal_file_append_entry(three, &ts, iovec, 2, NULL, NULL, NULL) == 0);
else {
if (i % 3 == 0)
assert_se(journal_file_append_entry(two, &ts, iovec, 2, NULL, NULL, NULL) == 0);
assert_se(journal_file_append_entry(one, &ts, iovec, 2, NULL, NULL, NULL) == 0);
}
free(p);
free(q);
}
journal_file_close(one);
journal_file_close(two);
journal_file_close(three);
assert_se(sd_journal_open_directory(&j, t, 0) >= 0);
assert_se(sd_journal_add_match(j, "MAGIC=quux", 0) >= 0);
SD_JOURNAL_FOREACH_BACKWARDS(j) {
_cleanup_free_ char *c = NULL;
assert_se(sd_journal_get_data(j, "NUMBER", &data, &l) >= 0);
printf("\t%.*s\n", (int) l, (const char*) data);
assert_se(sd_journal_get_cursor(j, &c) >= 0);
assert_se(sd_journal_test_cursor(j, c) > 0);
}
SD_JOURNAL_FOREACH(j) {
_cleanup_free_ char *c = NULL;
assert_se(sd_journal_get_data(j, "NUMBER", &data, &l) >= 0);
printf("\t%.*s\n", (int) l, (const char*) data);
assert_se(sd_journal_get_cursor(j, &c) >= 0);
assert_se(sd_journal_test_cursor(j, c) > 0);
}
sd_journal_flush_matches(j);
verify_contents(j, 1);
printf("NEXT TEST\n");
assert_se(sd_journal_add_match(j, "MAGIC=quux", 0) >= 0);
assert_se(z = journal_make_match_string(j));
printf("resulting match expression is: %s\n", z);
free(z);
verify_contents(j, 5);
printf("NEXT TEST\n");
sd_journal_flush_matches(j);
assert_se(sd_journal_add_match(j, "MAGIC=waldo", 0) >= 0);
assert_se(sd_journal_add_match(j, "NUMBER=10", 0) >= 0);
assert_se(sd_journal_add_match(j, "NUMBER=11", 0) >= 0);
assert_se(sd_journal_add_match(j, "NUMBER=12", 0) >= 0);
assert_se(z = journal_make_match_string(j));
printf("resulting match expression is: %s\n", z);
free(z);
verify_contents(j, 0);
assert_se(sd_journal_query_unique(j, "NUMBER") >= 0);
SD_JOURNAL_FOREACH_UNIQUE(j, data, l)
printf("%.*s\n", (int) l, (const char*) data);
assert_se(rm_rf_dangerous(t, false, true, false) >= 0);
return 0;
}
int main(int argc, char *argv[]) {
char t[] = "/tmp/journal-XXXXXX";
unsigned n;
JournalFile *f;
const char *verification_key = argv[1];
usec_t from = 0, to = 0, total = 0;
char a[FORMAT_TIMESTAMP_MAX];
char b[FORMAT_TIMESTAMP_MAX];
char c[FORMAT_TIMESPAN_MAX];
struct stat st;
uint64_t p;
/* journal_file_open requires a valid machine id */
if (access("/etc/machine-id", F_OK) != 0)
return EXIT_TEST_SKIP;
log_set_max_level(LOG_DEBUG);
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
log_info("Generating...");
assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, true, !!verification_key, NULL, NULL, NULL, &f) == 0);
for (n = 0; n < N_ENTRIES; n++) {
struct iovec iovec;
struct dual_timestamp ts;
char *test;
dual_timestamp_get(&ts);
assert_se(asprintf(&test, "RANDOM=%lu", random() % RANDOM_RANGE));
iovec.iov_base = (void*) test;
iovec.iov_len = strlen(test);
assert_se(journal_file_append_entry(f, &ts, &iovec, 1, NULL, NULL, NULL) == 0);
free(test);
}
journal_file_close(f);
log_info("Verifying...");
assert_se(journal_file_open("test.journal", O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, &f) == 0);
/* journal_file_print_header(f); */
journal_file_dump(f);
assert_se(journal_file_verify(f, verification_key, &from, &to, &total, true) >= 0);
if (verification_key && JOURNAL_HEADER_SEALED(f->header))
log_info("=> Validated from %s to %s, %s missing",
format_timestamp(a, sizeof(a), from),
format_timestamp(b, sizeof(b), to),
format_timespan(c, sizeof(c), total > to ? total - to : 0, 0));
journal_file_close(f);
if (verification_key) {
log_info("Toggling bits...");
assert_se(stat("test.journal", &st) >= 0);
for (p = 38448*8+0; p < ((uint64_t) st.st_size * 8); p ++) {
bit_toggle("test.journal", p);
log_info("[ %"PRIu64"+%"PRIu64"]", p / 8, p % 8);
if (raw_verify("test.journal", verification_key) >= 0)
log_notice(ANSI_HIGHLIGHT_RED ">>>> %"PRIu64" (bit %"PRIu64") can be toggled without detection." ANSI_NORMAL, p / 8, p % 8);
bit_toggle("test.journal", p);
}
}
log_info("Exiting...");
assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
return 0;
}
static JournalFile *test_open (const char *name)
{
JournalFile *f;
assert_ret(journal_file_open(name, O_RDWR|O_CREAT, 0644, true, false, NULL, NULL, NULL, &f));
return f;
}
static void test_sequence_numbers(void) {
char t[] = "/tmp/journal-seq-XXXXXX";
JournalFile *one, *two;
uint64_t seqnum = 0;
sd_id128_t seqnum_id;
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0644,
true, false, NULL, NULL, NULL, &one) == 0);
append_number(one, 1, &seqnum);
printf("seqnum=%"PRIu64"\n", seqnum);
assert(seqnum == 1);
append_number(one, 2, &seqnum);
printf("seqnum=%"PRIu64"\n", seqnum);
assert(seqnum == 2);
assert(one->header->state == STATE_ONLINE);
assert(!sd_id128_equal(one->header->file_id, one->header->machine_id));
assert(!sd_id128_equal(one->header->file_id, one->header->boot_id));
assert(sd_id128_equal(one->header->file_id, one->header->seqnum_id));
memcpy(&seqnum_id, &one->header->seqnum_id, sizeof(sd_id128_t));
assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0644,
true, false, NULL, NULL, one, &two) == 0);
assert(two->header->state == STATE_ONLINE);
assert(!sd_id128_equal(two->header->file_id, one->header->file_id));
assert(sd_id128_equal(one->header->machine_id, one->header->machine_id));
assert(sd_id128_equal(one->header->boot_id, one->header->boot_id));
assert(sd_id128_equal(one->header->seqnum_id, one->header->seqnum_id));
append_number(two, 3, &seqnum);
printf("seqnum=%"PRIu64"\n", seqnum);
assert(seqnum == 3);
append_number(two, 4, &seqnum);
printf("seqnum=%"PRIu64"\n", seqnum);
assert(seqnum == 4);
test_close(two);
append_number(one, 5, &seqnum);
printf("seqnum=%"PRIu64"\n", seqnum);
assert(seqnum == 5);
append_number(one, 6, &seqnum);
printf("seqnum=%"PRIu64"\n", seqnum);
assert(seqnum == 6);
test_close(one);
/* restart server */
seqnum = 0;
assert_se(journal_file_open("two.journal", O_RDWR, 0,
true, false, NULL, NULL, NULL, &two) == 0);
assert(sd_id128_equal(two->header->seqnum_id, seqnum_id));
append_number(two, 7, &seqnum);
printf("seqnum=%"PRIu64"\n", seqnum);
assert(seqnum == 5);
/* So..., here we have the same seqnum in two files with the
* same seqnum_id. */
test_close(two);
log_info("Done...");
if (arg_keep)
log_info("Not removing %s", t);
else {
journal_directory_vacuum(".", 3000000, 0, 0, NULL);
assert_se(rm_rf_dangerous(t, false, true, false) >= 0);
}
}
static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) {
dual_timestamp ts;
JournalFile *f;
struct iovec iovec;
Object *o;
uint64_t p;
char t[] = "/tmp/journal-XXXXXX";
char data[2048] = {0};
bool is_compressed;
int r;
assert_se(data_size <= sizeof(data));
test_setup_logging(LOG_DEBUG);
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0);
dual_timestamp_get(&ts);
iovec.iov_base = (void*) data;
iovec.iov_len = data_size;
assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0);
#if HAVE_GCRYPT
journal_file_append_tag(f);
#endif
journal_file_dump(f);
/* We have to partially reimplement some of the dump logic, because the normal next_entry does the
* decompression for us. */
p = le64toh(f->header->header_size);
for (;;) {
r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o);
assert_se(r == 0);
if (o->object.type == OBJECT_DATA)
break;
assert_se(p < le64toh(f->header->tail_object_offset));
p = p + ALIGN64(le64toh(o->object.size));
}
is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0;
(void) journal_file_close(f);
log_info("Done...");
if (arg_keep)
log_info("Not removing %s", t);
else {
journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
}
puts("------------------------------------------------------------");
return is_compressed;
}
static void test_non_empty(void) {
dual_timestamp ts;
JournalFile *f;
struct iovec iovec;
static const char test[] = "TEST1=1", test2[] = "TEST2=2";
Object *o;
uint64_t p;
sd_id128_t fake_boot_id;
char t[] = "/tmp/journal-XXXXXX";
test_setup_logging(LOG_DEBUG);
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f) == 0);
assert_se(dual_timestamp_get(&ts));
assert_se(sd_id128_randomize(&fake_boot_id) == 0);
iovec.iov_base = (void*) test;
iovec.iov_len = strlen(test);
assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0);
iovec.iov_base = (void*) test2;
iovec.iov_len = strlen(test2);
assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0);
iovec.iov_base = (void*) test;
iovec.iov_len = strlen(test);
assert_se(journal_file_append_entry(f, &ts, &fake_boot_id, &iovec, 1, NULL, NULL, NULL) == 0);
#if HAVE_GCRYPT
journal_file_append_tag(f);
#endif
journal_file_dump(f);
assert_se(journal_file_next_entry(f, 0, DIRECTION_DOWN, &o, &p) == 1);
assert_se(le64toh(o->entry.seqnum) == 1);
assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 1);
assert_se(le64toh(o->entry.seqnum) == 2);
assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 1);
assert_se(le64toh(o->entry.seqnum) == 3);
assert_se(sd_id128_equal(o->entry.boot_id, fake_boot_id));
assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 0);
assert_se(journal_file_next_entry(f, 0, DIRECTION_DOWN, &o, &p) == 1);
assert_se(le64toh(o->entry.seqnum) == 1);
assert_se(journal_file_find_data_object(f, test, strlen(test), NULL, &p) == 1);
assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1);
assert_se(le64toh(o->entry.seqnum) == 1);
assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1);
assert_se(le64toh(o->entry.seqnum) == 3);
assert_se(journal_file_find_data_object(f, test2, strlen(test2), NULL, &p) == 1);
assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1);
assert_se(le64toh(o->entry.seqnum) == 2);
assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1);
assert_se(le64toh(o->entry.seqnum) == 2);
assert_se(journal_file_find_data_object(f, "quux", 4, NULL, &p) == 0);
assert_se(journal_file_move_to_entry_by_seqnum(f, 1, DIRECTION_DOWN, &o, NULL) == 1);
assert_se(le64toh(o->entry.seqnum) == 1);
assert_se(journal_file_move_to_entry_by_seqnum(f, 3, DIRECTION_DOWN, &o, NULL) == 1);
assert_se(le64toh(o->entry.seqnum) == 3);
assert_se(journal_file_move_to_entry_by_seqnum(f, 2, DIRECTION_DOWN, &o, NULL) == 1);
assert_se(le64toh(o->entry.seqnum) == 2);
assert_se(journal_file_move_to_entry_by_seqnum(f, 10, DIRECTION_DOWN, &o, NULL) == 0);
journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
(void) journal_file_close(f);
log_info("Done...");
if (arg_keep)
log_info("Not removing %s", t);
else {
journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
}
puts("------------------------------------------------------------");
}
int
main(int argc, char **argv)
{
gfarm_error_t e;
int exit_code = EXIT_SUCCESS;
int opt_list = 0;
int opt_max_seqnum_only = 0;
int c, eof;
const char *path;
journal_post_read_op_t post_read = post_read_aggregate;
if (argc > 0)
program_name = basename(argv[0]);
gflog_initialize();
while ((c = getopt(argc, argv, "dhlmrv?")) != -1) {
switch (c) {
case 'd':
gflog_set_message_verbose(2);
gflog_set_priority_level(LOG_DEBUG);
break;
case 'l':
opt_list = 1;
post_read = post_read_list;
break;
case 'm':
opt_max_seqnum_only = 1;
break;
case 'v':
opt_verbose = 1;
break;
case 'r':
opt_record_only = 1;
break;
case 'h':
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
if (argc < 1)
usage();
path = argv[0];
if ((e = journal_file_open(path, 0, GFARM_METADB_SERVER_SEQNUM_INVALID,
&jf, GFARM_JOURNAL_RDONLY)) != GFARM_ERR_NO_ERROR) {
fprintf(stderr, "%s : %s\n", program_name,
gfarm_error_string(e));
exit_code = EXIT_FAILURE;
goto end;
}
reader = journal_file_main_reader(jf);
if (opt_list && !opt_record_only && !opt_max_seqnum_only) {
printf(
"seqnum operation "
"length ");
if (opt_verbose)
printf("offset ");
printf("argument\n");
}
while ((e = db_journal_read(reader, reader, post_read, NULL, &eof))
== GFARM_ERR_NO_ERROR && eof == 0)
;
ave_reclen = num_rec > 0 ? ave_reclen / num_rec : 0;
if (num_rec == 0)
min_seqnum = 0;
if (opt_max_seqnum_only)
printf("%" GFARM_PRId64 "\n", max_seqnum);
else if (!opt_record_only) {
if (opt_list)
printf("\n");
printf("records seqnum(min/max) "
"record length(min/max/ave)\n"
"%7lu %12" GFARM_PRId64 "/%12" GFARM_PRId64
" %7lu/%7lu/%7lu\n",
(unsigned long)num_rec, min_seqnum, max_seqnum,
(unsigned long)min_reclen, (unsigned long)max_reclen,
(unsigned long)ave_reclen);
}
if (e != GFARM_ERR_NO_ERROR) {
fprintf(stderr, "%s : %s\n", program_name,
gfarm_error_string(e));
exit_code = EXIT_FAILURE;
}
end:
journal_file_close(jf);
gflog_terminate();
return (exit_code);
}
