int main(int argc, char** argv) { struct arguments args; set_default_args(&args); if (parse_args(argc, argv, &args)) { return 1; } PersistEntryHandle handle; struct journald_state* journald_state = NULL; msg_init(TRUE); PersistState* state = persist_state_new(args.filename); gsize state_size; guint8 persist_version; persist_state_start_dump(state); switch (args.action) { case LIST: persist_state_foreach_entry(state, &handle_entry, NULL); break; case SHOW_JOURNALD_CURSOR: handle = persist_state_lookup_entry(state, systemd_journal, &state_size, &persist_version); if (handle == 0) break; journald_state = persist_state_map_entry(state, handle); printf("journald cursor: %s\n", journald_state->cursor); persist_state_unmap_entry(state, handle); break; case SAVE_JOURNALD_CURSOR: handle = persist_state_alloc_entry(state, systemd_journal, sizeof(struct journald_state)); if (handle == 0) break; journald_state = persist_state_map_entry(state, handle); journald_state->header.version = 0; journald_state->header.big_endian = (G_BYTE_ORDER == G_BIG_ENDIAN); strncpy(journald_state->cursor, args.action_arg.journald_cursor, MAX_CURSOR_LENGTH); persist_state_unmap_entry(state, handle); persist_state_commit(state); break; default:; } persist_state_free(state); msg_deinit(); return 0; }
static void _foreach_entry_func(gpointer key, gpointer value, gpointer userdata) { PersistStateKeysForeachData *data = (PersistStateKeysForeachData *) userdata; PersistEntry *entry = (PersistEntry *) value; gchar *name = (gchar *) key; PersistValueHeader *header = persist_state_map_entry(data->storage, entry->ofs - sizeof(PersistValueHeader)); gint size = GUINT32_FROM_BE(header->size); persist_state_unmap_entry(data->storage, entry->ofs); gpointer *state = persist_state_map_entry(data->storage, entry->ofs); data->func(name, size, state, data->userdata); persist_state_unmap_entry(data->storage, entry->ofs); }
/* easier to use string based interface */ gchar * persist_state_lookup_string(PersistState *self, const gchar *key, gsize *length, guint8 *version) { PersistEntryHandle handle; gpointer block; SerializeArchive *sa; gchar *result; gsize result_len, size; guint8 result_version; gboolean success; if (!(handle = persist_state_lookup_entry(self, key, &size, &result_version))) return NULL; block = persist_state_map_entry(self, handle); sa = serialize_buffer_archive_new(block, size); success = serialize_read_cstring(sa, &result, &result_len); serialize_archive_free(sa); persist_state_unmap_entry(self, handle); if (!success) return NULL; if (length) *length = result_len; if (version) *version = result_version; return result; }
void persist_state_alloc_string(PersistState *self, const gchar *persist_name, const gchar *value, gssize len) { PersistEntryHandle handle; SerializeArchive *sa; GString *buf; gboolean success; gpointer block; if (len < 0) len = strlen(value); buf = g_string_sized_new(len + 5); sa = serialize_string_archive_new(buf); success = serialize_write_cstring(sa, value, len); g_assert(success == TRUE); serialize_archive_free(sa); handle = persist_state_alloc_entry(self, persist_name, buf->len); block = persist_state_map_entry(self, handle); memcpy(block, buf->str, buf->len); persist_state_unmap_entry(self, handle); g_string_free(buf, TRUE); }
static PersistEntryHandle _alloc_value(PersistState *self, guint32 orig_size, gboolean in_use, guint8 version) { PersistEntryHandle result; PersistValueHeader *header; guint32 size = orig_size; /* round up size to 8 bytes boundary */ if ((size & 0x7)) size = ((size >> 3) + 1) << 3; if (self->current_ofs + size + sizeof(PersistValueHeader) > self->current_size) { if (!_grow_store(self, self->current_size + sizeof(PersistValueHeader) + size)) return 0; } result = self->current_ofs + sizeof(PersistValueHeader); /* fill value header */ header = (PersistValueHeader *) persist_state_map_entry(self, self->current_ofs); header->size = GUINT32_TO_BE(orig_size); header->in_use = in_use; header->version = version; persist_state_unmap_entry(self, self->current_ofs); self->current_ofs += size + sizeof(PersistValueHeader); return result; }
static void _reader_save_state(Bookmark *bookmark) { JournalBookmarkData *bookmark_data = (JournalBookmarkData *)(&bookmark->container); JournalReaderState *state = persist_state_map_entry(bookmark->persist_state, bookmark_data->persist_handle); strcpy(state->cursor, bookmark_data->cursor); persist_state_unmap_entry(bookmark->persist_state, bookmark_data->persist_handle); }
static void _alloc_state(JournalReader *self) { self->persist_handle = persist_state_alloc_entry(self->persist_state, self->persist_name, sizeof(JournalReaderState)); JournalReaderState *state = persist_state_map_entry(self->persist_state, self->persist_handle); state->header.version = 0; state->header.big_endian = (G_BYTE_ORDER == G_BIG_ENDIAN); persist_state_unmap_entry(self->persist_state, self->persist_handle); }
void write_test_file_for_test_in_use_handle(gboolean in_use_handle) { PersistState *state = clean_and_create_persist_state_for_test("test_in_use.persist"); PersistEntryHandle handle = persist_state_alloc_entry(state, "alma", sizeof(TestState)); TestState *test_state = (TestState*) persist_state_map_entry(state, handle); test_state->value = 0xDEADBEEF; persist_state_unmap_entry(state, handle); if (!in_use_handle) persist_state_remove_entry(state, "alma"); commit_and_free_persist_state(state); }
void test_persist_state_foreach_entry(void) { PersistState *state = clean_and_create_persist_state_for_test("test_persist_foreach.persist"); PersistEntryHandle handle = persist_state_alloc_entry(state, "test", sizeof(TestState)); TestState *test_state = persist_state_map_entry(state, handle); test_state->value = 3; persist_state_unmap_entry(state, handle); persist_state_foreach_entry(state, _foreach_callback_assertions, "test_userdata"); cancel_and_destroy_persist_state(state); }
LogProtoBufferedServerState * log_proto_buffered_server_get_state(LogProtoBufferedServer *self) { if (self->persist_state) { g_assert(self->persist_handle != 0); return persist_state_map_entry(self->persist_state, self->persist_handle); } if (G_UNLIKELY(!self->state1)) { self->state1 = g_new0(LogProtoBufferedServerState, 1); } return self->state1; }
static void create_persist_file_with_hostid(const gchar *persist_file, guint32 hostid) { PersistState *state; PersistEntryHandle handle; HostIdState *host_id_state; unlink(persist_file); state = create_persist_state(persist_file); handle = persist_state_alloc_entry(state, HOST_ID_PERSIST_KEY, sizeof(HostIdState)); host_id_state = persist_state_map_entry(state, handle); host_id_state->host_id = hostid; persist_state_unmap_entry(state, handle); persist_state_commit(state); persist_state_free(state); }
static void _buffered_server_bookmark_save(Bookmark *bookmark) { BufferedServerBookmarkData *bookmark_data = (BufferedServerBookmarkData *)(&bookmark->container); LogProtoBufferedServerState *state = persist_state_map_entry(bookmark->persist_state, bookmark_data->persist_handle); state->buffer_pos = bookmark_data->pending_buffer_pos; state->raw_stream_pos = bookmark_data->pending_raw_stream_pos; state->raw_buffer_size = bookmark_data->pending_raw_buffer_size; msg_trace("Last message got confirmed", evt_tag_int("raw_stream_pos", state->raw_stream_pos), evt_tag_int("raw_buffer_len", state->raw_buffer_size), evt_tag_int("buffer_pos", state->buffer_pos), evt_tag_int("buffer_end", state->pending_buffer_end)); persist_state_unmap_entry(bookmark->persist_state, bookmark_data->persist_handle); }
static guint32 load_hostid_from_persist(const gchar *persist_file) { PersistState *state; PersistEntryHandle handle; HostIdState *host_id_state; gsize size; guint8 version; guint32 result; state = create_persist_state(persist_file); handle = persist_state_lookup_entry(state, HOST_ID_PERSIST_KEY, &size, &version); assert_true(handle != 0, "cannot find hostid in persist file"); host_id_state = persist_state_map_entry(state, handle); result = host_id_state->host_id; persist_state_unmap_entry(state, handle); persist_state_free(state); return result; }
static PersistEntryHandle log_proto_buffered_server_alloc_state(LogProtoBufferedServer *self, PersistState *persist_state, const gchar *persist_name) { LogProtoBufferedServerState *state; PersistEntryHandle handle; handle = persist_state_alloc_entry(persist_state, persist_name, sizeof(LogProtoBufferedServerState)); if (handle) { state = persist_state_map_entry(persist_state, handle); state->header.version = 0; state->header.big_endian = (G_BYTE_ORDER == G_BIG_ENDIAN); persist_state_unmap_entry(persist_state, handle); } return handle; }
/* function to load v2 and v3 format persistent files */ static gboolean _load_v23(PersistState *self, gint version, SerializeArchive *sa) { gchar *key, *value; while (serialize_read_cstring(sa, &key, NULL)) { gsize len; guint32 str_len; if (key[0] && serialize_read_cstring(sa, &value, &len)) { gpointer new_block; PersistEntryHandle new_handle; /* add length of the string */ new_handle = _alloc_value(self, len + sizeof(str_len), FALSE, version); new_block = persist_state_map_entry(self, new_handle); /* NOTE: we add an extra length field to the old value, as our * persist_state_lookup_string() needs that. * persist_state_lookup_string is used to fetch disk queue file * names. It could have been solved somewhat better, but now it * doesn't justify a persist-state file format change. */ str_len = GUINT32_TO_BE(len); memcpy(new_block, &str_len, sizeof(str_len)); memcpy(new_block + sizeof(str_len), value, len); persist_state_unmap_entry(self, new_handle); /* add key to the current file */ _add_key(self, key, new_handle); g_free(value); g_free(key); } else { g_free(key); break; } } return TRUE; }
static inline gboolean _seek_to_saved_state(JournalReader *self) { JournalReaderState *state = persist_state_map_entry(self->persist_state, self->persist_handle); gint rc = journald_seek_cursor(self->journal, state->cursor); persist_state_unmap_entry(self->persist_state, self->persist_handle); if (rc != 0) { msg_warning("Failed to seek journal to the saved cursor position", evt_tag_str("cursor", state->cursor), evt_tag_errno("error", errno)); return _seek_to_head(self); } else { msg_debug("Seeking the journal to the last cursor position", evt_tag_str("cursor", state->cursor)); } journald_next(self->journal); return TRUE; }
static PersistValueHeader * _map_header_of_entry_from_handle(PersistState *self, PersistEntryHandle handle) { PersistValueHeader *header; if (handle > self->current_size) { msg_error("Corrupted handle in persist_state_lookup_entry, handle value too large", evt_tag_printf("handle", "%08x", handle), NULL); return NULL; } header = (PersistValueHeader *) persist_state_map_entry(self, handle - sizeof(PersistValueHeader)); if (GUINT32_FROM_BE(header->size) + handle > self->current_size) { msg_error("Corrupted entry header found in persist_state_lookup_entry, size too large", evt_tag_printf("handle", "%08x", handle), evt_tag_int("size", GUINT32_FROM_BE(header->size)), evt_tag_int("file_size", self->current_size), NULL); return NULL; } return header; }
/* * NOTE: can only be called from the main thread (e.g. log_pipe_init/deinit). */ static gboolean _add_key(PersistState *self, const gchar *key, PersistEntryHandle handle) { PersistEntry *entry; gpointer key_area; gboolean new_block_created = FALSE; SerializeArchive *sa; g_assert(key[0] != 0); entry = g_new(PersistEntry, 1); entry->ofs = handle; g_hash_table_insert(self->keys, g_strdup(key), entry); /* we try to insert the key into the current block first, then if it doesn't fit, we create a new block */ while (1) { /* the size of the key block chain part, 4 byte for the empty string length, guint32 for the link to the next block */ guint32 chain_size = sizeof(guint32) + sizeof(guint32); gboolean success; key_area = persist_state_map_entry(self, self->current_key_block); /* we reserve space for the next area pointer */ sa = serialize_buffer_archive_new(key_area + self->current_key_ofs, self->current_key_size - self->current_key_ofs - chain_size); sa->silent = TRUE; success = serialize_write_cstring(sa, key, -1) && serialize_write_uint32(sa, handle); if (!success) { serialize_archive_free(sa); if (!new_block_created) { PersistEntryHandle new_block; /* we unmap the key_area as otherwise we can't grow because of the pending maps */ persist_state_unmap_entry(self, self->current_key_block); /* ah, we couldn't fit into the current block, create a new one and link it off the old one */ new_block = _alloc_value(self, PERSIST_STATE_KEY_BLOCK_SIZE, TRUE, 0); if (!new_block) { msg_error("Unable to allocate space in the persistent file for key store", NULL); return FALSE; } key_area = persist_state_map_entry(self, self->current_key_block); sa = serialize_buffer_archive_new(key_area + self->current_key_ofs, self->current_key_size - self->current_key_ofs); if (!serialize_write_cstring(sa, "", 0) || !serialize_write_uint32(sa, new_block)) { /* hmmm. now this is bad, we couldn't write the tail of the block even though we always reserved space for it, this is a programming error somewhere in this function. */ g_assert_not_reached(); } serialize_archive_free(sa); persist_state_unmap_entry(self, self->current_key_block); self->current_key_block = new_block; self->current_key_size = PERSIST_STATE_KEY_BLOCK_SIZE; self->current_key_ofs = 0; new_block_created = TRUE; } else { /* if this happens, that means that the current key * entry won't fit even into a freshly initialized key * block, this means that the key is too large. */ msg_error("Persistent key too large, it cannot be larger than somewhat less than 4k", evt_tag_str("key", key), NULL); persist_state_unmap_entry(self, self->current_key_block); return FALSE; } } else { self->header->key_count = GUINT32_TO_BE(GUINT32_FROM_BE(self->header->key_count) + 1); self->current_key_ofs += serialize_buffer_archive_get_pos(sa); serialize_archive_free(sa); persist_state_unmap_entry(self, self->current_key_block); return TRUE; } } g_assert_not_reached(); }
gboolean log_proto_buffered_server_restart_with_state(LogProtoServer *s, PersistState *persist_state, const gchar *persist_name) { LogProtoBufferedServer *self = (LogProtoBufferedServer *) s; guint8 persist_version; PersistEntryHandle old_state_handle; gpointer old_state; gsize old_state_size; PersistEntryHandle new_state_handle = 0; gpointer new_state = NULL; gboolean success; self->pos_tracking = TRUE; self->persist_state = persist_state; old_state_handle = persist_state_lookup_entry(persist_state, persist_name, &old_state_size, &persist_version); if (!old_state_handle) { new_state_handle = log_proto_buffered_server_alloc_state(self, persist_state, persist_name); if (!new_state_handle) goto fallback_non_persistent; log_proto_buffered_server_apply_state(self, new_state_handle, persist_name); return TRUE; } if (persist_version < 4) { new_state_handle = log_proto_buffered_server_alloc_state(self, persist_state, persist_name); if (!new_state_handle) goto fallback_non_persistent; old_state = persist_state_map_entry(persist_state, old_state_handle); new_state = persist_state_map_entry(persist_state, new_state_handle); success = log_proto_buffered_server_convert_state(self, persist_version, old_state, old_state_size, new_state); persist_state_unmap_entry(persist_state, old_state_handle); persist_state_unmap_entry(persist_state, new_state_handle); /* we're using the newly allocated state structure regardless if * conversion succeeded. If the conversion went wrong then * new_state is still in its freshly initialized form since the * convert function will not touch the state in the error * branches. */ log_proto_buffered_server_apply_state(self, new_state_handle, persist_name); return success; } else if (persist_version == 4) { LogProtoBufferedServerState *state; old_state = persist_state_map_entry(persist_state, old_state_handle); state = old_state; if ((state->header.big_endian && G_BYTE_ORDER == G_LITTLE_ENDIAN) || (!state->header.big_endian && G_BYTE_ORDER == G_BIG_ENDIAN)) { /* byte order conversion in order to avoid the hassle with scattered byte order conversions in the code */ state->header.big_endian = !state->header.big_endian; state->buffer_pos = GUINT32_SWAP_LE_BE(state->buffer_pos); state->pending_buffer_pos = GUINT32_SWAP_LE_BE(state->pending_buffer_pos); state->pending_buffer_end = GUINT32_SWAP_LE_BE(state->pending_buffer_end); state->buffer_size = GUINT32_SWAP_LE_BE(state->buffer_size); state->raw_stream_pos = GUINT64_SWAP_LE_BE(state->raw_stream_pos); state->raw_buffer_size = GUINT32_SWAP_LE_BE(state->raw_buffer_size); state->pending_raw_stream_pos = GUINT64_SWAP_LE_BE(state->pending_raw_stream_pos); state->pending_raw_buffer_size = GUINT32_SWAP_LE_BE(state->pending_raw_buffer_size); state->file_size = GUINT64_SWAP_LE_BE(state->file_size); state->file_inode = GUINT64_SWAP_LE_BE(state->file_inode); } if (state->header.version > 0) { msg_error("Internal error restoring log reader state, stored data is too new", evt_tag_int("version", state->header.version)); goto error; } persist_state_unmap_entry(persist_state, old_state_handle); log_proto_buffered_server_apply_state(self, old_state_handle, persist_name); return TRUE; } else { msg_error("Internal error restoring log reader state, stored data is too new", evt_tag_int("version", persist_version)); goto error; } return TRUE; fallback_non_persistent: new_state = g_new0(LogProtoBufferedServerState, 1); error: if (!new_state) { new_state_handle = log_proto_buffered_server_alloc_state(self, persist_state, persist_name); if (!new_state_handle) goto fallback_non_persistent; new_state = persist_state_map_entry(persist_state, new_state_handle); } if (new_state) { LogProtoBufferedServerState *state = new_state; /* error happened, restart the file from the beginning */ state->raw_stream_pos = 0; state->file_inode = 0; state->file_size = 0; if (new_state_handle) log_proto_buffered_server_apply_state(self, new_state_handle, persist_name); else self->state1 = new_state; } if (new_state_handle) { persist_state_unmap_entry(persist_state, new_state_handle); } return FALSE; }
void test_values(void) { PersistState *state; gint i, j; gchar *data; state = clean_and_create_persist_state_for_test("test_values.persist"); for (i = 0; i < 1000; i++) { gchar buf[16]; PersistEntryHandle handle; g_snprintf(buf, sizeof(buf), "testkey%d", i); if (!(handle = persist_state_alloc_entry(state, buf, 128))) { fprintf(stderr, "Error allocating value in the persist file: %s\n", buf); exit(1); } data = persist_state_map_entry(state, handle); for (j = 0; j < 128; j++) { data[j] = (i % 26) + 'A'; } persist_state_unmap_entry(state, handle); } for (i = 0; i < 1000; i++) { gchar buf[16]; PersistEntryHandle handle; gsize size; guint8 version; g_snprintf(buf, sizeof(buf), "testkey%d", i); if (!(handle = persist_state_lookup_entry(state, buf, &size, &version))) { fprintf(stderr, "Error retrieving value from the persist file: %s\n", buf); exit(1); } data = persist_state_map_entry(state, handle); for (j = 0; j < 128; j++) { if (data[j] != (i % 26) + 'A') { fprintf(stderr, "Invalid data in persistent entry\n"); exit(1); } } persist_state_unmap_entry(state, handle); } state = restart_persist_state(state); for (i = 0; i < 1000; i++) { gchar buf[16]; PersistEntryHandle handle; gsize size; guint8 version; g_snprintf(buf, sizeof(buf), "testkey%d", i); if (!(handle = persist_state_lookup_entry(state, buf, &size, &version))) { fprintf(stderr, "Error retrieving value from the persist file: %s\n", buf); exit(1); } if (size != 128 || version != 4) { fprintf(stderr, "Error retrieving value from the persist file: %s, invalid size (%d) or version (%d)\n", buf, (gint) size, version); exit(1); } data = persist_state_map_entry(state, handle); for (j = 0; j < 128; j++) { if (data[j] != (i % 26) + 'A') { fprintf(stderr, "Invalid data in persistent entry\n"); exit(1); } } persist_state_unmap_entry(state, handle); } cancel_and_destroy_persist_state(state); }
static gboolean _load_v4(PersistState *self, gboolean load_all_entries) { gint fd; gint64 file_size; gpointer map; gpointer key_block; guint32 key_size; PersistFileHeader *header; gint key_count, i; fd = open(self->commited_filename, O_RDONLY); if (fd < 0) { /* no previous data found */ return TRUE; } file_size = lseek(fd, 0, SEEK_END); if (file_size > ((1LL << 31) - 1)) { msg_error("Persistent file too large", evt_tag_str("filename", self->commited_filename), evt_tag_printf("size", "%" G_GINT64_FORMAT, file_size), NULL); return FALSE; } map = mmap(NULL, file_size, PROT_READ, MAP_SHARED, fd, 0); close(fd); if (map == MAP_FAILED) { msg_error("Error mapping persistent file into memory", evt_tag_str("filename", self->commited_filename), evt_tag_errno("error", errno), NULL); return FALSE; } header = (PersistFileHeader *) map; key_block = ((gchar *) map) + offsetof(PersistFileHeader, initial_key_store); key_size = sizeof((((PersistFileHeader *) NULL))->initial_key_store); key_count = GUINT32_FROM_BE(header->key_count); i = 0; while (i < key_count) { gchar *name; guint32 entry_ofs, chain_ofs; SerializeArchive *sa; sa = serialize_buffer_archive_new(key_block, key_size); while (i < key_count) { if (!serialize_read_cstring(sa, &name, NULL)) { serialize_archive_free(sa); msg_error("Persistent file format error, unable to fetch key name", NULL); goto free_and_exit; } if (name[0]) { if (serialize_read_uint32(sa, &entry_ofs)) { PersistValueHeader *value_header; i++; if (entry_ofs < sizeof(PersistFileHeader) || entry_ofs > file_size) { serialize_archive_free(sa); g_free(name); msg_error("Persistent file format error, entry offset is out of bounds", NULL); goto free_and_exit; } value_header = (PersistValueHeader *) ((gchar *) map + entry_ofs - sizeof(PersistValueHeader)); if ((value_header->in_use) || load_all_entries) { gpointer new_block; PersistEntryHandle new_handle; new_handle = _alloc_value(self, GUINT32_FROM_BE(value_header->size), FALSE, value_header->version); new_block = persist_state_map_entry(self, new_handle); memcpy(new_block, value_header + 1, GUINT32_FROM_BE(value_header->size)); persist_state_unmap_entry(self, new_handle); /* add key to the current file */ _add_key(self, name, new_handle); } g_free(name); } else { /* bad format */ serialize_archive_free(sa); g_free(name); msg_error("Persistent file format error, unable to fetch key name", NULL); goto free_and_exit; } } else { g_free(name); if (serialize_read_uint32(sa, &chain_ofs)) { /* end of block, chain to the next one */ if (chain_ofs == 0 || chain_ofs > file_size) { msg_error("Persistent file format error, key block chain offset is too large or zero", evt_tag_printf("key_block", "%08lx", (gulong) ((gchar *) key_block - (gchar *) map)), evt_tag_printf("key_size", "%d", key_size), evt_tag_int("ofs", chain_ofs), NULL); serialize_archive_free(sa); goto free_and_exit; } key_block = ((gchar *) map) + chain_ofs; key_size = GUINT32_FROM_BE(*(guint32 *) (((gchar *) key_block) - sizeof(PersistValueHeader))); if (chain_ofs + key_size > file_size) { msg_error("Persistent file format error, key block size is too large", evt_tag_int("key_size", key_size), NULL); serialize_archive_free(sa); goto free_and_exit; } break; } else { serialize_archive_free(sa); msg_error("Persistent file format error, unable to fetch chained key block offset", NULL); goto free_and_exit; } } } serialize_archive_free(sa); } free_and_exit: munmap(map, file_size); return TRUE; }