systemd, a popular user-space system/service management suite on Linux, offers logging functionality, storing incoming logs in a binary journal format.
On live Linux system running systemd, these journals are typically located at:
This page hosts a formal specification of systemd journal file using Kaitai Struct. This specification can be automatically translated into a variety of programming languages to get a parsing library.
All parsing code for C++11/STL generated by Kaitai Struct depends on the C++/STL runtime library. You have to install it before you can parse data.
For C++, the easiest way is to clone the runtime library sources and build them along with your project.
Using Kaitai Struct in C++/STL usually consists of 3 steps.
std::istream). One can open local file for that, or use existing std::string or char* buffer.
#include <fstream>
std::ifstream is("path/to/local/file.journal", std::ifstream::binary);
#include <sstream>
std::istringstream is(str);
#include <sstream>
const char buf[] = { ... };
std::string str(buf, sizeof buf);
std::istringstream is(str);
#include "kaitai/kaitaistream.h"
kaitai::kstream ks(&is);
systemd_journal_t data(&ks);
After that, one can get various attributes from the structure by invoking getter methods like:
data.len_header() // => Header length is used to set substream size, as it thus required
prior to declaration of header.
#pragma once
// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
#include "kaitai/kaitaistruct.h"
#include <stdint.h>
#include <memory>
#include <vector>
#if KAITAI_STRUCT_VERSION < 9000L
#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required"
#endif
/**
* systemd, a popular user-space system/service management suite on Linux,
* offers logging functionality, storing incoming logs in a binary journal
* format.
*
* On live Linux system running systemd, these journals are typically located at:
*
* * /run/log/journal/machine-id/*.journal (volatile, lost after reboot)
* * /var/log/journal/machine-id/*.journal (persistent, but disabled by default on Debian / Ubuntu)
* \sa https://www.freedesktop.org/wiki/Software/systemd/journal-files/ Source
*/
class systemd_journal_t : public kaitai::kstruct {
public:
class header_t;
class journal_object_t;
class data_object_t;
enum state_t {
STATE_OFFLINE = 0,
STATE_ONLINE = 1,
STATE_ARCHIVED = 2
};
systemd_journal_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = nullptr, systemd_journal_t* p__root = nullptr);
private:
void _read();
void _clean_up();
public:
~systemd_journal_t();
class header_t : public kaitai::kstruct {
public:
header_t(kaitai::kstream* p__io, systemd_journal_t* p__parent = nullptr, systemd_journal_t* p__root = nullptr);
private:
void _read();
void _clean_up();
public:
~header_t();
private:
std::string m_signature;
uint32_t m_compatible_flags;
uint32_t m_incompatible_flags;
state_t m_state;
std::string m_reserved;
std::string m_file_id;
std::string m_machine_id;
std::string m_boot_id;
std::string m_seqnum_id;
uint64_t m_len_header;
uint64_t m_len_arena;
uint64_t m_ofs_data_hash_table;
uint64_t m_len_data_hash_table;
uint64_t m_ofs_field_hash_table;
uint64_t m_len_field_hash_table;
uint64_t m_ofs_tail_object;
uint64_t m_num_objects;
uint64_t m_num_entries;
uint64_t m_tail_entry_seqnum;
uint64_t m_head_entry_seqnum;
uint64_t m_ofs_entry_array;
uint64_t m_head_entry_realtime;
uint64_t m_tail_entry_realtime;
uint64_t m_tail_entry_monotonic;
uint64_t m_num_data;
bool n_num_data;
public:
bool _is_null_num_data() { num_data(); return n_num_data; };
private:
uint64_t m_num_fields;
bool n_num_fields;
public:
bool _is_null_num_fields() { num_fields(); return n_num_fields; };
private:
uint64_t m_num_tags;
bool n_num_tags;
public:
bool _is_null_num_tags() { num_tags(); return n_num_tags; };
private:
uint64_t m_num_entry_arrays;
bool n_num_entry_arrays;
public:
bool _is_null_num_entry_arrays() { num_entry_arrays(); return n_num_entry_arrays; };
private:
systemd_journal_t* m__root;
systemd_journal_t* m__parent;
public:
std::string signature() const { return m_signature; }
uint32_t compatible_flags() const { return m_compatible_flags; }
uint32_t incompatible_flags() const { return m_incompatible_flags; }
state_t state() const { return m_state; }
std::string reserved() const { return m_reserved; }
std::string file_id() const { return m_file_id; }
std::string machine_id() const { return m_machine_id; }
std::string boot_id() const { return m_boot_id; }
std::string seqnum_id() const { return m_seqnum_id; }
uint64_t len_header() const { return m_len_header; }
uint64_t len_arena() const { return m_len_arena; }
uint64_t ofs_data_hash_table() const { return m_ofs_data_hash_table; }
uint64_t len_data_hash_table() const { return m_len_data_hash_table; }
uint64_t ofs_field_hash_table() const { return m_ofs_field_hash_table; }
uint64_t len_field_hash_table() const { return m_len_field_hash_table; }
uint64_t ofs_tail_object() const { return m_ofs_tail_object; }
uint64_t num_objects() const { return m_num_objects; }
uint64_t num_entries() const { return m_num_entries; }
uint64_t tail_entry_seqnum() const { return m_tail_entry_seqnum; }
uint64_t head_entry_seqnum() const { return m_head_entry_seqnum; }
uint64_t ofs_entry_array() const { return m_ofs_entry_array; }
uint64_t head_entry_realtime() const { return m_head_entry_realtime; }
uint64_t tail_entry_realtime() const { return m_tail_entry_realtime; }
uint64_t tail_entry_monotonic() const { return m_tail_entry_monotonic; }
uint64_t num_data() const { return m_num_data; }
uint64_t num_fields() const { return m_num_fields; }
uint64_t num_tags() const { return m_num_tags; }
uint64_t num_entry_arrays() const { return m_num_entry_arrays; }
systemd_journal_t* _root() const { return m__root; }
systemd_journal_t* _parent() const { return m__parent; }
};
/**
* \sa https://www.freedesktop.org/wiki/Software/systemd/journal-files/#objects Source
*/
class journal_object_t : public kaitai::kstruct {
public:
enum object_types_t {
OBJECT_TYPES_UNUSED = 0,
OBJECT_TYPES_DATA = 1,
OBJECT_TYPES_FIELD = 2,
OBJECT_TYPES_ENTRY = 3,
OBJECT_TYPES_DATA_HASH_TABLE = 4,
OBJECT_TYPES_FIELD_HASH_TABLE = 5,
OBJECT_TYPES_ENTRY_ARRAY = 6,
OBJECT_TYPES_TAG = 7
};
journal_object_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = nullptr, systemd_journal_t* p__root = nullptr);
private:
void _read();
void _clean_up();
public:
~journal_object_t();
private:
std::string m_padding;
object_types_t m_object_type;
uint8_t m_flags;
std::string m_reserved;
uint64_t m_len_object;
std::unique_ptr<data_object_t> m_payload;
bool n_payload;
public:
bool _is_null_payload() { payload(); return n_payload; };
private:
systemd_journal_t* m__root;
kaitai::kstruct* m__parent;
std::string m__raw_payload;
std::unique_ptr<kaitai::kstream> m__io__raw_payload;
public:
std::string padding() const { return m_padding; }
object_types_t object_type() const { return m_object_type; }
uint8_t flags() const { return m_flags; }
std::string reserved() const { return m_reserved; }
uint64_t len_object() const { return m_len_object; }
data_object_t* payload() const { return m_payload.get(); }
systemd_journal_t* _root() const { return m__root; }
kaitai::kstruct* _parent() const { return m__parent; }
std::string _raw_payload() const { return m__raw_payload; }
kaitai::kstream* _io__raw_payload() const { return m__io__raw_payload.get(); }
};
/**
* Data objects are designed to carry log payload, typically in
* form of a "key=value" string in `payload` attribute.
* \sa https://www.freedesktop.org/wiki/Software/systemd/journal-files/#dataobjects Source
*/
class data_object_t : public kaitai::kstruct {
public:
data_object_t(kaitai::kstream* p__io, systemd_journal_t::journal_object_t* p__parent = nullptr, systemd_journal_t* p__root = nullptr);
private:
void _read();
void _clean_up();
public:
~data_object_t();
private:
bool f_next_hash;
std::unique_ptr<journal_object_t> m_next_hash;
bool n_next_hash;
public:
bool _is_null_next_hash() { next_hash(); return n_next_hash; };
private:
public:
journal_object_t* next_hash();
private:
bool f_head_field;
std::unique_ptr<journal_object_t> m_head_field;
bool n_head_field;
public:
bool _is_null_head_field() { head_field(); return n_head_field; };
private:
public:
journal_object_t* head_field();
private:
bool f_entry;
std::unique_ptr<journal_object_t> m_entry;
bool n_entry;
public:
bool _is_null_entry() { entry(); return n_entry; };
private:
public:
journal_object_t* entry();
private:
bool f_entry_array;
std::unique_ptr<journal_object_t> m_entry_array;
bool n_entry_array;
public:
bool _is_null_entry_array() { entry_array(); return n_entry_array; };
private:
public:
journal_object_t* entry_array();
private:
uint64_t m_hash;
uint64_t m_ofs_next_hash;
uint64_t m_ofs_head_field;
uint64_t m_ofs_entry;
uint64_t m_ofs_entry_array;
uint64_t m_num_entries;
std::string m_payload;
systemd_journal_t* m__root;
systemd_journal_t::journal_object_t* m__parent;
public:
uint64_t hash() const { return m_hash; }
uint64_t ofs_next_hash() const { return m_ofs_next_hash; }
uint64_t ofs_head_field() const { return m_ofs_head_field; }
uint64_t ofs_entry() const { return m_ofs_entry; }
uint64_t ofs_entry_array() const { return m_ofs_entry_array; }
uint64_t num_entries() const { return m_num_entries; }
std::string payload() const { return m_payload; }
systemd_journal_t* _root() const { return m__root; }
systemd_journal_t::journal_object_t* _parent() const { return m__parent; }
};
private:
bool f_len_header;
uint64_t m_len_header;
public:
/**
* Header length is used to set substream size, as it thus required
* prior to declaration of header.
*/
uint64_t len_header();
private:
bool f_data_hash_table;
std::string m_data_hash_table;
public:
std::string data_hash_table();
private:
bool f_field_hash_table;
std::string m_field_hash_table;
public:
std::string field_hash_table();
private:
std::unique_ptr<header_t> m_header;
std::unique_ptr<std::vector<std::unique_ptr<journal_object_t>>> m_objects;
systemd_journal_t* m__root;
kaitai::kstruct* m__parent;
std::string m__raw_header;
std::unique_ptr<kaitai::kstream> m__io__raw_header;
public:
header_t* header() const { return m_header.get(); }
std::vector<std::unique_ptr<journal_object_t>>* objects() const { return m_objects.get(); }
systemd_journal_t* _root() const { return m__root; }
kaitai::kstruct* _parent() const { return m__parent; }
std::string _raw_header() const { return m__raw_header; }
kaitai::kstream* _io__raw_header() const { return m__io__raw_header.get(); }
};
// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
#include "systemd_journal.h"
#include "kaitai/exceptions.h"
systemd_journal_t::systemd_journal_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, systemd_journal_t* p__root) : kaitai::kstruct(p__io) {
m__parent = p__parent;
m__root = this;
m_header = nullptr;
m__io__raw_header = nullptr;
m_objects = nullptr;
f_len_header = false;
f_data_hash_table = false;
f_field_hash_table = false;
_read();
}
void systemd_journal_t::_read() {
m__raw_header = m__io->read_bytes(len_header());
m__io__raw_header = std::unique_ptr<kaitai::kstream>(new kaitai::kstream(m__raw_header));
m_header = std::unique_ptr<header_t>(new header_t(m__io__raw_header.get(), this, m__root));
m_objects = std::unique_ptr<std::vector<std::unique_ptr<journal_object_t>>>(new std::vector<std::unique_ptr<journal_object_t>>());
const int l_objects = header()->num_objects();
for (int i = 0; i < l_objects; i++) {
m_objects->push_back(std::move(std::unique_ptr<journal_object_t>(new journal_object_t(m__io, this, m__root))));
}
}
systemd_journal_t::~systemd_journal_t() {
_clean_up();
}
void systemd_journal_t::_clean_up() {
if (f_len_header) {
}
if (f_data_hash_table) {
}
if (f_field_hash_table) {
}
}
systemd_journal_t::header_t::header_t(kaitai::kstream* p__io, systemd_journal_t* p__parent, systemd_journal_t* p__root) : kaitai::kstruct(p__io) {
m__parent = p__parent;
m__root = p__root;
_read();
}
void systemd_journal_t::header_t::_read() {
m_signature = m__io->read_bytes(8);
if (!(signature() == std::string("\x4C\x50\x4B\x53\x48\x48\x52\x48", 8))) {
throw kaitai::validation_not_equal_error<std::string>(std::string("\x4C\x50\x4B\x53\x48\x48\x52\x48", 8), signature(), _io(), std::string("/types/header/seq/0"));
}
m_compatible_flags = m__io->read_u4le();
m_incompatible_flags = m__io->read_u4le();
m_state = static_cast<systemd_journal_t::state_t>(m__io->read_u1());
m_reserved = m__io->read_bytes(7);
m_file_id = m__io->read_bytes(16);
m_machine_id = m__io->read_bytes(16);
m_boot_id = m__io->read_bytes(16);
m_seqnum_id = m__io->read_bytes(16);
m_len_header = m__io->read_u8le();
m_len_arena = m__io->read_u8le();
m_ofs_data_hash_table = m__io->read_u8le();
m_len_data_hash_table = m__io->read_u8le();
m_ofs_field_hash_table = m__io->read_u8le();
m_len_field_hash_table = m__io->read_u8le();
m_ofs_tail_object = m__io->read_u8le();
m_num_objects = m__io->read_u8le();
m_num_entries = m__io->read_u8le();
m_tail_entry_seqnum = m__io->read_u8le();
m_head_entry_seqnum = m__io->read_u8le();
m_ofs_entry_array = m__io->read_u8le();
m_head_entry_realtime = m__io->read_u8le();
m_tail_entry_realtime = m__io->read_u8le();
m_tail_entry_monotonic = m__io->read_u8le();
n_num_data = true;
if (!(_io()->is_eof())) {
n_num_data = false;
m_num_data = m__io->read_u8le();
}
n_num_fields = true;
if (!(_io()->is_eof())) {
n_num_fields = false;
m_num_fields = m__io->read_u8le();
}
n_num_tags = true;
if (!(_io()->is_eof())) {
n_num_tags = false;
m_num_tags = m__io->read_u8le();
}
n_num_entry_arrays = true;
if (!(_io()->is_eof())) {
n_num_entry_arrays = false;
m_num_entry_arrays = m__io->read_u8le();
}
}
systemd_journal_t::header_t::~header_t() {
_clean_up();
}
void systemd_journal_t::header_t::_clean_up() {
if (!n_num_data) {
}
if (!n_num_fields) {
}
if (!n_num_tags) {
}
if (!n_num_entry_arrays) {
}
}
systemd_journal_t::journal_object_t::journal_object_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, systemd_journal_t* p__root) : kaitai::kstruct(p__io) {
m__parent = p__parent;
m__root = p__root;
m__io__raw_payload = nullptr;
_read();
}
void systemd_journal_t::journal_object_t::_read() {
m_padding = m__io->read_bytes(kaitai::kstream::mod((8 - _io()->pos()), 8));
m_object_type = static_cast<systemd_journal_t::journal_object_t::object_types_t>(m__io->read_u1());
m_flags = m__io->read_u1();
m_reserved = m__io->read_bytes(6);
m_len_object = m__io->read_u8le();
n_payload = true;
switch (object_type()) {
case systemd_journal_t::journal_object_t::OBJECT_TYPES_DATA: {
n_payload = false;
m__raw_payload = m__io->read_bytes((len_object() - 16));
m__io__raw_payload = std::unique_ptr<kaitai::kstream>(new kaitai::kstream(m__raw_payload));
m_payload = std::unique_ptr<data_object_t>(new data_object_t(m__io__raw_payload.get(), this, m__root));
break;
}
default: {
m__raw_payload = m__io->read_bytes((len_object() - 16));
break;
}
}
}
systemd_journal_t::journal_object_t::~journal_object_t() {
_clean_up();
}
void systemd_journal_t::journal_object_t::_clean_up() {
if (!n_payload) {
}
}
systemd_journal_t::data_object_t::data_object_t(kaitai::kstream* p__io, systemd_journal_t::journal_object_t* p__parent, systemd_journal_t* p__root) : kaitai::kstruct(p__io) {
m__parent = p__parent;
m__root = p__root;
m_next_hash = nullptr;
m_head_field = nullptr;
m_entry = nullptr;
m_entry_array = nullptr;
f_next_hash = false;
f_head_field = false;
f_entry = false;
f_entry_array = false;
_read();
}
void systemd_journal_t::data_object_t::_read() {
m_hash = m__io->read_u8le();
m_ofs_next_hash = m__io->read_u8le();
m_ofs_head_field = m__io->read_u8le();
m_ofs_entry = m__io->read_u8le();
m_ofs_entry_array = m__io->read_u8le();
m_num_entries = m__io->read_u8le();
m_payload = m__io->read_bytes_full();
}
systemd_journal_t::data_object_t::~data_object_t() {
_clean_up();
}
void systemd_journal_t::data_object_t::_clean_up() {
if (f_next_hash && !n_next_hash) {
}
if (f_head_field && !n_head_field) {
}
if (f_entry && !n_entry) {
}
if (f_entry_array && !n_entry_array) {
}
}
systemd_journal_t::journal_object_t* systemd_journal_t::data_object_t::next_hash() {
if (f_next_hash)
return m_next_hash.get();
n_next_hash = true;
if (ofs_next_hash() != 0) {
n_next_hash = false;
kaitai::kstream *io = _root()->_io();
std::streampos _pos = io->pos();
io->seek(ofs_next_hash());
m_next_hash = std::unique_ptr<journal_object_t>(new journal_object_t(io, this, m__root));
io->seek(_pos);
f_next_hash = true;
}
return m_next_hash.get();
}
systemd_journal_t::journal_object_t* systemd_journal_t::data_object_t::head_field() {
if (f_head_field)
return m_head_field.get();
n_head_field = true;
if (ofs_head_field() != 0) {
n_head_field = false;
kaitai::kstream *io = _root()->_io();
std::streampos _pos = io->pos();
io->seek(ofs_head_field());
m_head_field = std::unique_ptr<journal_object_t>(new journal_object_t(io, this, m__root));
io->seek(_pos);
f_head_field = true;
}
return m_head_field.get();
}
systemd_journal_t::journal_object_t* systemd_journal_t::data_object_t::entry() {
if (f_entry)
return m_entry.get();
n_entry = true;
if (ofs_entry() != 0) {
n_entry = false;
kaitai::kstream *io = _root()->_io();
std::streampos _pos = io->pos();
io->seek(ofs_entry());
m_entry = std::unique_ptr<journal_object_t>(new journal_object_t(io, this, m__root));
io->seek(_pos);
f_entry = true;
}
return m_entry.get();
}
systemd_journal_t::journal_object_t* systemd_journal_t::data_object_t::entry_array() {
if (f_entry_array)
return m_entry_array.get();
n_entry_array = true;
if (ofs_entry_array() != 0) {
n_entry_array = false;
kaitai::kstream *io = _root()->_io();
std::streampos _pos = io->pos();
io->seek(ofs_entry_array());
m_entry_array = std::unique_ptr<journal_object_t>(new journal_object_t(io, this, m__root));
io->seek(_pos);
f_entry_array = true;
}
return m_entry_array.get();
}
uint64_t systemd_journal_t::len_header() {
if (f_len_header)
return m_len_header;
std::streampos _pos = m__io->pos();
m__io->seek(88);
m_len_header = m__io->read_u8le();
m__io->seek(_pos);
f_len_header = true;
return m_len_header;
}
std::string systemd_journal_t::data_hash_table() {
if (f_data_hash_table)
return m_data_hash_table;
std::streampos _pos = m__io->pos();
m__io->seek(header()->ofs_data_hash_table());
m_data_hash_table = m__io->read_bytes(header()->len_data_hash_table());
m__io->seek(_pos);
f_data_hash_table = true;
return m_data_hash_table;
}
std::string systemd_journal_t::field_hash_table() {
if (f_field_hash_table)
return m_field_hash_table;
std::streampos _pos = m__io->pos();
m__io->seek(header()->ofs_field_hash_table());
m_field_hash_table = m__io->read_bytes(header()->len_field_hash_table());
m__io->seek(_pos);
f_field_hash_table = true;
return m_field_hash_table;
}