systemd journal file: Go parsing library

systemd, a popular user-space system/service management suite on Linux, offers logging functionality, storing incoming logs in a binary journal format.

On live Linux system running systemd, these journals are typically located at:

  • /run/log/journal/machine-id/*.journal (volatile, lost after reboot)
  • /var/log/journal/machine-id/*.journal (persistent, but disabled by default on Debian / Ubuntu)

File extension

journal

KS implementation details

License: CC0-1.0

References

This page hosts a formal specification of systemd journal file using Kaitai Struct. This specification can be automatically translated into a variety of programming languages to get a parsing library.

Go source code to parse systemd journal file

systemd_journal.go

// Code generated by kaitai-struct-compiler from a .ksy source file. DO NOT EDIT.

import (
	"github.com/kaitai-io/kaitai_struct_go_runtime/kaitai"
	"bytes"
	"io"
)


/**
 * systemd, a popular user-space system/service management suite on Linux,
 * offers logging functionality, storing incoming logs in a binary journal
 * format.
 * 
 * On live Linux system running systemd, these journals are typically located at:
 * 
 * * /run/log/journal/machine-id/*.journal (volatile, lost after reboot)
 * * /var/log/journal/machine-id/*.journal (persistent, but disabled by default on Debian / Ubuntu)
 * @see <a href="https://www.freedesktop.org/wiki/Software/systemd/journal-files/">Source</a>
 */

type SystemdJournal_State int
const (
	SystemdJournal_State__Offline SystemdJournal_State = 0
	SystemdJournal_State__Online SystemdJournal_State = 1
	SystemdJournal_State__Archived SystemdJournal_State = 2
)
var values_SystemdJournal_State = map[SystemdJournal_State]struct{}{0: {}, 1: {}, 2: {}}
func (v SystemdJournal_State) isDefined() bool {
	_, ok := values_SystemdJournal_State[v]
	return ok
}
type SystemdJournal struct {
	Header *SystemdJournal_Header
	Objects []*SystemdJournal_JournalObject
	_io *kaitai.Stream
	_root *SystemdJournal
	_parent kaitai.Struct
	_raw_Header []byte
	_f_dataHashTable bool
	dataHashTable []byte
	_f_fieldHashTable bool
	fieldHashTable []byte
	_f_lenHeader bool
	lenHeader uint64
}
func NewSystemdJournal() *SystemdJournal {
	return &SystemdJournal{
	}
}

func (this SystemdJournal) IO_() *kaitai.Stream {
	return this._io
}

func (this *SystemdJournal) Read(io *kaitai.Stream, parent kaitai.Struct, root *SystemdJournal) (err error) {
	this._io = io
	this._parent = parent
	this._root = root

	tmp1, err := this.LenHeader()
	if err != nil {
		return err
	}
	tmp2, err := this._io.ReadBytes(int(tmp1))
	if err != nil {
		return err
	}
	tmp2 = tmp2
	this._raw_Header = tmp2
	_io__raw_Header := kaitai.NewStream(bytes.NewReader(this._raw_Header))
	tmp3 := NewSystemdJournal_Header()
	err = tmp3.Read(_io__raw_Header, this, this._root)
	if err != nil {
		return err
	}
	this.Header = tmp3
	for i := 0; i < int(this.Header.NumObjects); i++ {
		_ = i
		tmp4 := NewSystemdJournal_JournalObject()
		err = tmp4.Read(this._io, this, this._root)
		if err != nil {
			return err
		}
		this.Objects = append(this.Objects, tmp4)
	}
	return err
}
func (this *SystemdJournal) DataHashTable() (v []byte, err error) {
	if (this._f_dataHashTable) {
		return this.dataHashTable, nil
	}
	this._f_dataHashTable = true
	_pos, err := this._io.Pos()
	if err != nil {
		return nil, err
	}
	_, err = this._io.Seek(int64(this.Header.OfsDataHashTable), io.SeekStart)
	if err != nil {
		return nil, err
	}
	tmp5, err := this._io.ReadBytes(int(this.Header.LenDataHashTable))
	if err != nil {
		return nil, err
	}
	tmp5 = tmp5
	this.dataHashTable = tmp5
	_, err = this._io.Seek(_pos, io.SeekStart)
	if err != nil {
		return nil, err
	}
	return this.dataHashTable, nil
}
func (this *SystemdJournal) FieldHashTable() (v []byte, err error) {
	if (this._f_fieldHashTable) {
		return this.fieldHashTable, nil
	}
	this._f_fieldHashTable = true
	_pos, err := this._io.Pos()
	if err != nil {
		return nil, err
	}
	_, err = this._io.Seek(int64(this.Header.OfsFieldHashTable), io.SeekStart)
	if err != nil {
		return nil, err
	}
	tmp6, err := this._io.ReadBytes(int(this.Header.LenFieldHashTable))
	if err != nil {
		return nil, err
	}
	tmp6 = tmp6
	this.fieldHashTable = tmp6
	_, err = this._io.Seek(_pos, io.SeekStart)
	if err != nil {
		return nil, err
	}
	return this.fieldHashTable, nil
}

/**
 * Header length is used to set substream size, as it thus required
 * prior to declaration of header.
 */
func (this *SystemdJournal) LenHeader() (v uint64, err error) {
	if (this._f_lenHeader) {
		return this.lenHeader, nil
	}
	this._f_lenHeader = true
	_pos, err := this._io.Pos()
	if err != nil {
		return 0, err
	}
	_, err = this._io.Seek(int64(88), io.SeekStart)
	if err != nil {
		return 0, err
	}
	tmp7, err := this._io.ReadU8le()
	if err != nil {
		return 0, err
	}
	this.lenHeader = tmp7
	_, err = this._io.Seek(_pos, io.SeekStart)
	if err != nil {
		return 0, err
	}
	return this.lenHeader, nil
}

/**
 * Data objects are designed to carry log payload, typically in
 * form of a "key=value" string in `payload` attribute.
 * @see <a href="https://www.freedesktop.org/wiki/Software/systemd/journal-files/#dataobjects">Source</a>
 */
type SystemdJournal_DataObject struct {
	Hash uint64
	OfsNextHash uint64
	OfsHeadField uint64
	OfsEntry uint64
	OfsEntryArray uint64
	NumEntries uint64
	Payload []byte
	_io *kaitai.Stream
	_root *SystemdJournal
	_parent *SystemdJournal_JournalObject
	_f_entry bool
	entry *SystemdJournal_JournalObject
	_f_entryArray bool
	entryArray *SystemdJournal_JournalObject
	_f_headField bool
	headField *SystemdJournal_JournalObject
	_f_nextHash bool
	nextHash *SystemdJournal_JournalObject
}
func NewSystemdJournal_DataObject() *SystemdJournal_DataObject {
	return &SystemdJournal_DataObject{
	}
}

func (this SystemdJournal_DataObject) IO_() *kaitai.Stream {
	return this._io
}

func (this *SystemdJournal_DataObject) Read(io *kaitai.Stream, parent *SystemdJournal_JournalObject, root *SystemdJournal) (err error) {
	this._io = io
	this._parent = parent
	this._root = root

	tmp8, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.Hash = uint64(tmp8)
	tmp9, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.OfsNextHash = uint64(tmp9)
	tmp10, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.OfsHeadField = uint64(tmp10)
	tmp11, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.OfsEntry = uint64(tmp11)
	tmp12, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.OfsEntryArray = uint64(tmp12)
	tmp13, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.NumEntries = uint64(tmp13)
	tmp14, err := this._io.ReadBytesFull()
	if err != nil {
		return err
	}
	tmp14 = tmp14
	this.Payload = tmp14
	return err
}
func (this *SystemdJournal_DataObject) Entry() (v *SystemdJournal_JournalObject, err error) {
	if (this._f_entry) {
		return this.entry, nil
	}
	this._f_entry = true
	if (this.OfsEntry != 0) {
		thisIo := this._root._io
		_pos, err := thisIo.Pos()
		if err != nil {
			return nil, err
		}
		_, err = thisIo.Seek(int64(this.OfsEntry), io.SeekStart)
		if err != nil {
			return nil, err
		}
		tmp15 := NewSystemdJournal_JournalObject()
		err = tmp15.Read(thisIo, this, this._root)
		if err != nil {
			return nil, err
		}
		this.entry = tmp15
		_, err = thisIo.Seek(_pos, io.SeekStart)
		if err != nil {
			return nil, err
		}
	}
	return this.entry, nil
}
func (this *SystemdJournal_DataObject) EntryArray() (v *SystemdJournal_JournalObject, err error) {
	if (this._f_entryArray) {
		return this.entryArray, nil
	}
	this._f_entryArray = true
	if (this.OfsEntryArray != 0) {
		thisIo := this._root._io
		_pos, err := thisIo.Pos()
		if err != nil {
			return nil, err
		}
		_, err = thisIo.Seek(int64(this.OfsEntryArray), io.SeekStart)
		if err != nil {
			return nil, err
		}
		tmp16 := NewSystemdJournal_JournalObject()
		err = tmp16.Read(thisIo, this, this._root)
		if err != nil {
			return nil, err
		}
		this.entryArray = tmp16
		_, err = thisIo.Seek(_pos, io.SeekStart)
		if err != nil {
			return nil, err
		}
	}
	return this.entryArray, nil
}
func (this *SystemdJournal_DataObject) HeadField() (v *SystemdJournal_JournalObject, err error) {
	if (this._f_headField) {
		return this.headField, nil
	}
	this._f_headField = true
	if (this.OfsHeadField != 0) {
		thisIo := this._root._io
		_pos, err := thisIo.Pos()
		if err != nil {
			return nil, err
		}
		_, err = thisIo.Seek(int64(this.OfsHeadField), io.SeekStart)
		if err != nil {
			return nil, err
		}
		tmp17 := NewSystemdJournal_JournalObject()
		err = tmp17.Read(thisIo, this, this._root)
		if err != nil {
			return nil, err
		}
		this.headField = tmp17
		_, err = thisIo.Seek(_pos, io.SeekStart)
		if err != nil {
			return nil, err
		}
	}
	return this.headField, nil
}
func (this *SystemdJournal_DataObject) NextHash() (v *SystemdJournal_JournalObject, err error) {
	if (this._f_nextHash) {
		return this.nextHash, nil
	}
	this._f_nextHash = true
	if (this.OfsNextHash != 0) {
		thisIo := this._root._io
		_pos, err := thisIo.Pos()
		if err != nil {
			return nil, err
		}
		_, err = thisIo.Seek(int64(this.OfsNextHash), io.SeekStart)
		if err != nil {
			return nil, err
		}
		tmp18 := NewSystemdJournal_JournalObject()
		err = tmp18.Read(thisIo, this, this._root)
		if err != nil {
			return nil, err
		}
		this.nextHash = tmp18
		_, err = thisIo.Seek(_pos, io.SeekStart)
		if err != nil {
			return nil, err
		}
	}
	return this.nextHash, nil
}
type SystemdJournal_Header struct {
	Signature []byte
	CompatibleFlags uint32
	IncompatibleFlags uint32
	State SystemdJournal_State
	Reserved []byte
	FileId []byte
	MachineId []byte
	BootId []byte
	SeqnumId []byte
	LenHeader uint64
	LenArena uint64
	OfsDataHashTable uint64
	LenDataHashTable uint64
	OfsFieldHashTable uint64
	LenFieldHashTable uint64
	OfsTailObject uint64
	NumObjects uint64
	NumEntries uint64
	TailEntrySeqnum uint64
	HeadEntrySeqnum uint64
	OfsEntryArray uint64
	HeadEntryRealtime uint64
	TailEntryRealtime uint64
	TailEntryMonotonic uint64
	NumData uint64
	NumFields uint64
	NumTags uint64
	NumEntryArrays uint64
	_io *kaitai.Stream
	_root *SystemdJournal
	_parent *SystemdJournal
}
func NewSystemdJournal_Header() *SystemdJournal_Header {
	return &SystemdJournal_Header{
	}
}

func (this SystemdJournal_Header) IO_() *kaitai.Stream {
	return this._io
}

func (this *SystemdJournal_Header) Read(io *kaitai.Stream, parent *SystemdJournal, root *SystemdJournal) (err error) {
	this._io = io
	this._parent = parent
	this._root = root

	tmp19, err := this._io.ReadBytes(int(8))
	if err != nil {
		return err
	}
	tmp19 = tmp19
	this.Signature = tmp19
	if !(bytes.Equal(this.Signature, []uint8{76, 80, 75, 83, 72, 72, 82, 72})) {
		return kaitai.NewValidationNotEqualError([]uint8{76, 80, 75, 83, 72, 72, 82, 72}, this.Signature, this._io, "/types/header/seq/0")
	}
	tmp20, err := this._io.ReadU4le()
	if err != nil {
		return err
	}
	this.CompatibleFlags = uint32(tmp20)
	tmp21, err := this._io.ReadU4le()
	if err != nil {
		return err
	}
	this.IncompatibleFlags = uint32(tmp21)
	tmp22, err := this._io.ReadU1()
	if err != nil {
		return err
	}
	this.State = SystemdJournal_State(tmp22)
	tmp23, err := this._io.ReadBytes(int(7))
	if err != nil {
		return err
	}
	tmp23 = tmp23
	this.Reserved = tmp23
	tmp24, err := this._io.ReadBytes(int(16))
	if err != nil {
		return err
	}
	tmp24 = tmp24
	this.FileId = tmp24
	tmp25, err := this._io.ReadBytes(int(16))
	if err != nil {
		return err
	}
	tmp25 = tmp25
	this.MachineId = tmp25
	tmp26, err := this._io.ReadBytes(int(16))
	if err != nil {
		return err
	}
	tmp26 = tmp26
	this.BootId = tmp26
	tmp27, err := this._io.ReadBytes(int(16))
	if err != nil {
		return err
	}
	tmp27 = tmp27
	this.SeqnumId = tmp27
	tmp28, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.LenHeader = uint64(tmp28)
	tmp29, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.LenArena = uint64(tmp29)
	tmp30, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.OfsDataHashTable = uint64(tmp30)
	tmp31, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.LenDataHashTable = uint64(tmp31)
	tmp32, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.OfsFieldHashTable = uint64(tmp32)
	tmp33, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.LenFieldHashTable = uint64(tmp33)
	tmp34, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.OfsTailObject = uint64(tmp34)
	tmp35, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.NumObjects = uint64(tmp35)
	tmp36, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.NumEntries = uint64(tmp36)
	tmp37, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.TailEntrySeqnum = uint64(tmp37)
	tmp38, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.HeadEntrySeqnum = uint64(tmp38)
	tmp39, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.OfsEntryArray = uint64(tmp39)
	tmp40, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.HeadEntryRealtime = uint64(tmp40)
	tmp41, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.TailEntryRealtime = uint64(tmp41)
	tmp42, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.TailEntryMonotonic = uint64(tmp42)
	tmp43, err := this._io.EOF()
	if err != nil {
		return err
	}
	if (!(tmp43)) {
		tmp44, err := this._io.ReadU8le()
		if err != nil {
			return err
		}
		this.NumData = uint64(tmp44)
	}
	tmp45, err := this._io.EOF()
	if err != nil {
		return err
	}
	if (!(tmp45)) {
		tmp46, err := this._io.ReadU8le()
		if err != nil {
			return err
		}
		this.NumFields = uint64(tmp46)
	}
	tmp47, err := this._io.EOF()
	if err != nil {
		return err
	}
	if (!(tmp47)) {
		tmp48, err := this._io.ReadU8le()
		if err != nil {
			return err
		}
		this.NumTags = uint64(tmp48)
	}
	tmp49, err := this._io.EOF()
	if err != nil {
		return err
	}
	if (!(tmp49)) {
		tmp50, err := this._io.ReadU8le()
		if err != nil {
			return err
		}
		this.NumEntryArrays = uint64(tmp50)
	}
	return err
}

/**
 * @see <a href="https://www.freedesktop.org/wiki/Software/systemd/journal-files/#objects">Source</a>
 */

type SystemdJournal_JournalObject_ObjectTypes int
const (
	SystemdJournal_JournalObject_ObjectTypes__Unused SystemdJournal_JournalObject_ObjectTypes = 0
	SystemdJournal_JournalObject_ObjectTypes__Data SystemdJournal_JournalObject_ObjectTypes = 1
	SystemdJournal_JournalObject_ObjectTypes__Field SystemdJournal_JournalObject_ObjectTypes = 2
	SystemdJournal_JournalObject_ObjectTypes__Entry SystemdJournal_JournalObject_ObjectTypes = 3
	SystemdJournal_JournalObject_ObjectTypes__DataHashTable SystemdJournal_JournalObject_ObjectTypes = 4
	SystemdJournal_JournalObject_ObjectTypes__FieldHashTable SystemdJournal_JournalObject_ObjectTypes = 5
	SystemdJournal_JournalObject_ObjectTypes__EntryArray SystemdJournal_JournalObject_ObjectTypes = 6
	SystemdJournal_JournalObject_ObjectTypes__Tag SystemdJournal_JournalObject_ObjectTypes = 7
)
var values_SystemdJournal_JournalObject_ObjectTypes = map[SystemdJournal_JournalObject_ObjectTypes]struct{}{0: {}, 1: {}, 2: {}, 3: {}, 4: {}, 5: {}, 6: {}, 7: {}}
func (v SystemdJournal_JournalObject_ObjectTypes) isDefined() bool {
	_, ok := values_SystemdJournal_JournalObject_ObjectTypes[v]
	return ok
}
type SystemdJournal_JournalObject struct {
	Padding []byte
	ObjectType SystemdJournal_JournalObject_ObjectTypes
	Flags uint8
	Reserved []byte
	LenObject uint64
	Payload interface{}
	_io *kaitai.Stream
	_root *SystemdJournal
	_parent kaitai.Struct
	_raw_Payload []byte
}
func NewSystemdJournal_JournalObject() *SystemdJournal_JournalObject {
	return &SystemdJournal_JournalObject{
	}
}

func (this SystemdJournal_JournalObject) IO_() *kaitai.Stream {
	return this._io
}

func (this *SystemdJournal_JournalObject) Read(io *kaitai.Stream, parent kaitai.Struct, root *SystemdJournal) (err error) {
	this._io = io
	this._parent = parent
	this._root = root

	tmp52, err := this._io.Pos()
	if err != nil {
		return err
	}
	tmp51 := (8 - tmp52) % 8
	if tmp51 < 0 {
		tmp51 += 8
	}
	tmp53, err := this._io.ReadBytes(int(tmp51))
	if err != nil {
		return err
	}
	tmp53 = tmp53
	this.Padding = tmp53
	tmp54, err := this._io.ReadU1()
	if err != nil {
		return err
	}
	this.ObjectType = SystemdJournal_JournalObject_ObjectTypes(tmp54)
	tmp55, err := this._io.ReadU1()
	if err != nil {
		return err
	}
	this.Flags = tmp55
	tmp56, err := this._io.ReadBytes(int(6))
	if err != nil {
		return err
	}
	tmp56 = tmp56
	this.Reserved = tmp56
	tmp57, err := this._io.ReadU8le()
	if err != nil {
		return err
	}
	this.LenObject = uint64(tmp57)
	switch (this.ObjectType) {
	case SystemdJournal_JournalObject_ObjectTypes__Data:
		tmp58, err := this._io.ReadBytes(int(this.LenObject - 16))
		if err != nil {
			return err
		}
		tmp58 = tmp58
		this._raw_Payload = tmp58
		_io__raw_Payload := kaitai.NewStream(bytes.NewReader(this._raw_Payload))
		tmp59 := NewSystemdJournal_DataObject()
		err = tmp59.Read(_io__raw_Payload, this, this._root)
		if err != nil {
			return err
		}
		this.Payload = tmp59
	default:
		tmp60, err := this._io.ReadBytes(int(this.LenObject - 16))
		if err != nil {
			return err
		}
		tmp60 = tmp60
		this._raw_Payload = tmp60
	}
	return err
}