InfluxDB is a scalable database optimized for storage of time series, real-time application metrics, operations monitoring events, etc, written in Go.
Data is stored in .tsm files, which are kept pretty simple conceptually. Each .tsm file contains a header and footer, which stores offset to an index. Index is used to find a data block for a requested time boundary.
This page hosts a formal specification of InfluxDB TSM file using Kaitai Struct. This specification can be automatically translated into a variety of programming languages to get a parsing library.
All parsing code for C# generated by Kaitai Struct depends on the C# runtime library. You have to install it before you can parse data.
The C# runtime library is available in the NuGet Gallery. Installation instructions can also be found there.
Parse a local file and get structure in memory:
var data = Tsm.FromFile("path/to/local/file.tsm");
Or parse structure from a byte array:
byte[] someArray = new byte[] { ... };
var data = new Tsm(new KaitaiStream(someArray));
After that, one can get various attributes from the structure by accessing properties like:
data.Header // => get header
// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
using System.Collections.Generic;
namespace Kaitai
{
/// <summary>
/// InfluxDB is a scalable database optimized for storage of time
/// series, real-time application metrics, operations monitoring events,
/// etc, written in Go.
///
/// Data is stored in .tsm files, which are kept pretty simple
/// conceptually. Each .tsm file contains a header and footer, which
/// stores offset to an index. Index is used to find a data block for a
/// requested time boundary.
/// </summary>
public partial class Tsm : KaitaiStruct
{
public static Tsm FromFile(string fileName)
{
return new Tsm(new KaitaiStream(fileName));
}
public Tsm(KaitaiStream p__io, KaitaiStruct p__parent = null, Tsm p__root = null) : base(p__io)
{
m_parent = p__parent;
m_root = p__root ?? this;
f_index = false;
_read();
}
private void _read()
{
_header = new Header(m_io, this, m_root);
}
public partial class Header : KaitaiStruct
{
public static Header FromFile(string fileName)
{
return new Header(new KaitaiStream(fileName));
}
public Header(KaitaiStream p__io, Tsm p__parent = null, Tsm p__root = null) : base(p__io)
{
m_parent = p__parent;
m_root = p__root;
_read();
}
private void _read()
{
_magic = m_io.ReadBytes(4);
if (!((KaitaiStream.ByteArrayCompare(Magic, new byte[] { 22, 209, 22, 209 }) == 0)))
{
throw new ValidationNotEqualError(new byte[] { 22, 209, 22, 209 }, Magic, M_Io, "/types/header/seq/0");
}
_version = m_io.ReadU1();
}
private byte[] _magic;
private byte _version;
private Tsm m_root;
private Tsm m_parent;
public byte[] Magic { get { return _magic; } }
public byte Version { get { return _version; } }
public Tsm M_Root { get { return m_root; } }
public Tsm M_Parent { get { return m_parent; } }
}
public partial class Index : KaitaiStruct
{
public static Index FromFile(string fileName)
{
return new Index(new KaitaiStream(fileName));
}
public Index(KaitaiStream p__io, Tsm p__parent = null, Tsm p__root = null) : base(p__io)
{
m_parent = p__parent;
m_root = p__root;
f_entries = false;
_read();
}
private void _read()
{
_offset = m_io.ReadU8be();
}
public partial class IndexHeader : KaitaiStruct
{
public static IndexHeader FromFile(string fileName)
{
return new IndexHeader(new KaitaiStream(fileName));
}
public IndexHeader(KaitaiStream p__io, Tsm.Index p__parent = null, Tsm p__root = null) : base(p__io)
{
m_parent = p__parent;
m_root = p__root;
_read();
}
private void _read()
{
_keyLen = m_io.ReadU2be();
_key = System.Text.Encoding.GetEncoding("UTF-8").GetString(m_io.ReadBytes(KeyLen));
_type = m_io.ReadU1();
_entryCount = m_io.ReadU2be();
_indexEntries = new List<IndexEntry>();
for (var i = 0; i < EntryCount; i++)
{
_indexEntries.Add(new IndexEntry(m_io, this, m_root));
}
}
public partial class IndexEntry : KaitaiStruct
{
public static IndexEntry FromFile(string fileName)
{
return new IndexEntry(new KaitaiStream(fileName));
}
public IndexEntry(KaitaiStream p__io, Tsm.Index.IndexHeader p__parent = null, Tsm p__root = null) : base(p__io)
{
m_parent = p__parent;
m_root = p__root;
f_block = false;
_read();
}
private void _read()
{
_minTime = m_io.ReadU8be();
_maxTime = m_io.ReadU8be();
_blockOffset = m_io.ReadU8be();
_blockSize = m_io.ReadU4be();
}
public partial class BlockEntry : KaitaiStruct
{
public static BlockEntry FromFile(string fileName)
{
return new BlockEntry(new KaitaiStream(fileName));
}
public BlockEntry(KaitaiStream p__io, Tsm.Index.IndexHeader.IndexEntry p__parent = null, Tsm p__root = null) : base(p__io)
{
m_parent = p__parent;
m_root = p__root;
_read();
}
private void _read()
{
_crc32 = m_io.ReadU4be();
_data = m_io.ReadBytes((M_Parent.BlockSize - 4));
}
private uint _crc32;
private byte[] _data;
private Tsm m_root;
private Tsm.Index.IndexHeader.IndexEntry m_parent;
public uint Crc32 { get { return _crc32; } }
public byte[] Data { get { return _data; } }
public Tsm M_Root { get { return m_root; } }
public Tsm.Index.IndexHeader.IndexEntry M_Parent { get { return m_parent; } }
}
private bool f_block;
private BlockEntry _block;
public BlockEntry Block
{
get
{
if (f_block)
return _block;
KaitaiStream io = M_Root.M_Io;
long _pos = io.Pos;
io.Seek(BlockOffset);
_block = new BlockEntry(io, this, m_root);
io.Seek(_pos);
f_block = true;
return _block;
}
}
private ulong _minTime;
private ulong _maxTime;
private ulong _blockOffset;
private uint _blockSize;
private Tsm m_root;
private Tsm.Index.IndexHeader m_parent;
public ulong MinTime { get { return _minTime; } }
public ulong MaxTime { get { return _maxTime; } }
public ulong BlockOffset { get { return _blockOffset; } }
public uint BlockSize { get { return _blockSize; } }
public Tsm M_Root { get { return m_root; } }
public Tsm.Index.IndexHeader M_Parent { get { return m_parent; } }
}
private ushort _keyLen;
private string _key;
private byte _type;
private ushort _entryCount;
private List<IndexEntry> _indexEntries;
private Tsm m_root;
private Tsm.Index m_parent;
public ushort KeyLen { get { return _keyLen; } }
public string Key { get { return _key; } }
public byte Type { get { return _type; } }
public ushort EntryCount { get { return _entryCount; } }
public List<IndexEntry> IndexEntries { get { return _indexEntries; } }
public Tsm M_Root { get { return m_root; } }
public Tsm.Index M_Parent { get { return m_parent; } }
}
private bool f_entries;
private List<IndexHeader> _entries;
public List<IndexHeader> Entries
{
get
{
if (f_entries)
return _entries;
long _pos = m_io.Pos;
m_io.Seek(Offset);
_entries = new List<IndexHeader>();
{
var i = 0;
IndexHeader M_;
do {
M_ = new IndexHeader(m_io, this, m_root);
_entries.Add(M_);
i++;
} while (!(M_Io.Pos == (M_Io.Size - 8)));
}
m_io.Seek(_pos);
f_entries = true;
return _entries;
}
}
private ulong _offset;
private Tsm m_root;
private Tsm m_parent;
public ulong Offset { get { return _offset; } }
public Tsm M_Root { get { return m_root; } }
public Tsm M_Parent { get { return m_parent; } }
}
private bool f_index;
private Index _index;
public Index Index
{
get
{
if (f_index)
return _index;
long _pos = m_io.Pos;
m_io.Seek((M_Io.Size - 8));
_index = new Index(m_io, this, m_root);
m_io.Seek(_pos);
f_index = true;
return _index;
}
}
private Header _header;
private Tsm m_root;
private KaitaiStruct m_parent;
public Header Header { get { return _header; } }
public Tsm M_Root { get { return m_root; } }
public KaitaiStruct M_Parent { get { return m_parent; } }
}
}