InfluxDB TSM file: JavaScript parsing library

InfluxDB is a scalable database optimized for storage of time series, real-time application metrics, operations monitoring events, etc, written in Go.

Data is stored in .tsm files, which are kept pretty simple conceptually. Each .tsm file contains a header and footer, which stores offset to an index. Index is used to find a data block for a requested time boundary.

Application

InfluxDB

File extension

tsm

KS implementation details

License: MIT

This page hosts a formal specification of InfluxDB TSM file using Kaitai Struct. This specification can be automatically translated into a variety of programming languages to get a parsing library.

Usage

See the usage examples in the JavaScript notes.

Parse structure from an ArrayBuffer:

var arrayBuffer = ...;
var data = new Tsm(new KaitaiStream(arrayBuffer));

After that, one can get various attributes from the structure by accessing fields or properties like:

data.header // => get header

JavaScript source code to parse InfluxDB TSM file

Tsm.js

// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild

(function (root, factory) {
  if (typeof define === 'function' && define.amd) {
    define(['kaitai-struct/KaitaiStream'], factory);
  } else if (typeof module === 'object' && module.exports) {
    module.exports = factory(require('kaitai-struct/KaitaiStream'));
  } else {
    root.Tsm = factory(root.KaitaiStream);
  }
}(this, function (KaitaiStream) {
/**
 * InfluxDB is a scalable database optimized for storage of time
 * series, real-time application metrics, operations monitoring events,
 * etc, written in Go.
 * 
 * Data is stored in .tsm files, which are kept pretty simple
 * conceptually. Each .tsm file contains a header and footer, which
 * stores offset to an index. Index is used to find a data block for a
 * requested time boundary.
 */

var Tsm = (function() {
  function Tsm(_io, _parent, _root) {
    this._io = _io;
    this._parent = _parent;
    this._root = _root || this;

    this._read();
  }
  Tsm.prototype._read = function() {
    this.header = new Header(this._io, this, this._root);
  }

  var Header = Tsm.Header = (function() {
    function Header(_io, _parent, _root) {
      this._io = _io;
      this._parent = _parent;
      this._root = _root || this;

      this._read();
    }
    Header.prototype._read = function() {
      this.magic = this._io.ensureFixedContents([22, 209, 22, 209]);
      this.version = this._io.readU1();
    }

    return Header;
  })();

  var Index = Tsm.Index = (function() {
    function Index(_io, _parent, _root) {
      this._io = _io;
      this._parent = _parent;
      this._root = _root || this;

      this._read();
    }
    Index.prototype._read = function() {
      this.offset = this._io.readU8be();
    }

    var IndexHeader = Index.IndexHeader = (function() {
      function IndexHeader(_io, _parent, _root) {
        this._io = _io;
        this._parent = _parent;
        this._root = _root || this;

        this._read();
      }
      IndexHeader.prototype._read = function() {
        this.keyLen = this._io.readU2be();
        this.key = KaitaiStream.bytesToStr(this._io.readBytes(this.keyLen), "UTF-8");
        this.type = this._io.readU1();
        this.entryCount = this._io.readU2be();
        this.indexEntries = new Array(this.entryCount);
        for (var i = 0; i < this.entryCount; i++) {
          this.indexEntries[i] = new IndexEntry(this._io, this, this._root);
        }
      }

      var IndexEntry = IndexHeader.IndexEntry = (function() {
        function IndexEntry(_io, _parent, _root) {
          this._io = _io;
          this._parent = _parent;
          this._root = _root || this;

          this._read();
        }
        IndexEntry.prototype._read = function() {
          this.minTime = this._io.readU8be();
          this.maxTime = this._io.readU8be();
          this.blockOffset = this._io.readU8be();
          this.blockSize = this._io.readU4be();
        }

        var BlockEntry = IndexEntry.BlockEntry = (function() {
          function BlockEntry(_io, _parent, _root) {
            this._io = _io;
            this._parent = _parent;
            this._root = _root || this;

            this._read();
          }
          BlockEntry.prototype._read = function() {
            this.crc32 = this._io.readU4be();
            this.data = this._io.readBytes((this._parent.blockSize - 4));
          }

          return BlockEntry;
        })();
        Object.defineProperty(IndexEntry.prototype, 'block', {
          get: function() {
            if (this._m_block !== undefined)
              return this._m_block;
            var io = this._root._io;
            var _pos = io.pos;
            io.seek(this.blockOffset);
            this._m_block = new BlockEntry(io, this, this._root);
            io.seek(_pos);
            return this._m_block;
          }
        });

        return IndexEntry;
      })();

      return IndexHeader;
    })();
    Object.defineProperty(Index.prototype, 'entries', {
      get: function() {
        if (this._m_entries !== undefined)
          return this._m_entries;
        var _pos = this._io.pos;
        this._io.seek(this.offset);
        this._m_entries = []
        var i = 0;
        do {
          var _ = new IndexHeader(this._io, this, this._root);
          this._m_entries.push(_);
          i++;
        } while (!(this._io.pos == (this._io.size - 8)));
        this._io.seek(_pos);
        return this._m_entries;
      }
    });

    return Index;
  })();
  Object.defineProperty(Tsm.prototype, 'index', {
    get: function() {
      if (this._m_index !== undefined)
        return this._m_index;
      var _pos = this._io.pos;
      this._io.seek((this._io.size - 8));
      this._m_index = new Index(this._io, this, this._root);
      this._io.seek(_pos);
      return this._m_index;
    }
  });

  return Tsm;
})();
return Tsm;
}));