MessagePack (msgpack) serialization format: C++/STL parsing library

MessagePack (msgpack) is a system to serialize arbitrary structured data into a compact binary stream.

KS implementation details

License: CC0-1.0

This page hosts a formal specification of MessagePack (msgpack) serialization format using Kaitai Struct. This specification can be automatically translated into a variety of programming languages to get a parsing library.

Usage

Using Kaitai Struct in C++/STL usually consists of 3 steps.

  1. We need to create an STL input stream (std::istream).
    • One can open a stream for reading from a local file:
      #include <fstream>
      
      std::ifstream is("path/to/local/file.msgpack", std::ifstream::binary);
    • Or one can prepare a stream for reading from existing std::string str:
      #include <sstream>
      
      std::istringstream is(str);
    • Or one can parse arbitrary char* buffer in memory, given that we know its size:
      #include <sstream>
      
      const char buf[] = { ... };
      std::string str(buf, sizeof buf);
      std::istringstream is(str);
  2. We need to wrap our input stream into Kaitai stream:
    #include <kaitai/kaitaistream.h>
    
    kaitai::kstream ks(&is);
  3. And finally, we can invoke the parsing:
    msgpack_t data(&ks);

After that, one can get various attributes from the structure by invoking getter methods like:

data.b1() // => First byte is msgpack message is either a piece of data by
itself or determines types of further, more complex data
structures.

C++/STL source code to parse MessagePack (msgpack) serialization format

msgpack.h

#ifndef MSGPACK_H_
#define MSGPACK_H_

// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild

#include "kaitai/kaitaistruct.h"

#include <stdint.h>
#include <vector>

#if KAITAI_STRUCT_VERSION < 7000L
#error "Incompatible Kaitai Struct C++/STL API: version 0.7 or later is required"
#endif

/**
 * MessagePack (msgpack) is a system to serialize arbitrary structured
 * data into a compact binary stream.
 * \sa Source
 */

class msgpack_t : public kaitai::kstruct {

public:
    class map_tuple_t;

    msgpack_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = 0, msgpack_t* p__root = 0);

private:
    void _read();

public:
    ~msgpack_t();

    class map_tuple_t : public kaitai::kstruct {

    public:

        map_tuple_t(kaitai::kstream* p__io, msgpack_t* p__parent = 0, msgpack_t* p__root = 0);

    private:
        void _read();

    public:
        ~map_tuple_t();

    private:
        msgpack_t* m_key;
        msgpack_t* m_value;
        msgpack_t* m__root;
        msgpack_t* m__parent;

    public:
        msgpack_t* key() const { return m_key; }
        msgpack_t* value() const { return m_value; }
        msgpack_t* _root() const { return m__root; }
        msgpack_t* _parent() const { return m__parent; }
    };

private:
    bool f_is_array_32;
    bool m_is_array_32;

public:

    /**
     * \sa Source
     */
    bool is_array_32();

private:
    bool f_int_value;
    int32_t m_int_value;
    bool n_int_value;

public:
    bool _is_null_int_value() { int_value(); return n_int_value; };

private:

public:
    int32_t int_value();

private:
    bool f_str_len;
    int32_t m_str_len;
    bool n_str_len;

public:
    bool _is_null_str_len() { str_len(); return n_str_len; };

private:

public:
    int32_t str_len();

private:
    bool f_is_fix_array;
    bool m_is_fix_array;

public:

    /**
     * \sa Source
     */
    bool is_fix_array();

private:
    bool f_is_map;
    bool m_is_map;

public:

    /**
     * \sa Source
     */
    bool is_map();

private:
    bool f_is_array;
    bool m_is_array;

public:

    /**
     * \sa Source
     */
    bool is_array();

private:
    bool f_is_float;
    bool m_is_float;

public:
    bool is_float();

private:
    bool f_is_str_8;
    bool m_is_str_8;

public:

    /**
     * \sa Source
     */
    bool is_str_8();

private:
    bool f_is_fix_map;
    bool m_is_fix_map;

public:

    /**
     * \sa Source
     */
    bool is_fix_map();

private:
    bool f_is_int;
    bool m_is_int;

public:
    bool is_int();

private:
    bool f_is_bool;
    bool m_is_bool;

public:
    bool is_bool();

private:
    bool f_is_str_16;
    bool m_is_str_16;

public:

    /**
     * \sa Source
     */
    bool is_str_16();

private:
    bool f_is_float_64;
    bool m_is_float_64;

public:

    /**
     * \sa Source
     */
    bool is_float_64();

private:
    bool f_is_map_16;
    bool m_is_map_16;

public:

    /**
     * \sa Source
     */
    bool is_map_16();

private:
    bool f_is_neg_int5;
    bool m_is_neg_int5;

public:
    bool is_neg_int5();

private:
    bool f_pos_int7_value;
    uint8_t m_pos_int7_value;
    bool n_pos_int7_value;

public:
    bool _is_null_pos_int7_value() { pos_int7_value(); return n_pos_int7_value; };

private:

public:
    uint8_t pos_int7_value();

private:
    bool f_is_nil;
    bool m_is_nil;

public:

    /**
     * \sa Source
     */
    bool is_nil();

private:
    bool f_float_value;
    double m_float_value;
    bool n_float_value;

public:
    bool _is_null_float_value() { float_value(); return n_float_value; };

private:

public:
    double float_value();

private:
    bool f_num_array_elements;
    int32_t m_num_array_elements;
    bool n_num_array_elements;

public:
    bool _is_null_num_array_elements() { num_array_elements(); return n_num_array_elements; };

private:

public:

    /**
     * \sa Source
     */
    int32_t num_array_elements();

private:
    bool f_neg_int5_value;
    int32_t m_neg_int5_value;
    bool n_neg_int5_value;

public:
    bool _is_null_neg_int5_value() { neg_int5_value(); return n_neg_int5_value; };

private:

public:
    int32_t neg_int5_value();

private:
    bool f_bool_value;
    bool m_bool_value;
    bool n_bool_value;

public:
    bool _is_null_bool_value() { bool_value(); return n_bool_value; };

private:

public:

    /**
     * \sa Source
     */
    bool bool_value();

private:
    bool f_is_pos_int7;
    bool m_is_pos_int7;

public:
    bool is_pos_int7();

private:
    bool f_is_array_16;
    bool m_is_array_16;

public:

    /**
     * \sa Source
     */
    bool is_array_16();

private:
    bool f_is_str;
    bool m_is_str;

public:
    bool is_str();

private:
    bool f_is_fix_str;
    bool m_is_fix_str;

public:

    /**
     * \sa Source
     */
    bool is_fix_str();

private:
    bool f_is_str_32;
    bool m_is_str_32;

public:

    /**
     * \sa Source
     */
    bool is_str_32();

private:
    bool f_num_map_elements;
    int32_t m_num_map_elements;
    bool n_num_map_elements;

public:
    bool _is_null_num_map_elements() { num_map_elements(); return n_num_map_elements; };

private:

public:

    /**
     * \sa Source
     */
    int32_t num_map_elements();

private:
    bool f_is_float_32;
    bool m_is_float_32;

public:

    /**
     * \sa Source
     */
    bool is_float_32();

private:
    bool f_is_map_32;
    bool m_is_map_32;

public:

    /**
     * \sa Source
     */
    bool is_map_32();

private:
    uint8_t m_b1;
    int32_t m_int_extra;
    bool n_int_extra;

public:
    bool _is_null_int_extra() { int_extra(); return n_int_extra; };

private:
    float m_float_32_value;
    bool n_float_32_value;

public:
    bool _is_null_float_32_value() { float_32_value(); return n_float_32_value; };

private:
    double m_float_64_value;
    bool n_float_64_value;

public:
    bool _is_null_float_64_value() { float_64_value(); return n_float_64_value; };

private:
    uint8_t m_str_len_8;
    bool n_str_len_8;

public:
    bool _is_null_str_len_8() { str_len_8(); return n_str_len_8; };

private:
    uint16_t m_str_len_16;
    bool n_str_len_16;

public:
    bool _is_null_str_len_16() { str_len_16(); return n_str_len_16; };

private:
    uint32_t m_str_len_32;
    bool n_str_len_32;

public:
    bool _is_null_str_len_32() { str_len_32(); return n_str_len_32; };

private:
    std::string m_str_value;
    bool n_str_value;

public:
    bool _is_null_str_value() { str_value(); return n_str_value; };

private:
    uint16_t m_num_array_elements_16;
    bool n_num_array_elements_16;

public:
    bool _is_null_num_array_elements_16() { num_array_elements_16(); return n_num_array_elements_16; };

private:
    uint32_t m_num_array_elements_32;
    bool n_num_array_elements_32;

public:
    bool _is_null_num_array_elements_32() { num_array_elements_32(); return n_num_array_elements_32; };

private:
    std::vector<msgpack_t*>* m_array_elements;
    bool n_array_elements;

public:
    bool _is_null_array_elements() { array_elements(); return n_array_elements; };

private:
    uint16_t m_num_map_elements_16;
    bool n_num_map_elements_16;

public:
    bool _is_null_num_map_elements_16() { num_map_elements_16(); return n_num_map_elements_16; };

private:
    uint32_t m_num_map_elements_32;
    bool n_num_map_elements_32;

public:
    bool _is_null_num_map_elements_32() { num_map_elements_32(); return n_num_map_elements_32; };

private:
    std::vector<map_tuple_t*>* m_map_elements;
    bool n_map_elements;

public:
    bool _is_null_map_elements() { map_elements(); return n_map_elements; };

private:
    msgpack_t* m__root;
    kaitai::kstruct* m__parent;

public:

    /**
     * First byte is msgpack message is either a piece of data by
     * itself or determines types of further, more complex data
     * structures.
     */
    uint8_t b1() const { return m_b1; }
    int32_t int_extra() const { return m_int_extra; }
    float float_32_value() const { return m_float_32_value; }
    double float_64_value() const { return m_float_64_value; }
    uint8_t str_len_8() const { return m_str_len_8; }
    uint16_t str_len_16() const { return m_str_len_16; }
    uint32_t str_len_32() const { return m_str_len_32; }
    std::string str_value() const { return m_str_value; }
    uint16_t num_array_elements_16() const { return m_num_array_elements_16; }
    uint32_t num_array_elements_32() const { return m_num_array_elements_32; }
    std::vector<msgpack_t*>* array_elements() const { return m_array_elements; }
    uint16_t num_map_elements_16() const { return m_num_map_elements_16; }
    uint32_t num_map_elements_32() const { return m_num_map_elements_32; }
    std::vector<map_tuple_t*>* map_elements() const { return m_map_elements; }
    msgpack_t* _root() const { return m__root; }
    kaitai::kstruct* _parent() const { return m__parent; }
};

#endif  // MSGPACK_H_

msgpack.cpp

// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild

#include "msgpack.h"



msgpack_t::msgpack_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, msgpack_t* p__root) : kaitai::kstruct(p__io) {
    m__parent = p__parent;
    m__root = this;
    f_is_array_32 = false;
    f_int_value = false;
    f_str_len = false;
    f_is_fix_array = false;
    f_is_map = false;
    f_is_array = false;
    f_is_float = false;
    f_is_str_8 = false;
    f_is_fix_map = false;
    f_is_int = false;
    f_is_bool = false;
    f_is_str_16 = false;
    f_is_float_64 = false;
    f_is_map_16 = false;
    f_is_neg_int5 = false;
    f_pos_int7_value = false;
    f_is_nil = false;
    f_float_value = false;
    f_num_array_elements = false;
    f_neg_int5_value = false;
    f_bool_value = false;
    f_is_pos_int7 = false;
    f_is_array_16 = false;
    f_is_str = false;
    f_is_fix_str = false;
    f_is_str_32 = false;
    f_num_map_elements = false;
    f_is_float_32 = false;
    f_is_map_32 = false;
    _read();
}

void msgpack_t::_read() {
    m_b1 = m__io->read_u1();
    n_int_extra = true;
    switch (b1()) {
    case 211: {
        n_int_extra = false;
        m_int_extra = m__io->read_s8be();
        break;
    }
    case 209: {
        n_int_extra = false;
        m_int_extra = m__io->read_s2be();
        break;
    }
    case 210: {
        n_int_extra = false;
        m_int_extra = m__io->read_s4be();
        break;
    }
    case 208: {
        n_int_extra = false;
        m_int_extra = m__io->read_s1();
        break;
    }
    case 205: {
        n_int_extra = false;
        m_int_extra = m__io->read_u2be();
        break;
    }
    case 207: {
        n_int_extra = false;
        m_int_extra = m__io->read_u8be();
        break;
    }
    case 204: {
        n_int_extra = false;
        m_int_extra = m__io->read_u1();
        break;
    }
    case 206: {
        n_int_extra = false;
        m_int_extra = m__io->read_u4be();
        break;
    }
    }
    n_float_32_value = true;
    if (is_float_32()) {
        n_float_32_value = false;
        m_float_32_value = m__io->read_f4be();
    }
    n_float_64_value = true;
    if (is_float_64()) {
        n_float_64_value = false;
        m_float_64_value = m__io->read_f8be();
    }
    n_str_len_8 = true;
    if (is_str_8()) {
        n_str_len_8 = false;
        m_str_len_8 = m__io->read_u1();
    }
    n_str_len_16 = true;
    if (is_str_16()) {
        n_str_len_16 = false;
        m_str_len_16 = m__io->read_u2be();
    }
    n_str_len_32 = true;
    if (is_str_32()) {
        n_str_len_32 = false;
        m_str_len_32 = m__io->read_u4be();
    }
    n_str_value = true;
    if (is_str()) {
        n_str_value = false;
        m_str_value = kaitai::kstream::bytes_to_str(m__io->read_bytes(str_len()), std::string("UTF-8"));
    }
    n_num_array_elements_16 = true;
    if (is_array_16()) {
        n_num_array_elements_16 = false;
        m_num_array_elements_16 = m__io->read_u2be();
    }
    n_num_array_elements_32 = true;
    if (is_array_32()) {
        n_num_array_elements_32 = false;
        m_num_array_elements_32 = m__io->read_u4be();
    }
    n_array_elements = true;
    if (is_array()) {
        n_array_elements = false;
        int l_array_elements = num_array_elements();
        m_array_elements = new std::vector<msgpack_t*>();
        m_array_elements->reserve(l_array_elements);
        for (int i = 0; i < l_array_elements; i++) {
            m_array_elements->push_back(new msgpack_t(m__io));
        }
    }
    n_num_map_elements_16 = true;
    if (is_map_16()) {
        n_num_map_elements_16 = false;
        m_num_map_elements_16 = m__io->read_u2be();
    }
    n_num_map_elements_32 = true;
    if (is_map_32()) {
        n_num_map_elements_32 = false;
        m_num_map_elements_32 = m__io->read_u4be();
    }
    n_map_elements = true;
    if (is_map()) {
        n_map_elements = false;
        int l_map_elements = num_map_elements();
        m_map_elements = new std::vector<map_tuple_t*>();
        m_map_elements->reserve(l_map_elements);
        for (int i = 0; i < l_map_elements; i++) {
            m_map_elements->push_back(new map_tuple_t(m__io, this, m__root));
        }
    }
}

msgpack_t::~msgpack_t() {
    if (!n_int_extra) {
    }
    if (!n_float_32_value) {
    }
    if (!n_float_64_value) {
    }
    if (!n_str_len_8) {
    }
    if (!n_str_len_16) {
    }
    if (!n_str_len_32) {
    }
    if (!n_str_value) {
    }
    if (!n_num_array_elements_16) {
    }
    if (!n_num_array_elements_32) {
    }
    if (!n_array_elements) {
        for (std::vector<msgpack_t*>::iterator it = m_array_elements->begin(); it != m_array_elements->end(); ++it) {
            delete *it;
        }
        delete m_array_elements;
    }
    if (!n_num_map_elements_16) {
    }
    if (!n_num_map_elements_32) {
    }
    if (!n_map_elements) {
        for (std::vector<map_tuple_t*>::iterator it = m_map_elements->begin(); it != m_map_elements->end(); ++it) {
            delete *it;
        }
        delete m_map_elements;
    }
}

msgpack_t::map_tuple_t::map_tuple_t(kaitai::kstream* p__io, msgpack_t* p__parent, msgpack_t* p__root) : kaitai::kstruct(p__io) {
    m__parent = p__parent;
    m__root = p__root;
    _read();
}

void msgpack_t::map_tuple_t::_read() {
    m_key = new msgpack_t(m__io);
    m_value = new msgpack_t(m__io);
}

msgpack_t::map_tuple_t::~map_tuple_t() {
    delete m_key;
    delete m_value;
}

bool msgpack_t::is_array_32() {
    if (f_is_array_32)
        return m_is_array_32;
    m_is_array_32 = b1() == 221;
    f_is_array_32 = true;
    return m_is_array_32;
}

int32_t msgpack_t::int_value() {
    if (f_int_value)
        return m_int_value;
    n_int_value = true;
    if (is_int()) {
        n_int_value = false;
        m_int_value = ((is_pos_int7()) ? (pos_int7_value()) : (((is_neg_int5()) ? (neg_int5_value()) : (4919))));
    }
    f_int_value = true;
    return m_int_value;
}

int32_t msgpack_t::str_len() {
    if (f_str_len)
        return m_str_len;
    n_str_len = true;
    if (is_str()) {
        n_str_len = false;
        m_str_len = ((is_fix_str()) ? ((b1() & 31)) : (((is_str_8()) ? (str_len_8()) : (((is_str_16()) ? (str_len_16()) : (str_len_32()))))));
    }
    f_str_len = true;
    return m_str_len;
}

bool msgpack_t::is_fix_array() {
    if (f_is_fix_array)
        return m_is_fix_array;
    m_is_fix_array = (b1() & 240) == 144;
    f_is_fix_array = true;
    return m_is_fix_array;
}

bool msgpack_t::is_map() {
    if (f_is_map)
        return m_is_map;
    m_is_map =  ((is_fix_map()) || (is_map_16()) || (is_map_32())) ;
    f_is_map = true;
    return m_is_map;
}

bool msgpack_t::is_array() {
    if (f_is_array)
        return m_is_array;
    m_is_array =  ((is_fix_array()) || (is_array_16()) || (is_array_32())) ;
    f_is_array = true;
    return m_is_array;
}

bool msgpack_t::is_float() {
    if (f_is_float)
        return m_is_float;
    m_is_float =  ((is_float_32()) || (is_float_64())) ;
    f_is_float = true;
    return m_is_float;
}

bool msgpack_t::is_str_8() {
    if (f_is_str_8)
        return m_is_str_8;
    m_is_str_8 = b1() == 217;
    f_is_str_8 = true;
    return m_is_str_8;
}

bool msgpack_t::is_fix_map() {
    if (f_is_fix_map)
        return m_is_fix_map;
    m_is_fix_map = (b1() & 240) == 128;
    f_is_fix_map = true;
    return m_is_fix_map;
}

bool msgpack_t::is_int() {
    if (f_is_int)
        return m_is_int;
    m_is_int =  ((is_pos_int7()) || (is_neg_int5())) ;
    f_is_int = true;
    return m_is_int;
}

bool msgpack_t::is_bool() {
    if (f_is_bool)
        return m_is_bool;
    m_is_bool =  ((b1() == 194) || (b1() == 195)) ;
    f_is_bool = true;
    return m_is_bool;
}

bool msgpack_t::is_str_16() {
    if (f_is_str_16)
        return m_is_str_16;
    m_is_str_16 = b1() == 218;
    f_is_str_16 = true;
    return m_is_str_16;
}

bool msgpack_t::is_float_64() {
    if (f_is_float_64)
        return m_is_float_64;
    m_is_float_64 = b1() == 203;
    f_is_float_64 = true;
    return m_is_float_64;
}

bool msgpack_t::is_map_16() {
    if (f_is_map_16)
        return m_is_map_16;
    m_is_map_16 = b1() == 222;
    f_is_map_16 = true;
    return m_is_map_16;
}

bool msgpack_t::is_neg_int5() {
    if (f_is_neg_int5)
        return m_is_neg_int5;
    m_is_neg_int5 = (b1() & 224) == 224;
    f_is_neg_int5 = true;
    return m_is_neg_int5;
}

uint8_t msgpack_t::pos_int7_value() {
    if (f_pos_int7_value)
        return m_pos_int7_value;
    n_pos_int7_value = true;
    if (is_pos_int7()) {
        n_pos_int7_value = false;
        m_pos_int7_value = b1();
    }
    f_pos_int7_value = true;
    return m_pos_int7_value;
}

bool msgpack_t::is_nil() {
    if (f_is_nil)
        return m_is_nil;
    m_is_nil = b1() == 192;
    f_is_nil = true;
    return m_is_nil;
}

double msgpack_t::float_value() {
    if (f_float_value)
        return m_float_value;
    n_float_value = true;
    if (is_float()) {
        n_float_value = false;
        m_float_value = ((is_float_32()) ? (float_32_value()) : (float_64_value()));
    }
    f_float_value = true;
    return m_float_value;
}

int32_t msgpack_t::num_array_elements() {
    if (f_num_array_elements)
        return m_num_array_elements;
    n_num_array_elements = true;
    if (is_array()) {
        n_num_array_elements = false;
        m_num_array_elements = ((is_fix_array()) ? ((b1() & 15)) : (((is_array_16()) ? (num_array_elements_16()) : (num_array_elements_32()))));
    }
    f_num_array_elements = true;
    return m_num_array_elements;
}

int32_t msgpack_t::neg_int5_value() {
    if (f_neg_int5_value)
        return m_neg_int5_value;
    n_neg_int5_value = true;
    if (is_neg_int5()) {
        n_neg_int5_value = false;
        m_neg_int5_value = -((b1() & 31));
    }
    f_neg_int5_value = true;
    return m_neg_int5_value;
}

bool msgpack_t::bool_value() {
    if (f_bool_value)
        return m_bool_value;
    n_bool_value = true;
    if (is_bool()) {
        n_bool_value = false;
        m_bool_value = b1() == 194;
    }
    f_bool_value = true;
    return m_bool_value;
}

bool msgpack_t::is_pos_int7() {
    if (f_is_pos_int7)
        return m_is_pos_int7;
    m_is_pos_int7 = (b1() & 128) == 0;
    f_is_pos_int7 = true;
    return m_is_pos_int7;
}

bool msgpack_t::is_array_16() {
    if (f_is_array_16)
        return m_is_array_16;
    m_is_array_16 = b1() == 220;
    f_is_array_16 = true;
    return m_is_array_16;
}

bool msgpack_t::is_str() {
    if (f_is_str)
        return m_is_str;
    m_is_str =  ((is_fix_str()) || (is_str_8()) || (is_str_16()) || (is_str_32())) ;
    f_is_str = true;
    return m_is_str;
}

bool msgpack_t::is_fix_str() {
    if (f_is_fix_str)
        return m_is_fix_str;
    m_is_fix_str = (b1() & 224) == 160;
    f_is_fix_str = true;
    return m_is_fix_str;
}

bool msgpack_t::is_str_32() {
    if (f_is_str_32)
        return m_is_str_32;
    m_is_str_32 = b1() == 219;
    f_is_str_32 = true;
    return m_is_str_32;
}

int32_t msgpack_t::num_map_elements() {
    if (f_num_map_elements)
        return m_num_map_elements;
    n_num_map_elements = true;
    if (is_map()) {
        n_num_map_elements = false;
        m_num_map_elements = ((is_fix_map()) ? ((b1() & 15)) : (((is_map_16()) ? (num_map_elements_16()) : (num_map_elements_32()))));
    }
    f_num_map_elements = true;
    return m_num_map_elements;
}

bool msgpack_t::is_float_32() {
    if (f_is_float_32)
        return m_is_float_32;
    m_is_float_32 = b1() == 202;
    f_is_float_32 = true;
    return m_is_float_32;
}

bool msgpack_t::is_map_32() {
    if (f_is_map_32)
        return m_is_map_32;
    m_is_map_32 = b1() == 223;
    f_is_map_32 = true;
    return m_is_map_32;
}