Android sparse image: C++11/STL parsing library

The Android sparse format is a format to more efficiently store files for for example firmware updates to save on bandwidth. Files in sparse format first have to be converted back to their original format.

A tool to create images for testing can be found in the Android source code tree:

https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse - img2simg.c

Note: this is not the same as the Android sparse data image format.

File extension

img

KS implementation details

License: CC0-1.0
Minimal Kaitai Struct required: 0.9

This page hosts a formal specification of Android sparse image using Kaitai Struct. This specification can be automatically translated into a variety of programming languages to get a parsing library.

Usage

Runtime library

All parsing code for C++11/STL generated by Kaitai Struct depends on the C++/STL runtime library. You have to install it before you can parse data.

For C++, the easiest way is to clone the runtime library sources and build them along with your project.

Code

Using Kaitai Struct in C++/STL usually consists of 3 steps.

  1. We need to create an STL input stream (std::istream). One can open local file for that, or use existing std::string or char* buffer.
    #include <fstream>
    
    std::ifstream is("path/to/local/file.img", std::ifstream::binary);
    
    #include <sstream>
    
    std::istringstream is(str);
    
    #include <sstream>
    
    const char buf[] = { ... };
    std::string str(buf, sizeof buf);
    std::istringstream is(str);
    
  2. We need to wrap our input stream into Kaitai stream:
    #include "kaitai/kaitaistream.h"
    
    kaitai::kstream ks(&is);
    
  3. And finally, we can invoke the parsing:
    android_sparse_t data(&ks);
    

After that, one can get various attributes from the structure by invoking getter methods like:

data.header_prefix() // => internal; access `_root.header` instead

C++11/STL source code to parse Android sparse image

android_sparse.h

#pragma once

// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild

#include "kaitai/kaitaistruct.h"
#include <stdint.h>
#include <memory>
#include <vector>

#if KAITAI_STRUCT_VERSION < 9000L
#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required"
#endif

/**
 * The Android sparse format is a format to more efficiently store files
 * for for example firmware updates to save on bandwidth. Files in sparse
 * format first have to be converted back to their original format.
 * 
 * A tool to create images for testing can be found in the Android source code tree:
 * 
 * <https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse> - `img2simg.c`
 * 
 * Note: this is not the same as the Android sparse data image format.
 * \sa https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_format.h Source
 * \sa https://source.android.com/devices/bootloader/images#sparse-image-format Source
 */

class android_sparse_t : public kaitai::kstruct {

public:
    class file_header_prefix_t;
    class file_header_t;
    class chunk_t;
    class version_t;

    enum chunk_types_t {
        CHUNK_TYPES_RAW = 51905,
        CHUNK_TYPES_FILL = 51906,
        CHUNK_TYPES_DONT_CARE = 51907,
        CHUNK_TYPES_CRC32 = 51908
    };

    android_sparse_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = nullptr, android_sparse_t* p__root = nullptr);

private:
    void _read();
    void _clean_up();

public:
    ~android_sparse_t();

    class file_header_prefix_t : public kaitai::kstruct {

    public:

        file_header_prefix_t(kaitai::kstream* p__io, android_sparse_t* p__parent = nullptr, android_sparse_t* p__root = nullptr);

    private:
        void _read();
        void _clean_up();

    public:
        ~file_header_prefix_t();

    private:
        std::string m_magic;
        std::unique_ptr<version_t> m_version;
        uint16_t m_len_header;
        android_sparse_t* m__root;
        android_sparse_t* m__parent;

    public:
        std::string magic() const { return m_magic; }

        /**
         * internal; access `_root.header.version` instead
         */
        version_t* version() const { return m_version.get(); }

        /**
         * internal; access `_root.header.len_header` instead
         */
        uint16_t len_header() const { return m_len_header; }
        android_sparse_t* _root() const { return m__root; }
        android_sparse_t* _parent() const { return m__parent; }
    };

    class file_header_t : public kaitai::kstruct {

    public:

        file_header_t(kaitai::kstream* p__io, android_sparse_t* p__parent = nullptr, android_sparse_t* p__root = nullptr);

    private:
        void _read();
        void _clean_up();

    public:
        ~file_header_t();

    private:
        bool f_version;
        version_t* m_version;

    public:
        version_t* version();

    private:
        bool f_len_header;
        uint16_t m_len_header;

    public:

        /**
         * size of file header, should be 28
         */
        uint16_t len_header();

    private:
        uint16_t m_len_chunk_header;
        uint32_t m_block_size;
        uint32_t m_num_blocks;
        uint32_t m_num_chunks;
        uint32_t m_checksum;
        android_sparse_t* m__root;
        android_sparse_t* m__parent;

    public:

        /**
         * size of chunk header, should be 12
         */
        uint16_t len_chunk_header() const { return m_len_chunk_header; }

        /**
         * block size in bytes, must be a multiple of 4
         */
        uint32_t block_size() const { return m_block_size; }

        /**
         * blocks in the original data
         */
        uint32_t num_blocks() const { return m_num_blocks; }
        uint32_t num_chunks() const { return m_num_chunks; }

        /**
         * CRC32 checksum of the original data
         * 
         * In practice always 0; if checksum writing is requested, a CRC32 chunk is written
         * at the end of the file instead. The canonical `libsparse` implementation does this
         * and other implementations tend to follow it, see
         * <https://gitlab.com/ra_kete/android-sparse-rs/-/blob/57c2577/src/write.rs#L112-114>
         */
        uint32_t checksum() const { return m_checksum; }
        android_sparse_t* _root() const { return m__root; }
        android_sparse_t* _parent() const { return m__parent; }
    };

    class chunk_t : public kaitai::kstruct {

    public:
        class chunk_header_t;

        chunk_t(kaitai::kstream* p__io, android_sparse_t* p__parent = nullptr, android_sparse_t* p__root = nullptr);

    private:
        void _read();
        void _clean_up();

    public:
        ~chunk_t();

        class chunk_header_t : public kaitai::kstruct {

        public:

            chunk_header_t(kaitai::kstream* p__io, android_sparse_t::chunk_t* p__parent = nullptr, android_sparse_t* p__root = nullptr);

        private:
            void _read();
            void _clean_up();

        public:
            ~chunk_header_t();

        private:
            bool f_len_body;
            int32_t m_len_body;

        public:
            int32_t len_body();

        private:
            bool f_len_body_expected;
            int32_t m_len_body_expected;

        public:

            /**
             * \sa https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#184 Source
             * \sa https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#215 Source
             * \sa https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#249 Source
             * \sa https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#270 Source
             */
            int32_t len_body_expected();

        private:
            chunk_types_t m_chunk_type;
            uint16_t m_reserved1;
            uint32_t m_num_body_blocks;
            uint32_t m_len_chunk;
            android_sparse_t* m__root;
            android_sparse_t::chunk_t* m__parent;

        public:
            chunk_types_t chunk_type() const { return m_chunk_type; }
            uint16_t reserved1() const { return m_reserved1; }

            /**
             * size of the chunk body in blocks in output image
             */
            uint32_t num_body_blocks() const { return m_num_body_blocks; }

            /**
             * in bytes of chunk input file including chunk header and data
             */
            uint32_t len_chunk() const { return m_len_chunk; }
            android_sparse_t* _root() const { return m__root; }
            android_sparse_t::chunk_t* _parent() const { return m__parent; }
        };

    private:
        std::unique_ptr<chunk_header_t> m_header;
        uint32_t m_body;
        bool n_body;

    public:
        bool _is_null_body() { body(); return n_body; };

    private:
        android_sparse_t* m__root;
        android_sparse_t* m__parent;
        std::string m__raw_header;
        std::unique_ptr<kaitai::kstream> m__io__raw_header;

    public:
        chunk_header_t* header() const { return m_header.get(); }
        uint32_t body() const { return m_body; }
        android_sparse_t* _root() const { return m__root; }
        android_sparse_t* _parent() const { return m__parent; }
        std::string _raw_header() const { return m__raw_header; }
        kaitai::kstream* _io__raw_header() const { return m__io__raw_header.get(); }
    };

    class version_t : public kaitai::kstruct {

    public:

        version_t(kaitai::kstream* p__io, android_sparse_t::file_header_prefix_t* p__parent = nullptr, android_sparse_t* p__root = nullptr);

    private:
        void _read();
        void _clean_up();

    public:
        ~version_t();

    private:
        uint16_t m_major;
        uint16_t m_minor;
        android_sparse_t* m__root;
        android_sparse_t::file_header_prefix_t* m__parent;

    public:
        uint16_t major() const { return m_major; }
        uint16_t minor() const { return m_minor; }
        android_sparse_t* _root() const { return m__root; }
        android_sparse_t::file_header_prefix_t* _parent() const { return m__parent; }
    };

private:
    std::unique_ptr<file_header_prefix_t> m_header_prefix;
    std::unique_ptr<file_header_t> m_header;
    std::unique_ptr<std::vector<std::unique_ptr<chunk_t>>> m_chunks;
    android_sparse_t* m__root;
    kaitai::kstruct* m__parent;
    std::string m__raw_header;
    std::unique_ptr<kaitai::kstream> m__io__raw_header;

public:

    /**
     * internal; access `_root.header` instead
     */
    file_header_prefix_t* header_prefix() const { return m_header_prefix.get(); }
    file_header_t* header() const { return m_header.get(); }
    std::vector<std::unique_ptr<chunk_t>>* chunks() const { return m_chunks.get(); }
    android_sparse_t* _root() const { return m__root; }
    kaitai::kstruct* _parent() const { return m__parent; }
    std::string _raw_header() const { return m__raw_header; }
    kaitai::kstream* _io__raw_header() const { return m__io__raw_header.get(); }
};

android_sparse.cpp

// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild

#include "android_sparse.h"
#include "kaitai/exceptions.h"

android_sparse_t::android_sparse_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
    m__parent = p__parent;
    m__root = this;
    m_header_prefix = nullptr;
    m_header = nullptr;
    m__io__raw_header = nullptr;
    m_chunks = nullptr;
    _read();
}

void android_sparse_t::_read() {
    m_header_prefix = std::unique_ptr<file_header_prefix_t>(new file_header_prefix_t(m__io, this, m__root));
    m__raw_header = m__io->read_bytes((header_prefix()->len_header() - 10));
    m__io__raw_header = std::unique_ptr<kaitai::kstream>(new kaitai::kstream(m__raw_header));
    m_header = std::unique_ptr<file_header_t>(new file_header_t(m__io__raw_header.get(), this, m__root));
    int l_chunks = header()->num_chunks();
    m_chunks = std::unique_ptr<std::vector<std::unique_ptr<chunk_t>>>(new std::vector<std::unique_ptr<chunk_t>>());
    m_chunks->reserve(l_chunks);
    for (int i = 0; i < l_chunks; i++) {
        m_chunks->push_back(std::move(std::unique_ptr<chunk_t>(new chunk_t(m__io, this, m__root))));
    }
}

android_sparse_t::~android_sparse_t() {
    _clean_up();
}

void android_sparse_t::_clean_up() {
}

android_sparse_t::file_header_prefix_t::file_header_prefix_t(kaitai::kstream* p__io, android_sparse_t* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
    m__parent = p__parent;
    m__root = p__root;
    m_version = nullptr;
    _read();
}

void android_sparse_t::file_header_prefix_t::_read() {
    m_magic = m__io->read_bytes(4);
    if (!(magic() == std::string("\x3A\xFF\x26\xED", 4))) {
        throw kaitai::validation_not_equal_error<std::string>(std::string("\x3A\xFF\x26\xED", 4), magic(), _io(), std::string("/types/file_header_prefix/seq/0"));
    }
    m_version = std::unique_ptr<version_t>(new version_t(m__io, this, m__root));
    m_len_header = m__io->read_u2le();
}

android_sparse_t::file_header_prefix_t::~file_header_prefix_t() {
    _clean_up();
}

void android_sparse_t::file_header_prefix_t::_clean_up() {
}

android_sparse_t::file_header_t::file_header_t(kaitai::kstream* p__io, android_sparse_t* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
    m__parent = p__parent;
    m__root = p__root;
    f_version = false;
    f_len_header = false;
    _read();
}

void android_sparse_t::file_header_t::_read() {
    m_len_chunk_header = m__io->read_u2le();
    m_block_size = m__io->read_u4le();
    {
        uint32_t _ = block_size();
        if (!(kaitai::kstream::mod(_, 4) == 0)) {
            throw kaitai::validation_expr_error<uint32_t>(block_size(), _io(), std::string("/types/file_header/seq/1"));
        }
    }
    m_num_blocks = m__io->read_u4le();
    m_num_chunks = m__io->read_u4le();
    m_checksum = m__io->read_u4le();
}

android_sparse_t::file_header_t::~file_header_t() {
    _clean_up();
}

void android_sparse_t::file_header_t::_clean_up() {
}

android_sparse_t::version_t* android_sparse_t::file_header_t::version() {
    if (f_version)
        return m_version;
    m_version = _root()->header_prefix()->version();
    f_version = true;
    return m_version;
}

uint16_t android_sparse_t::file_header_t::len_header() {
    if (f_len_header)
        return m_len_header;
    m_len_header = _root()->header_prefix()->len_header();
    f_len_header = true;
    return m_len_header;
}

android_sparse_t::chunk_t::chunk_t(kaitai::kstream* p__io, android_sparse_t* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
    m__parent = p__parent;
    m__root = p__root;
    m_header = nullptr;
    m__io__raw_header = nullptr;
    _read();
}

void android_sparse_t::chunk_t::_read() {
    m__raw_header = m__io->read_bytes(_root()->header()->len_chunk_header());
    m__io__raw_header = std::unique_ptr<kaitai::kstream>(new kaitai::kstream(m__raw_header));
    m_header = std::unique_ptr<chunk_header_t>(new chunk_header_t(m__io__raw_header.get(), this, m__root));
    n_body = true;
    switch (header()->chunk_type()) {
    case android_sparse_t::CHUNK_TYPES_CRC32: {
        n_body = false;
        m_body = m__io->read_u4le();
        break;
    }
    default: {
        m__raw_body = m__io->read_bytes(header()->len_body());
        break;
    }
    }
}

android_sparse_t::chunk_t::~chunk_t() {
    _clean_up();
}

void android_sparse_t::chunk_t::_clean_up() {
    if (!n_body) {
    }
}

android_sparse_t::chunk_t::chunk_header_t::chunk_header_t(kaitai::kstream* p__io, android_sparse_t::chunk_t* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
    m__parent = p__parent;
    m__root = p__root;
    f_len_body = false;
    f_len_body_expected = false;
    _read();
}

void android_sparse_t::chunk_t::chunk_header_t::_read() {
    m_chunk_type = static_cast<android_sparse_t::chunk_types_t>(m__io->read_u2le());
    m_reserved1 = m__io->read_u2le();
    m_num_body_blocks = m__io->read_u4le();
    m_len_chunk = m__io->read_u4le();
    if (!(len_chunk() == ((len_body_expected() != -1) ? ((_root()->header()->len_chunk_header() + len_body_expected())) : (len_chunk())))) {
        throw kaitai::validation_not_equal_error<uint32_t>(((len_body_expected() != -1) ? ((_root()->header()->len_chunk_header() + len_body_expected())) : (len_chunk())), len_chunk(), _io(), std::string("/types/chunk/types/chunk_header/seq/3"));
    }
}

android_sparse_t::chunk_t::chunk_header_t::~chunk_header_t() {
    _clean_up();
}

void android_sparse_t::chunk_t::chunk_header_t::_clean_up() {
}

int32_t android_sparse_t::chunk_t::chunk_header_t::len_body() {
    if (f_len_body)
        return m_len_body;
    m_len_body = (len_chunk() - _root()->header()->len_chunk_header());
    f_len_body = true;
    return m_len_body;
}

int32_t android_sparse_t::chunk_t::chunk_header_t::len_body_expected() {
    if (f_len_body_expected)
        return m_len_body_expected;
    m_len_body_expected = ((chunk_type() == android_sparse_t::CHUNK_TYPES_RAW) ? ((_root()->header()->block_size() * num_body_blocks())) : (((chunk_type() == android_sparse_t::CHUNK_TYPES_FILL) ? (4) : (((chunk_type() == android_sparse_t::CHUNK_TYPES_DONT_CARE) ? (0) : (((chunk_type() == android_sparse_t::CHUNK_TYPES_CRC32) ? (4) : (-1))))))));
    f_len_body_expected = true;
    return m_len_body_expected;
}

android_sparse_t::version_t::version_t(kaitai::kstream* p__io, android_sparse_t::file_header_prefix_t* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
    m__parent = p__parent;
    m__root = p__root;
    _read();
}

void android_sparse_t::version_t::_read() {
    m_major = m__io->read_u2le();
    if (!(major() == 1)) {
        throw kaitai::validation_not_equal_error<uint16_t>(1, major(), _io(), std::string("/types/version/seq/0"));
    }
    m_minor = m__io->read_u2le();
}

android_sparse_t::version_t::~version_t() {
    _clean_up();
}

void android_sparse_t::version_t::_clean_up() {
}