The Android sparse format is a format to more efficiently store files for for example firmware updates to save on bandwidth. Files in sparse format first have to be converted back to their original format.
A tool to create images for testing can be found in the Android source code tree:
https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse - img2simg.c
Note: this is not the same as the Android sparse data image format.
This page hosts a formal specification of Android sparse image using Kaitai Struct. This specification can be automatically translated into a variety of programming languages to get a parsing library.
All parsing code for C++11/STL generated by Kaitai Struct depends on the C++/STL runtime library. You have to install it before you can parse data.
For C++, the easiest way is to clone the runtime library sources and build them along with your project.
Using Kaitai Struct in C++/STL usually consists of 3 steps.
std::istream). One can open local file for that, or use existing std::string or char* buffer.
#include <fstream>
std::ifstream is("path/to/local/file.img", std::ifstream::binary);
#include <sstream>
std::istringstream is(str);
#include <sstream>
const char buf[] = { ... };
std::string str(buf, sizeof buf);
std::istringstream is(str);
#include "kaitai/kaitaistream.h"
kaitai::kstream ks(&is);
android_sparse_t data(&ks);
After that, one can get various attributes from the structure by invoking getter methods like:
data.header_prefix() // => internal; access `_root.header` instead
#pragma once
// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
#include "kaitai/kaitaistruct.h"
#include <stdint.h>
#include <memory>
#include <vector>
#if KAITAI_STRUCT_VERSION < 9000L
#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required"
#endif
/**
* The Android sparse format is a format to more efficiently store files
* for for example firmware updates to save on bandwidth. Files in sparse
* format first have to be converted back to their original format.
*
* A tool to create images for testing can be found in the Android source code tree:
*
* <https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse> - `img2simg.c`
*
* Note: this is not the same as the Android sparse data image format.
* \sa https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_format.h Source
* \sa https://web.archive.org/web/20220322054458/https://source.android.com/devices/bootloader/images#sparse-image-format Source
*/
class android_sparse_t : public kaitai::kstruct {
public:
class file_header_prefix_t;
class file_header_t;
class chunk_t;
class version_t;
enum chunk_types_t {
CHUNK_TYPES_RAW = 51905,
CHUNK_TYPES_FILL = 51906,
CHUNK_TYPES_DONT_CARE = 51907,
CHUNK_TYPES_CRC32 = 51908
};
android_sparse_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = nullptr, android_sparse_t* p__root = nullptr);
private:
void _read();
void _clean_up();
public:
~android_sparse_t();
class file_header_prefix_t : public kaitai::kstruct {
public:
file_header_prefix_t(kaitai::kstream* p__io, android_sparse_t* p__parent = nullptr, android_sparse_t* p__root = nullptr);
private:
void _read();
void _clean_up();
public:
~file_header_prefix_t();
private:
std::string m_magic;
std::unique_ptr<version_t> m_version;
uint16_t m_len_header;
android_sparse_t* m__root;
android_sparse_t* m__parent;
public:
std::string magic() const { return m_magic; }
/**
* internal; access `_root.header.version` instead
*/
version_t* version() const { return m_version.get(); }
/**
* internal; access `_root.header.len_header` instead
*/
uint16_t len_header() const { return m_len_header; }
android_sparse_t* _root() const { return m__root; }
android_sparse_t* _parent() const { return m__parent; }
};
class file_header_t : public kaitai::kstruct {
public:
file_header_t(kaitai::kstream* p__io, android_sparse_t* p__parent = nullptr, android_sparse_t* p__root = nullptr);
private:
void _read();
void _clean_up();
public:
~file_header_t();
private:
bool f_version;
version_t* m_version;
public:
version_t* version();
private:
bool f_len_header;
uint16_t m_len_header;
public:
/**
* size of file header, should be 28
*/
uint16_t len_header();
private:
uint16_t m_len_chunk_header;
uint32_t m_block_size;
uint32_t m_num_blocks;
uint32_t m_num_chunks;
uint32_t m_checksum;
android_sparse_t* m__root;
android_sparse_t* m__parent;
public:
/**
* size of chunk header, should be 12
*/
uint16_t len_chunk_header() const { return m_len_chunk_header; }
/**
* block size in bytes, must be a multiple of 4
*/
uint32_t block_size() const { return m_block_size; }
/**
* blocks in the original data
*/
uint32_t num_blocks() const { return m_num_blocks; }
uint32_t num_chunks() const { return m_num_chunks; }
/**
* CRC32 checksum of the original data
*
* In practice always 0; if checksum writing is requested, a CRC32 chunk is written
* at the end of the file instead. The canonical `libsparse` implementation does this
* and other implementations tend to follow it, see
* <https://gitlab.com/teskje/android-sparse-rs/-/blob/57c2577/src/write.rs#L112-114>
*/
uint32_t checksum() const { return m_checksum; }
android_sparse_t* _root() const { return m__root; }
android_sparse_t* _parent() const { return m__parent; }
};
class chunk_t : public kaitai::kstruct {
public:
class chunk_header_t;
chunk_t(kaitai::kstream* p__io, android_sparse_t* p__parent = nullptr, android_sparse_t* p__root = nullptr);
private:
void _read();
void _clean_up();
public:
~chunk_t();
class chunk_header_t : public kaitai::kstruct {
public:
chunk_header_t(kaitai::kstream* p__io, android_sparse_t::chunk_t* p__parent = nullptr, android_sparse_t* p__root = nullptr);
private:
void _read();
void _clean_up();
public:
~chunk_header_t();
private:
bool f_len_body;
int32_t m_len_body;
public:
int32_t len_body();
private:
bool f_len_body_expected;
int32_t m_len_body_expected;
public:
/**
* \sa https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#184 Source
* \sa https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#215 Source
* \sa https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#249 Source
* \sa https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#270 Source
*/
int32_t len_body_expected();
private:
chunk_types_t m_chunk_type;
uint16_t m_reserved1;
uint32_t m_num_body_blocks;
uint32_t m_len_chunk;
android_sparse_t* m__root;
android_sparse_t::chunk_t* m__parent;
public:
chunk_types_t chunk_type() const { return m_chunk_type; }
uint16_t reserved1() const { return m_reserved1; }
/**
* size of the chunk body in blocks in output image
*/
uint32_t num_body_blocks() const { return m_num_body_blocks; }
/**
* in bytes of chunk input file including chunk header and data
*/
uint32_t len_chunk() const { return m_len_chunk; }
android_sparse_t* _root() const { return m__root; }
android_sparse_t::chunk_t* _parent() const { return m__parent; }
};
private:
std::unique_ptr<chunk_header_t> m_header;
uint32_t m_body;
bool n_body;
public:
bool _is_null_body() { body(); return n_body; };
private:
android_sparse_t* m__root;
android_sparse_t* m__parent;
std::string m__raw_header;
std::unique_ptr<kaitai::kstream> m__io__raw_header;
public:
chunk_header_t* header() const { return m_header.get(); }
uint32_t body() const { return m_body; }
android_sparse_t* _root() const { return m__root; }
android_sparse_t* _parent() const { return m__parent; }
std::string _raw_header() const { return m__raw_header; }
kaitai::kstream* _io__raw_header() const { return m__io__raw_header.get(); }
};
class version_t : public kaitai::kstruct {
public:
version_t(kaitai::kstream* p__io, android_sparse_t::file_header_prefix_t* p__parent = nullptr, android_sparse_t* p__root = nullptr);
private:
void _read();
void _clean_up();
public:
~version_t();
private:
uint16_t m_major;
uint16_t m_minor;
android_sparse_t* m__root;
android_sparse_t::file_header_prefix_t* m__parent;
public:
uint16_t major() const { return m_major; }
uint16_t minor() const { return m_minor; }
android_sparse_t* _root() const { return m__root; }
android_sparse_t::file_header_prefix_t* _parent() const { return m__parent; }
};
private:
std::unique_ptr<file_header_prefix_t> m_header_prefix;
std::unique_ptr<file_header_t> m_header;
std::unique_ptr<std::vector<std::unique_ptr<chunk_t>>> m_chunks;
android_sparse_t* m__root;
kaitai::kstruct* m__parent;
std::string m__raw_header;
std::unique_ptr<kaitai::kstream> m__io__raw_header;
public:
/**
* internal; access `_root.header` instead
*/
file_header_prefix_t* header_prefix() const { return m_header_prefix.get(); }
file_header_t* header() const { return m_header.get(); }
std::vector<std::unique_ptr<chunk_t>>* chunks() const { return m_chunks.get(); }
android_sparse_t* _root() const { return m__root; }
kaitai::kstruct* _parent() const { return m__parent; }
std::string _raw_header() const { return m__raw_header; }
kaitai::kstream* _io__raw_header() const { return m__io__raw_header.get(); }
};
// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
#include "android_sparse.h"
#include "kaitai/exceptions.h"
android_sparse_t::android_sparse_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
m__parent = p__parent;
m__root = this;
m_header_prefix = nullptr;
m_header = nullptr;
m__io__raw_header = nullptr;
m_chunks = nullptr;
_read();
}
void android_sparse_t::_read() {
m_header_prefix = std::unique_ptr<file_header_prefix_t>(new file_header_prefix_t(m__io, this, m__root));
m__raw_header = m__io->read_bytes((header_prefix()->len_header() - 10));
m__io__raw_header = std::unique_ptr<kaitai::kstream>(new kaitai::kstream(m__raw_header));
m_header = std::unique_ptr<file_header_t>(new file_header_t(m__io__raw_header.get(), this, m__root));
m_chunks = std::unique_ptr<std::vector<std::unique_ptr<chunk_t>>>(new std::vector<std::unique_ptr<chunk_t>>());
const int l_chunks = header()->num_chunks();
for (int i = 0; i < l_chunks; i++) {
m_chunks->push_back(std::move(std::unique_ptr<chunk_t>(new chunk_t(m__io, this, m__root))));
}
}
android_sparse_t::~android_sparse_t() {
_clean_up();
}
void android_sparse_t::_clean_up() {
}
android_sparse_t::file_header_prefix_t::file_header_prefix_t(kaitai::kstream* p__io, android_sparse_t* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
m__parent = p__parent;
m__root = p__root;
m_version = nullptr;
_read();
}
void android_sparse_t::file_header_prefix_t::_read() {
m_magic = m__io->read_bytes(4);
if (!(magic() == std::string("\x3A\xFF\x26\xED", 4))) {
throw kaitai::validation_not_equal_error<std::string>(std::string("\x3A\xFF\x26\xED", 4), magic(), _io(), std::string("/types/file_header_prefix/seq/0"));
}
m_version = std::unique_ptr<version_t>(new version_t(m__io, this, m__root));
m_len_header = m__io->read_u2le();
}
android_sparse_t::file_header_prefix_t::~file_header_prefix_t() {
_clean_up();
}
void android_sparse_t::file_header_prefix_t::_clean_up() {
}
android_sparse_t::file_header_t::file_header_t(kaitai::kstream* p__io, android_sparse_t* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
m__parent = p__parent;
m__root = p__root;
f_version = false;
f_len_header = false;
_read();
}
void android_sparse_t::file_header_t::_read() {
m_len_chunk_header = m__io->read_u2le();
m_block_size = m__io->read_u4le();
{
uint32_t _ = block_size();
if (!(kaitai::kstream::mod(_, 4) == 0)) {
throw kaitai::validation_expr_error<uint32_t>(block_size(), _io(), std::string("/types/file_header/seq/1"));
}
}
m_num_blocks = m__io->read_u4le();
m_num_chunks = m__io->read_u4le();
m_checksum = m__io->read_u4le();
}
android_sparse_t::file_header_t::~file_header_t() {
_clean_up();
}
void android_sparse_t::file_header_t::_clean_up() {
}
android_sparse_t::version_t* android_sparse_t::file_header_t::version() {
if (f_version)
return m_version;
m_version = _root()->header_prefix()->version();
f_version = true;
return m_version;
}
uint16_t android_sparse_t::file_header_t::len_header() {
if (f_len_header)
return m_len_header;
m_len_header = _root()->header_prefix()->len_header();
f_len_header = true;
return m_len_header;
}
android_sparse_t::chunk_t::chunk_t(kaitai::kstream* p__io, android_sparse_t* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
m__parent = p__parent;
m__root = p__root;
m_header = nullptr;
m__io__raw_header = nullptr;
_read();
}
void android_sparse_t::chunk_t::_read() {
m__raw_header = m__io->read_bytes(_root()->header()->len_chunk_header());
m__io__raw_header = std::unique_ptr<kaitai::kstream>(new kaitai::kstream(m__raw_header));
m_header = std::unique_ptr<chunk_header_t>(new chunk_header_t(m__io__raw_header.get(), this, m__root));
n_body = true;
switch (header()->chunk_type()) {
case android_sparse_t::CHUNK_TYPES_CRC32: {
n_body = false;
m_body = m__io->read_u4le();
break;
}
default: {
m__raw_body = m__io->read_bytes(header()->len_body());
break;
}
}
}
android_sparse_t::chunk_t::~chunk_t() {
_clean_up();
}
void android_sparse_t::chunk_t::_clean_up() {
if (!n_body) {
}
}
android_sparse_t::chunk_t::chunk_header_t::chunk_header_t(kaitai::kstream* p__io, android_sparse_t::chunk_t* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
m__parent = p__parent;
m__root = p__root;
f_len_body = false;
f_len_body_expected = false;
_read();
}
void android_sparse_t::chunk_t::chunk_header_t::_read() {
m_chunk_type = static_cast<android_sparse_t::chunk_types_t>(m__io->read_u2le());
m_reserved1 = m__io->read_u2le();
m_num_body_blocks = m__io->read_u4le();
m_len_chunk = m__io->read_u4le();
if (!(len_chunk() == ((len_body_expected() != -1) ? ((_root()->header()->len_chunk_header() + len_body_expected())) : (len_chunk())))) {
throw kaitai::validation_not_equal_error<uint32_t>(((len_body_expected() != -1) ? ((_root()->header()->len_chunk_header() + len_body_expected())) : (len_chunk())), len_chunk(), _io(), std::string("/types/chunk/types/chunk_header/seq/3"));
}
}
android_sparse_t::chunk_t::chunk_header_t::~chunk_header_t() {
_clean_up();
}
void android_sparse_t::chunk_t::chunk_header_t::_clean_up() {
}
int32_t android_sparse_t::chunk_t::chunk_header_t::len_body() {
if (f_len_body)
return m_len_body;
m_len_body = (len_chunk() - _root()->header()->len_chunk_header());
f_len_body = true;
return m_len_body;
}
int32_t android_sparse_t::chunk_t::chunk_header_t::len_body_expected() {
if (f_len_body_expected)
return m_len_body_expected;
m_len_body_expected = ((chunk_type() == android_sparse_t::CHUNK_TYPES_RAW) ? ((_root()->header()->block_size() * num_body_blocks())) : (((chunk_type() == android_sparse_t::CHUNK_TYPES_FILL) ? (4) : (((chunk_type() == android_sparse_t::CHUNK_TYPES_DONT_CARE) ? (0) : (((chunk_type() == android_sparse_t::CHUNK_TYPES_CRC32) ? (4) : (-1))))))));
f_len_body_expected = true;
return m_len_body_expected;
}
android_sparse_t::version_t::version_t(kaitai::kstream* p__io, android_sparse_t::file_header_prefix_t* p__parent, android_sparse_t* p__root) : kaitai::kstruct(p__io) {
m__parent = p__parent;
m__root = p__root;
_read();
}
void android_sparse_t::version_t::_read() {
m_major = m__io->read_u2le();
if (!(major() == 1)) {
throw kaitai::validation_not_equal_error<uint16_t>(1, major(), _io(), std::string("/types/version/seq/0"));
}
m_minor = m__io->read_u2le();
}
android_sparse_t::version_t::~version_t() {
_clean_up();
}
void android_sparse_t::version_t::_clean_up() {
}