Android sparse image: Go parsing library

The Android sparse format is a format to more efficiently store files for for example firmware updates to save on bandwidth. Files in sparse format first have to be converted back to their original format.

A tool to create images for testing can be found in the Android source code tree:

https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse - img2simg.c

Note: this is not the same as the Android sparse data image format.

File extension

img

KS implementation details

License: CC0-1.0
Minimal Kaitai Struct required: 0.9

This page hosts a formal specification of Android sparse image using Kaitai Struct. This specification can be automatically translated into a variety of programming languages to get a parsing library.

Go source code to parse Android sparse image

android_sparse.go

// Code generated by kaitai-struct-compiler from a .ksy source file. DO NOT EDIT.

import (
	"github.com/kaitai-io/kaitai_struct_go_runtime/kaitai"
	"bytes"
)


/**
 * The Android sparse format is a format to more efficiently store files
 * for for example firmware updates to save on bandwidth. Files in sparse
 * format first have to be converted back to their original format.
 * 
 * A tool to create images for testing can be found in the Android source code tree:
 * 
 * <https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse> - `img2simg.c`
 * 
 * Note: this is not the same as the Android sparse data image format.
 * @see <a href="https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_format.h">Source</a>
 * @see <a href="https://web.archive.org/web/20220322054458/https://source.android.com/devices/bootloader/images#sparse-image-format">Source</a>
 */

type AndroidSparse_ChunkTypes int
const (
	AndroidSparse_ChunkTypes__Raw AndroidSparse_ChunkTypes = 51905
	AndroidSparse_ChunkTypes__Fill AndroidSparse_ChunkTypes = 51906
	AndroidSparse_ChunkTypes__DontCare AndroidSparse_ChunkTypes = 51907
	AndroidSparse_ChunkTypes__Crc32 AndroidSparse_ChunkTypes = 51908
)
var values_AndroidSparse_ChunkTypes = map[AndroidSparse_ChunkTypes]struct{}{51905: {}, 51906: {}, 51907: {}, 51908: {}}
func (v AndroidSparse_ChunkTypes) isDefined() bool {
	_, ok := values_AndroidSparse_ChunkTypes[v]
	return ok
}
type AndroidSparse struct {
	HeaderPrefix *AndroidSparse_FileHeaderPrefix
	Header *AndroidSparse_FileHeader
	Chunks []*AndroidSparse_Chunk
	_io *kaitai.Stream
	_root *AndroidSparse
	_parent kaitai.Struct
	_raw_Header []byte
}
func NewAndroidSparse() *AndroidSparse {
	return &AndroidSparse{
	}
}

func (this AndroidSparse) IO_() *kaitai.Stream {
	return this._io
}

func (this *AndroidSparse) Read(io *kaitai.Stream, parent kaitai.Struct, root *AndroidSparse) (err error) {
	this._io = io
	this._parent = parent
	this._root = root

	tmp1 := NewAndroidSparse_FileHeaderPrefix()
	err = tmp1.Read(this._io, this, this._root)
	if err != nil {
		return err
	}
	this.HeaderPrefix = tmp1
	tmp2, err := this._io.ReadBytes(int(this.HeaderPrefix.LenHeader - 10))
	if err != nil {
		return err
	}
	tmp2 = tmp2
	this._raw_Header = tmp2
	_io__raw_Header := kaitai.NewStream(bytes.NewReader(this._raw_Header))
	tmp3 := NewAndroidSparse_FileHeader()
	err = tmp3.Read(_io__raw_Header, this, this._root)
	if err != nil {
		return err
	}
	this.Header = tmp3
	for i := 0; i < int(this.Header.NumChunks); i++ {
		_ = i
		tmp4 := NewAndroidSparse_Chunk()
		err = tmp4.Read(this._io, this, this._root)
		if err != nil {
			return err
		}
		this.Chunks = append(this.Chunks, tmp4)
	}
	return err
}

/**
 * internal; access `_root.header` instead
 */
type AndroidSparse_Chunk struct {
	Header *AndroidSparse_Chunk_ChunkHeader
	Body interface{}
	_io *kaitai.Stream
	_root *AndroidSparse
	_parent *AndroidSparse
	_raw_Header []byte
}
func NewAndroidSparse_Chunk() *AndroidSparse_Chunk {
	return &AndroidSparse_Chunk{
	}
}

func (this AndroidSparse_Chunk) IO_() *kaitai.Stream {
	return this._io
}

func (this *AndroidSparse_Chunk) Read(io *kaitai.Stream, parent *AndroidSparse, root *AndroidSparse) (err error) {
	this._io = io
	this._parent = parent
	this._root = root

	tmp5, err := this._io.ReadBytes(int(this._root.Header.LenChunkHeader))
	if err != nil {
		return err
	}
	tmp5 = tmp5
	this._raw_Header = tmp5
	_io__raw_Header := kaitai.NewStream(bytes.NewReader(this._raw_Header))
	tmp6 := NewAndroidSparse_Chunk_ChunkHeader()
	err = tmp6.Read(_io__raw_Header, this, this._root)
	if err != nil {
		return err
	}
	this.Header = tmp6
	switch (this.Header.ChunkType) {
	case AndroidSparse_ChunkTypes__Crc32:
		tmp7, err := this._io.ReadU4le()
		if err != nil {
			return err
		}
		this.Body = tmp7
	default:
		tmp8, err := this.Header.LenBody()
		if err != nil {
			return err
		}
		tmp9, err := this._io.ReadBytes(int(tmp8))
		if err != nil {
			return err
		}
		tmp9 = tmp9
		this._raw_Body = tmp9
	}
	return err
}
type AndroidSparse_Chunk_ChunkHeader struct {
	ChunkType AndroidSparse_ChunkTypes
	Reserved1 uint16
	NumBodyBlocks uint32
	LenChunk uint32
	_io *kaitai.Stream
	_root *AndroidSparse
	_parent *AndroidSparse_Chunk
	_f_lenBody bool
	lenBody int
	_f_lenBodyExpected bool
	lenBodyExpected int
}
func NewAndroidSparse_Chunk_ChunkHeader() *AndroidSparse_Chunk_ChunkHeader {
	return &AndroidSparse_Chunk_ChunkHeader{
	}
}

func (this AndroidSparse_Chunk_ChunkHeader) IO_() *kaitai.Stream {
	return this._io
}

func (this *AndroidSparse_Chunk_ChunkHeader) Read(io *kaitai.Stream, parent *AndroidSparse_Chunk, root *AndroidSparse) (err error) {
	this._io = io
	this._parent = parent
	this._root = root

	tmp10, err := this._io.ReadU2le()
	if err != nil {
		return err
	}
	this.ChunkType = AndroidSparse_ChunkTypes(tmp10)
	tmp11, err := this._io.ReadU2le()
	if err != nil {
		return err
	}
	this.Reserved1 = uint16(tmp11)
	tmp12, err := this._io.ReadU4le()
	if err != nil {
		return err
	}
	this.NumBodyBlocks = uint32(tmp12)
	tmp13, err := this._io.ReadU4le()
	if err != nil {
		return err
	}
	this.LenChunk = uint32(tmp13)
	var tmp14 int;
	tmp15, err := this.LenBodyExpected()
	if err != nil {
		return err
	}
	if (tmp15 != -1) {
		tmp16, err := this.LenBodyExpected()
		if err != nil {
			return err
		}
		tmp14 = this._root.Header.LenChunkHeader + tmp16
	} else {
		tmp14 = this.LenChunk
	}
	var tmp17 int;
	tmp18, err := this.LenBodyExpected()
	if err != nil {
		return err
	}
	if (tmp18 != -1) {
		tmp19, err := this.LenBodyExpected()
		if err != nil {
			return err
		}
		tmp17 = this._root.Header.LenChunkHeader + tmp19
	} else {
		tmp17 = this.LenChunk
	}
	if !(this.LenChunk == tmp14) {
		return kaitai.NewValidationNotEqualError(tmp17, this.LenChunk, this._io, "/types/chunk/types/chunk_header/seq/3")
	}
	return err
}
func (this *AndroidSparse_Chunk_ChunkHeader) LenBody() (v int, err error) {
	if (this._f_lenBody) {
		return this.lenBody, nil
	}
	this._f_lenBody = true
	this.lenBody = int(this.LenChunk - this._root.Header.LenChunkHeader)
	return this.lenBody, nil
}

/**
 * @see <a href="https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#184">Source</a>
 * @see <a href="https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#215">Source</a>
 * @see <a href="https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#249">Source</a>
 * @see <a href="https://android.googlesource.com/platform/system/core/+/e8d02c50d7/libsparse/sparse_read.cpp#270">Source</a>
 */
func (this *AndroidSparse_Chunk_ChunkHeader) LenBodyExpected() (v int, err error) {
	if (this._f_lenBodyExpected) {
		return this.lenBodyExpected, nil
	}
	this._f_lenBodyExpected = true
	var tmp20 int;
	if (this.ChunkType == AndroidSparse_ChunkTypes__Raw) {
		tmp20 = this._root.Header.BlockSize * this.NumBodyBlocks
	} else {
		var tmp21 int8;
		if (this.ChunkType == AndroidSparse_ChunkTypes__Fill) {
			tmp21 = 4
		} else {
			var tmp22 int8;
			if (this.ChunkType == AndroidSparse_ChunkTypes__DontCare) {
				tmp22 = 0
			} else {
				var tmp23 int8;
				if (this.ChunkType == AndroidSparse_ChunkTypes__Crc32) {
					tmp23 = 4
				} else {
					tmp23 = -1
				}
				tmp22 = tmp23
			}
			tmp21 = tmp22
		}
		tmp20 = tmp21
	}
	this.lenBodyExpected = int(tmp20)
	return this.lenBodyExpected, nil
}

/**
 * size of the chunk body in blocks in output image
 */

/**
 * in bytes of chunk input file including chunk header and data
 */
type AndroidSparse_FileHeader struct {
	LenChunkHeader uint16
	BlockSize uint32
	NumBlocks uint32
	NumChunks uint32
	Checksum uint32
	_io *kaitai.Stream
	_root *AndroidSparse
	_parent *AndroidSparse
	_f_lenHeader bool
	lenHeader uint16
	_f_version bool
	version *AndroidSparse_Version
}
func NewAndroidSparse_FileHeader() *AndroidSparse_FileHeader {
	return &AndroidSparse_FileHeader{
	}
}

func (this AndroidSparse_FileHeader) IO_() *kaitai.Stream {
	return this._io
}

func (this *AndroidSparse_FileHeader) Read(io *kaitai.Stream, parent *AndroidSparse, root *AndroidSparse) (err error) {
	this._io = io
	this._parent = parent
	this._root = root

	tmp24, err := this._io.ReadU2le()
	if err != nil {
		return err
	}
	this.LenChunkHeader = uint16(tmp24)
	tmp25, err := this._io.ReadU4le()
	if err != nil {
		return err
	}
	this.BlockSize = uint32(tmp25)
	{
		_it := this.BlockSize
		tmp26 := _it % 4
		if tmp26 < 0 {
			tmp26 += 4
		}
		if !(tmp26 == 0) {
			return kaitai.NewValidationExprError(this.BlockSize, this._io, "/types/file_header/seq/1")
		}
	}
	tmp27, err := this._io.ReadU4le()
	if err != nil {
		return err
	}
	this.NumBlocks = uint32(tmp27)
	tmp28, err := this._io.ReadU4le()
	if err != nil {
		return err
	}
	this.NumChunks = uint32(tmp28)
	tmp29, err := this._io.ReadU4le()
	if err != nil {
		return err
	}
	this.Checksum = uint32(tmp29)
	return err
}

/**
 * size of file header, should be 28
 */
func (this *AndroidSparse_FileHeader) LenHeader() (v uint16, err error) {
	if (this._f_lenHeader) {
		return this.lenHeader, nil
	}
	this._f_lenHeader = true
	this.lenHeader = uint16(this._root.HeaderPrefix.LenHeader)
	return this.lenHeader, nil
}
func (this *AndroidSparse_FileHeader) Version() (v *AndroidSparse_Version, err error) {
	if (this._f_version) {
		return this.version, nil
	}
	this._f_version = true
	this.version = this._root.HeaderPrefix.Version
	return this.version, nil
}

/**
 * size of chunk header, should be 12
 */

/**
 * block size in bytes, must be a multiple of 4
 */

/**
 * blocks in the original data
 */

/**
 * CRC32 checksum of the original data
 * 
 * In practice always 0; if checksum writing is requested, a CRC32 chunk is written
 * at the end of the file instead. The canonical `libsparse` implementation does this
 * and other implementations tend to follow it, see
 * <https://gitlab.com/teskje/android-sparse-rs/-/blob/57c2577/src/write.rs#L112-114>
 */
type AndroidSparse_FileHeaderPrefix struct {
	Magic []byte
	Version *AndroidSparse_Version
	LenHeader uint16
	_io *kaitai.Stream
	_root *AndroidSparse
	_parent *AndroidSparse
}
func NewAndroidSparse_FileHeaderPrefix() *AndroidSparse_FileHeaderPrefix {
	return &AndroidSparse_FileHeaderPrefix{
	}
}

func (this AndroidSparse_FileHeaderPrefix) IO_() *kaitai.Stream {
	return this._io
}

func (this *AndroidSparse_FileHeaderPrefix) Read(io *kaitai.Stream, parent *AndroidSparse, root *AndroidSparse) (err error) {
	this._io = io
	this._parent = parent
	this._root = root

	tmp30, err := this._io.ReadBytes(int(4))
	if err != nil {
		return err
	}
	tmp30 = tmp30
	this.Magic = tmp30
	if !(bytes.Equal(this.Magic, []uint8{58, 255, 38, 237})) {
		return kaitai.NewValidationNotEqualError([]uint8{58, 255, 38, 237}, this.Magic, this._io, "/types/file_header_prefix/seq/0")
	}
	tmp31 := NewAndroidSparse_Version()
	err = tmp31.Read(this._io, this, this._root)
	if err != nil {
		return err
	}
	this.Version = tmp31
	tmp32, err := this._io.ReadU2le()
	if err != nil {
		return err
	}
	this.LenHeader = uint16(tmp32)
	return err
}

/**
 * internal; access `_root.header.version` instead
 */

/**
 * internal; access `_root.header.len_header` instead
 */
type AndroidSparse_Version struct {
	Major uint16
	Minor uint16
	_io *kaitai.Stream
	_root *AndroidSparse
	_parent *AndroidSparse_FileHeaderPrefix
}
func NewAndroidSparse_Version() *AndroidSparse_Version {
	return &AndroidSparse_Version{
	}
}

func (this AndroidSparse_Version) IO_() *kaitai.Stream {
	return this._io
}

func (this *AndroidSparse_Version) Read(io *kaitai.Stream, parent *AndroidSparse_FileHeaderPrefix, root *AndroidSparse) (err error) {
	this._io = io
	this._parent = parent
	this._root = root

	tmp33, err := this._io.ReadU2le()
	if err != nil {
		return err
	}
	this.Major = uint16(tmp33)
	if !(this.Major == 1) {
		return kaitai.NewValidationNotEqualError(1, this.Major, this._io, "/types/version/seq/0")
	}
	tmp34, err := this._io.ReadU2le()
	if err != nil {
		return err
	}
	this.Minor = uint16(tmp34)
	return err
}