UTF-8 is a popular character encoding scheme that allows to represent strings as sequence of code points defined in Unicode standard. Its features are:
WARNING: For the vast majority of practical purposes of format
definitions in Kaitai Struct, you'd likely NOT want to use this and
rather just use type: str
with encoding: utf-8
. That will use
native string implementations, which are most likely more efficient
and will give you native language strings, rather than an array of
individual codepoints. This format definition is provided mostly
for educational / research purposes.
This page hosts a formal specification of UTF-8-encoded string using Kaitai Struct. This specification can be automatically translated into a variety of programming languages to get a parsing library.
// Code generated by kaitai-struct-compiler from a .ksy source file. DO NOT EDIT.
import (
"github.com/kaitai-io/kaitai_struct_go_runtime/kaitai"
"io"
)
/**
* UTF-8 is a popular character encoding scheme that allows to
* represent strings as sequence of code points defined in Unicode
* standard. Its features are:
*
* * variable width (i.e. one code point might be represented by 1 to 4
* bytes)
* * backward compatiblity with ASCII
* * basic validity checking (and thus distinguishing from other legacy
* 8-bit encodings)
* * maintaining sort order of codepoints if sorted as a byte array
*
* WARNING: For the vast majority of practical purposes of format
* definitions in Kaitai Struct, you'd likely NOT want to use this and
* rather just use `type: str` with `encoding: utf-8`. That will use
* native string implementations, which are most likely more efficient
* and will give you native language strings, rather than an array of
* individual codepoints. This format definition is provided mostly
* for educational / research purposes.
*/
type Utf8String struct {
Codepoints []*Utf8String_Utf8Codepoint
_io *kaitai.Stream
_root *Utf8String
_parent kaitai.Struct
}
func NewUtf8String() *Utf8String {
return &Utf8String{
}
}
func (this Utf8String) IO_() *kaitai.Stream {
return this._io
}
func (this *Utf8String) Read(io *kaitai.Stream, parent kaitai.Struct, root *Utf8String) (err error) {
this._io = io
this._parent = parent
this._root = root
for i := 0;; i++ {
tmp1, err := this._io.EOF()
if err != nil {
return err
}
if tmp1 {
break
}
tmp2, err := this._io.Pos()
if err != nil {
return err
}
tmp3 := NewUtf8String_Utf8Codepoint(tmp2)
err = tmp3.Read(this._io, this, this._root)
if err != nil {
return err
}
this.Codepoints = append(this.Codepoints, tmp3)
}
return err
}
type Utf8String_Utf8Codepoint struct {
Bytes []byte
Ofs uint64
_io *kaitai.Stream
_root *Utf8String
_parent *Utf8String
_f_byte0 bool
byte0 uint8
_f_lenBytes bool
lenBytes int
_f_raw0 bool
raw0 int
_f_raw1 bool
raw1 int
_f_raw2 bool
raw2 int
_f_raw3 bool
raw3 int
_f_valueAsInt bool
valueAsInt int
}
func NewUtf8String_Utf8Codepoint(ofs uint64) *Utf8String_Utf8Codepoint {
return &Utf8String_Utf8Codepoint{
Ofs: ofs,
}
}
func (this Utf8String_Utf8Codepoint) IO_() *kaitai.Stream {
return this._io
}
func (this *Utf8String_Utf8Codepoint) Read(io *kaitai.Stream, parent *Utf8String, root *Utf8String) (err error) {
this._io = io
this._parent = parent
this._root = root
tmp4, err := this.LenBytes()
if err != nil {
return err
}
tmp5, err := this._io.ReadBytes(int(tmp4))
if err != nil {
return err
}
tmp5 = tmp5
this.Bytes = tmp5
return err
}
func (this *Utf8String_Utf8Codepoint) Byte0() (v uint8, err error) {
if (this._f_byte0) {
return this.byte0, nil
}
this._f_byte0 = true
_pos, err := this._io.Pos()
if err != nil {
return 0, err
}
_, err = this._io.Seek(int64(this.Ofs), io.SeekStart)
if err != nil {
return 0, err
}
tmp6, err := this._io.ReadU1()
if err != nil {
return 0, err
}
this.byte0 = tmp6
_, err = this._io.Seek(_pos, io.SeekStart)
if err != nil {
return 0, err
}
return this.byte0, nil
}
func (this *Utf8String_Utf8Codepoint) LenBytes() (v int, err error) {
if (this._f_lenBytes) {
return this.lenBytes, nil
}
this._f_lenBytes = true
var tmp7 int8;
tmp8, err := this.Byte0()
if err != nil {
return 0, err
}
if (tmp8 & 128 == 0) {
tmp7 = 1
} else {
var tmp9 int8;
tmp10, err := this.Byte0()
if err != nil {
return 0, err
}
if (tmp10 & 224 == 192) {
tmp9 = 2
} else {
var tmp11 int8;
tmp12, err := this.Byte0()
if err != nil {
return 0, err
}
if (tmp12 & 240 == 224) {
tmp11 = 3
} else {
var tmp13 int8;
tmp14, err := this.Byte0()
if err != nil {
return 0, err
}
if (tmp14 & 248 == 240) {
tmp13 = 4
} else {
tmp13 = -1
}
tmp11 = tmp13
}
tmp9 = tmp11
}
tmp7 = tmp9
}
this.lenBytes = int(tmp7)
return this.lenBytes, nil
}
func (this *Utf8String_Utf8Codepoint) Raw0() (v int, err error) {
if (this._f_raw0) {
return this.raw0, nil
}
this._f_raw0 = true
var tmp15 int8;
tmp16, err := this.LenBytes()
if err != nil {
return 0, err
}
if (tmp16 == 1) {
tmp15 = 127
} else {
var tmp17 int8;
tmp18, err := this.LenBytes()
if err != nil {
return 0, err
}
if (tmp18 == 2) {
tmp17 = 31
} else {
var tmp19 int8;
tmp20, err := this.LenBytes()
if err != nil {
return 0, err
}
if (tmp20 == 3) {
tmp19 = 15
} else {
var tmp21 int8;
tmp22, err := this.LenBytes()
if err != nil {
return 0, err
}
if (tmp22 == 4) {
tmp21 = 7
} else {
tmp21 = 0
}
tmp19 = tmp21
}
tmp17 = tmp19
}
tmp15 = tmp17
}
this.raw0 = int(this.Bytes[0] & tmp15)
return this.raw0, nil
}
func (this *Utf8String_Utf8Codepoint) Raw1() (v int, err error) {
if (this._f_raw1) {
return this.raw1, nil
}
this._f_raw1 = true
tmp23, err := this.LenBytes()
if err != nil {
return 0, err
}
if (tmp23 >= 2) {
this.raw1 = int(this.Bytes[1] & 63)
}
return this.raw1, nil
}
func (this *Utf8String_Utf8Codepoint) Raw2() (v int, err error) {
if (this._f_raw2) {
return this.raw2, nil
}
this._f_raw2 = true
tmp24, err := this.LenBytes()
if err != nil {
return 0, err
}
if (tmp24 >= 3) {
this.raw2 = int(this.Bytes[2] & 63)
}
return this.raw2, nil
}
func (this *Utf8String_Utf8Codepoint) Raw3() (v int, err error) {
if (this._f_raw3) {
return this.raw3, nil
}
this._f_raw3 = true
tmp25, err := this.LenBytes()
if err != nil {
return 0, err
}
if (tmp25 >= 4) {
this.raw3 = int(this.Bytes[3] & 63)
}
return this.raw3, nil
}
func (this *Utf8String_Utf8Codepoint) ValueAsInt() (v int, err error) {
if (this._f_valueAsInt) {
return this.valueAsInt, nil
}
this._f_valueAsInt = true
var tmp26 int;
tmp27, err := this.LenBytes()
if err != nil {
return 0, err
}
if (tmp27 == 1) {
tmp28, err := this.Raw0()
if err != nil {
return 0, err
}
tmp26 = tmp28
} else {
var tmp29 int;
tmp30, err := this.LenBytes()
if err != nil {
return 0, err
}
if (tmp30 == 2) {
tmp31, err := this.Raw0()
if err != nil {
return 0, err
}
tmp32, err := this.Raw1()
if err != nil {
return 0, err
}
tmp29 = tmp31 << 6 | tmp32
} else {
var tmp33 int;
tmp34, err := this.LenBytes()
if err != nil {
return 0, err
}
if (tmp34 == 3) {
tmp35, err := this.Raw0()
if err != nil {
return 0, err
}
tmp36, err := this.Raw1()
if err != nil {
return 0, err
}
tmp37, err := this.Raw2()
if err != nil {
return 0, err
}
tmp33 = (tmp35 << 12 | tmp36 << 6) | tmp37
} else {
var tmp38 int;
tmp39, err := this.LenBytes()
if err != nil {
return 0, err
}
if (tmp39 == 4) {
tmp40, err := this.Raw0()
if err != nil {
return 0, err
}
tmp41, err := this.Raw1()
if err != nil {
return 0, err
}
tmp42, err := this.Raw2()
if err != nil {
return 0, err
}
tmp43, err := this.Raw3()
if err != nil {
return 0, err
}
tmp38 = ((tmp40 << 18 | tmp41 << 12) | tmp42 << 6) | tmp43
} else {
tmp38 = -1
}
tmp33 = tmp38
}
tmp29 = tmp33
}
tmp26 = tmp29
}
this.valueAsInt = int(tmp26)
return this.valueAsInt, nil
}