UTF-8-encoded string: GraphViz block diagram (.dot) source

UTF-8 is a popular character encoding scheme that allows to represent strings as sequence of code points defined in Unicode standard. Its features are:

variable width (i.e. one code point might be represented by 1 to 4 bytes)
backward compatiblity with ASCII
basic validity checking (and thus distinguishing from other legacy 8-bit encodings)
maintaining sort order of codepoints if sorted as a byte array

WARNING: For the vast majority of practical purposes of format definitions in Kaitai Struct, you'd likely NOT want to use this and rather just use type: str with encoding: utf-8. That will use native string implementations, which are most likely more efficient and will give you native language strings, rather than an array of individual codepoints. This format definition is provided mostly for educational / research purposes.

File extension

txt

KS implementation details

License: CC0-1.0

References

Wikidata Q193537

This page hosts a formal specification of UTF-8-encoded string using Kaitai Struct. This specification can be automatically translated into a variety of programming languages to get a parsing library.

GraphViz block diagram source

utf8_string.dot

Download

digraph {
	rankdir=LR;
	node [shape=plaintext];
	subgraph cluster__utf8_string {
		label="Utf8String";
		graph[style=dotted];

		utf8_string__seq [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
			<TR><TD BGCOLOR="#E0FFE0">pos</TD><TD BGCOLOR="#E0FFE0">size</TD><TD BGCOLOR="#E0FFE0">type</TD><TD BGCOLOR="#E0FFE0">id</TD></TR>
			<TR><TD PORT="codepoints_pos">0</TD><TD PORT="codepoints_size">...</TD><TD>Utf8Codepoint</TD><TD PORT="codepoints_type">codepoints</TD></TR>
			<TR><TD COLSPAN="4" PORT="codepoints__repeat">repeat to end of stream</TD></TR>
		</TABLE>>];
		subgraph cluster__utf8_codepoint {
			label="Utf8String::Utf8Codepoint";
			graph[style=dotted];

			utf8_codepoint__seq [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
				<TR><TD BGCOLOR="#E0FFE0">pos</TD><TD BGCOLOR="#E0FFE0">size</TD><TD BGCOLOR="#E0FFE0">type</TD><TD BGCOLOR="#E0FFE0">id</TD></TR>
				<TR><TD PORT="bytes_pos">0</TD><TD PORT="bytes_size">len_bytes</TD><TD></TD><TD PORT="bytes_type">bytes</TD></TR>
			</TABLE>>];
			utf8_codepoint__inst__raw1 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
				<TR><TD BGCOLOR="#E0FFE0">id</TD><TD BGCOLOR="#E0FFE0">value</TD></TR>
				<TR><TD>raw1</TD><TD>(bytes[1].ord &amp; 63)</TD></TR>
			</TABLE>>];
			utf8_codepoint__inst__len_bytes [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
				<TR><TD BGCOLOR="#E0FFE0">id</TD><TD BGCOLOR="#E0FFE0">value</TD></TR>
				<TR><TD>len_bytes</TD><TD>((byte0 &amp; 128) == 0 ? 1 : ((byte0 &amp; 224) == 192 ? 2 : ((byte0 &amp; 240) == 224 ? 3 : ((byte0 &amp; 248) == 240 ? 4 : -1))))</TD></TR>
			</TABLE>>];
			utf8_codepoint__inst__raw3 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
				<TR><TD BGCOLOR="#E0FFE0">id</TD><TD BGCOLOR="#E0FFE0">value</TD></TR>
				<TR><TD>raw3</TD><TD>(bytes[3].ord &amp; 63)</TD></TR>
			</TABLE>>];
			utf8_codepoint__inst__value_as_int [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
				<TR><TD BGCOLOR="#E0FFE0">id</TD><TD BGCOLOR="#E0FFE0">value</TD></TR>
				<TR><TD>value_as_int</TD><TD>(len_bytes == 1 ? raw0 : (len_bytes == 2 ? ((raw0 &lt;&lt; 6) | raw1) : (len_bytes == 3 ? (((raw0 &lt;&lt; 12) | (raw1 &lt;&lt; 6)) | raw2) : (len_bytes == 4 ? ((((raw0 &lt;&lt; 18) | (raw1 &lt;&lt; 12)) | (raw2 &lt;&lt; 6)) | raw3) : -1))))</TD></TR>
			</TABLE>>];
			utf8_codepoint__inst__raw0 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
				<TR><TD BGCOLOR="#E0FFE0">id</TD><TD BGCOLOR="#E0FFE0">value</TD></TR>
				<TR><TD>raw0</TD><TD>(bytes[0].ord &amp; (len_bytes == 1 ? 127 : (len_bytes == 2 ? 31 : (len_bytes == 3 ? 15 : (len_bytes == 4 ? 7 : 0)))))</TD></TR>
			</TABLE>>];
			utf8_codepoint__inst__byte0 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
				<TR><TD BGCOLOR="#E0FFE0">pos</TD><TD BGCOLOR="#E0FFE0">size</TD><TD BGCOLOR="#E0FFE0">type</TD><TD BGCOLOR="#E0FFE0">id</TD></TR>
				<TR><TD PORT="byte0_pos">ofs</TD><TD PORT="byte0_size">1</TD><TD>u1</TD><TD PORT="byte0_type">byte0</TD></TR>
			</TABLE>>];
			utf8_codepoint__inst__raw2 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
				<TR><TD BGCOLOR="#E0FFE0">id</TD><TD BGCOLOR="#E0FFE0">value</TD></TR>
				<TR><TD>raw2</TD><TD>(bytes[2].ord &amp; 63)</TD></TR>
			</TABLE>>];
		}
	}
	utf8_string__seq:codepoints_type -> utf8_codepoint__seq [style=bold];
	utf8_codepoint__inst__len_bytes:len_bytes_type -> utf8_codepoint__seq:bytes_size [color="#404040"];
	utf8_codepoint__seq:bytes_type -> utf8_codepoint__inst__raw1 [color="#404040"];
	utf8_codepoint__inst__byte0:byte0_type -> utf8_codepoint__inst__len_bytes [color="#404040"];
	utf8_codepoint__inst__byte0:byte0_type -> utf8_codepoint__inst__len_bytes [color="#404040"];
	utf8_codepoint__inst__byte0:byte0_type -> utf8_codepoint__inst__len_bytes [color="#404040"];
	utf8_codepoint__inst__byte0:byte0_type -> utf8_codepoint__inst__len_bytes [color="#404040"];
	utf8_codepoint__seq:bytes_type -> utf8_codepoint__inst__raw3 [color="#404040"];
	utf8_codepoint__inst__len_bytes:len_bytes_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__raw0:raw0_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__len_bytes:len_bytes_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__raw0:raw0_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__raw1:raw1_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__len_bytes:len_bytes_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__raw0:raw0_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__raw1:raw1_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__raw2:raw2_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__len_bytes:len_bytes_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__raw0:raw0_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__raw1:raw1_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__raw2:raw2_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__inst__raw3:raw3_type -> utf8_codepoint__inst__value_as_int [color="#404040"];
	utf8_codepoint__seq:bytes_type -> utf8_codepoint__inst__raw0 [color="#404040"];
	utf8_codepoint__inst__len_bytes:len_bytes_type -> utf8_codepoint__inst__raw0 [color="#404040"];
	utf8_codepoint__inst__len_bytes:len_bytes_type -> utf8_codepoint__inst__raw0 [color="#404040"];
	utf8_codepoint__inst__len_bytes:len_bytes_type -> utf8_codepoint__inst__raw0 [color="#404040"];
	utf8_codepoint__inst__len_bytes:len_bytes_type -> utf8_codepoint__inst__raw0 [color="#404040"];
	utf8_codepoint__params:ofs_type -> utf8_codepoint__inst__byte0:byte0_pos [color="#404040"];
	utf8_codepoint__seq:bytes_type -> utf8_codepoint__inst__raw2 [color="#404040"];
}