Gotweb

Blame

Date:: Sat Mar 30 17:21:23 2024 UTC
Message:: got-notify-http: fix unicode handling JSON strings are made of UNICODE codepoints, of which only \, " and control characters have to be escaped, and the whole document MUST be encoded in UTF-8. The current code generates invalid strings for non-ASCII characters, so it has to be made UTF-8 aware. tedu' isu8cont() can't be used since it allows surrogate pairs and overlong sequences which will cause decoding errors on the receiving side. Similarly, mbtowc() depends on the current locale and could cause issues in -portable. Instead, bundle Björn Höhrmann's "Flexible and Economical UTF-8 Decoder" and use it to parse the text. Decoding errors results in the replacement character U+FFFD being emitted and the bytes considered so far to be discarded; the decoder is then restarted with the next byte. Git commit messages don't carry the notion of the encoding, but it's reasonable to expect UTF-8 (which is a superset of ASCII). For other more esotic encodings, the commit id can be used to manually extract the data. ok stsp@
Actions:: History | Blob | Raw File

 1 fcfdd0a1 2024-03-30 thomas /*
 2 fcfdd0a1 2024-03-30 thomas  * Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
 3 fcfdd0a1 2024-03-30 thomas  *
 4 fcfdd0a1 2024-03-30 thomas  * Permission is hereby granted, free of charge, to any person obtaining a
 5 fcfdd0a1 2024-03-30 thomas  * copy of this software and associated documentation files (the "Software"),
 6 fcfdd0a1 2024-03-30 thomas  * to deal in the Software without restriction, including without limitation
 7 fcfdd0a1 2024-03-30 thomas  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 8 fcfdd0a1 2024-03-30 thomas  * and/or sell copies of the Software, and to permit persons to whom the
 9 fcfdd0a1 2024-03-30 thomas  * Software is furnished to do so, subject to the following conditions:
10 fcfdd0a1 2024-03-30 thomas  *
11 fcfdd0a1 2024-03-30 thomas  * The above copyright notice and this permission notice shall be included in
12 fcfdd0a1 2024-03-30 thomas  * all copies or substantial portions of the Software.
13 fcfdd0a1 2024-03-30 thomas  *
14 fcfdd0a1 2024-03-30 thomas  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 fcfdd0a1 2024-03-30 thomas  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 fcfdd0a1 2024-03-30 thomas  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 fcfdd0a1 2024-03-30 thomas  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 fcfdd0a1 2024-03-30 thomas  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 fcfdd0a1 2024-03-30 thomas  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 fcfdd0a1 2024-03-30 thomas  * DEALINGS IN THE SOFTWARE.
21 fcfdd0a1 2024-03-30 thomas  */
22 fcfdd0a1 2024-03-30 thomas 
23 fcfdd0a1 2024-03-30 thomas // See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
24 fcfdd0a1 2024-03-30 thomas 
25 fcfdd0a1 2024-03-30 thomas #define UTF8_ACCEPT 0
26 fcfdd0a1 2024-03-30 thomas #define UTF8_REJECT 1
27 fcfdd0a1 2024-03-30 thomas 
28 fcfdd0a1 2024-03-30 thomas static const uint8_t utf8d[] = {
29 fcfdd0a1 2024-03-30 thomas   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f
30 fcfdd0a1 2024-03-30 thomas   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f
31 fcfdd0a1 2024-03-30 thomas   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f
32 fcfdd0a1 2024-03-30 thomas   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f
33 fcfdd0a1 2024-03-30 thomas   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f
34 fcfdd0a1 2024-03-30 thomas   7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf
35 fcfdd0a1 2024-03-30 thomas   8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df
36 fcfdd0a1 2024-03-30 thomas   0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef
37 fcfdd0a1 2024-03-30 thomas   0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff
38 fcfdd0a1 2024-03-30 thomas   0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0
39 fcfdd0a1 2024-03-30 thomas   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2
40 fcfdd0a1 2024-03-30 thomas   1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4
41 fcfdd0a1 2024-03-30 thomas   1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6
42 fcfdd0a1 2024-03-30 thomas   1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8
43 fcfdd0a1 2024-03-30 thomas };
44 fcfdd0a1 2024-03-30 thomas 
45 fcfdd0a1 2024-03-30 thomas static uint32_t inline
46 fcfdd0a1 2024-03-30 thomas decode(uint32_t* state, uint32_t* codep, uint32_t byte) {
47 fcfdd0a1 2024-03-30 thomas   uint32_t type = utf8d[byte];
48 fcfdd0a1 2024-03-30 thomas 
49 fcfdd0a1 2024-03-30 thomas   *codep = (*state != UTF8_ACCEPT) ?
50 fcfdd0a1 2024-03-30 thomas     (byte & 0x3fu) | (*codep << 6) :
51 fcfdd0a1 2024-03-30 thomas     (0xff >> type) & (byte);
52 fcfdd0a1 2024-03-30 thomas 
53 fcfdd0a1 2024-03-30 thomas   *state = utf8d[256 + *state*16 + type];
54 fcfdd0a1 2024-03-30 thomas   return *state;
55 fcfdd0a1 2024-03-30 thomas }