Blob

Date:
Message:
got-notify-http: fix unicode handling JSON strings are made of UNICODE codepoints, of which only \, " and control characters have to be escaped, and the whole document MUST be encoded in UTF-8. The current code generates invalid strings for non-ASCII characters, so it has to be made UTF-8 aware. tedu' isu8cont() can't be used since it allows surrogate pairs and overlong sequences which will cause decoding errors on the receiving side. Similarly, mbtowc() depends on the current locale and could cause issues in -portable. Instead, bundle Björn Höhrmann's "Flexible and Economical UTF-8 Decoder" and use it to parse the text. Decoding errors results in the replacement character U+FFFD being emitted and the bytes considered so far to be discarded; the decoder is then restarted with the next byte. Git commit messages don't carry the notion of the encoding, but it's reasonable to expect UTF-8 (which is a superset of ASCII). For other more esotic encodings, the commit id can be used to manually extract the data. ok stsp@
Actions:
History | Blame | Raw File
1 /*

2  * Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>

3  *

4  * Permission is hereby granted, free of charge, to any person obtaining a

5  * copy of this software and associated documentation files (the "Software"),

6  * to deal in the Software without restriction, including without limitation

7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,

8  * and/or sell copies of the Software, and to permit persons to whom the

9  * Software is furnished to do so, subject to the following conditions:

10  *

11  * The above copyright notice and this permission notice shall be included in

12  * all copies or substantial portions of the Software.

13  *

14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL

17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING

19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER

20  * DEALINGS IN THE SOFTWARE.

21  */

23 // See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.

25 #define UTF8_ACCEPT 0

26 #define UTF8_REJECT 1

28 static const uint8_t utf8d[] = {

29   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f

30   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f

31   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f

32   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f

33   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f

34   7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf

35   8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df

36   0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef

37   0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff

38   0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0

39   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2

40   1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4

41   1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6

42   1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8

43 };

45 static uint32_t inline

46 decode(uint32_t* state, uint32_t* codep, uint32_t byte) {

47   uint32_t type = utf8d[byte];

49   *codep = (*state != UTF8_ACCEPT) ?

50     (byte & 0x3fu) | (*codep << 6) :

51     (0xff >> type) & (byte);

53   *state = utf8d[256 + *state*16 + type];

54   return *state;

55 }