1 /* $NetBSD: c8rtomb.c,v 1.9 2024/10/12 16:44:44 rillig Exp $ */
2
3 /*-
4 * Copyright (c) 2024 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /*
30 * c8rtomb(s, c8, ps)
31 *
32 * Encode the Unicode UTF-8 code unit c8 into the multibyte buffer
33 * s under the current locale, using multibyte encoding state ps.
34 *
35 * If c8 is not the last byte of a UTF-8 scalar value sequence, no
36 * output will be produced, but c8 will be remembered; this must
37 * be followed by another call passing the following bytes.
38 *
39 * Return the number of bytes stored on success, or (size_t)-1 on
40 * error with errno set to EILSEQ.
41 *
42 * At most MB_CUR_MAX bytes will be stored.
43 *
44 * References:
45 *
46 * The Unicode Standard, Version 15.0 -- Core Specification, The
47 * Unicode Consortium, Sec. 3.9 `Unicode Encoding Forms': UTF-8,
48 * p. 124.
49 * https://www.unicode.org/versions/Unicode15.0.0/UnicodeStandard-15.0.pdf#page=150
50 * https://web.archive.org/web/20240718101254/https://www.unicode.org/versions/Unicode15.0.0/UnicodeStandard-15.0.pdf#page=150
51 *
52 * F. Yergeau, `UTF-8, a transformation format of ISO 10646',
53 * RFC 3629, Internet Engineering Task Force, November 2003.
54 * https://datatracker.ietf.org/doc/html/rfc3629
55 */
56
57 #include <sys/cdefs.h>
58 __RCSID("$NetBSD: c8rtomb.c,v 1.9 2024/10/12 16:44:44 rillig Exp $");
59
60 #include "namespace.h"
61
62 #include <assert.h>
63 #include <errno.h>
64 #include <limits.h>
65 #include <locale.h>
66 #include <stdalign.h>
67 #include <stddef.h>
68 #include <stdint.h>
69 #include <uchar.h>
70
71 #include "c32rtomb.h"
72 #include "setlocale_local.h"
73
74 struct c8rtombstate {
75 char32_t state_c32; /* 8-bit state and 24-bit buffer */
76 mbstate_t mbs;
77 };
78 __CTASSERT(offsetof(struct c8rtombstate, mbs) <= sizeof(mbstate_t));
79 __CTASSERT(sizeof(struct c32rtombstate) <= sizeof(mbstate_t) -
80 offsetof(struct c8rtombstate, mbs));
81 __CTASSERT(alignof(struct c8rtombstate) <= alignof(mbstate_t));
82
83 /*
84 * UTF-8 validation, inspired by Bjoern Hoermann's UTF-8 decoder at
85 * <http://bjoern.hoehrmann.de/utf-8/decoder/dfa/>, but reimplemented
86 * from scratch.
87 */
88
89 #define UTF8_ACCEPT 0
90 #define UTF8_REJECT 96
91
92 typedef uint8_t utf8_class_t;
93 typedef uint8_t utf8_state_t;
94
95 static const uint8_t utf8_classtab[] = {
96 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
97 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
98 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
99 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
100 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
101 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
102 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
103 11,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 7,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
104 };
105
106 static const uint8_t utf8_statetab[] = {
107 0,96,12,36,48,84,72,60,96,96,96,24, 96, 0,96,96,96,96,96,96, 0, 0,96,96,
108 96,12,96,96,96,96,96,96,96,96,96,96, 96,12,96,96,96,96,96,96,12,12,96,96,
109 96,96,96,96,96,96,96,96,12,12,96,96, 96,36,96,96,96,96,96,96,96,36,96,96,
110 96,36,96,96,96,96,96,96,36,36,96,96, 96,96,96,96,96,96,96,96,36,96,96,96,
111 96,96,96,96,96,96,96,96,96,96,96,96,
112 };
113
114 static utf8_state_t
utf8_decode_step(utf8_state_t state,char8_t c8,char32_t * pc32)115 utf8_decode_step(utf8_state_t state, char8_t c8, char32_t *pc32)
116 {
117 const utf8_class_t class = utf8_classtab[c8];
118
119 *pc32 = (state == UTF8_ACCEPT
120 ? (c8 & (0xff >> class))
121 : ((c8 & 0x3f) | (*pc32 << 6)));
122
123 return utf8_statetab[state + class];
124 }
125
126 #ifdef __weak_alias
__weak_alias(c8rtomb_l,_c8rtomb_l)127 __weak_alias(c8rtomb_l,_c8rtomb_l)
128 #endif
129
130 size_t
131 c8rtomb(char *restrict s, char8_t c8, mbstate_t *restrict ps)
132 {
133
134 return c8rtomb_l(s, c8, ps, _current_locale());
135 }
136
137 size_t
c8rtomb_l(char * restrict s,char8_t c8,mbstate_t * restrict ps,locale_t loc)138 c8rtomb_l(char *restrict s, char8_t c8, mbstate_t *restrict ps, locale_t loc)
139 {
140 static mbstate_t psbuf;
141 char buf[MB_LEN_MAX];
142 struct c8rtombstate *S;
143 utf8_state_t state;
144 char32_t c32;
145
146 /*
147 * `If ps is a null pointer, each function uses its own
148 * internal mbstate_t object instead, which is initialized at
149 * program startup to the initial conversion state; the
150 * functions are not required to avoid data races with other
151 * calls to the same function in this case. The
152 * implementation behaves as if no library function calls
153 * these functions with a null pointer for ps.'
154 */
155 if (ps == NULL)
156 ps = &psbuf;
157
158 /*
159 * `If s is a null pointer, the c8rtomb function is equivalent
160 * to the call
161 *
162 * c8rtomb(buf, u8'\0', ps)
163 *
164 * where buf is an internal buffer.
165 */
166 if (s == NULL) {
167 s = buf;
168 c8 = 0; /* XXX u8'\0' */
169 }
170
171 /*
172 * Open the private UTF-8 decoding state.
173 */
174 S = (struct c8rtombstate *)(void *)ps;
175
176 /*
177 * `If c8 is a null character, a null byte is stored, preceded
178 * by any shift sequence needed to restore the initial shift
179 * state; the resulting state described is the initial
180 * conversion state.'
181 *
182 * So if c8 is null, discard any buffered input -- there's
183 * nothing we can legitimately do with it -- and convert a null
184 * scalar value, which by definition of c32rtomb writes out any
185 * shift sequence reset followed by a null byte.
186 */
187 if (c8 == '\0') {
188 c32 = 0;
189 goto accept;
190 }
191
192 /*
193 * Get the current state and buffer.
194 */
195 __CTASSERT(UTF8_ACCEPT == 0); /* initial conversion state */
196 state = __SHIFTOUT(S->state_c32, __BITS(31,24));
197 c32 = __SHIFTOUT(S->state_c32, __BITS(23,0));
198
199 /*
200 * Feed the byte into the state machine to update the state.
201 */
202 state = utf8_decode_step(state, c8, &c32);
203 switch (state) {
204 case UTF8_REJECT:
205 /*
206 * Invalid UTF-8. Fail with EILSEQ.
207 */
208 errno = EILSEQ;
209 return (size_t)-1;
210 default:
211 /*
212 * Valid UTF-8 so far but incomplete. Update state and
213 * output nothing.
214 */
215 S->state_c32 =
216 __SHIFTIN(state, __BITS(31,24)) |
217 __SHIFTIN(c32, __BITS(23,0));
218 return 0;
219 case UTF8_ACCEPT:
220 accept:
221 /*
222 * We have a scalar value. Clear the state and output
223 * the scalar value.
224 */
225 S->state_c32 = 0;
226 return c32rtomb_l(s, c32, &S->mbs, loc);
227 }
228 }
229