1 /* $NetBSD: tokenizer.c,v 1.28 2016/04/11 18:56:31 christos Exp $ */
2
3 /*-
4 * Copyright (c) 1992, 1993
5 * The Regents of the University of California. All rights reserved.
6 *
7 * This code is derived from software contributed to Berkeley by
8 * Christos Zoulas of Cornell University.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #ifndef NARROWCHAR
36 #include "config.h"
37 #endif
38
39 #if !defined(lint) && !defined(SCCSID)
40 #if 0
41 static char sccsid[] = "@(#)tokenizer.c 8.1 (Berkeley) 6/4/93";
42 #else
43 __RCSID("$NetBSD: tokenizer.c,v 1.28 2016/04/11 18:56:31 christos Exp $");
44 #endif
45 #endif /* not lint && not SCCSID */
46
47 /* We build this file twice, once as NARROW, once as WIDE. */
48 /*
49 * tokenize.c: Bourne shell like tokenizer
50 */
51 #include <stdlib.h>
52 #include <string.h>
53
54 #include "histedit.h"
55
56 typedef enum {
57 Q_none, Q_single, Q_double, Q_one, Q_doubleone
58 } quote_t;
59
60 #define TOK_KEEP 1
61 #define TOK_EAT 2
62
63 #define WINCR 20
64 #define AINCR 10
65
66 #define IFS STR("\t \n")
67
68 #define tok_malloc(a) malloc(a)
69 #define tok_free(a) free(a)
70 #define tok_realloc(a, b) realloc(a, b)
71
72 #ifdef NARROWCHAR
73 #define Char char
74 #define FUN(prefix, rest) prefix ## _ ## rest
75 #define TYPE(type) type
76 #define STR(x) x
77 #define Strchr(s, c) strchr(s, c)
78 #define tok_strdup(s) strdup(s)
79 #else
80 #define Char wchar_t
81 #define FUN(prefix, rest) prefix ## _w ## rest
82 #define TYPE(type) type ## W
83 #define STR(x) L ## x
84 #define Strchr(s, c) wcschr(s, c)
85 #define tok_strdup(s) wcsdup(s)
86 #endif
87
TYPE(tokenizer)88 struct TYPE(tokenizer) {
89 Char *ifs; /* In field separator */
90 size_t argc, amax; /* Current and maximum number of args */
91 Char **argv; /* Argument list */
92 Char *wptr, *wmax; /* Space and limit on the word buffer */
93 Char *wstart; /* Beginning of next word */
94 Char *wspace; /* Space of word buffer */
95 quote_t quote; /* Quoting state */
96 int flags; /* flags; */
97 };
98
99
100 static void FUN(tok,finish)(TYPE(Tokenizer) *);
101
102
103 /* FUN(tok,finish)():
104 * Finish a word in the tokenizer.
105 */
106 static void
FUN(tok,finish)107 FUN(tok,finish)(TYPE(Tokenizer) *tok)
108 {
109
110 *tok->wptr = '\0';
111 if ((tok->flags & TOK_KEEP) || tok->wptr != tok->wstart) {
112 tok->argv[tok->argc++] = tok->wstart;
113 tok->argv[tok->argc] = NULL;
114 tok->wstart = ++tok->wptr;
115 }
116 tok->flags &= ~TOK_KEEP;
117 }
118
119
120 /* FUN(tok,init)():
121 * Initialize the tokenizer
122 */
TYPE(Tokenizer)123 TYPE(Tokenizer) *
124 FUN(tok,init)(const Char *ifs)
125 {
126 TYPE(Tokenizer) *tok = tok_malloc(sizeof(*tok));
127
128 if (tok == NULL)
129 return NULL;
130 tok->ifs = tok_strdup(ifs ? ifs : IFS);
131 if (tok->ifs == NULL) {
132 tok_free(tok);
133 return NULL;
134 }
135 tok->argc = 0;
136 tok->amax = AINCR;
137 tok->argv = tok_malloc(sizeof(*tok->argv) * tok->amax);
138 if (tok->argv == NULL) {
139 tok_free(tok->ifs);
140 tok_free(tok);
141 return NULL;
142 }
143 tok->argv[0] = NULL;
144 tok->wspace = tok_malloc(WINCR * sizeof(*tok->wspace));
145 if (tok->wspace == NULL) {
146 tok_free(tok->argv);
147 tok_free(tok->ifs);
148 tok_free(tok);
149 return NULL;
150 }
151 tok->wmax = tok->wspace + WINCR;
152 tok->wstart = tok->wspace;
153 tok->wptr = tok->wspace;
154 tok->flags = 0;
155 tok->quote = Q_none;
156
157 return tok;
158 }
159
160
161 /* FUN(tok,reset)():
162 * Reset the tokenizer
163 */
164 void
FUN(tok,reset)165 FUN(tok,reset)(TYPE(Tokenizer) *tok)
166 {
167
168 tok->argc = 0;
169 tok->wstart = tok->wspace;
170 tok->wptr = tok->wspace;
171 tok->flags = 0;
172 tok->quote = Q_none;
173 }
174
175
176 /* FUN(tok,end)():
177 * Clean up
178 */
179 void
FUN(tok,end)180 FUN(tok,end)(TYPE(Tokenizer) *tok)
181 {
182
183 tok_free(tok->ifs);
184 tok_free(tok->wspace);
185 tok_free(tok->argv);
186 tok_free(tok);
187 }
188
189
190
191 /* FUN(tok,line)():
192 * Bourne shell (sh(1)) like tokenizing
193 * Arguments:
194 * tok current tokenizer state (setup with FUN(tok,init)())
195 * line line to parse
196 * Returns:
197 * -1 Internal error
198 * 3 Quoted return
199 * 2 Unmatched double quote
200 * 1 Unmatched single quote
201 * 0 Ok
202 * Modifies (if return value is 0):
203 * argc number of arguments
204 * argv argument array
205 * cursorc if !NULL, argv element containing cursor
206 * cursorv if !NULL, offset in argv[cursorc] of cursor
207 */
208 int
FUN(tok,line)209 FUN(tok,line)(TYPE(Tokenizer) *tok, const TYPE(LineInfo) *line,
210 int *argc, const Char ***argv, int *cursorc, int *cursoro)
211 {
212 const Char *ptr;
213 int cc, co;
214
215 cc = co = -1;
216 ptr = line->buffer;
217 for (ptr = line->buffer; ;ptr++) {
218 if (ptr >= line->lastchar)
219 ptr = STR("");
220 if (ptr == line->cursor) {
221 cc = (int)tok->argc;
222 co = (int)(tok->wptr - tok->wstart);
223 }
224 switch (*ptr) {
225 case '\'':
226 tok->flags |= TOK_KEEP;
227 tok->flags &= ~TOK_EAT;
228 switch (tok->quote) {
229 case Q_none:
230 tok->quote = Q_single; /* Enter single quote
231 * mode */
232 break;
233
234 case Q_single: /* Exit single quote mode */
235 tok->quote = Q_none;
236 break;
237
238 case Q_one: /* Quote this ' */
239 tok->quote = Q_none;
240 *tok->wptr++ = *ptr;
241 break;
242
243 case Q_double: /* Stay in double quote mode */
244 *tok->wptr++ = *ptr;
245 break;
246
247 case Q_doubleone: /* Quote this ' */
248 tok->quote = Q_double;
249 *tok->wptr++ = *ptr;
250 break;
251
252 default:
253 return -1;
254 }
255 break;
256
257 case '"':
258 tok->flags &= ~TOK_EAT;
259 tok->flags |= TOK_KEEP;
260 switch (tok->quote) {
261 case Q_none: /* Enter double quote mode */
262 tok->quote = Q_double;
263 break;
264
265 case Q_double: /* Exit double quote mode */
266 tok->quote = Q_none;
267 break;
268
269 case Q_one: /* Quote this " */
270 tok->quote = Q_none;
271 *tok->wptr++ = *ptr;
272 break;
273
274 case Q_single: /* Stay in single quote mode */
275 *tok->wptr++ = *ptr;
276 break;
277
278 case Q_doubleone: /* Quote this " */
279 tok->quote = Q_double;
280 *tok->wptr++ = *ptr;
281 break;
282
283 default:
284 return -1;
285 }
286 break;
287
288 case '\\':
289 tok->flags |= TOK_KEEP;
290 tok->flags &= ~TOK_EAT;
291 switch (tok->quote) {
292 case Q_none: /* Quote next character */
293 tok->quote = Q_one;
294 break;
295
296 case Q_double: /* Quote next character */
297 tok->quote = Q_doubleone;
298 break;
299
300 case Q_one: /* Quote this, restore state */
301 *tok->wptr++ = *ptr;
302 tok->quote = Q_none;
303 break;
304
305 case Q_single: /* Stay in single quote mode */
306 *tok->wptr++ = *ptr;
307 break;
308
309 case Q_doubleone: /* Quote this \ */
310 tok->quote = Q_double;
311 *tok->wptr++ = *ptr;
312 break;
313
314 default:
315 return -1;
316 }
317 break;
318
319 case '\n':
320 tok->flags &= ~TOK_EAT;
321 switch (tok->quote) {
322 case Q_none:
323 goto tok_line_outok;
324
325 case Q_single:
326 case Q_double:
327 *tok->wptr++ = *ptr; /* Add the return */
328 break;
329
330 case Q_doubleone: /* Back to double, eat the '\n' */
331 tok->flags |= TOK_EAT;
332 tok->quote = Q_double;
333 break;
334
335 case Q_one: /* No quote, more eat the '\n' */
336 tok->flags |= TOK_EAT;
337 tok->quote = Q_none;
338 break;
339
340 default:
341 return 0;
342 }
343 break;
344
345 case '\0':
346 switch (tok->quote) {
347 case Q_none:
348 /* Finish word and return */
349 if (tok->flags & TOK_EAT) {
350 tok->flags &= ~TOK_EAT;
351 return 3;
352 }
353 goto tok_line_outok;
354
355 case Q_single:
356 return 1;
357
358 case Q_double:
359 return 2;
360
361 case Q_doubleone:
362 tok->quote = Q_double;
363 *tok->wptr++ = *ptr;
364 break;
365
366 case Q_one:
367 tok->quote = Q_none;
368 *tok->wptr++ = *ptr;
369 break;
370
371 default:
372 return -1;
373 }
374 break;
375
376 default:
377 tok->flags &= ~TOK_EAT;
378 switch (tok->quote) {
379 case Q_none:
380 if (Strchr(tok->ifs, *ptr) != NULL)
381 FUN(tok,finish)(tok);
382 else
383 *tok->wptr++ = *ptr;
384 break;
385
386 case Q_single:
387 case Q_double:
388 *tok->wptr++ = *ptr;
389 break;
390
391
392 case Q_doubleone:
393 *tok->wptr++ = '\\';
394 tok->quote = Q_double;
395 *tok->wptr++ = *ptr;
396 break;
397
398 case Q_one:
399 tok->quote = Q_none;
400 *tok->wptr++ = *ptr;
401 break;
402
403 default:
404 return -1;
405
406 }
407 break;
408 }
409
410 if (tok->wptr >= tok->wmax - 4) {
411 size_t size = (size_t)(tok->wmax - tok->wspace + WINCR);
412 Char *s = tok_realloc(tok->wspace,
413 size * sizeof(*s));
414 if (s == NULL)
415 return -1;
416
417 if (s != tok->wspace) {
418 size_t i;
419 for (i = 0; i < tok->argc; i++) {
420 tok->argv[i] =
421 (tok->argv[i] - tok->wspace) + s;
422 }
423 tok->wptr = (tok->wptr - tok->wspace) + s;
424 tok->wstart = (tok->wstart - tok->wspace) + s;
425 tok->wspace = s;
426 }
427 tok->wmax = s + size;
428 }
429 if (tok->argc >= tok->amax - 4) {
430 Char **p;
431 tok->amax += AINCR;
432 p = tok_realloc(tok->argv, tok->amax * sizeof(*p));
433 if (p == NULL) {
434 tok->amax -= AINCR;
435 return -1;
436 }
437 tok->argv = p;
438 }
439 }
440 tok_line_outok:
441 if (cc == -1 && co == -1) {
442 cc = (int)tok->argc;
443 co = (int)(tok->wptr - tok->wstart);
444 }
445 if (cursorc != NULL)
446 *cursorc = cc;
447 if (cursoro != NULL)
448 *cursoro = co;
449 FUN(tok,finish)(tok);
450 *argv = (const Char **)tok->argv;
451 *argc = (int)tok->argc;
452 return 0;
453 }
454
455 /* FUN(tok,str)():
456 * Simpler version of tok_line, taking a NUL terminated line
457 * and splitting into words, ignoring cursor state.
458 */
459 int
FUN(tok,str)460 FUN(tok,str)(TYPE(Tokenizer) *tok, const Char *line, int *argc,
461 const Char ***argv)
462 {
463 TYPE(LineInfo) li;
464
465 memset(&li, 0, sizeof(li));
466 li.buffer = line;
467 li.cursor = li.lastchar = Strchr(line, '\0');
468 return FUN(tok,line)(tok, &li, argc, argv, NULL, NULL);
469 }
470