"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "libedit/tokenizer.c" between
tnftp-20151004.tar.gz and tnftp-20200705.tar.gz

About: tnftp is an enhanced ftp client (prior name "lukemftp").

tokenizer.c  (tnftp-20151004):tokenizer.c  (tnftp-20200705)
/* $NetBSD: tokenizer.c,v 1.5 2005/06/09 16:48:58 lukem Exp $ */ /* $NetBSD: tokenizer.c,v 1.8 2020/07/04 13:43:21 lukem Exp $ */
/* from NetBSD: tokenizer.c,v 1.14 2003/12/05 13:37:48 lukem Exp * /* from NetBSD: tokenizer.c,v 1.28 2016/04/11 18:56:31 christos Exp *
/ /
/*- /*-
* Copyright (c) 1992, 1993 * Copyright (c) 1992, 1993
* The Regents of the University of California. All rights reserved. * The Regents of the University of California. All rights reserved.
* *
* This code is derived from software contributed to Berkeley by * This code is derived from software contributed to Berkeley by
* Christos Zoulas of Cornell University. * Christos Zoulas of Cornell University.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions * modification, are permitted provided that the following conditions
skipping to change at line 36 skipping to change at line 36
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE. * SUCH DAMAGE.
*/ */
#include "tnftp.h" #include "config.h"
#include "sys.h"
#if 0 /* tnftp */
#if !defined(lint) && !defined(SCCSID)
#if 0
static char sccsid[] = "@(#)tokenizer.c 8.1 (Berkeley) 6/4/93";
#else
__RCSID(" NetBSD: tokenizer.c,v 1.28 2016/04/11 18:56:31 christos Exp ");
#endif
#endif /* not lint && not SCCSID */
#endif /* tnftp */
/* We build this file twice, once as NARROW, once as WIDE. */
/* /*
* tokenize.c: Bourne shell like tokenizer * tokenize.c: Bourne shell like tokenizer
*/ */
#include <string.h> #if 0 /* tnftp */
#include <stdlib.h> #include <stdlib.h>
#include <string.h>
#endif /* tnftp */
#include "histedit.h" #include "histedit.h"
typedef enum { typedef enum {
Q_none, Q_single, Q_double, Q_one, Q_doubleone Q_none, Q_single, Q_double, Q_one, Q_doubleone
} quote_t; } quote_t;
#define IFS "\t \n"
#define TOK_KEEP 1 #define TOK_KEEP 1
#define TOK_EAT 2 #define TOK_EAT 2
#define WINCR 20 #define WINCR 20
#define AINCR 10 #define AINCR 10
#define tok_strdup(a) strdup(a) #define IFS STR("\t \n")
#define tok_malloc(a) malloc(a) #define tok_malloc(a) malloc(a)
#define tok_free(a) free(a) #define tok_free(a) free(a)
#define tok_realloc(a, b) realloc(a, b) #define tok_realloc(a, b) realloc(a, b)
struct tokenizer { #ifdef NARROWCHAR
char *ifs; /* In field separator */ #define Char char
int argc, amax; /* Current and maximum number of args */ #define FUN(prefix, rest) prefix ## _ ## rest
char **argv; /* Argument list */ #define TYPE(type) type
char *wptr, *wmax; /* Space and limit on the word buffer */ #define STR(x) x
char *wstart; /* Beginning of next word */ #define Strchr(s, c) strchr(s, c)
char *wspace; /* Space of word buffer */ #define tok_strdup(s) strdup(s)
#else
#define Char wchar_t
#define FUN(prefix, rest) prefix ## _w ## rest
#define TYPE(type) type ## W
#define STR(x) L ## x
#define Strchr(s, c) wcschr(s, c)
#define tok_strdup(s) wcsdup(s)
#endif
struct TYPE(tokenizer) {
Char *ifs; /* In field separator */
size_t argc, amax; /* Current and maximum number of args */
Char **argv; /* Argument list */
Char *wptr, *wmax; /* Space and limit on the word buffer */
Char *wstart; /* Beginning of next word */
Char *wspace; /* Space of word buffer */
quote_t quote; /* Quoting state */ quote_t quote; /* Quoting state */
int flags; /* flags; */ int flags; /* flags; */
}; };
private void tok_finish(Tokenizer *); static void FUN(tok,finish)(TYPE(Tokenizer) *);
/* tok_finish(): /* FUN(tok,finish)():
* Finish a word in the tokenizer. * Finish a word in the tokenizer.
*/ */
private void static void
tok_finish(Tokenizer *tok) FUN(tok,finish)(TYPE(Tokenizer) *tok)
{ {
*tok->wptr = '\0'; *tok->wptr = '\0';
if ((tok->flags & TOK_KEEP) || tok->wptr != tok->wstart) { if ((tok->flags & TOK_KEEP) || tok->wptr != tok->wstart) {
tok->argv[tok->argc++] = tok->wstart; tok->argv[tok->argc++] = tok->wstart;
tok->argv[tok->argc] = NULL; tok->argv[tok->argc] = NULL;
tok->wstart = ++tok->wptr; tok->wstart = ++tok->wptr;
} }
tok->flags &= ~TOK_KEEP; tok->flags &= ~TOK_KEEP;
} }
/* tok_init(): /* FUN(tok,init)():
* Initialize the tokenizer * Initialize the tokenizer
*/ */
public Tokenizer * TYPE(Tokenizer) *
tok_init(const char *ifs) FUN(tok,init)(const Char *ifs)
{ {
Tokenizer *tok = (Tokenizer *) tok_malloc(sizeof(Tokenizer)); TYPE(Tokenizer) *tok = tok_malloc(sizeof(*tok));
if (tok == NULL) if (tok == NULL)
return NULL; return NULL;
tok->ifs = tok_strdup(ifs ? ifs : IFS); tok->ifs = tok_strdup(ifs ? ifs : IFS);
if (tok->ifs == NULL) { if (tok->ifs == NULL) {
tok_free((ptr_t)tok); tok_free(tok);
return NULL; return NULL;
} }
tok->argc = 0; tok->argc = 0;
tok->amax = AINCR; tok->amax = AINCR;
tok->argv = (char **) tok_malloc(sizeof(char *) * tok->amax); tok->argv = tok_malloc(sizeof(*tok->argv) * tok->amax);
if (tok->argv == NULL) { if (tok->argv == NULL) {
tok_free((ptr_t)tok->ifs); tok_free(tok->ifs);
tok_free((ptr_t)tok); tok_free(tok);
return NULL; return NULL;
} }
tok->argv[0] = NULL; tok->argv[0] = NULL;
tok->wspace = (char *) tok_malloc(WINCR); tok->wspace = tok_malloc(WINCR * sizeof(*tok->wspace));
if (tok->wspace == NULL) { if (tok->wspace == NULL) {
tok_free((ptr_t)tok->argv); tok_free(tok->argv);
tok_free((ptr_t)tok->ifs); tok_free(tok->ifs);
tok_free((ptr_t)tok); tok_free(tok);
return NULL; return NULL;
} }
tok->wmax = tok->wspace + WINCR; tok->wmax = tok->wspace + WINCR;
tok->wstart = tok->wspace; tok->wstart = tok->wspace;
tok->wptr = tok->wspace; tok->wptr = tok->wspace;
tok->flags = 0; tok->flags = 0;
tok->quote = Q_none; tok->quote = Q_none;
return (tok); return tok;
} }
/* tok_reset(): /* FUN(tok,reset)():
* Reset the tokenizer * Reset the tokenizer
*/ */
public void void
tok_reset(Tokenizer *tok) FUN(tok,reset)(TYPE(Tokenizer) *tok)
{ {
tok->argc = 0; tok->argc = 0;
tok->wstart = tok->wspace; tok->wstart = tok->wspace;
tok->wptr = tok->wspace; tok->wptr = tok->wspace;
tok->flags = 0; tok->flags = 0;
tok->quote = Q_none; tok->quote = Q_none;
} }
/* tok_end(): /* FUN(tok,end)():
* Clean up * Clean up
*/ */
public void void
tok_end(Tokenizer *tok) FUN(tok,end)(TYPE(Tokenizer) *tok)
{ {
tok_free((ptr_t) tok->ifs); tok_free(tok->ifs);
tok_free((ptr_t) tok->wspace); tok_free(tok->wspace);
tok_free((ptr_t) tok->argv); tok_free(tok->argv);
tok_free((ptr_t) tok); tok_free(tok);
} }
/* tok_line(): /* FUN(tok,line)():
* Bourne shell (sh(1)) like tokenizing * Bourne shell (sh(1)) like tokenizing
* Arguments: * Arguments:
* tok current tokenizer state (setup with tok_init()) * tok current tokenizer state (setup with FUN(tok,init)())
* line line to parse * line line to parse
* Returns: * Returns:
* -1 Internal error * -1 Internal error
* 3 Quoted return * 3 Quoted return
* 2 Unmatched double quote * 2 Unmatched double quote
* 1 Unmatched single quote * 1 Unmatched single quote
* 0 Ok * 0 Ok
* Modifies (if return value is 0): * Modifies (if return value is 0):
* argc number of arguments * argc number of arguments
* argv argument array * argv argument array
* cursorc if !NULL, argv element containing cursor * cursorc if !NULL, argv element containing cursor
* cursorv if !NULL, offset in argv[cursorc] of cursor * cursorv if !NULL, offset in argv[cursorc] of cursor
*/ */
public int int
tok_line(Tokenizer *tok, const LineInfo *line, FUN(tok,line)(TYPE(Tokenizer) *tok, const TYPE(LineInfo) *line,
int *argc, const char ***argv, int *cursorc, int *cursoro) int *argc, const Char ***argv, int *cursorc, int *cursoro)
{ {
const char *ptr; const Char *ptr;
int cc, co; int cc, co;
cc = co = -1; cc = co = -1;
ptr = line->buffer; ptr = line->buffer;
for (ptr = line->buffer; ;ptr++) { for (ptr = line->buffer; ;ptr++) {
if (ptr >= line->lastchar) if (ptr >= line->lastchar)
ptr = ""; ptr = STR("");
if (ptr == line->cursor) { if (ptr == line->cursor) {
cc = tok->argc; cc = (int)tok->argc;
co = tok->wptr - tok->wstart; co = (int)(tok->wptr - tok->wstart);
} }
switch (*ptr) { switch (*ptr) {
case '\'': case '\'':
tok->flags |= TOK_KEEP; tok->flags |= TOK_KEEP;
tok->flags &= ~TOK_EAT; tok->flags &= ~TOK_EAT;
switch (tok->quote) { switch (tok->quote) {
case Q_none: case Q_none:
tok->quote = Q_single; /* Enter single quote tok->quote = Q_single; /* Enter single quote
* mode */ * mode */
break; break;
skipping to change at line 221 skipping to change at line 249
case Q_double: /* Stay in double quote mode */ case Q_double: /* Stay in double quote mode */
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
case Q_doubleone: /* Quote this ' */ case Q_doubleone: /* Quote this ' */
tok->quote = Q_double; tok->quote = Q_double;
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
default: default:
return (-1); return -1;
} }
break; break;
case '"': case '"':
tok->flags &= ~TOK_EAT; tok->flags &= ~TOK_EAT;
tok->flags |= TOK_KEEP; tok->flags |= TOK_KEEP;
switch (tok->quote) { switch (tok->quote) {
case Q_none: /* Enter double quote mode */ case Q_none: /* Enter double quote mode */
tok->quote = Q_double; tok->quote = Q_double;
break; break;
skipping to change at line 252 skipping to change at line 280
case Q_single: /* Stay in single quote mode */ case Q_single: /* Stay in single quote mode */
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
case Q_doubleone: /* Quote this " */ case Q_doubleone: /* Quote this " */
tok->quote = Q_double; tok->quote = Q_double;
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
default: default:
return (-1); return -1;
} }
break; break;
case '\\': case '\\':
tok->flags |= TOK_KEEP; tok->flags |= TOK_KEEP;
tok->flags &= ~TOK_EAT; tok->flags &= ~TOK_EAT;
switch (tok->quote) { switch (tok->quote) {
case Q_none: /* Quote next character */ case Q_none: /* Quote next character */
tok->quote = Q_one; tok->quote = Q_one;
break; break;
skipping to change at line 283 skipping to change at line 311
case Q_single: /* Stay in single quote mode */ case Q_single: /* Stay in single quote mode */
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
case Q_doubleone: /* Quote this \ */ case Q_doubleone: /* Quote this \ */
tok->quote = Q_double; tok->quote = Q_double;
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
default: default:
return (-1); return -1;
} }
break; break;
case '\n': case '\n':
tok->flags &= ~TOK_EAT; tok->flags &= ~TOK_EAT;
switch (tok->quote) { switch (tok->quote) {
case Q_none: case Q_none:
goto tok_line_outok; goto tok_line_outok;
case Q_single: case Q_single:
skipping to change at line 309 skipping to change at line 337
tok->flags |= TOK_EAT; tok->flags |= TOK_EAT;
tok->quote = Q_double; tok->quote = Q_double;
break; break;
case Q_one: /* No quote, more eat the '\n' */ case Q_one: /* No quote, more eat the '\n' */
tok->flags |= TOK_EAT; tok->flags |= TOK_EAT;
tok->quote = Q_none; tok->quote = Q_none;
break; break;
default: default:
return (0); return 0;
} }
break; break;
case '\0': case '\0':
switch (tok->quote) { switch (tok->quote) {
case Q_none: case Q_none:
/* Finish word and return */ /* Finish word and return */
if (tok->flags & TOK_EAT) { if (tok->flags & TOK_EAT) {
tok->flags &= ~TOK_EAT; tok->flags &= ~TOK_EAT;
return (3); return 3;
} }
goto tok_line_outok; goto tok_line_outok;
case Q_single: case Q_single:
return (1); return 1;
case Q_double: case Q_double:
return (2); return 2;
case Q_doubleone: case Q_doubleone:
tok->quote = Q_double; tok->quote = Q_double;
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
case Q_one: case Q_one:
tok->quote = Q_none; tok->quote = Q_none;
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
default: default:
return (-1); return -1;
} }
break; break;
default: default:
tok->flags &= ~TOK_EAT; tok->flags &= ~TOK_EAT;
switch (tok->quote) { switch (tok->quote) {
case Q_none: case Q_none:
if (strchr(tok->ifs, *ptr) != NULL) if (Strchr(tok->ifs, *ptr) != NULL)
tok_finish(tok); FUN(tok,finish)(tok);
else else
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
case Q_single: case Q_single:
case Q_double: case Q_double:
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
case Q_doubleone: case Q_doubleone:
skipping to change at line 371 skipping to change at line 399
tok->quote = Q_double; tok->quote = Q_double;
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
case Q_one: case Q_one:
tok->quote = Q_none; tok->quote = Q_none;
*tok->wptr++ = *ptr; *tok->wptr++ = *ptr;
break; break;
default: default:
return (-1); return -1;
} }
break; break;
} }
if (tok->wptr >= tok->wmax - 4) { if (tok->wptr >= tok->wmax - 4) {
size_t size = tok->wmax - tok->wspace + WINCR; size_t size = (size_t)(tok->wmax - tok->wspace + WINCR);
char *s = (char *) tok_realloc(tok->wspace, size); Char *s = tok_realloc(tok->wspace,
size * sizeof(*s));
if (s == NULL) if (s == NULL)
return (-1); return -1;
if (s != tok->wspace) { if (s != tok->wspace) {
int i; size_t i;
for (i = 0; i < tok->argc; i++) { for (i = 0; i < tok->argc; i++) {
tok->argv[i] = tok->argv[i] =
(tok->argv[i] - tok->wspace) + s; (tok->argv[i] - tok->wspace) + s;
} }
tok->wptr = (tok->wptr - tok->wspace) + s; tok->wptr = (tok->wptr - tok->wspace) + s;
tok->wstart = (tok->wstart - tok->wspace) + s; tok->wstart = (tok->wstart - tok->wspace) + s;
tok->wspace = s; tok->wspace = s;
} }
tok->wmax = s + size; tok->wmax = s + size;
} }
if (tok->argc >= tok->amax - 4) { if (tok->argc >= tok->amax - 4) {
char **p; Char **p;
tok->amax += AINCR; tok->amax += AINCR;
p = (char **) tok_realloc(tok->argv, p = tok_realloc(tok->argv, tok->amax * sizeof(*p));
tok->amax * sizeof(char *)); if (p == NULL) {
if (p == NULL) tok->amax -= AINCR;
return (-1); return -1;
}
tok->argv = p; tok->argv = p;
} }
} }
tok_line_outok: tok_line_outok:
if (cc == -1 && co == -1) { if (cc == -1 && co == -1) {
cc = tok->argc; cc = (int)tok->argc;
co = tok->wptr - tok->wstart; co = (int)(tok->wptr - tok->wstart);
} }
if (cursorc != NULL) if (cursorc != NULL)
*cursorc = cc; *cursorc = cc;
if (cursoro != NULL) if (cursoro != NULL)
*cursoro = co; *cursoro = co;
tok_finish(tok); FUN(tok,finish)(tok);
*argv = (const char **)tok->argv; *argv = (const Char **)tok->argv;
*argc = tok->argc; *argc = (int)tok->argc;
return (0); return 0;
} }
/* tok_str(): /* FUN(tok,str)():
* Simpler version of tok_line, taking a NUL terminated line * Simpler version of tok_line, taking a NUL terminated line
* and splitting into words, ignoring cursor state. * and splitting into words, ignoring cursor state.
*/ */
public int int
tok_str(Tokenizer *tok, const char *line, int *argc, const char ***argv) FUN(tok,str)(TYPE(Tokenizer) *tok, const Char *line, int *argc,
const Char ***argv)
{ {
LineInfo li; TYPE(LineInfo) li;
memset(&li, 0, sizeof(li)); memset(&li, 0, sizeof(li));
li.buffer = line; li.buffer = line;
li.cursor = li.lastchar = strchr(line, '\0'); li.cursor = li.lastchar = Strchr(line, '\0');
return (tok_line(tok, &li, argc, argv, NULL, NULL)); return FUN(tok,line)(tok, &li, argc, argv, NULL, NULL);
} }
 End of changes. 52 change blocks. 
84 lines changed or deleted 115 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)