Mercurial > projects > pwm

changeset 27:722a45b4028b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add define command for defining macros Macros are parsed when they are defined with the D command and can subsequently be used as arguments for other commands. Handle out of memory errors directly in tok.c.
author Guido Berhoerster <guido+pwm@berhoerster.name>
date Mon, 25 Sep 2017 21:21:25 +0200
parents 5bdea77d0c1d
children e3db02d7f1f4
files Makefile cmd.c macro.c macro.h pwm.1.xml pwm.c pwm.h tok.c tok.h
diffstat 9 files changed, 495 insertions(+), 83 deletions(-) [+]
line wrap: on
line diff
--- a/Makefile	Thu Sep 21 09:45:59 2017 +0200
+++ b/Makefile	Mon Sep 25 21:21:25 2017 +0200
@@ -162,6 +162,7 @@
 
 PWM_OBJS =		cmd.o \
 			io.o \
+			macro.o \
 			pager.o \
 			proc.o \
 			pw.o \
--- a/cmd.c	Thu Sep 21 09:45:59 2017 +0200
+++ b/cmd.c	Mon Sep 25 21:21:25 2017 +0200
@@ -37,10 +37,12 @@
 
 #include "cmd.h"
 #include "io.h"
+#include "macro.h"
 #include "pager.h"
 #include "proc.h"
 #include "pw.h"
 #include "pwfile.h"
+#include "tok.h"
 #include "util.h"
 
 #define	TIME_FORMAT	"%Y-%m-%dT%TZ"
@@ -82,6 +84,7 @@
 	CHARCLASS_GRAPH
 };
 
+static enum cmd_return	cmd_define(struct pwm_ctx *, int, char *[]);
 static enum cmd_return	cmd_status(struct pwm_ctx *, int, char *[]);
 static enum cmd_return	cmd_info(struct pwm_ctx *, int, char *[]);
 static enum cmd_return	cmd_list(struct pwm_ctx *, int, char *[]);
@@ -152,6 +155,7 @@
 };
 
 static struct cmd cmds[] = {
+    { "D", "define", "define name=value", "Define a macro", cmd_define },
     { "t", "status", "status", "Redisplay an error message of the previous "
     "command and unsaved changes", cmd_status },
     { "i", "info", "info", "Show metadata information about the current file",
@@ -217,6 +221,66 @@
 }
 
 static enum cmd_return
+cmd_define(struct pwm_ctx *ctx, int argc, char *argv[])
+{
+	int		retval = CMD_ERR;
+	const char	*value;
+	char		*name = NULL;
+	size_t		tokenc = 0;
+	union tok	**tokenv = NULL;
+	struct macro_entry *macro_entry;
+
+	if (argc != 2) {
+		return (CMD_USAGE);
+	}
+
+	/* split into name and value */
+	value = strchr(argv[1], '=');
+	if (value == NULL) {
+		pwm_err(ctx, "bad macro definition \"%s\"", argv[1]);
+		goto out;
+	}
+	xasprintf(&name, "%.*s", value - argv[1], argv[1]);
+	value++;
+
+	/* tokenize macro value */
+	switch (tok_tokenize(value, &tokenc, &tokenv)) {
+	case TOK_ERR_UNTERMINATED_QUOTE:
+		pwm_err(ctx, "unterminated quote in macro");
+		goto out;
+	case TOK_ERR_TRAILING_BACKSLASH:
+		pwm_err(ctx, "trailing backslash in macro");
+		goto out;
+	case TOK_ERR_INVALID_MACRO_NAME:
+		pwm_err(ctx, "invalid macro name referenced in macro");
+		goto out;
+	}
+
+	/* parse macro definition */
+	switch (macro_parse(name, tokenc, tokenv, ctx->macro_head,
+	    &macro_entry)) {
+	case MACRO_ERR_INVALID_NAME:
+		pwm_err(ctx, "invalid macro name \"%s\"", name);
+		goto out;
+	case MACRO_ERR_UNDEFINED_MACRO:
+		pwm_err(ctx, "macro definition references undefined macro");
+		goto out;
+	case MACRO_ERR_RECURSIVE:
+		pwm_err(ctx, "macro definition must not be recursive");
+		goto out;
+	}
+
+	macro_add(ctx->macro_head, macro_entry);
+	retval = CMD_OK;
+
+out:
+	tok_free(tokenv);
+	free(name);
+
+	return (retval);
+}
+
+static enum cmd_return
 cmd_status(struct pwm_ctx *ctx, int argc, char *argv[])
 {
 	if (argc != 1) {
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/macro.c	Mon Sep 25 21:21:25 2017 +0200
@@ -0,0 +1,202 @@
+/*
+ * Copyright (C) 2017 Guido Berhoerster <guido+pwm@berhoerster.name>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "compat.h"
+
+#include <ctype.h>
+#ifdef	HAVE_QUEUE_H
+#include <sys/queue.h>
+#endif /* HAVE_QUEUE_H */
+#include <stdlib.h>
+#include <string.h>
+
+#include "macro.h"
+#include "tok.h"
+#include "util.h"
+
+TAILQ_HEAD(macro_head, macro_entry);
+
+struct macro_entry {
+	TAILQ_ENTRY(macro_entry) entry;
+	char		*name;
+	int		argc;
+	char		**argv;
+};
+
+static struct macro_entry *
+match_macro(struct macro_head *head, const char *name)
+{
+	struct macro_entry *entry;
+
+	TAILQ_FOREACH(entry, head, entry) {
+		if (strcmp(entry->name, name) == 0) {
+			return (entry);
+		}
+	}
+
+	return (NULL);
+}
+
+void
+macro_init(struct macro_head **headp)
+{
+	struct macro_head *head;
+
+	head = xmalloc(sizeof (struct macro_head));
+	TAILQ_INIT(head);
+	*headp = head;
+}
+
+static void
+free_macro_entry(struct macro_entry *entry)
+{
+	int	i;
+
+	if (entry == NULL) {
+		return;
+	}
+
+	free(entry->name);
+	for (i = 0; i < entry->argc; i++) {
+		free(entry->argv[i]);
+	}
+	free(entry->argv);
+	free(entry);
+}
+
+void
+macro_destroy(struct macro_head *head)
+{
+	struct macro_entry *entry;
+	struct macro_entry *tmp_entry;
+
+	if (head == NULL) {
+		return;
+	}
+
+	TAILQ_FOREACH_SAFE(entry, head, entry, tmp_entry) {
+		TAILQ_REMOVE(head, entry, entry);
+		free_macro_entry(entry);
+	}
+	free(head);
+}
+
+enum macro_err
+macro_parse(const char *name, size_t tokenc, union tok **tokenv,
+    struct macro_head *head, struct macro_entry **macro_entryp)
+{
+	const char	*p = name;
+	int		argc;
+	char		**argv;
+	struct macro_entry *entry;
+
+	/* validate macro name */
+	for (p = name; *p != '\0'; p++) {
+		/*
+		 * macro names must contain only ASCII alphanumeric characters
+		 * and underscores
+		 */
+		if (!isascii(*p) || (!isalnum(*p) && (*p != '_'))) {
+			return (MACRO_ERR_INVALID_NAME);
+		}
+	}
+
+	/* expand macros */
+	if (macro_expand_macros(head, tokenc, tokenv, &argc, &argv) != 0) {
+		return (MACRO_ERR_UNDEFINED_MACRO);
+	}
+
+	/* create macro */
+	entry = xmalloc(sizeof (struct macro_entry));
+	entry->name = xstrdup(name);
+	entry->argc = argc;
+	entry->argv = argv;
+	*macro_entryp = entry;
+
+	return (MACRO_ERR_OK);
+}
+
+void
+macro_add(struct macro_head *head, struct macro_entry *entry)
+{
+	struct macro_entry *old_entry;
+
+	/* remove existing macro with the same name */
+	old_entry = match_macro(head, entry->name);
+	if (old_entry != NULL) {
+		TAILQ_REMOVE(head, old_entry, entry);
+		free_macro_entry(old_entry);
+	}
+
+	TAILQ_INSERT_TAIL(head, entry, entry);
+}
+
+int
+macro_expand_macros(struct macro_head *head, size_t tokenc, union tok **tokenv,
+    int *argcp, char ***argvp)
+{
+	int		retval = -1;
+	char		**argv = NULL;
+	int		argc = 0;
+	size_t		argv_size = tokenc + 1;
+	size_t		i;
+	struct macro_entry *entry;
+	int		j;
+
+	argv = xmalloc(argv_size * sizeof (char *));
+	argv[0] = NULL;
+
+	for (i = 0; i < tokenc; i++) {
+		switch (tokenv[i]->any.type) {
+		case TOK_ARG:
+			argv[argc++] = xstrdup(tokenv[i]->arg.value);
+			argv[argc] = NULL;
+			break;
+		case TOK_MACRO:
+			entry = match_macro(head, tokenv[i]->macro.name);
+			if (entry == NULL) {
+				goto out;
+			}
+			argv_size += entry->argc;
+			argv = xrealloc(argv, argv_size * sizeof (char *));
+			for (j = 0; j < entry->argc; j++) {
+				argv[argc++] = xstrdup(entry->argv[j]);
+				argv[argc] = NULL;
+			}
+			break;
+		}
+	}
+	*argvp = argv;
+	*argcp = argc;
+	retval = 0;
+
+out:
+	if (retval != 0) {
+		for (j = 0; j < argc; j++) {
+			free(argv[j]);
+		}
+		free(argv);
+	}
+
+	return (retval);
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/macro.h	Mon Sep 25 21:21:25 2017 +0200
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2017 Guido Berhoerster <guido+pwm@berhoerster.name>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef	MACRO_H
+#define	MACRO_H
+
+#include "tok.h"
+
+enum macro_err {
+	MACRO_ERR_OK,
+	MACRO_ERR_INVALID_NAME = -1,
+	MACRO_ERR_UNDEFINED_MACRO = -2,
+	MACRO_ERR_RECURSIVE = -3
+};
+
+struct macro_head;
+struct macro_entry;
+
+void		macro_init(struct macro_head **);
+void		macro_destroy(struct macro_head *);
+enum macro_err	macro_parse(const char *, size_t, union tok **,
+    struct macro_head *, struct macro_entry **);
+void		macro_add(struct macro_head *, struct macro_entry *);
+int		macro_expand_macros(struct macro_head *, size_t, union tok **,
+    int *, char ***);
+
+#endif /* !MACRO_H */
--- a/pwm.1.xml	Thu Sep 21 09:45:59 2017 +0200
+++ b/pwm.1.xml	Mon Sep 25 21:21:25 2017 +0200
@@ -34,7 +34,7 @@
       <email>guido+pwm@berhoerster.name</email>
       <personblurb/>
     </author>
-    <date>17 September, 2017</date>
+    <date>25 September, 2017</date>
   </info>
   <refmeta>
     <refentrytitle>pwm</refentrytitle>
@@ -482,6 +482,34 @@
           </listitem>
         </varlistentry>
         <varlistentry>
+          <term>Define macro</term>
+          <listitem>
+            <cmdsynopsis>
+              <command>define</command>
+              <arg choice="plain">
+                <replaceable>name</replaceable>=<replaceable>value</replaceable>
+              </arg>
+            </cmdsynopsis>
+            <cmdsynopsis>
+              <command>D</command>
+              <arg choice="plain">
+                <replaceable>name</replaceable>=<replaceable>value</replaceable>
+              </arg>
+              <sbr/>
+            </cmdsynopsis>
+            <para>Define the macro <replaceable>name</replaceable> to expand to
+            <replaceable>value</replaceable>.  A Macro is a shorthand term for
+            a command and/or command arguments.  Macros are tokenized when they
+            are defined, thus if a macro contains other macros, these other
+            macros are expanded once at the time the macro is defined and not
+            each time a macro is expanded later.  A macro can be used as like a
+            command or as part of the command's argument by prefixing the macro
+            name by a &quot;$&quot;. It is expanded by substituting the
+            previously tokenized contents before the command is
+            evaluated.</para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
           <term>Display help text</term>
           <listitem>
             <cmdsynopsis>
--- a/pwm.c	Thu Sep 21 09:45:59 2017 +0200
+++ b/pwm.c	Mon Sep 25 21:21:25 2017 +0200
@@ -42,6 +42,7 @@
 #include "pwm.h"
 #include "cmd.h"
 #include "io.h"
+#include "macro.h"
 #include "pwfile.h"
 #include "tok.h"
 #include "util.h"
@@ -108,6 +109,8 @@
 	GetLine		*gl = NULL;
 	char		buf[PWM_LINE_MAX + 1];
 	int		io_retval;
+	size_t		tokenc = 0;
+	union tok	**tokenv = NULL;
 	int		argc = 0;
 	char		**argv = NULL;
 	struct cmd	*cmd;
@@ -159,9 +162,7 @@
 		}
 
 		/* tokenize line */
-		switch (tok_tokenize(buf, &argc, &argv)) {
-		case TOK_ERR_SYSTEM_ERROR:
-			err(1, "tok_tokenize");
+		switch (tok_tokenize(buf, &tokenc, &tokenv)) {
 		case TOK_ERR_UNTERMINATED_QUOTE:
 			fprintf(stderr, "unterminated quote\n");
 			if (!ctx->is_interactive) {
@@ -174,6 +175,22 @@
 				goto out;
 			}
 			goto next;
+		case TOK_ERR_INVALID_MACRO_NAME:
+			fprintf(stderr, "invalid macro name\n");
+			if (!ctx->is_interactive) {
+				goto out;
+			}
+			goto next;
+		}
+
+		/* expand macros */
+		if (macro_expand_macros(ctx->macro_head, tokenc, tokenv, &argc,
+		    &argv) != 0) {
+			fprintf(stderr, "undefined macro\n");
+			if (!ctx->is_interactive) {
+				goto out;
+			}
+			goto next;
 		}
 
 		if (argc == 0) {
@@ -216,6 +233,9 @@
 		free(argv);
 		argc = 0;
 		argv = NULL;
+		tok_free(tokenv);
+		tokenc = 0;
+		tokenv = NULL;
 	}
 
 quit:
@@ -226,6 +246,7 @@
 		free(argv[i]);
 	}
 	free(argv);
+	tok_free(tokenv);
 	del_GetLine(gl);
 
 	return (retval);
@@ -397,6 +418,7 @@
 	}
 
 	pwfile_init(&ctx);
+	macro_init(&ctx.macro_head);
 
 	fp = fopen(ctx.filename, "r");
 	if ((fp == NULL) && (errno != ENOENT)) {
@@ -424,6 +446,7 @@
 	status = (run_input_loop(&ctx) != 0);
 
 out:
+	macro_destroy(ctx.macro_head);
 	pwfile_destroy(&ctx);
 	if (fp != NULL) {
 		fclose(fp);
--- a/pwm.h	Thu Sep 21 09:45:59 2017 +0200
+++ b/pwm.h	Mon Sep 25 21:21:25 2017 +0200
@@ -55,6 +55,7 @@
 	int		unsaved_changes;
 	unsigned int	next_id;
 	char		password[PWS3_MAX_PASSWORD_LEN + 1];
+	struct macro_head *macro_head;
 };
 
 void	pwm_err(struct pwm_ctx *, char *, ...);
--- a/tok.c	Thu Sep 21 09:45:59 2017 +0200
+++ b/tok.c	Mon Sep 25 21:21:25 2017 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 Guido Berhoerster <guido+pwm@berhoerster.name>
+ * Copyright (C) 2017 Guido Berhoerster <guido+pwm@berhoerster.name>
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -23,22 +23,24 @@
 
 #include "compat.h"
 
-#include <errno.h>
+#include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include "tok.h"
+#include "util.h"
 
 enum tok_states {
 	STATE_INITIAL,
 	STATE_IN_WORD,
 	STATE_IN_QUOTE,
 	STATE_IN_WORD_ESCAPE,
-	STATE_IN_QUOTE_ESCAPE
+	STATE_IN_QUOTE_ESCAPE,
+	STATE_IN_MACRO
 };
 
-static inline int
+static inline void
 strbuf_appendc(char **bufp, size_t *buf_sizep, int c)
 {
 	char	*buf = *bufp;
@@ -50,10 +52,7 @@
 	/* allocate buffer if *bufp is NULL and *buf_sizep is 0 */
 	if (buf_size < len + (c >= 0) + 1) {
 		buf_size = (buf_size * 2 > BUFSIZ) ? buf_size * 2 : BUFSIZ;
-		buf = realloc(buf, buf_size);
-		if (buf == NULL) {
-			return (-1);
-		}
+		buf = xrealloc(buf, buf_size);
 	}
 
 	/* append character to string buffer or reset buffer if c is -1 */
@@ -64,35 +63,48 @@
 
 	*bufp = buf;
 	*buf_sizep = buf_size;
+}
 
-	return (0);
+void
+tok_free(union tok **tokenv)
+{
+	size_t	i;
+
+	if (tokenv == NULL) {
+		return;
+	}
+
+	for (i = 0; tokenv[i] != NULL; i++) {
+		switch (tokenv[i]->any.type) {
+		case TOK_MACRO:
+			free(tokenv[i]->macro.name);
+			break;
+		case TOK_ARG:
+			free(tokenv[i]->arg.value);
+			break;
+		}
+		free(tokenv[i]);
+	}
+	free(tokenv);
 }
 
 enum tok_err
-tok_tokenize(const char *s, int *tokencp, char ***tokenvp)
+tok_tokenize(const char *s, size_t *tokencp, union tok ***tokenvp)
 {
-	int		retval = TOK_ERR_SYSTEM_ERROR;
-	int		saved_errno = 0;
-	char		**tokenv;
+	int		retval = TOK_ERR_OK;
+	union tok	**tokenv;
 	size_t		tokenc = 0;
 	const char	*p = s;
 	enum tok_states	state = STATE_INITIAL;
 	char		quote;
 	char		*buf = NULL;
 	size_t		buf_size = 0;
-	char		*token;
-	size_t		i;
+	char		*value;
+	char		*name;
 
-	/*
-	 * allocate maximum number of tokens including the terminating NULL
-	 * pointer: ceil(length / 2) + 1
-	 */
-	tokenv = malloc(((strlen(s) + 2 - 1) / 2 + 1) * sizeof (char *));
-	if (tokenv == NULL) {
-		saved_errno = errno;
-		goto out;
-	}
-	tokenv[0] = NULL;
+	/* allocate maximum number of tokens: ceil(length / 2) */
+	tokenv = xmalloc((((strlen(s) + 2 - 1) / 2) + 1) *
+	    sizeof (union tok *));
 
 	for (;;) {
 		switch (state) {
@@ -108,34 +120,27 @@
 				/* start quoted part of token */
 				state = STATE_IN_QUOTE;
 				quote = *p;
-				if (strbuf_appendc(&buf, &buf_size, -1) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
+				strbuf_appendc(&buf, &buf_size, -1);
 				break;
 			case '\\':
 				/* start token with a backslash escape */
 				state = STATE_IN_WORD_ESCAPE;
-				if (strbuf_appendc(&buf, &buf_size, -1) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
+				strbuf_appendc(&buf, &buf_size, -1);
+				break;
+			case '$':
+				/* start macro token */
+				state = STATE_IN_MACRO;
+				strbuf_appendc(&buf, &buf_size, -1);
 				break;
 			case '\0':
 				/* end of input */
-				retval = 0;
+				retval = TOK_ERR_OK;
 				goto out;
 			default:
 				/* start token with a word */
 				state = STATE_IN_WORD;
-				if (strbuf_appendc(&buf, &buf_size, -1) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
-				if (strbuf_appendc(&buf, &buf_size, *p) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
+				strbuf_appendc(&buf, &buf_size, -1);
+				strbuf_appendc(&buf, &buf_size, *p);
 			}
 			break;
 		case STATE_IN_WORD:
@@ -145,15 +150,14 @@
 			case '\n':	/* FALLTHROUGH */
 			case '\0':
 				/* end of token */
-				token = strdup(buf);
-				if (token == NULL) {
-					saved_errno = errno;
-					goto out;
-				}
-				tokenv[tokenc++] = token;
-				tokenv[tokenc] = NULL;
+				value = xstrdup(buf);
+				tokenv[tokenc] = xmalloc(sizeof (union tok));
+				tokenv[tokenc]->arg.type = TOK_ARG;
+				tokenv[tokenc]->arg.value = value;
+				tokenc++;
+
 				if (*p == '\0') {
-					retval = 0;
+					retval = TOK_ERR_OK;
 					goto out;
 				}
 				state = STATE_INITIAL;
@@ -170,10 +174,7 @@
 				break;
 			default:
 				/* regular character */
-				if (strbuf_appendc(&buf, &buf_size, *p) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
+				strbuf_appendc(&buf, &buf_size, *p);
 			}
 			break;
 		case STATE_IN_QUOTE:
@@ -185,11 +186,7 @@
 					state = STATE_IN_WORD;
 				} else {
 					/* quote quote character */
-					if (strbuf_appendc(&buf, &buf_size,
-					    *p) != 0) {
-						saved_errno = errno;
-						goto out;
-					}
+					strbuf_appendc(&buf, &buf_size, *p);
 				}
 				break;
 			case '\\':
@@ -202,10 +199,7 @@
 				goto out;
 			default:
 				/* regular character */
-				if (strbuf_appendc(&buf, &buf_size, *p) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
+				strbuf_appendc(&buf, &buf_size, *p);
 			}
 			break;
 		case STATE_IN_WORD_ESCAPE:	/* FALLTHROUGH */
@@ -218,9 +212,38 @@
 			/* escaped character */
 			state = (state == STATE_IN_WORD_ESCAPE) ?
 			    STATE_IN_WORD : STATE_IN_QUOTE;
-			if (strbuf_appendc(&buf, &buf_size, *p) != 0) {
-				saved_errno = errno;
-				goto out;
+			strbuf_appendc(&buf, &buf_size, *p);
+			break;
+		case STATE_IN_MACRO:
+			switch (*p) {
+			case ' ':	/* FALLTHROUGH */
+			case '\t':	/* FALLTHROUGH */
+			case '\n':	/* FALLTHROUGH */
+			case '\0':
+				/* end of token */
+				name = xstrdup(buf);
+				tokenv[tokenc] = xmalloc(sizeof (union tok));
+				tokenv[tokenc]->macro.type = TOK_MACRO;
+				tokenv[tokenc]->macro.name = name;
+				tokenc++;
+
+				if (*p == '\0') {
+					retval = TOK_ERR_OK;
+					goto out;
+				}
+				state = STATE_INITIAL;
+				break;
+			default:
+				/*
+				 * macro names must only contain alphanumeric
+				 * characters and underscores
+				 */
+				if (!isascii(*p) || (!isalnum(*p) &&
+				    (*p != '_'))) {
+					retval = TOK_ERR_INVALID_MACRO_NAME;
+					goto out;
+				}
+				strbuf_appendc(&buf, &buf_size, *p);
 			}
 			break;
 		}
@@ -229,18 +252,14 @@
 
 out:
 	if (retval < 0) {
-		for (i = 0; i < tokenc; i++) {
-			free(tokenv[i]);
-		}
-		free(tokenv);
+		tok_free(tokenv);
 	} else {
+		tokenv[tokenc] = NULL;
 		*tokencp = tokenc;
-		*tokenvp = realloc(tokenv, (tokenc + 1) * sizeof (char *));
+		*tokenvp = xrealloc(tokenv, (tokenc + 1) *
+		    sizeof (union tok *));
 	}
 	free(buf);
-	if (retval < 0) {
-		errno = saved_errno;
-	}
 
 	return (retval);
 }
--- a/tok.h	Thu Sep 21 09:45:59 2017 +0200
+++ b/tok.h	Mon Sep 25 21:21:25 2017 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 Guido Berhoerster <guido+pwm@berhoerster.name>
+ * Copyright (C) 2017 Guido Berhoerster <guido+pwm@berhoerster.name>
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -25,11 +25,38 @@
 #define	TOK_H
 
 enum tok_err {
-	TOK_ERR_SYSTEM_ERROR = -1,
-	TOK_ERR_UNTERMINATED_QUOTE = -2,
-	TOK_ERR_TRAILING_BACKSLASH = -3
+	TOK_ERR_OK,
+	TOK_ERR_UNTERMINATED_QUOTE = -1,
+	TOK_ERR_TRAILING_BACKSLASH = -2,
+	TOK_ERR_INVALID_MACRO_NAME = -3
+};
+
+enum tok_type {
+	TOK_ARG,
+	TOK_MACRO
+};
+
+struct tok_any {
+	enum tok_type	type;
 };
 
-enum tok_err	tok_tokenize(const char *, int *, char ***);
+struct tok_arg {
+	enum tok_type	type;
+	char		*value;
+};
+
+struct tok_macro {
+	enum tok_type	type;
+	char		*name;
+};
+
+union tok {
+	struct tok_any	any;
+	struct tok_arg	arg;
+	struct tok_macro macro;
+};
+
+void		tok_free(union tok **);
+enum tok_err	tok_tokenize(const char *, size_t *, union tok ***);
 
 #endif /* !TOK_H */