--- a/tok.c	Thu Sep 21 09:45:59 2017 +0200
+++ b/tok.c	Mon Sep 25 21:21:25 2017 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 Guido Berhoerster <guido+pwm@berhoerster.name>
+ * Copyright (C) 2017 Guido Berhoerster <guido+pwm@berhoerster.name>
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -23,22 +23,24 @@
 
 #include "compat.h"
 
-#include <errno.h>
+#include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include "tok.h"
+#include "util.h"
 
 enum tok_states {
 	STATE_INITIAL,
 	STATE_IN_WORD,
 	STATE_IN_QUOTE,
 	STATE_IN_WORD_ESCAPE,
-	STATE_IN_QUOTE_ESCAPE
+	STATE_IN_QUOTE_ESCAPE,
+	STATE_IN_MACRO
 };
 
-static inline int
+static inline void
 strbuf_appendc(char **bufp, size_t *buf_sizep, int c)
 {
 	char	*buf = *bufp;
@@ -50,10 +52,7 @@
 	/* allocate buffer if *bufp is NULL and *buf_sizep is 0 */
 	if (buf_size < len + (c >= 0) + 1) {
 		buf_size = (buf_size * 2 > BUFSIZ) ? buf_size * 2 : BUFSIZ;
-		buf = realloc(buf, buf_size);
-		if (buf == NULL) {
-			return (-1);
-		}
+		buf = xrealloc(buf, buf_size);
 	}
 
 	/* append character to string buffer or reset buffer if c is -1 */
@@ -64,35 +63,48 @@
 
 	*bufp = buf;
 	*buf_sizep = buf_size;
+}
 
-	return (0);
+void
+tok_free(union tok **tokenv)
+{
+	size_t	i;
+
+	if (tokenv == NULL) {
+		return;
+	}
+
+	for (i = 0; tokenv[i] != NULL; i++) {
+		switch (tokenv[i]->any.type) {
+		case TOK_MACRO:
+			free(tokenv[i]->macro.name);
+			break;
+		case TOK_ARG:
+			free(tokenv[i]->arg.value);
+			break;
+		}
+		free(tokenv[i]);
+	}
+	free(tokenv);
 }
 
 enum tok_err
-tok_tokenize(const char *s, int *tokencp, char ***tokenvp)
+tok_tokenize(const char *s, size_t *tokencp, union tok ***tokenvp)
 {
-	int		retval = TOK_ERR_SYSTEM_ERROR;
-	int		saved_errno = 0;
-	char		**tokenv;
+	int		retval = TOK_ERR_OK;
+	union tok	**tokenv;
 	size_t		tokenc = 0;
 	const char	*p = s;
 	enum tok_states	state = STATE_INITIAL;
 	char		quote;
 	char		*buf = NULL;
 	size_t		buf_size = 0;
-	char		*token;
-	size_t		i;
+	char		*value;
+	char		*name;
 
-	/*
-	 * allocate maximum number of tokens including the terminating NULL
-	 * pointer: ceil(length / 2) + 1
-	 */
-	tokenv = malloc(((strlen(s) + 2 - 1) / 2 + 1) * sizeof (char *));
-	if (tokenv == NULL) {
-		saved_errno = errno;
-		goto out;
-	}
-	tokenv[0] = NULL;
+	/* allocate maximum number of tokens: ceil(length / 2) */
+	tokenv = xmalloc((((strlen(s) + 2 - 1) / 2) + 1) *
+	    sizeof (union tok *));
 
 	for (;;) {
 		switch (state) {
@@ -108,34 +120,27 @@
 				/* start quoted part of token */
 				state = STATE_IN_QUOTE;
 				quote = *p;
-				if (strbuf_appendc(&buf, &buf_size, -1) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
+				strbuf_appendc(&buf, &buf_size, -1);
 				break;
 			case '\\':
 				/* start token with a backslash escape */
 				state = STATE_IN_WORD_ESCAPE;
-				if (strbuf_appendc(&buf, &buf_size, -1) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
+				strbuf_appendc(&buf, &buf_size, -1);
+				break;
+			case '$':
+				/* start macro token */
+				state = STATE_IN_MACRO;
+				strbuf_appendc(&buf, &buf_size, -1);
 				break;
 			case '\0':
 				/* end of input */
-				retval = 0;
+				retval = TOK_ERR_OK;
 				goto out;
 			default:
 				/* start token with a word */
 				state = STATE_IN_WORD;
-				if (strbuf_appendc(&buf, &buf_size, -1) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
-				if (strbuf_appendc(&buf, &buf_size, *p) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
+				strbuf_appendc(&buf, &buf_size, -1);
+				strbuf_appendc(&buf, &buf_size, *p);
 			}
 			break;
 		case STATE_IN_WORD:
@@ -145,15 +150,14 @@
 			case '\n':	/* FALLTHROUGH */
 			case '\0':
 				/* end of token */
-				token = strdup(buf);
-				if (token == NULL) {
-					saved_errno = errno;
-					goto out;
-				}
-				tokenv[tokenc++] = token;
-				tokenv[tokenc] = NULL;
+				value = xstrdup(buf);
+				tokenv[tokenc] = xmalloc(sizeof (union tok));
+				tokenv[tokenc]->arg.type = TOK_ARG;
+				tokenv[tokenc]->arg.value = value;
+				tokenc++;
+
 				if (*p == '\0') {
-					retval = 0;
+					retval = TOK_ERR_OK;
 					goto out;
 				}
 				state = STATE_INITIAL;
@@ -170,10 +174,7 @@
 				break;
 			default:
 				/* regular character */
-				if (strbuf_appendc(&buf, &buf_size, *p) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
+				strbuf_appendc(&buf, &buf_size, *p);
 			}
 			break;
 		case STATE_IN_QUOTE:
@@ -185,11 +186,7 @@
 					state = STATE_IN_WORD;
 				} else {
 					/* quote quote character */
-					if (strbuf_appendc(&buf, &buf_size,
-					    *p) != 0) {
-						saved_errno = errno;
-						goto out;
-					}
+					strbuf_appendc(&buf, &buf_size, *p);
 				}
 				break;
 			case '\\':
@@ -202,10 +199,7 @@
 				goto out;
 			default:
 				/* regular character */
-				if (strbuf_appendc(&buf, &buf_size, *p) != 0) {
-					saved_errno = errno;
-					goto out;
-				}
+				strbuf_appendc(&buf, &buf_size, *p);
 			}
 			break;
 		case STATE_IN_WORD_ESCAPE:	/* FALLTHROUGH */
@@ -218,9 +212,38 @@
 			/* escaped character */
 			state = (state == STATE_IN_WORD_ESCAPE) ?
 			    STATE_IN_WORD : STATE_IN_QUOTE;
-			if (strbuf_appendc(&buf, &buf_size, *p) != 0) {
-				saved_errno = errno;
-				goto out;
+			strbuf_appendc(&buf, &buf_size, *p);
+			break;
+		case STATE_IN_MACRO:
+			switch (*p) {
+			case ' ':	/* FALLTHROUGH */
+			case '\t':	/* FALLTHROUGH */
+			case '\n':	/* FALLTHROUGH */
+			case '\0':
+				/* end of token */
+				name = xstrdup(buf);
+				tokenv[tokenc] = xmalloc(sizeof (union tok));
+				tokenv[tokenc]->macro.type = TOK_MACRO;
+				tokenv[tokenc]->macro.name = name;
+				tokenc++;
+
+				if (*p == '\0') {
+					retval = TOK_ERR_OK;
+					goto out;
+				}
+				state = STATE_INITIAL;
+				break;
+			default:
+				/*
+				 * macro names must only contain alphanumeric
+				 * characters and underscores
+				 */
+				if (!isascii(*p) || (!isalnum(*p) &&
+				    (*p != '_'))) {
+					retval = TOK_ERR_INVALID_MACRO_NAME;
+					goto out;
+				}
+				strbuf_appendc(&buf, &buf_size, *p);
 			}
 			break;
 		}
@@ -229,18 +252,14 @@
 
 out:
 	if (retval < 0) {
-		for (i = 0; i < tokenc; i++) {
-			free(tokenv[i]);
-		}
-		free(tokenv);
+		tok_free(tokenv);
 	} else {
+		tokenv[tokenc] = NULL;
 		*tokencp = tokenc;
-		*tokenvp = realloc(tokenv, (tokenc + 1) * sizeof (char *));
+		*tokenvp = xrealloc(tokenv, (tokenc + 1) *
+		    sizeof (union tok *));
 	}
 	free(buf);
-	if (retval < 0) {
-		errno = saved_errno;
-	}
 
 	return (retval);
 }