token: Add TOTHER for other non-whitespace tokens

This is for any non-whitespace character that doesn't match any
other token categories, and could be valid if stringified, or the
preprocessor is used by itself.

1 files changed, 5 insertions, 1 deletions

diff --git a/token.c b/token.c
index dc2df02..b3673a8 100644
--- a/token.c
+++ b/token.c
@@ -1,4 +1,5 @@
 #include <assert.h>
+#include <ctype.h>
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stdint.h>
@@ -149,7 +150,8 @@ tokendesc(char *buf, size_t len, enum tokenkind kind, const char *lit)
 	case TNUMBER:    class = "number";     quote = true;  break;
 	case TCHARCONST: class = "character";  quote = false; break;
 	case TSTRINGLIT: class = "string";     quote = false; break;
-	case TNEWLINE:   class = "newline";    quote = true;  break;
+	case TNEWLINE:   class = "newline";                   break;
+	case TOTHER:     class = NULL;                        break;
 	default:
 		class = NULL;
 		lit = kind < LEN(tokstr) ? tokstr[kind] : NULL;
@@ -158,6 +160,8 @@ tokendesc(char *buf, size_t len, enum tokenkind kind, const char *lit)
 		snprintf(buf, len, quote ? "%s '%s'" : "%s %s", class, lit);
 	else if (class)
 		snprintf(buf, len, "%s", class);
+	else if (kind == TOTHER && !isprint(lit[0]))
+		snprintf(buf, len, "<U+%04x>", lit[0]);
 	else if (lit)
 		snprintf(buf, len, "'%s'", lit);
 	else