From 65aba167033019994e35f43b819d8fbd3620e9ff Mon Sep 17 00:00:00 2001
From: John Hodge <tpg@mutabah.net>
Date: Thu, 18 Aug 2016 13:18:48 +0800
Subject: Parse - Fix lexing bug with unicode whitespace

---
 src/parse/lex.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp
index 62e25942..42e36574 100644
--- a/src/parse/lex.cpp
+++ b/src/parse/lex.cpp
@@ -232,7 +232,7 @@ bool issym(Codepoint ch)
     if( ch == '_' )
         return true;
     if( ch.v >= 128 )
-        return true;
+        return !ch.isspace();
     return false;
 }
 
@@ -1170,6 +1170,11 @@ bool Codepoint::isspace() const {
     case '\r':
     case '\n':
     case ' ':
+    case 0xC:   // ^L
+    case 0x85:
+    case 0x200E ... 0x200F: // LTR / RTL markers
+    case 0x2028:    // Line Separator
+    case 0x2029:    // Paragrah Separator
         return true;
     default:
         return false;
-- 
cgit v1.2.3