Cleanup and new syntax

4 files changed, 164 insertions, 62 deletions

diff --git a/src/parse/eTokenType.enum.h b/src/parse/eTokenType.enum.h
index 69f00f02..2adfb042 100644
--- a/src/parse/eTokenType.enum.h
+++ b/src/parse/eTokenType.enum.h
@@ -13,6 +13,7 @@ _(TOK_INTEGER)
 _(TOK_CHAR)
 _(TOK_FLOAT)
 _(TOK_STRING)
+_(TOK_BYTESTRING)
 
 _(TOK_CATTR_OPEN)
 _(TOK_ATTR_OPEN)
@@ -120,7 +121,6 @@ _(TOK_RWORD_SUPER)
 
 _(TOK_RWORD_PROC)
 _(TOK_RWORD_MOVE)
-_(TOK_RWORD_ONCE)
 
 _(TOK_RWORD_ABSTRACT)
 _(TOK_RWORD_FINAL)
diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp
index a12eb526..15e983f7 100644
--- a/src/parse/expr.cpp
+++ b/src/parse/expr.cpp
@@ -755,14 +755,8 @@ LEFTASSOC(Parse_Expr9, Parse_Expr10,
         rv = NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::SUB, ::std::move(rv), next(lex));

         break;

 )

-// 10: Cast

+// 10: Times / Divide / Modulo

 LEFTASSOC(Parse_Expr10, Parse_Expr11,

-    case TOK_RWORD_AS:

-        rv = NEWNODE( AST::ExprNode_Cast, ::std::move(rv), Parse_Type(lex) );

-        break;

-)

-// 11: Times / Divide / Modulo

-LEFTASSOC(Parse_Expr11, Parse_Expr12,

     case TOK_STAR:

         rv = NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::MULTIPLY, ::std::move(rv), next(lex));

         break;

@@ -773,6 +767,12 @@ LEFTASSOC(Parse_Expr11, Parse_Expr12,
         rv = NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::MODULO, ::std::move(rv), next(lex));

         break;

 )

+// 11: Cast

+LEFTASSOC(Parse_Expr11, Parse_Expr12,

+    case TOK_RWORD_AS:

+        rv = NEWNODE( AST::ExprNode_Cast, ::std::move(rv), Parse_Type(lex) );

+        break;

+)

 // 12: Unaries

 ExprNodeP Parse_ExprFC(TokenStream& lex);

 ExprNodeP Parse_Expr12(TokenStream& lex)

@@ -891,7 +891,7 @@ ExprNodeP Parse_ExprVal_StructLiteral(TokenStream& lex, AST::Path path)
     return NEWNODE( AST::ExprNode_StructLiteral, path, ::std::move(base_val), ::std::move(items) );

 }

 

-ExprNodeP Parse_ExprVal_Closure(TokenStream& lex)

+ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move)

 {

     TRACE_FUNCTION;

     Token   tok;

@@ -1041,16 +1041,37 @@ ExprNodeP Parse_ExprVal(TokenStream& lex)
             lex.putback(tok);

             return NEWNODE( AST::ExprNode_NamedValue, ::std::move(path) );

         }

+    case TOK_RWORD_MOVE:

+        // TODO: Annotate closure as move

+        GET_TOK(tok, lex);

+        if(tok.type() == TOK_PIPE)

+            return Parse_ExprVal_Closure(lex, true);

+        else if(tok.type() == TOK_DOUBLE_PIPE) {

+            lex.putback(Token(TOK_PIPE));

+            return Parse_ExprVal_Closure(lex, true);

+        }

+        else {

+            CHECK_TOK(tok, TOK_PIPE);

+        }

     case TOK_DOUBLE_PIPE:

         lex.putback(Token(TOK_PIPE));

     case TOK_PIPE:

-        return Parse_ExprVal_Closure(lex);

+        return Parse_ExprVal_Closure(lex, false);

     case TOK_INTEGER:

         return NEWNODE( AST::ExprNode_Integer, tok.intval(), tok.datatype() );

     case TOK_FLOAT:

         return NEWNODE( AST::ExprNode_Float, tok.floatval(), tok.datatype() );

     case TOK_STRING:

         return NEWNODE( AST::ExprNode_String, tok.str() );

+    case TOK_BYTESTRING: {

+        ::std::vector<ExprNodeP>    items;

+        for(char b: tok.str()) {

+            items.push_back( NEWNODE( AST::ExprNode_Integer, b, CORETYPE_U8 ) );

+        }

+        return NEWNODE( AST::ExprNode_Array, ::std::move(items) );

+        }

+        // TODO: Correct type here

+        return NEWNODE( AST::ExprNode_String, tok.str() );

     case TOK_RWORD_TRUE:

         return NEWNODE( AST::ExprNode_Bool, true );

     case TOK_RWORD_FALSE:

diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp
index 419a04d5..cac8f511 100644
--- a/src/parse/lex.cpp
+++ b/src/parse/lex.cpp
@@ -97,6 +97,8 @@ static const struct {
   TOKENT("^",  TOK_CARET),
   TOKENT("^=", TOK_CARET_EQUAL),
   TOKENT("`",  TOK_BACKTICK),
+  // a-z :: Elsewhere
+  //TOKENT("b\"", DOUBLEQUOTE),
 
   TOKENT("{",  TOK_BRACE_OPEN),
   TOKENT("|",  TOK_PIPE),
@@ -139,7 +141,6 @@ static const struct {
   TOKENT("move",    TOK_RWORD_MOVE),
   TOKENT("mut",     TOK_RWORD_MUT),
   TOKENT("offsetof",TOK_RWORD_OFFSETOF),
-  TOKENT("once",    TOK_RWORD_ONCE),
   TOKENT("override",TOK_RWORD_OVERRIDE),
   TOKENT("priv",    TOK_RWORD_PRIV),
   TOKENT("proc",    TOK_RWORD_PROC),
@@ -401,6 +402,7 @@ Token Lexer::getTokenInt()
                     return Token(val, num_type);
                 }
             }
+            // Symbols
             else if( issym(ch) )
             {
                 ::std::string   str;
@@ -416,19 +418,34 @@ Token Lexer::getTokenInt()
                 }
                 else
                 {
-                    if( str == "b" && ch == '\'' ) {
-                        // Byte constant
-                        ch = this->getc();
-                        if( ch == '\\' ) {
-                            uint32_t val = this->parseEscape('\'');
-                            if( this->getc() != '\'' )
-                                throw ParseError::Generic(*this, "Multi-byte character literal");
-                            return Token((uint64_t)val, CORETYPE_U8);
+                    if( str == "b" )
+                    {
+                        if( ch == '\'' ) {
+                            // Byte constant
+                            ch = this->getc();
+                            if( ch == '\\' ) {
+                                uint32_t val = this->parseEscape('\'');
+                                if( this->getc() != '\'' )
+                                    throw ParseError::Generic(*this, "Multi-byte character literal");
+                                return Token((uint64_t)val, CORETYPE_U8);
+                            }
+                            else {
+                                if( this->getc() != '\'' )
+                                    throw ParseError::Generic(*this, "Multi-byte character literal");
+                                return Token((uint64_t)ch, CORETYPE_U8);
+                            }
+                        }
+                        else if( ch == '"') {
+                            ::std::string str;
+                            while( (ch = this->getc()) != '"' )
+                            {
+                                if( ch == '\\' )
+                                    ch = this->parseEscape('"');
+                                str.push_back(ch);
+                            }
+                            return Token(TOK_BYTESTRING, str);
                         }
                         else {
-                            if( this->getc() != '\'' )
-                                throw ParseError::Generic(*this, "Multi-byte character literal");
-                            return Token((uint64_t)ch, CORETYPE_U8);
                         }
                     }
                 
@@ -719,6 +736,7 @@ enum eTokenType Token::typefromstr(const ::std::string& s)
     case TOK_CHAR:      return FMT("'\\u{"<< ::std::hex << m_intval << "}");
     case TOK_FLOAT:     return FMT(m_floatval);
     case TOK_STRING:    return "\"" + m_str + "\"";
+    case TOK_BYTESTRING:return "b\"" + m_str + "\"";
     case TOK_CATTR_OPEN:return "#![";
     case TOK_ATTR_OPEN: return "#[";
     case TOK_UNDERSCORE:return "_";
@@ -829,7 +847,6 @@ enum eTokenType Token::typefromstr(const ::std::string& s)
 
     case TOK_RWORD_PROC:    return "proc";
     case TOK_RWORD_MOVE:    return "move";
-    case TOK_RWORD_ONCE:    return "once";
 
     case TOK_RWORD_ABSTRACT:return "abstract";
     case TOK_RWORD_FINAL:   return "final";
diff --git a/src/parse/root.cpp b/src/parse/root.cpp
index 2ae39d91..1fd62741 100644
--- a/src/parse/root.cpp
+++ b/src/parse/root.cpp
@@ -821,6 +821,12 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl)
         impl.add_type(is_public, name, Parse_Type(lex));

         GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);

         break; }

+    case TOK_RWORD_CONST:

+        if( GET_TOK(tok, lex) != TOK_RWORD_FN )

+        {

+            throw ParseError::Todo("Associated const");

+        }

+        if( 0 )

     case TOK_RWORD_EXTERN:

         {

             abi = "C";

@@ -883,6 +889,22 @@ void Parse_Use_Wildcard(const AST::Path& base_path, ::std::function<void(AST::Pa
 {

     fcn(base_path, ""); // HACK! Empty path indicates wilcard import

 }

+void Parse_Use_Set(TokenStream& lex, const AST::Path& base_path, ::std::function<void(AST::Path, ::std::string)> fcn)

+{

+    TRACE_FUNCTION;

+

+    Token   tok;

+    do {

+        if( GET_TOK(tok, lex) == TOK_RWORD_SELF ) {

+            fcn(base_path, base_path[base_path.size()-1].name());

+        }

+        else {

+            CHECK_TOK(tok, TOK_IDENT);

+            fcn(base_path + AST::PathNode(tok.str(), {}), tok.str());

+        }

+    } while( GET_TOK(tok, lex) == TOK_COMMA );

+    lex.putback(tok);

+}

 

 void Parse_Use(TokenStream& lex, ::std::function<void(AST::Path, ::std::string)> fcn)

 {

@@ -916,6 +938,10 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::Path, ::std::string)>
             lex.putback(tok);

         }

         break;

+    case TOK_BRACE_OPEN:

+        Parse_Use_Set(lex, path, fcn);

+        GET_CHECK_TOK(tok, lex, TOK_BRACE_CLOSE);

+        return;

     default:

         throw ParseError::Unexpected(lex, tok);

     }

@@ -931,16 +957,8 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::Path, ::std::string)>
             switch( tok.type() )

             {

             case TOK_BRACE_OPEN:

-                do {

-                    if( GET_TOK(tok, lex) == TOK_RWORD_SELF ) {

-                        fcn(path, path[path.size()-1].name());

-                    }

-                    else {

-                        CHECK_TOK(tok, TOK_IDENT);

-                        fcn(path + AST::PathNode(tok.str(), {}), tok.str());

-                    }

-                } while( GET_TOK(tok, lex) == TOK_COMMA );

-                CHECK_TOK(tok, TOK_BRACE_CLOSE);

+                Parse_Use_Set(lex, path, fcn);

+                GET_CHECK_TOK(tok, lex, TOK_BRACE_CLOSE);

                 break ;

             case TOK_STAR:

                 Parse_Use_Wildcard(path, fcn);

@@ -1244,8 +1262,10 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Crate& crate, AST::Module& mod,
         lex.putback(tok);

         DEBUG("meta_items = " << meta_items);

 

+        // root-level macros

         if( GET_TOK(tok, lex) == TOK_MACRO )

         {

+            // `macro_rules! ...`

             if( tok.str() == "macro_rules" )

             {

                 Parse_MacroRules(lex, mod, mv$(meta_items));

@@ -1275,19 +1295,17 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Crate& crate, AST::Module& mod,
     

         // Module visibility

         bool    is_public = false;

-        if( GET_TOK(tok, lex) == TOK_RWORD_PUB )

-        {

+        if( GET_TOK(tok, lex) == TOK_RWORD_PUB ) {

             is_public = true;

         }

-        else

-        {

+        else {

             lex.putback(tok);

         }

 

         // The actual item!

         switch( GET_TOK(tok, lex) )

         {

-        

+        // `use ...`

         case TOK_RWORD_USE:

             Parse_Use(lex, [&mod,is_public,&path](AST::Path p, std::string s) {

                     DEBUG(path << " - use " << p << " as '" << s << "'");

@@ -1299,41 +1317,58 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Crate& crate, AST::Module& mod,
         case TOK_RWORD_EXTERN:

             switch( GET_TOK(tok, lex) )

             {

+            // `extern "<ABI>" fn ...`

+            // `extern "<ABI>" { ...`

             case TOK_STRING: {

                 ::std::string abi = tok.str();

                 switch(GET_TOK(tok, lex))

                 {

+                // `extern "<ABI>" fn ...`

                 case TOK_RWORD_FN:

-                    throw ParseError::Todo(lex, "'extern \"<ABI>\" fn'");

+                    GET_CHECK_TOK(tok, lex, TOK_IDENT);

+                    mod.add_function(is_public, tok.str(), Parse_FunctionDefWithCode(lex, abi, ::std::move(meta_items), false));

+                    break;

+                // `extern "<ABI>" { ...`

                 case TOK_BRACE_OPEN:

                     Parse_ExternBlock(lex, mod, ::std::move(abi));

                     break;

                 default:

-                    throw ParseError::Unexpected(lex, tok);

-                }

+                    throw ParseError::Unexpected(lex, tok, {TOK_RWORD_FN, TOK_BRACE_OPEN});

                 }

+                break; }

+            // `extern fn ...`

+            case TOK_RWORD_FN:

+                throw ParseError::Todo(lex, "'extern fn'");

                 break;

+            // `extern { ...`

+            case TOK_BRACE_OPEN:

+                Parse_ExternBlock(lex, mod, "C");

+                break;

+            // `extern crate "crate-name" as crate_name;`

+            // `extern crate crate_name;`

             case TOK_RWORD_CRATE: {

                 ::std::string   path, name;

-                // TODO: Handle #[macro_use]/#[macro_use(...)]

-                if( GET_TOK(tok, lex) == TOK_STRING )

+                switch( GET_TOK(tok, lex) )

                 {

+                // `extern crate "crate-name" as crate_name;`

+                // TODO: rustc no longer supports this feature

+                case TOK_STRING:

                     path = tok.str();

                     GET_CHECK_TOK(tok, lex, TOK_RWORD_AS);

                     GET_CHECK_TOK(tok, lex, TOK_IDENT);

                     name = tok.str();

-                }

-                else if( tok.type() == TOK_IDENT )

-                {

+                    break;

+                // `extern crate crate_name;`

+                case TOK_IDENT:

                     path = name = tok.str();

-                }

-                else

-                {

-                    throw ParseError::Unexpected(lex, tok);

+                    break;

+                default:

+                    throw ParseError::Unexpected(lex, tok, {TOK_STRING, TOK_IDENT});

                 }

                 crate.load_extern_crate(path);

                 mod.add_ext_crate(path, name);

             

+                // Handle #[macro_use]/#[macro_use(...)]

                 auto at = meta_items.get("macro_use");

                 if( at )

                 {

@@ -1350,21 +1385,37 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Crate& crate, AST::Module& mod,
                 GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);

                 break; }

             default:

-                throw ParseError::Unexpected(lex, tok);

+                throw ParseError::Unexpected(lex, tok, {TOK_STRING, TOK_RWORD_FN, TOK_BRACE_OPEN, TOK_RWORD_CRATE});

             }

             break;

         

+        // `const NAME`

+        // `const fn`

         case TOK_RWORD_CONST: {

-            GET_CHECK_TOK(tok, lex, TOK_IDENT);

-            ::std::string name = tok.str();

+            switch( GET_TOK(tok, lex) )

+            {

+            case TOK_IDENT: {

+                ::std::string name = tok.str();

 

-            GET_CHECK_TOK(tok, lex, TOK_COLON);

-            TypeRef type = Parse_Type(lex);

-            GET_CHECK_TOK(tok, lex, TOK_EQUAL);

-            AST::Expr val = Parse_Expr(lex, true);

-            GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);

-            mod.add_static(is_public, name, AST::Static(::std::move(meta_items), AST::Static::CONST, type, val));

+                GET_CHECK_TOK(tok, lex, TOK_COLON);

+                TypeRef type = Parse_Type(lex);

+                GET_CHECK_TOK(tok, lex, TOK_EQUAL);

+                AST::Expr val = Parse_Expr(lex, true);

+                GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);

+                mod.add_static(is_public, name, AST::Static(::std::move(meta_items), AST::Static::CONST, type, val));

+                break; }

+            case TOK_RWORD_FN: {

+                GET_CHECK_TOK(tok, lex, TOK_IDENT);

+                // - self not allowed, not prototype

+                // TODO: Mark as const

+                mod.add_function(is_public, tok.str(), Parse_FunctionDefWithCode(lex, "rust", ::std::move(meta_items), false));

+                break; }

+            default:

+                throw ParseError::Unexpected(lex, tok, {TOK_IDENT, TOK_RWORD_FN});

+            }

             break; }

+        // `static NAME`

+        // `static mut NAME`

         case TOK_RWORD_STATIC: {

             tok = lex.getToken();

             bool is_mut = false;

@@ -1388,52 +1439,64 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Crate& crate, AST::Module& mod,
                 );

             break; }

 

+        // `unsafe fn`

+        // `unsafe trait`

+        // `unsafe impl`

         case TOK_RWORD_UNSAFE:

             meta_items.push_back( AST::MetaItem("#UNSAFE") );

             switch(GET_TOK(tok, lex))

             {

+            // `unsafe fn`

             case TOK_RWORD_FN:

                 GET_CHECK_TOK(tok, lex, TOK_IDENT);

                 // - self not allowed, not prototype

+                // TODO: Mark as unsafe

                 mod.add_function(is_public, tok.str(), Parse_FunctionDefWithCode(lex, "rust", ::std::move(meta_items), false));

                 break;

+            // `unsafe trait`

             case TOK_RWORD_TRAIT: {

                 GET_CHECK_TOK(tok, lex, TOK_IDENT);

                 ::std::string name = tok.str();

                 mod.add_trait(is_public, name, Parse_TraitDef(lex, meta_items));

-                break;

-                }

+                break; }

+            // `unsafe impl`

             case TOK_RWORD_IMPL:

                 Parse_Impl(lex, mod, true);

                 break;

             default:

-                throw ParseError::Unexpected(lex, tok);

+                throw ParseError::Unexpected(lex, tok, {TOK_RWORD_FN, TOK_RWORD_TRAIT, TOK_RWORD_IMPL});

             }

             break;

+        // `fn`

         case TOK_RWORD_FN: {

             GET_CHECK_TOK(tok, lex, TOK_IDENT);

             ::std::string name = tok.str();

             // - self not allowed, not prototype

             mod.add_function(is_public, name, Parse_FunctionDefWithCode(lex, "rust", ::std::move(meta_items), false));

             break; }

+        // `type`

         case TOK_RWORD_TYPE: {

             GET_CHECK_TOK(tok, lex, TOK_IDENT);

             ::std::string name = tok.str();

             mod.add_typealias(is_public, name, Parse_TypeAlias(lex, meta_items));

             break; }

+        // `struct`

         case TOK_RWORD_STRUCT: {

             GET_CHECK_TOK(tok, lex, TOK_IDENT);

             ::std::string name = tok.str();

             mod.add_struct( is_public, name, Parse_Struct(lex, meta_items) );

             break; }

+        // `enum`

         case TOK_RWORD_ENUM: {

             GET_CHECK_TOK(tok, lex, TOK_IDENT);

             ::std::string name = tok.str();

             mod.add_enum(is_public, name, Parse_EnumDef(lex, meta_items));

             break; }

+        // `impl`

         case TOK_RWORD_IMPL:

             Parse_Impl(lex, mod);

             break;

+        // `trait`

         case TOK_RWORD_TRAIT: {

             GET_CHECK_TOK(tok, lex, TOK_IDENT);

             ::std::string name = tok.str();

@@ -1443,6 +1506,7 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Crate& crate, AST::Module& mod,
         case TOK_RWORD_MOD: {

             GET_CHECK_TOK(tok, lex, TOK_IDENT);

             ::std::string name = tok.str();

+            

             // TODO: Remove this copy, by keeping record of macro_use()

             AST::Module submod(meta_items, name);

             DEBUG("Sub module '"<<name<<"'");