Parses a nice set of files once more

4 files changed, 76 insertions, 40 deletions

diff --git a/bnf/Makefile b/bnf/Makefile
index 02cafc3a..ed5aab70 100644
--- a/bnf/Makefile
+++ b/bnf/Makefile
@@ -4,10 +4,11 @@
 all: test.bin
 
 TSTFILE := ../samples/1.rs
-TSTFILE := ../../rust_os/libcore/lib.rs ../../rust_os/libcore/str/mod.rs ../../rust_os/libcore/str/pattern.rs
+TSTFILE := $(addprefix ../../rust_os/libcore/, lib.rs str/mod.rs str/pattern.rs borrow.rs any.rs array.rs)
 
 test: test.bin $(TSTFILE)
-	for f in $(TSTFILE); do ./test.bin "$$f"; done
+	$(foreach f,$(TSTFILE), ./test.bin "$f" &&) true
+#	for f in $(TSTFILE); do ./test.bin "$$f"; done
 
 test.bin: rust.tab.c rust.lex.c
 	gcc rust.tab.c rust.lex.c -o $@
diff --git a/bnf/rust.lex b/bnf/rust.lex
index 300742e9..14ca0709 100644
--- a/bnf/rust.lex
+++ b/bnf/rust.lex
@@ -10,6 +10,7 @@ extern int yydebug;
 int rustbnf_forcetoken = 0;
 #define YY_DECL	int yylex_inner()
 
+// Wrap the real yylex with one that can yeild a pushbacked token
 int yylex() {
 	if(rustbnf_forcetoken>0) {
 		int rv = rustbnf_forcetoken;
@@ -21,6 +22,9 @@ int yylex() {
 	}
 }
 
+void handle_block_comment();
+
+const char *gsCurrentFilename = "-";
 
 %}
 
@@ -31,7 +35,7 @@ ident_c	[a-zA-Z_]
 
 "//"[^/].*\n	{ }
 "///".*\n	{ /* TODO: Handle /// by desugaring */ }
-"/*"	{ comment(); /* TODO: Handle doc comments */ }
+"/*"	{ handle_block_comment(); /* TODO: Handle doc comments */ }
 \n	/* */
 \r	/* */
 [ \t]	/* */
@@ -92,6 +96,9 @@ ident_c	[a-zA-Z_]
 ".."	{ return DOUBLEDOT; }
 "..."	{ return TRIPLEDOT; }
 
+"#!"	{ return HASHBANG; }
+
+"?"	{ return *yytext; }
 "#"	{ return *yytext; }
 "$"	{ return *yytext; }
 "&"	{ return *yytext; }
@@ -112,20 +119,20 @@ ident_c	[a-zA-Z_]
 "+"	{ return *yytext; }
 "-"	{ return *yytext; }
 
-{ident_c}({ident_c}|[0-9])*	{ yylval.text = strdup(yytext); return IDENT; }
+{ident_c}({ident_c}|[0-9])*	{ if(*yytext == '_' && yytext[1] == 0) return '_'; else { yylval.text = strdup(yytext); return IDENT; } }
 {ident_c}({ident_c}|[0-9])*"!"	{ yylval.text = strdup(yytext); return MACRO; }
 '{ident_c}{ident_c}*	{ yylval.text = strdup(yytext+1); return LIFETIME; }
 [0-9]{dec_digit}*"."{dec_digit}+	{ yylval.realnum = strtod(yytext, NULL); return FLOAT; }
 [0-9]{dec_digit}*	{ yylval.integer = strtoull(yytext, NULL, 0); return INTEGER; }
-0x[0-9a-fA-F]*	{ yylval.integer = strtoull(yytext, NULL, 0); return INTEGER; }
+0x[0-9a-fA-F_]+	{ yylval.integer = strtoull(yytext, NULL, 0); return INTEGER; }
+0b[01_]+	{ yylval.integer = strtoull(yytext, NULL, 0); return INTEGER; }
 
 b?'(.|\\['rn])'	{ yylval.text = strdup(yytext); return CHARLIT; }
 \"([^"])*\"	{ yylval.text = strdup(yytext); return STRING; }
 
-.	{ fprintf(stderr, "\x1b[31m" "ERROR: Invalid character '%c' on line %i\x1b[0m\n", *yytext, yylineno); exit(1); }
+.	{ fprintf(stderr, "\x1b[31m" "ERROR: %s:%d: Invalid character '%c'\x1b[0m\n", gsCurrentFilename, yylineno, *yytext); exit(1); }
 
 %%
-const char *gsCurrentFilename = "-";
 int main(int argc, char *argv[]) {
 	if(argc < 2 || strcmp(argv[1], "-") == 0) {
 		yyin = stdin;
@@ -134,7 +141,7 @@ int main(int argc, char *argv[]) {
 		gsCurrentFilename = argv[1];
 		yyin = fopen(argv[1], "r");
 		if( !yyin ) {
-			fprintf(stderr, "ERROR: Unable to open '%s': '%s'", argv[1], strerror(errno));
+			fprintf(stderr, "ERROR: Unable to open '%s': '%s'\n", argv[1], strerror(errno));
 			return 1;
 		}
 	}
@@ -145,6 +152,7 @@ int main(int argc, char *argv[]) {
 }
 void yyerror(const char *s) {
 	fprintf(stderr, "\x1b[31mERROR: %s:%d: yyerror(%s)\x1b[0m\n", gsCurrentFilename, yylineno, s);
+	exit(1);
 }
 int yywrap(void) {
 	printf("done\n");
@@ -152,18 +160,19 @@ int yywrap(void) {
 }
 
 // Thanks stackoverflow: http://www.lysator.liu.se/c/ANSI-C-grammar-l.html
-void comment() {
+void handle_block_comment() {
     char c, c1;
 
 loop:
-    while ((c = input()) != '*' && c != 0)
-        putchar(c);
+    while ((c = input()) != '*' && c != 0) {
+    //    putchar(c);
+    }
 
     if ((c1 = input()) != '/' && c != 0) {
         unput(c1);
         goto loop;
     }
 
-    if (c != 0)
-        putchar(c1);
+    //if (c != 0)
+    //    putchar(c1);
 }
diff --git a/bnf/rust.y b/bnf/rust.y
index 1d31e963..24f445c3 100644
--- a/bnf/rust.y
+++ b/bnf/rust.y
@@ -2,6 +2,7 @@
 %token <integer> INTEGER CHARLIT
 %token <realnum> FLOAT 
 %token DOC_COMMENT SUPER_DOC_COMMENT
+%token HASHBANG
 %token DOUBLECOLON THINARROW FATARROW DOUBLEDOT TRIPLEDOT
 %token DOUBLEEQUAL EXCLAMEQUAL DOUBLEPIPE DOUBLEAMP
 %token GTEQUAL LTEQUAL
@@ -53,6 +54,8 @@ static void yyprint(FILE *outstream, int type, YYSTYPE value)
 	}
 }
 
+
+// semi-evil hack used to break '>>' apart into '>' '>'
 extern int rustbnf_forcetoken;
 %}
 
@@ -93,8 +96,8 @@ tt_group_item: tt_paren ';' | tt_brace | tt_square ';';
 super_attrs : | super_attrs super_attr;
 
 opt_pub
- : RWD_pub	{ bnf_trace("public"); }
- | /* mt */	{ bnf_trace("private"); }
+ : /* mt */	{ bnf_trace("private"); }
+ | RWD_pub	{ bnf_trace("public"); }
  ;
 opt_comma: | ',';
 opt_semicolon: | ';';
@@ -108,7 +111,7 @@ module_body
 attrs: attrs attr | ;
 
 super_attr
- : '#' '!' '[' meta_items ']'
+ : HASHBANG /*'#' '!'*/ '[' meta_items ']'
  | SUPER_DOC_COMMENT
  ;
 attr
@@ -129,20 +132,35 @@ Root Items
 ==========================
 */
 item
- : opt_pub RWD_mod module_def
- | opt_pub fn_qualifiers RWD_fn fn_def
- | opt_pub RWD_use use_def
- | opt_pub RWD_static static_def
- | opt_pub RWD_const const_def
- | opt_pub RWD_struct struct_def
- | opt_pub RWD_enum enum_def
- | opt_pub opt_unsafe RWD_trait trait_def
+ : RWD_pub vis_item	{ /* $2.set_pub(); */ }
+ |         vis_item
+ | RWD_pub RWD_unsafe unsafe_vis_item	{ /* $2.set_pub(); */ }
+ |         RWD_unsafe unsafe_item
+ | RWD_impl impl_def
  | RWD_extern extern_block
- | opt_unsafe RWD_impl impl_def
  | MACRO IDENT tt_brace
  | MACRO tt_brace
  | MACRO tt_paren ';'
  ;
+/* Items for which visibility is valid */
+vis_item
+ : RWD_mod module_def
+ | RWD_use use_def
+ | RWD_static static_def
+ | RWD_const const_def
+ | RWD_struct struct_def
+ | RWD_enum enum_def
+ | unsafe_vis_item
+ ;
+/* Possibily unsafe visibility item */
+unsafe_vis_item
+ : fn_qualifiers RWD_fn fn_def
+ | RWD_trait trait_def
+ ;
+unsafe_item
+ : unsafe_vis_item
+ | RWD_impl impl_def
+ ;
 
 extern_block: extern_abi | '{' extern_items '}';
 extern_abi: | STRING;
@@ -191,8 +209,8 @@ fn_def_arg_PROTO
 fn_qualifiers
  :
  | RWD_extern extern_abi
- | RWD_unsafe
  | RWD_const
+ | RWD_unsafe
  | RWD_unsafe RWD_const
  ;
 
@@ -264,15 +282,15 @@ enum_variant
 
 /* --- Trait --- */
 trait_def: IDENT generic_def trait_bounds '{' trait_items '}';
-trait_bounds: ':' type_path | ;
-trait_bound_list: trait_bound_list '+' trait_bound | trait_bound;
-trait_bound: type_path | LIFETIME;
+trait_bounds: ':' trait_bound_list | ;
+trait_bound_list: trait_bound_list '+' bound | bound;
+
 trait_items: | trait_items attrs trait_item;
 trait_item
  : RWD_type IDENT ';'
  | RWD_type IDENT ':' trait_bound_list ';'
- | fn_qualifiers RWD_fn fn_def_hdr_PROTO ';'
- | fn_qualifiers RWD_fn fn_def_hdr_PROTO code 
+ | opt_unsafe fn_qualifiers RWD_fn fn_def_hdr_PROTO ';'
+ | opt_unsafe fn_qualifiers RWD_fn fn_def_hdr_PROTO code 
  ;
 
 /* --- Impl --- */
@@ -284,7 +302,7 @@ impl_def_line
  ;
 impl_items: | impl_items attrs impl_item;
 impl_item
- : opt_pub fn_qualifiers RWD_fn fn_def
+ : opt_pub opt_unsafe fn_qualifiers RWD_fn fn_def
  | opt_pub RWD_type generic_def IDENT '=' type ';'
  | MACRO tt_group_item
  ;
@@ -309,7 +327,7 @@ where_clauses
 where_clause_ent
 	: type ':' bounds;
 bounds: bounds '+' bound | bound;
-bound: LIFETIME | type_path;
+bound: LIFETIME | '?' type_path | type_path;
 
 /*
 =========================================
@@ -365,13 +383,17 @@ Types
 =========================================
 */
 type
+ : trait_list
+ | type_ele
+ ;
+type_ele
  : type_path
- | '&' type
- | '&' LIFETIME type
- | '&' RWD_mut type
- | '&' LIFETIME RWD_mut type
- | '*' RWD_const type
- | '*' RWD_mut type
+ | '&' type_ele
+ | '&' LIFETIME type_ele
+ | '&' RWD_mut type_ele
+ | '&' LIFETIME RWD_mut type_ele
+ | '*' RWD_const type_ele
+ | '*' RWD_mut type_ele
  | '[' type ']'
  | '[' type ';' expr ']'
  | '(' ')'
@@ -379,6 +401,8 @@ type
  | '(' type ',' ')'
  | '(' type ',' type_list ')'
  ;
+trait_list: type_path '+' trait_list_inner;
+trait_list_inner: type_path | trait_list_inner '+' type_path;
 type_list: type_list ',' type | type;
 
 /*
@@ -397,7 +421,7 @@ struct_pattern_items: struct_pattern_items ',' struct_pattern_item | struct_patt
 
 pattern
  : /*IDENT	{ /* maybe bind * / }
- */| IDENT '@' nonbind_pattern
+ |*/ IDENT '@' nonbind_pattern
  | RWD_ref IDENT
  | RWD_ref IDENT '@' nonbind_pattern
  | RWD_mut IDENT
diff --git a/bnf/rust_expr.y_tree.h b/bnf/rust_expr.y_tree.h
index 489cc08b..e7358e2f 100644
--- a/bnf/rust_expr.y_tree.h
+++ b/bnf/rust_expr.y_tree.h
@@ -106,6 +106,8 @@ _(expr_value)
  | '(' expr ')'
  | '(' ')'
  | '(' expr ',' expr_list ')'
+ | '[' expr_list opt_comma ']'
+ | '[' expr ';' expr ']'
  | MACRO tt_paren	{ bnf_trace("Expr macro invocation"); }
  | '|' pattern_list '|' expr
  | DOUBLEPIPE expr