// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information. using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Web.Razor.Editor; using System.Web.Razor.Generator; using System.Web.Razor.Parser.SyntaxTree; using System.Web.Razor.Resources; using System.Web.Razor.Text; using System.Web.Razor.Tokenizer; using System.Web.Razor.Tokenizer.Symbols; using System.Web.Razor.Utils; namespace System.Web.Razor.Parser { public abstract partial class TokenizerBackedParser : ParserBase where TTokenizer : Tokenizer where TSymbol : SymbolBase { // Helpers [SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic", Justification = "This only occurs in Release builds, where this method is empty by design")] [Conditional("DEBUG")] internal void Assert(TSymbolType expectedType) { Debug.Assert(!EndOfFile && Equals(CurrentSymbol.Type, expectedType)); } protected internal void PutBack(TSymbol symbol) { if (symbol != null) { Tokenizer.PutBack(symbol); } } /// /// Put the specified symbols back in the input stream. The provided list MUST be in the ORDER THE SYMBOLS WERE READ. The /// list WILL be reversed and the Putback(TSymbol) will be called on each item. /// /// /// If a document contains symbols: a, b, c, d, e, f /// and AcceptWhile or AcceptUntil is used to collect until d /// the list returned by AcceptWhile/Until will contain: a, b, c IN THAT ORDER /// that is the correct format for providing to this method. The caller of this method would, /// in that case, want to put c, b and a back into the stream, so "a, b, c" is the CORRECT order /// protected internal void PutBack(IEnumerable symbols) { foreach (TSymbol symbol in symbols.Reverse()) { PutBack(symbol); } } protected internal void PutCurrentBack() { if (!EndOfFile && CurrentSymbol != null) { PutBack(CurrentSymbol); } } protected internal bool Balance(BalancingModes mode) { TSymbolType left = CurrentSymbol.Type; TSymbolType right = Language.FlipBracket(left); SourceLocation start = CurrentLocation; AcceptAndMoveNext(); if (EndOfFile && !mode.HasFlag(BalancingModes.NoErrorOnFailure)) { Context.OnError(start, RazorResources.ParseError_Expected_CloseBracket_Before_EOF, Language.GetSample(left), Language.GetSample(right)); } return Balance(mode, left, right, start); } protected internal bool Balance(BalancingModes mode, TSymbolType left, TSymbolType right, SourceLocation start) { int startPosition = CurrentLocation.AbsoluteIndex; int nesting = 1; if (!EndOfFile) { IList syms = new List(); do { if (IsAtEmbeddedTransition( mode.HasFlag(BalancingModes.AllowCommentsAndTemplates), mode.HasFlag(BalancingModes.AllowEmbeddedTransitions))) { Accept(syms); syms.Clear(); HandleEmbeddedTransition(); // Reset backtracking since we've already outputted some spans. startPosition = CurrentLocation.AbsoluteIndex; } if (At(left)) { nesting++; } else if (At(right)) { nesting--; } if (nesting > 0) { syms.Add(CurrentSymbol); } } while (nesting > 0 && NextToken()); if (nesting > 0) { if (!mode.HasFlag(BalancingModes.NoErrorOnFailure)) { Context.OnError(start, RazorResources.ParseError_Expected_CloseBracket_Before_EOF, Language.GetSample(left), Language.GetSample(right)); } if (mode.HasFlag(BalancingModes.BacktrackOnFailure)) { Context.Source.Position = startPosition; NextToken(); } else { Accept(syms); } } else { // Accept all the symbols we saw Accept(syms); } } return nesting == 0; } protected internal bool NextIs(TSymbolType type) { return NextIs(sym => sym != null && Equals(type, sym.Type)); } protected internal bool NextIs(params TSymbolType[] types) { return NextIs(sym => sym != null && types.Any(t => Equals(t, sym.Type))); } protected internal bool NextIs(Func condition) { TSymbol cur = CurrentSymbol; NextToken(); bool result = condition(CurrentSymbol); PutCurrentBack(); PutBack(cur); EnsureCurrent(); return result; } protected internal bool Was(TSymbolType type) { return PreviousSymbol != null && Equals(PreviousSymbol.Type, type); } protected internal bool At(TSymbolType type) { return !EndOfFile && CurrentSymbol != null && Equals(CurrentSymbol.Type, type); } protected internal bool AcceptAndMoveNext() { Accept(CurrentSymbol); return NextToken(); } protected TSymbol AcceptSingleWhiteSpaceCharacter() { if (Language.IsWhiteSpace(CurrentSymbol)) { Tuple pair = Language.SplitSymbol(CurrentSymbol, 1, Language.GetKnownSymbolType(KnownSymbolType.WhiteSpace)); Accept(pair.Item1); Span.EditHandler.AcceptedCharacters = AcceptedCharacters.None; NextToken(); return pair.Item2; } return null; } protected internal void Accept(IEnumerable symbols) { foreach (TSymbol symbol in symbols) { Accept(symbol); } } protected internal void Accept(TSymbol symbol) { if (symbol != null) { foreach (RazorError error in symbol.Errors) { Context.Errors.Add(error); } Span.Accept(symbol); } } protected internal bool AcceptAll(params TSymbolType[] types) { foreach (TSymbolType type in types) { if (CurrentSymbol == null || !Equals(CurrentSymbol.Type, type)) { return false; } AcceptAndMoveNext(); } return true; } protected internal void AddMarkerSymbolIfNecessary() { AddMarkerSymbolIfNecessary(CurrentLocation); } protected internal void AddMarkerSymbolIfNecessary(SourceLocation location) { if (Span.Symbols.Count == 0 && Context.LastAcceptedCharacters != AcceptedCharacters.Any) { Accept(Language.CreateMarkerSymbol(location)); } } protected internal void Output(SpanKind kind) { Configure(kind, null); Output(); } protected internal void Output(SpanKind kind, AcceptedCharacters accepts) { Configure(kind, accepts); Output(); } protected internal void Output(AcceptedCharacters accepts) { Configure(null, accepts); Output(); } private void Output() { if (Span.Symbols.Count > 0) { Context.AddSpan(Span.Build()); Initialize(Span); } } protected IDisposable PushSpanConfig() { return PushSpanConfig(newConfig: (Action>)null); } protected IDisposable PushSpanConfig(Action newConfig) { return PushSpanConfig(newConfig == null ? (Action>)null : (span, _) => newConfig(span)); } [SuppressMessage("Microsoft.Design", "CA1006:DoNotNestGenericTypesInMemberSignatures", Justification = "The Action parameters are preferred over custom delegates")] protected IDisposable PushSpanConfig(Action> newConfig) { Action old = SpanConfig; ConfigureSpan(newConfig); return new DisposableAction(() => SpanConfig = old); } protected void ConfigureSpan(Action config) { SpanConfig = config; Initialize(Span); } [SuppressMessage("Microsoft.Design", "CA1006:DoNotNestGenericTypesInMemberSignatures", Justification = "The Action parameters are preferred over custom delegates")] protected void ConfigureSpan(Action> config) { Action prev = SpanConfig; if (config == null) { SpanConfig = null; } else { SpanConfig = span => config(span, prev); } Initialize(Span); } protected internal void Expected(KnownSymbolType type) { Expected(Language.GetKnownSymbolType(type)); } [SuppressMessage("Microsoft.Usage", "CA1801:ReviewUnusedParameters", MessageId = "types", Justification = "It is used in debug builds")] protected internal void Expected(params TSymbolType[] types) { Debug.Assert(!EndOfFile && CurrentSymbol != null && types.Contains(CurrentSymbol.Type)); AcceptAndMoveNext(); } protected internal bool Optional(KnownSymbolType type) { return Optional(Language.GetKnownSymbolType(type)); } protected internal bool Optional(TSymbolType type) { if (At(type)) { AcceptAndMoveNext(); return true; } return false; } protected internal bool Required(TSymbolType expected, bool errorIfNotFound, string errorBase) { bool found = At(expected); if (!found && errorIfNotFound) { string error; if (Language.IsNewLine(CurrentSymbol)) { error = RazorResources.ErrorComponent_Newline; } else if (Language.IsWhiteSpace(CurrentSymbol)) { error = RazorResources.ErrorComponent_Whitespace; } else if (EndOfFile) { error = RazorResources.ErrorComponent_EndOfFile; } else { error = String.Format(CultureInfo.CurrentCulture, RazorResources.ErrorComponent_Character, CurrentSymbol.Content); } Context.OnError( CurrentLocation, errorBase, error); } return found; } protected bool EnsureCurrent() { if (CurrentSymbol == null) { return NextToken(); } return true; } protected internal void AcceptWhile(TSymbolType type) { AcceptWhile(sym => Equals(type, sym.Type)); } // We want to avoid array allocations and enumeration where possible, so we use the same technique as String.Format protected internal void AcceptWhile(TSymbolType type1, TSymbolType type2) { AcceptWhile(sym => Equals(type1, sym.Type) || Equals(type2, sym.Type)); } protected internal void AcceptWhile(TSymbolType type1, TSymbolType type2, TSymbolType type3) { AcceptWhile(sym => Equals(type1, sym.Type) || Equals(type2, sym.Type) || Equals(type3, sym.Type)); } protected internal void AcceptWhile(params TSymbolType[] types) { AcceptWhile(sym => types.Any(expected => Equals(expected, sym.Type))); } protected internal void AcceptUntil(TSymbolType type) { AcceptWhile(sym => !Equals(type, sym.Type)); } // We want to avoid array allocations and enumeration where possible, so we use the same technique as String.Format protected internal void AcceptUntil(TSymbolType type1, TSymbolType type2) { AcceptWhile(sym => !Equals(type1, sym.Type) && !Equals(type2, sym.Type)); } protected internal void AcceptUntil(TSymbolType type1, TSymbolType type2, TSymbolType type3) { AcceptWhile(sym => !Equals(type1, sym.Type) && !Equals(type2, sym.Type) && !Equals(type3, sym.Type)); } protected internal void AcceptUntil(params TSymbolType[] types) { AcceptWhile(sym => types.All(expected => !Equals(expected, sym.Type))); } protected internal void AcceptWhile(Func condition) { Accept(ReadWhileLazy(condition)); } protected internal IEnumerable ReadWhile(Func condition) { return ReadWhileLazy(condition).ToList(); } protected TSymbol AcceptWhiteSpaceInLines() { TSymbol lastWs = null; while (Language.IsWhiteSpace(CurrentSymbol) || Language.IsNewLine(CurrentSymbol)) { // Capture the previous whitespace node if (lastWs != null) { Accept(lastWs); } if (Language.IsWhiteSpace(CurrentSymbol)) { lastWs = CurrentSymbol; } else if (Language.IsNewLine(CurrentSymbol)) { // Accept newline and reset last whitespace tracker Accept(CurrentSymbol); lastWs = null; } Tokenizer.Next(); } return lastWs; } protected bool AtIdentifier(bool allowKeywords) { return CurrentSymbol != null && (Language.IsIdentifier(CurrentSymbol) || (allowKeywords && Language.IsKeyword(CurrentSymbol))); } // Don't open this to sub classes because it's lazy but it looks eager. // You have to advance the Enumerable to read the next characters. internal IEnumerable ReadWhileLazy(Func condition) { while (EnsureCurrent() && condition(CurrentSymbol)) { yield return CurrentSymbol; NextToken(); } } private void Configure(SpanKind? kind, AcceptedCharacters? accepts) { if (kind != null) { Span.Kind = kind.Value; } if (accepts != null) { Span.EditHandler.AcceptedCharacters = accepts.Value; } } protected virtual void OutputSpanBeforeRazorComment() { throw new InvalidOperationException(RazorResources.Language_Does_Not_Support_RazorComment); } private void CommentSpanConfig(SpanBuilder span) { span.CodeGenerator = SpanCodeGenerator.Null; span.EditHandler = SpanEditHandler.CreateDefault(Language.TokenizeString); } protected void RazorComment() { if (!Language.KnowsSymbolType(KnownSymbolType.CommentStart) || !Language.KnowsSymbolType(KnownSymbolType.CommentStar) || !Language.KnowsSymbolType(KnownSymbolType.CommentBody)) { throw new InvalidOperationException(RazorResources.Language_Does_Not_Support_RazorComment); } OutputSpanBeforeRazorComment(); using (PushSpanConfig(CommentSpanConfig)) { using (Context.StartBlock(BlockType.Comment)) { Context.CurrentBlock.CodeGenerator = new RazorCommentCodeGenerator(); SourceLocation start = CurrentLocation; Expected(KnownSymbolType.CommentStart); Output(SpanKind.Transition, AcceptedCharacters.None); Expected(KnownSymbolType.CommentStar); Output(SpanKind.MetaCode, AcceptedCharacters.None); Optional(KnownSymbolType.CommentBody); AddMarkerSymbolIfNecessary(); Output(SpanKind.Comment); bool errorReported = false; if (!Optional(KnownSymbolType.CommentStar)) { errorReported = true; Context.OnError(start, RazorResources.ParseError_RazorComment_Not_Terminated); } else { Output(SpanKind.MetaCode, AcceptedCharacters.None); } if (!Optional(KnownSymbolType.CommentStart)) { if (!errorReported) { errorReported = true; Context.OnError(start, RazorResources.ParseError_RazorComment_Not_Terminated); } } else { Output(SpanKind.Transition, AcceptedCharacters.None); } } } Initialize(Span); } } }