1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer;
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
using Assert = Microsoft.TestCommon.AssertEx;
namespace System.Web.Razor.Test.Tokenizer
{
public class CSharpTokenizerTest : CSharpTokenizerTestBase
{
[Fact]
public void Constructor_Throws_ArgNull_If_Null_Source_Provided()
{
Assert.ThrowsArgumentNull(() => new CSharpTokenizer(null), "source");
}
[Fact]
public void Next_Returns_Null_When_EOF_Reached()
{
TestTokenizer("");
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_CR()
{
TestTokenizer("\r\ra",
new CSharpSymbol(0, 0, 0, "\r", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\r", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_LF()
{
TestTokenizer("\n\na",
new CSharpSymbol(0, 0, 0, "\n", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\n", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_NEL()
{
// NEL: Unicode "Next Line" U+0085
TestTokenizer("\u0085\u0085a",
new CSharpSymbol(0, 0, 0, "\u0085", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\u0085", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_Line_Separator()
{
// Unicode "Line Separator" U+2028
TestTokenizer("\u2028\u2028a",
new CSharpSymbol(0, 0, 0, "\u2028", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\u2028", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_Paragraph_Separator()
{
// Unicode "Paragraph Separator" U+2029
TestTokenizer("\u2029\u2029a",
new CSharpSymbol(0, 0, 0, "\u2029", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\u2029", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Single_Newline_Token_For_CRLF()
{
TestTokenizer("\r\n\r\na",
new CSharpSymbol(0, 0, 0, "\r\n", CSharpSymbolType.NewLine),
new CSharpSymbol(2, 1, 0, "\r\n", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Token_For_Whitespace_Characters()
{
TestTokenizer(" \f\t\u000B \n ",
new CSharpSymbol(0, 0, 0, " \f\t\u000B ", CSharpSymbolType.WhiteSpace),
new CSharpSymbol(5, 0, 5, "\n", CSharpSymbolType.NewLine),
new CSharpSymbol(6, 1, 0, " ", CSharpSymbolType.WhiteSpace));
}
[Fact]
public void Transition_Is_Recognized()
{
TestSingleToken("@", CSharpSymbolType.Transition);
}
[Fact]
public void Transition_Is_Recognized_As_SingleCharacter()
{
TestTokenizer("@(",
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.Transition),
new CSharpSymbol(1, 0, 1, "(", CSharpSymbolType.LeftParenthesis));
}
}
}
|
