1 package de.fhdw.wtf.dsl.scanner.test;
2
3 import static org.junit.Assert.assertEquals;
4 import static org.junit.Assert.assertTrue;
5
6 import org.junit.Test;
7
8 import de.fhdw.wtf.common.stream.SimpleScannerInput;
9 import de.fhdw.wtf.common.stream.SimpleTokenStream;
10 import de.fhdw.wtf.common.stream.TokenStream;
11 import de.fhdw.wtf.common.token.CommentTextToken;
12 import de.fhdw.wtf.common.token.EndToken;
13 import de.fhdw.wtf.common.token.IdentifierToken;
14 import de.fhdw.wtf.common.token.InvalidToken;
15 import de.fhdw.wtf.common.token.Position;
16 import de.fhdw.wtf.common.token.WhitespaceToken;
17 import de.fhdw.wtf.common.token.keywords.AbstractToken;
18 import de.fhdw.wtf.common.token.keywords.ClassToken;
19 import de.fhdw.wtf.common.token.keywords.ExceptionToken;
20 import de.fhdw.wtf.common.token.keywords.FindableToken;
21 import de.fhdw.wtf.common.token.keywords.GroupToken;
22 import de.fhdw.wtf.common.token.keywords.MutableToken;
23 import de.fhdw.wtf.common.token.keywords.PriorToken;
24 import de.fhdw.wtf.common.token.keywords.ServiceToken;
25 import de.fhdw.wtf.common.token.keywords.SymmetricToken;
26 import de.fhdw.wtf.common.token.keywords.TransientToken;
27 import de.fhdw.wtf.common.token.symbols.ArrowToken;
28 import de.fhdw.wtf.common.token.symbols.AsteriskToken;
29 import de.fhdw.wtf.common.token.symbols.BracketCloseToken;
30 import de.fhdw.wtf.common.token.symbols.BracketOpenToken;
31 import de.fhdw.wtf.common.token.symbols.ColonToken;
32 import de.fhdw.wtf.common.token.symbols.CurlyBracketCloseToken;
33 import de.fhdw.wtf.common.token.symbols.CurlyBracketOpenToken;
34 import de.fhdw.wtf.common.token.symbols.DoubleSquareBracketCloseToken;
35 import de.fhdw.wtf.common.token.symbols.DoubleSquareBracketOpenToken;
36 import de.fhdw.wtf.common.token.symbols.EqualToken;
37 import de.fhdw.wtf.common.token.symbols.ExclamationToken;
38 import de.fhdw.wtf.common.token.symbols.GreaterSymbolToken;
39 import de.fhdw.wtf.common.token.symbols.PlusSymbolToken;
40 import de.fhdw.wtf.common.token.symbols.SemicolonToken;
41 import de.fhdw.wtf.common.token.symbols.SquareBracketCloseToken;
42 import de.fhdw.wtf.common.token.symbols.SquareBracketOpenToken;
43 import de.fhdw.wtf.dsl.scanner.common.Scanner;
44 import de.fhdw.wtf.dsl.scanner.modelScanner.ModelDslScanner;
45
46
47
48
49
50 public class ScannerTest {
51
52 @Test
53 public void digitIdentifierTest() {
54 final SimpleScannerInput input = new SimpleScannerInput("9Hallo");
55 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
56 final Scanner scanner = ModelDslScanner.create();
57 scanner.scan(input, output);
58 assertEquals(InvalidToken.create("9", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
59 assertEquals(IdentifierToken.create("Hallo", Position.create("", 0 + 1, 1 + 1, 1)), output.next());
60 assertEquals(EndToken.create(Position.create("", 0 + 1, 6 + 1, 6)), output.next());
61 }
62
63 @Test
64 public void singleSymol() {
65 final SimpleScannerInput input = new SimpleScannerInput("[]{}+:;=>*");
66 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
67 final Scanner scanner = ModelDslScanner.create();
68 scanner.scan(input, output);
69 assertEquals(true, output.hasNext());
70 assertEquals(SquareBracketOpenToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
71 assertEquals(SquareBracketCloseToken.create(Position.create("", 0 + 1, 1 + 1, 1)), output.next());
72 assertEquals(CurlyBracketOpenToken.create(Position.create("", 0 + 1, 2 + 1, 2)), output.next());
73 assertEquals(CurlyBracketCloseToken.create(Position.create("", 0 + 1, 3 + 1, 3)), output.next());
74 assertEquals(PlusSymbolToken.create(Position.create("", 0 + 1, 4 + 1, 4)), output.next());
75 assertEquals(ColonToken.create(Position.create("", 0 + 1, 5 + 1, 5)), output.next());
76 assertEquals(SemicolonToken.create(Position.create("", 0 + 1, 6 + 1, 6)), output.next());
77 assertEquals(EqualToken.create(Position.create("", 0 + 1, 7 + 1, 7)), output.next());
78 assertEquals(GreaterSymbolToken.create(Position.create("", 0 + 1, 8 + 1, 8)), output.next());
79 assertEquals(AsteriskToken.create(Position.create("", 0 + 1, 9 + 1, 9)), output.next());
80 assertEquals(EndToken.create(Position.create("", 0 + 1, 10 + 1, 10)), output.peek());
81 output.removeFirst();
82 assertEquals(false, output.hasNext());
83 }
84
85 @Test
86 public void keywords() {
87 final SimpleScannerInput input = new SimpleScannerInput("abstract class group prior");
88 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
89 final Scanner scanner = ModelDslScanner.create();
90 scanner.scan(input, output);
91 assertEquals(AbstractToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
92 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 8 + 1, 8)), output.next());
93 assertEquals(ClassToken.create(Position.create("", 0 + 1, 9 + 1, 9)), output.next());
94 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 14 + 1, 14)), output.next());
95 assertEquals(GroupToken.create(Position.create("", 0 + 1, 15 + 1, 15)), output.next());
96 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 20 + 1, 20)), output.next());
97 assertEquals(PriorToken.create(Position.create("", 0 + 1, 21 + 1, 21)), output.next());
98 assertEquals(EndToken.create(Position.create("", 0 + 1, 26 + 1, 26)), output.next());
99 }
100
101 @Test
102 public void identifiers() {
103 final SimpleScannerInput input =
104 new SimpleScannerInput(
105 "Bond007 what_is_the_answer_to_life_the_universe_and_everything ThreadsafeButton_Channel SimpleNodeTester");
106 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
107 final Scanner scanner = ModelDslScanner.create();
108 scanner.scan(input, output);
109 assertEquals(IdentifierToken.create("Bond007", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
110 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 7 + 1, 7)), output.next());
111 assertEquals(
112 IdentifierToken.create(
113 "what_is_the_answer_to_life_the_universe_and_everything",
114 Position.create("", 0 + 1, 8 + 1, 8)),
115 output.next());
116 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 62 + 1, 62)), output.next());
117 assertEquals(
118 IdentifierToken.create("ThreadsafeButton_Channel", Position.create("", 0 + 1, 64 + 1, 64)),
119 output.next());
120 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 88 + 1, 88)), output.next());
121 assertEquals(IdentifierToken.create("SimpleNodeTester", Position.create("", 0 + 1, 89 + 1, 89)), output.next());
122 assertEquals(EndToken.create(Position.create("", 0 + 1, 105 + 1, 105)), output.next());
123 }
124
125 @Test
126 public void classInGroup() {
127 final SimpleScannerInput input = new SimpleScannerInput("ichbingruppe:group=[IchBinEINEkLASSE:class={}]");
128 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
129 final Scanner scanner = ModelDslScanner.create();
130 scanner.scan(input, output);
131 assertEquals(IdentifierToken.create("ichbingruppe", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
132 assertEquals(ColonToken.create(Position.create("", 0 + 1, 12 + 1, 12)), output.next());
133 assertEquals(GroupToken.create(Position.create("", 0 + 1, 13 + 1, 13)), output.next());
134 assertEquals(EqualToken.create(Position.create("", 0 + 1, 18 + 1, 18)), output.next());
135 assertEquals(SquareBracketOpenToken.create(Position.create("", 0 + 1, 19 + 1, 19)), output.next());
136 assertEquals(IdentifierToken.create("IchBinEINEkLASSE", Position.create("", 0 + 1, 20 + 1, 20)), output.next());
137 assertEquals(ColonToken.create(Position.create("", 0 + 1, 36 + 1, 36)), output.next());
138 assertEquals(ClassToken.create(Position.create("", 0 + 1, 37 + 1, 37)), output.next());
139 assertEquals(EqualToken.create(Position.create("", 0 + 1, 42 + 1, 42)), output.next());
140 assertEquals(CurlyBracketOpenToken.create(Position.create("", 0 + 1, 43 + 1, 43)), output.next());
141 assertEquals(CurlyBracketCloseToken.create(Position.create("", 0 + 1, 44 + 1, 44)), output.next());
142 assertEquals(SquareBracketCloseToken.create(Position.create("", 0 + 1, 45 + 1, 45)), output.next());
143 assertEquals(EndToken.create(Position.create("", 0 + 1, 46 + 1, 46)), output.next());
144 }
145
146 @Test
147 public void testInvalidChar() {
148 final SimpleScannerInput input = new SimpleScannerInput("^ichbingruppe:group=[IchBinEINEkLASSE:class^={}]");
149 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
150 final Scanner scanner = ModelDslScanner.create();
151 scanner.scan(input, output);
152 assertEquals(InvalidToken.create('^', Position.create("", 0 + 1, 0 + 1, 0)), output.next());
153 assertEquals(IdentifierToken.create("ichbingruppe", Position.create("", 0 + 1, 1 + 1, 1)), output.next());
154 assertEquals(ColonToken.create(Position.create("", 0 + 1, 13 + 1, 13)), output.next());
155 assertEquals(GroupToken.create(Position.create("", 0 + 1, 14 + 1, 14)), output.next());
156 assertEquals(EqualToken.create(Position.create("", 0 + 1, 19 + 1, 19)), output.next());
157 assertEquals(SquareBracketOpenToken.create(Position.create("", 0 + 1, 20 + 1, 20)), output.next());
158 assertEquals(IdentifierToken.create("IchBinEINEkLASSE", Position.create("", 0 + 1, 21 + 1, 21)), output.next());
159 assertEquals(ColonToken.create(Position.create("", 0 + 1, 37 + 1, 37)), output.next());
160 assertEquals(ClassToken.create(Position.create("", 0 + 1, 38 + 1, 38)), output.next());
161 assertEquals(InvalidToken.create('^', Position.create("", 0 + 1, 43 + 1, 43)), output.next());
162 assertEquals(EqualToken.create(Position.create("", 0 + 1, 44 + 1, 44)), output.next());
163 assertEquals(CurlyBracketOpenToken.create(Position.create("", 0 + 1, 45 + 1, 45)), output.next());
164 assertEquals(CurlyBracketCloseToken.create(Position.create("", 0 + 1, 46 + 1, 46)), output.next());
165 assertEquals(SquareBracketCloseToken.create(Position.create("", 0 + 1, 47 + 1, 47)), output.next());
166 assertEquals(EndToken.create(Position.create("", 0 + 1, 48 + 1, 48)), output.next());
167 }
168
169 @Test
170 public void testClassNames() {
171 final SimpleScannerInput input = new SimpleScannerInput("IchBinEINEkLASSE:class={};className:class={};");
172 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
173 final Scanner scanner = ModelDslScanner.create();
174 scanner.scan(input, output);
175 assertEquals(IdentifierToken.create("IchBinEINEkLASSE", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
176 assertTrue(output.next() instanceof ColonToken);
177 assertTrue(output.next() instanceof ClassToken);
178 assertTrue(output.next() instanceof EqualToken);
179 assertTrue(output.next() instanceof CurlyBracketOpenToken);
180 assertTrue(output.next() instanceof CurlyBracketCloseToken);
181 assertTrue(output.next() instanceof SemicolonToken);
182 assertEquals(IdentifierToken.create("className", Position.create("", 0 + 1, 26 + 1, 26)), output.next());
183 assertTrue(output.next() instanceof ColonToken);
184 assertTrue(output.next() instanceof ClassToken);
185 assertTrue(output.next() instanceof EqualToken);
186 assertTrue(output.next() instanceof CurlyBracketOpenToken);
187 assertTrue(output.next() instanceof CurlyBracketCloseToken);
188 assertTrue(output.next() instanceof SemicolonToken);
189 assertEquals(EndToken.create(Position.create("", 0 + 1, 45 + 1, 45)), output.next());
190 }
191
192 @Test
193 public void invalid() {
194 final SimpleScannerInput input = new SimpleScannerInput("abstract &hallo");
195 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
196 final Scanner scanner = ModelDslScanner.create();
197 scanner.scan(input, output);
198 assertEquals(AbstractToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
199 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 8 + 1, 8)), output.next());
200 assertEquals(InvalidToken.create("&", Position.create("", 0 + 1, 10 + 1, 10)), output.next());
201 assertEquals(IdentifierToken.create("hallo", Position.create("", 0 + 1, 11 + 1, 11)), output.next());
202 assertEquals(EndToken.create(Position.create("", 0 + 1, 16 + 1, 16)), output.next());
203 }
204
205 @Test
206 public void testComment1() {
207 final SimpleScannerInput input = new SimpleScannerInput("#Ich bin ein Kommentar#");
208 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
209 final Scanner scanner = ModelDslScanner.create();
210 scanner.scan(input, output);
211 assertEquals(
212 CommentTextToken.create("#Ich bin ein Kommentar#", Position.create("", 0 + 1, 0 + 1, 0)),
213 output.next());
214 assertEquals(EndToken.create(Position.create("", 0 + 1, 23 + 1, 23)), output.next());
215 }
216
217 @Test
218 public void testComment2() {
219 final SimpleScannerInput input = new SimpleScannerInput("class#Ich bin ein Kommentar#;");
220 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
221 final Scanner scanner = ModelDslScanner.create();
222 scanner.scan(input, output);
223 assertTrue(output.next() instanceof ClassToken);
224 assertEquals(
225 CommentTextToken.create("#Ich bin ein Kommentar#", Position.create("", 0 + 1, 5 + 1, 5)),
226 output.next());
227 assertTrue(output.next() instanceof SemicolonToken);
228 assertEquals(EndToken.create(Position.create("", 0 + 1, 29 + 1, 29)), output.next());
229 }
230
231 @Test
232 public void testComment3() {
233 final SimpleScannerInput input = new SimpleScannerInput("#Ich bin ein Kommentar;");
234 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
235 final Scanner scanner = ModelDslScanner.create();
236 scanner.scan(input, output);
237 assertEquals(
238 CommentTextToken.create("#Ich bin ein Kommentar;#", Position.create("", 0 + 1, 0 + 1, 0)),
239 output.next());
240 assertEquals(EndToken.create(Position.create("", 0 + 1, 23 + 1, 23)), output.next());
241
242 }
243
244 @Test
245 public void testComment4() {
246 final SimpleScannerInput input = new SimpleScannerInput("#auto:class={};");
247 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
248 final Scanner scanner = ModelDslScanner.create();
249 scanner.scan(input, output);
250 assertEquals(CommentTextToken.create("#auto:class={};#", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
251 assertEquals(EndToken.create(Position.create("", 0 + 1, 15 + 1, 15)), output.next());
252
253 }
254
255 @Test
256 public void testComment5() {
257 final SimpleScannerInput input = new SimpleScannerInput("#######");
258 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
259 final Scanner scanner = ModelDslScanner.create();
260 scanner.scan(input, output);
261 assertEquals(CommentTextToken.create("##", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
262 assertEquals(CommentTextToken.create("##", Position.create("", 0 + 1, 2 + 1, 2)), output.next());
263 assertEquals(CommentTextToken.create("##", Position.create("", 0 + 1, 4 + 1, 4)), output.next());
264 assertEquals(CommentTextToken.create("##", Position.create("", 0 + 1, 6 + 1, 6)), output.next());
265 assertEquals(EndToken.create(Position.create("", 0 + 1, 7 + 1, 7)), output.next());
266
267 }
268
269 @Test
270 public void testComment6() {
271 final SimpleScannerInput input = new SimpleScannerInput("/*Kommentar;*/");
272 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
273 final Scanner scanner = ModelDslScanner.create();
274 scanner.scan(input, output);
275 assertEquals(InvalidToken.create("/", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
276 assertEquals(AsteriskToken.create(Position.create("", 0 + 1, 1 + 1, 1)), output.next());
277 assertEquals(IdentifierToken.create("Kommentar", Position.create("", 0 + 1, 2 + 1, 2)), output.next());
278 assertEquals(SemicolonToken.create(Position.create("", 0 + 1, 11 + 1, 11)), output.next());
279 assertEquals(AsteriskToken.create(Position.create("", 0 + 1, 12 + 1, 12)), output.next());
280 assertEquals(InvalidToken.create("/", Position.create("", 0 + 1, 13 + 1, 13)), output.next());
281 assertEquals(EndToken.create(Position.create("", 0 + 1, 14 + 1, 14)), output.next());
282
283 }
284
285 @Test
286 public void testCommentEmpty() {
287 final SimpleScannerInput input = new SimpleScannerInput("##");
288 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
289 final Scanner scanner = ModelDslScanner.create();
290 scanner.scan(input, output);
291 assertEquals(CommentTextToken.create("##", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
292 assertEquals(EndToken.create(Position.create("", 0 + 1, 2 + 1, 2)), output.next());
293 }
294
295 @Test
296 public void testWhitespace() {
297 final SimpleScannerInput input = new SimpleScannerInput("abstract hallo class");
298 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
299 final Scanner scanner = ModelDslScanner.create();
300 scanner.scan(input, output);
301 assertEquals(AbstractToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
302 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 8 + 1, 8)), output.next());
303 assertEquals(IdentifierToken.create("hallo", Position.create("", 0 + 1, 9 + 1, 9)), output.next());
304 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 14 + 1, 14)), output.next());
305 assertEquals(ClassToken.create(Position.create("", 0 + 1, 15 + 1, 15)), output.next());
306 assertEquals(EndToken.create(Position.create("", 0 + 1, 20 + 1, 20)), output.next());
307 }
308
309 @Test
310 public void testWhitespace2() {
311 final SimpleScannerInput input = new SimpleScannerInput(" abstract hallo class");
312 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
313 final Scanner scanner = ModelDslScanner.create();
314 scanner.scan(input, output);
315 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
316 assertEquals(AbstractToken.create(Position.create("", 0 + 1, 2 + 1, 2)), output.next());
317 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 10 + 1, 10)), output.next());
318 assertEquals(IdentifierToken.create("hallo", Position.create("", 0 + 1, 14 + 1, 14)), output.next());
319 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 19 + 1, 19)), output.next());
320 assertEquals(ClassToken.create(Position.create("", 0 + 1, 24 + 1, 24)), output.next());
321 assertEquals(EndToken.create(Position.create("", 0 + 1, 29 + 1, 29)), output.next());
322 }
323
324 @Test
325 public void testTab() {
326 final SimpleScannerInput input = new SimpleScannerInput(" abstract hallo class");
327 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
328 final Scanner scanner = ModelDslScanner.create();
329 scanner.scan(input, output);
330 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
331 assertEquals(AbstractToken.create(Position.create("", 0 + 1, 1 + 1, 1)), output.next());
332 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 9 + 1, 9)), output.next());
333 assertEquals(IdentifierToken.create("hallo", Position.create("", 0 + 1, 10 + 1, 10)), output.next());
334 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 15 + 1, 15)), output.next());
335 assertEquals(ClassToken.create(Position.create("", 0 + 1, 16 + 1, 16)), output.next());
336 assertEquals(EndToken.create(Position.create("", 0 + 1, 21 + 1, 21)), output.next());
337 }
338
339 @Test
340 public void testInvalidCharacterInKeyword() {
341 final SimpleScannerInput input = new SimpleScannerInput("cla&%ss");
342 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
343 final Scanner scanner = ModelDslScanner.create();
344 scanner.scan(input, output);
345 assertEquals(IdentifierToken.create("cla", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
346 assertEquals(InvalidToken.create("&%", Position.create("", 0 + 1, 3 + 1, 3)), output.next());
347 assertEquals(IdentifierToken.create("ss", Position.create("", 0 + 1, 5 + 1, 5)), output.next());
348 assertEquals(EndToken.create(Position.create("", 0 + 1, 7 + 1, 7)), output.next());
349 }
350
351 @Test
352 public void testExclamationCharacterInKeyword() {
353 final SimpleScannerInput input = new SimpleScannerInput("cla!ss");
354 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
355 final Scanner scanner = ModelDslScanner.create();
356 scanner.scan(input, output);
357 assertEquals(IdentifierToken.create("cla", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
358 assertTrue(output.next() instanceof ExclamationToken);
359 assertEquals(IdentifierToken.create("ss", Position.create("", 0 + 1, 4 + 1, 4)), output.next());
360 assertEquals(EndToken.create(Position.create("", 0 + 1, 6 + 1, 6)), output.next());
361 }
362
363 @Test
364 public void testCommentCharacterInKeyword() {
365 final SimpleScannerInput input = new SimpleScannerInput("cla#this is a class#ss");
366 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
367 final Scanner scanner = ModelDslScanner.create();
368 scanner.scan(input, output);
369 assertEquals(IdentifierToken.create("cla", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
370 assertEquals(CommentTextToken.create("#this is a class#", Position.create("", 0 + 1, 3 + 1, 3)), output.next());
371 assertEquals(IdentifierToken.create("ss", Position.create("", 0 + 1, 20 + 1, 20)), output.next());
372 assertEquals(EndToken.create(Position.create("", 0 + 1, 22 + 1, 22)), output.next());
373 }
374
375 @Test
376 public void testWhitespaceInKeyword() {
377 final SimpleScannerInput input = new SimpleScannerInput("cla ss");
378 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
379 final Scanner scanner = ModelDslScanner.create();
380 scanner.scan(input, output);
381 assertEquals(IdentifierToken.create("cla", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
382 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 3 + 1, 3)), output.next());
383 assertEquals(IdentifierToken.create("ss", Position.create("", 0 + 1, 6 + 1, 6)), output.next());
384 assertEquals(EndToken.create(Position.create("", 0 + 1, 8 + 1, 8)), output.next());
385 }
386
387 @Test
388 public void testExceptionAsKeyword() {
389 final SimpleScannerInput input = new SimpleScannerInput("exception");
390 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
391 final Scanner scanner = ModelDslScanner.create();
392 scanner.scan(input, output);
393 assertEquals(ExceptionToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
394 assertEquals(EndToken.create(Position.create("", 0 + 1, 9 + 1, 9)), output.next());
395 }
396
397 @Test
398 public void testServiceAsKeyword() {
399 final SimpleScannerInput input = new SimpleScannerInput("service");
400 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
401 final Scanner scanner = ModelDslScanner.create();
402 scanner.scan(input, output);
403 assertEquals(ServiceToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
404 assertEquals(EndToken.create(Position.create("", 0 + 1, 7 + 1, 7)), output.next());
405 }
406
407 @Test
408 public void testTransientAsKeyword() {
409 final SimpleScannerInput input = new SimpleScannerInput("transient");
410 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
411 final Scanner scanner = ModelDslScanner.create();
412 scanner.scan(input, output);
413 assertEquals(TransientToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
414 assertEquals(EndToken.create(Position.create("", 0 + 1, 9 + 1, 9)), output.next());
415 }
416
417 @Test
418 public void testFindableAsKeyword() {
419 final SimpleScannerInput input = new SimpleScannerInput("findable");
420 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
421 final Scanner scanner = ModelDslScanner.create();
422 scanner.scan(input, output);
423 assertEquals(FindableToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
424 assertEquals(EndToken.create(Position.create("", 0 + 1, 8 + 1, 8)), output.next());
425 }
426
427 @Test
428 public void testArrowAsToken() {
429 final SimpleScannerInput input = new SimpleScannerInput("->");
430 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
431 final Scanner scanner = ModelDslScanner.create();
432 scanner.scan(input, output);
433 assertEquals(ArrowToken.create(Position.create("", 1, 1, 0)), output.next());
434 assertEquals(EndToken.create(Position.create("", 1, 3, 2)), output.next());
435 }
436
437 @Test
438 public void testDoubleSquareBracketOpenAsToken() {
439 final SimpleScannerInput input = new SimpleScannerInput("[[");
440 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
441 final Scanner scanner = ModelDslScanner.create();
442 scanner.scan(input, output);
443 assertEquals(DoubleSquareBracketOpenToken.create(Position.create("", 1, 1, 0)), output.next());
444 assertEquals(EndToken.create(Position.create("", 0 + 1, 2 + 1, 2)), output.next());
445 }
446
447 @Test
448 public void testDoubleSquareBracketCloseAsToken() {
449 final SimpleScannerInput input = new SimpleScannerInput("]]");
450 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
451 final Scanner scanner = ModelDslScanner.create();
452 scanner.scan(input, output);
453 assertEquals(DoubleSquareBracketCloseToken.create(Position.create("", 1, 1, 0)), output.next());
454 assertEquals(EndToken.create(Position.create("", 1, 3, 2)), output.next());
455 }
456
457 @Test
458 public void testTripleSquareBracketCloseAsToken() {
459 final SimpleScannerInput input = new SimpleScannerInput("] ]]");
460 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
461 final Scanner scanner = ModelDslScanner.create();
462 scanner.scan(input, output);
463 assertEquals(SquareBracketCloseToken.create(Position.create("", 1, 1, 0)), output.next());
464 assertEquals(WhitespaceToken.create(" ", Position.create("", 1, 2, 1)), output.next());
465 assertEquals(DoubleSquareBracketCloseToken.create(Position.create("", 1, 3, 2)), output.next());
466 assertEquals(EndToken.create(Position.create("", 1, 5, 4)), output.next());
467 }
468
469 @Test
470 public void testAsteriskAsToken() {
471 final SimpleScannerInput input = new SimpleScannerInput("*");
472 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
473 final Scanner scanner = ModelDslScanner.create();
474 scanner.scan(input, output);
475 assertEquals(AsteriskToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
476 assertEquals(EndToken.create(Position.create("", 0 + 1, 1 + 1, 1)), output.next());
477 }
478
479 @Test
480 public void testMutableAsToken() {
481 final SimpleScannerInput input = new SimpleScannerInput("mutable");
482 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
483 final Scanner scanner = ModelDslScanner.create();
484 scanner.scan(input, output);
485 assertEquals(MutableToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
486 assertEquals(EndToken.create(Position.create("", 0 + 1, 7 + 1, 7)), output.next());
487 }
488
489 @Test
490 public void testSymmetricAsToken() {
491 final SimpleScannerInput input = new SimpleScannerInput("symmetric");
492 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
493 final Scanner scanner = ModelDslScanner.create();
494 scanner.scan(input, output);
495 assertEquals(SymmetricToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
496 assertEquals(EndToken.create(Position.create("", 0 + 1, 9 + 1, 9)), output.next());
497 }
498
499 @Test
500 public void testWhitespaceInIdentifier() {
501 final SimpleScannerInput input = new SimpleScannerInput("Hannover Hbf:class");
502 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
503 final Scanner scanner = ModelDslScanner.create();
504 scanner.scan(input, output);
505 assertEquals(IdentifierToken.create("Hannover", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
506 assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 8 + 1, 8)), output.next());
507 assertEquals(IdentifierToken.create("Hbf", Position.create("", 0 + 1, 9 + 1, 9)), output.next());
508 assertEquals(ColonToken.create(Position.create("", 0 + 1, 12 + 1, 12)), output.next());
509 assertEquals(ClassToken.create(Position.create("", 0 + 1, 13 + 1, 13)), output.next());
510 assertEquals(EndToken.create(Position.create("", 0 + 1, 18 + 1, 18)), output.next());
511 }
512
513
514
515
516 @Test
517 public void testExclamationToken() {
518 final SimpleScannerInput input =
519 new SimpleScannerInput("MyGroup:group=[MyClass:class={MyOperation:[[()->[String->Integer]!]];};];");
520 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
521 final Scanner scanner = ModelDslScanner.create();
522 scanner.scan(input, output);
523
524 final TokenStream stream = SimpleTokenStream.create();
525
526 final IdentifierToken myGroupToken = IdentifierToken.create("MyGroup", Position.create("", 1, 1, 0));
527 final IdentifierToken myClassToken = IdentifierToken.create("MyClass", Position.create("", 1, 16, 15));
528 final IdentifierToken myOperationToken = IdentifierToken.create("MyOperation", Position.create("", 1, 31, 30));
529
530 stream.add(myGroupToken);
531 stream.add(ColonToken.create(Position.create("", 1, 8, 7)));
532 stream.add(GroupToken.create(Position.create("", 1, 9, 8)));
533 stream.add(EqualToken.create(Position.create("", 1, 14, 13)));
534 stream.add(SquareBracketOpenToken.create(Position.create("", 1, 15, 14)));
535 stream.add(myClassToken);
536 stream.add(ColonToken.create(Position.create("", 1, 23, 22)));
537 stream.add(ClassToken.create(Position.create("", 1, 24, 23)));
538 stream.add(EqualToken.create(Position.create("", 1, 29, 28)));
539 stream.add(CurlyBracketOpenToken.create(Position.create("", 1, 30, 29)));
540 stream.add(myOperationToken);
541 stream.add(ColonToken.create(Position.create("", 1, 42, 41)));
542 stream.add(DoubleSquareBracketOpenToken.create(Position.create("", 1, 43, 42)));
543 stream.add(BracketOpenToken.create(Position.create("", 1, 45, 44)));
544 stream.add(BracketCloseToken.create(Position.create("", 1, 46, 45)));
545 stream.add(ArrowToken.create(Position.create("", 1, 47, 46)));
546 stream.add(SquareBracketOpenToken.create(Position.create("", 1, 49, 48)));
547 stream.add(IdentifierToken.create("String", Position.create("", 1, 50, 49)));
548 stream.add(ArrowToken.create(Position.create("", 1, 56, 55)));
549 stream.add(IdentifierToken.create("Integer", Position.create("", 1, 58, 57)));
550 stream.add(SquareBracketCloseToken.create(Position.create("", 1, 65, 64)));
551 stream.add(ExclamationToken.create(Position.create("", 1, 66, 65)));
552 stream.add(DoubleSquareBracketCloseToken.create(Position.create("", 1, 67, 66)));
553 stream.add(SemicolonToken.create(Position.create("", 1, 69, 68)));
554 stream.add(CurlyBracketCloseToken.create(Position.create("", 1, 70, 69)));
555 stream.add(SemicolonToken.create(Position.create("", 1, 71, 70)));
556 stream.add(SquareBracketCloseToken.create(Position.create("", 1, 72, 71)));
557 stream.add(SemicolonToken.create(Position.create("", 1, 73, 72)));
558 stream.add(EndToken.create(Position.create("", 1, 74, 73)));
559
560 this.assertEqualsTokenStream(stream, output);
561 }
562
563
564
565
566 @Test
567 public void testExclamationToken2() {
568 final SimpleScannerInput input =
569 new SimpleScannerInput(
570 "MyGroup:group=[MyClass:class={MyOperation:[[()->{TestException}!]];};TestException:exception={};];");
571 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
572 final Scanner scanner = ModelDslScanner.create();
573 scanner.scan(input, output);
574
575 final TokenStream stream = SimpleTokenStream.create();
576
577 final IdentifierToken myGroupToken = IdentifierToken.create("MyGroup", Position.create("", 1, 1, 0));
578 final IdentifierToken myClassToken = IdentifierToken.create("MyClass", Position.create("", 1, 16, 15));
579 final IdentifierToken myOperationToken = IdentifierToken.create("MyOperation", Position.create("", 1, 31, 30));
580 final IdentifierToken testExceptionReferenceToken =
581 IdentifierToken.create("TestException", Position.create("", 1, 50, 49));
582 final IdentifierToken testExceptionToken =
583 IdentifierToken.create("TestException", Position.create("", 1, 70, 69));
584 final CurlyBracketOpenToken sumTypeStart = CurlyBracketOpenToken.create(Position.create("", 1, 49, 48));
585
586 stream.add(myGroupToken);
587 stream.add(ColonToken.create(Position.create("", 1, 8, 7)));
588 stream.add(GroupToken.create(Position.create("", 1, 9, 8)));
589 stream.add(EqualToken.create(Position.create("", 1, 14, 13)));
590 stream.add(SquareBracketOpenToken.create(Position.create("", 1, 15, 14)));
591 stream.add(myClassToken);
592 stream.add(ColonToken.create(Position.create("", 1, 23, 22)));
593 stream.add(ClassToken.create(Position.create("", 1, 24, 23)));
594 stream.add(EqualToken.create(Position.create("", 1, 29, 28)));
595 stream.add(CurlyBracketOpenToken.create(Position.create("", 1, 30, 29)));
596 stream.add(myOperationToken);
597 stream.add(ColonToken.create(Position.create("", 1, 42, 41)));
598 stream.add(DoubleSquareBracketOpenToken.create(Position.create("", 1, 43, 42)));
599 stream.add(BracketOpenToken.create(Position.create("", 1, 45, 44)));
600 stream.add(BracketCloseToken.create(Position.create("", 1, 46, 45)));
601 stream.add(ArrowToken.create(Position.create("", 1, 47, 46)));
602 stream.add(sumTypeStart);
603 stream.add(testExceptionReferenceToken);
604 stream.add(CurlyBracketCloseToken.create(Position.create("", 1, 63, 62)));
605 stream.add(ExclamationToken.create(Position.create("", 1, 64, 63)));
606 stream.add(DoubleSquareBracketCloseToken.create(Position.create("", 1, 65, 64)));
607 stream.add(SemicolonToken.create(Position.create("", 1, 67, 66)));
608 stream.add(CurlyBracketCloseToken.create(Position.create("", 1, 68, 67)));
609 stream.add(SemicolonToken.create(Position.create("", 1, 69, 68)));
610 stream.add(testExceptionToken);
611 stream.add(ColonToken.create(Position.create("", 1, 83, 82)));
612 stream.add(ExceptionToken.create(Position.create("", 1, 84, 83)));
613 stream.add(EqualToken.create(Position.create("", 1, 93, 92)));
614 stream.add(CurlyBracketOpenToken.create(Position.create("", 1, 94, 93)));
615 stream.add(CurlyBracketCloseToken.create(Position.create("", 1, 95, 94)));
616 stream.add(SemicolonToken.create(Position.create("", 1, 96, 95)));
617 stream.add(SquareBracketCloseToken.create(Position.create("", 1, 97, 96)));
618 stream.add(SemicolonToken.create(Position.create("", 1, 98, 97)));
619 stream.add(EndToken.create(Position.create("", 1, 99, 98)));
620
621 this.assertEqualsTokenStream(stream, output);
622 }
623
624
625
626
627 @Test
628 public void testExclamationToken3() {
629 final SimpleScannerInput input = new SimpleScannerInput("MyGroup:group!!!!{};];");
630 final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
631 final Scanner scanner = ModelDslScanner.create();
632 scanner.scan(input, output);
633
634 final TokenStream stream = SimpleTokenStream.create();
635
636 final IdentifierToken myGroupToken = IdentifierToken.create("MyGroup", Position.create("", 1, 1, 0));
637
638 stream.add(myGroupToken);
639 stream.add(ColonToken.create(Position.create("", 1, 8, 7)));
640 stream.add(GroupToken.create(Position.create("", 1, 9, 8)));
641 stream.add(ExclamationToken.create(Position.create("", 1, 14, 13)));
642 stream.add(ExclamationToken.create(Position.create("", 1, 15, 14)));
643 stream.add(ExclamationToken.create(Position.create("", 1, 16, 15)));
644 stream.add(ExclamationToken.create(Position.create("", 1, 17, 16)));
645 stream.add(CurlyBracketOpenToken.create(Position.create("", 1, 18, 17)));
646 stream.add(CurlyBracketCloseToken.create(Position.create("", 1, 19, 18)));
647 stream.add(SemicolonToken.create(Position.create("", 1, 20, 19)));
648 stream.add(SquareBracketCloseToken.create(Position.create("", 1, 21, 20)));
649 stream.add(SemicolonToken.create(Position.create("", 1, 22, 21)));
650 stream.add(EndToken.create(Position.create("", 1, 23, 22)));
651
652 this.assertEqualsTokenStream(stream, output);
653 }
654
655
656
657
658
659
660
661
662
663 private void assertEqualsTokenStream(final TokenStream expected, final TokenStream actual) {
664
665 while (expected.hasNext()) {
666 assertTrue(actual.hasNext());
667 assertEquals(expected.next(), actual.next());
668 }
669
670 assertTrue(!actual.hasNext());
671 }
672 }