View Javadoc
1   package de.fhdw.wtf.dsl.scanner.test;
2   
3   import static org.junit.Assert.assertEquals;
4   import static org.junit.Assert.assertTrue;
5   
6   import org.junit.Test;
7   
8   import de.fhdw.wtf.common.stream.SimpleScannerInput;
9   import de.fhdw.wtf.common.stream.SimpleTokenStream;
10  import de.fhdw.wtf.common.stream.TokenStream;
11  import de.fhdw.wtf.common.token.CommentTextToken;
12  import de.fhdw.wtf.common.token.EndToken;
13  import de.fhdw.wtf.common.token.IdentifierToken;
14  import de.fhdw.wtf.common.token.InvalidToken;
15  import de.fhdw.wtf.common.token.Position;
16  import de.fhdw.wtf.common.token.WhitespaceToken;
17  import de.fhdw.wtf.common.token.keywords.AbstractToken;
18  import de.fhdw.wtf.common.token.keywords.ClassToken;
19  import de.fhdw.wtf.common.token.keywords.ExceptionToken;
20  import de.fhdw.wtf.common.token.keywords.FindableToken;
21  import de.fhdw.wtf.common.token.keywords.GroupToken;
22  import de.fhdw.wtf.common.token.keywords.MutableToken;
23  import de.fhdw.wtf.common.token.keywords.PriorToken;
24  import de.fhdw.wtf.common.token.keywords.ServiceToken;
25  import de.fhdw.wtf.common.token.keywords.SymmetricToken;
26  import de.fhdw.wtf.common.token.keywords.TransientToken;
27  import de.fhdw.wtf.common.token.symbols.ArrowToken;
28  import de.fhdw.wtf.common.token.symbols.AsteriskToken;
29  import de.fhdw.wtf.common.token.symbols.BracketCloseToken;
30  import de.fhdw.wtf.common.token.symbols.BracketOpenToken;
31  import de.fhdw.wtf.common.token.symbols.ColonToken;
32  import de.fhdw.wtf.common.token.symbols.CurlyBracketCloseToken;
33  import de.fhdw.wtf.common.token.symbols.CurlyBracketOpenToken;
34  import de.fhdw.wtf.common.token.symbols.DoubleSquareBracketCloseToken;
35  import de.fhdw.wtf.common.token.symbols.DoubleSquareBracketOpenToken;
36  import de.fhdw.wtf.common.token.symbols.EqualToken;
37  import de.fhdw.wtf.common.token.symbols.ExclamationToken;
38  import de.fhdw.wtf.common.token.symbols.GreaterSymbolToken;
39  import de.fhdw.wtf.common.token.symbols.PlusSymbolToken;
40  import de.fhdw.wtf.common.token.symbols.SemicolonToken;
41  import de.fhdw.wtf.common.token.symbols.SquareBracketCloseToken;
42  import de.fhdw.wtf.common.token.symbols.SquareBracketOpenToken;
43  import de.fhdw.wtf.dsl.scanner.common.Scanner;
44  import de.fhdw.wtf.dsl.scanner.modelScanner.ModelDslScanner;
45  
46  /**
47   * Test cases for operations provided by the Scanner.
48   * 
49   */
50  public class ScannerTest {
51  	
52  	@Test
53  	public void digitIdentifierTest() {
54  		final SimpleScannerInput input = new SimpleScannerInput("9Hallo");
55  		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
56  		final Scanner scanner = ModelDslScanner.create();
57  		scanner.scan(input, output);
58  		assertEquals(InvalidToken.create("9", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
59  		assertEquals(IdentifierToken.create("Hallo", Position.create("", 0 + 1, 1 + 1, 1)), output.next());
60  		assertEquals(EndToken.create(Position.create("", 0 + 1, 6 + 1, 6)), output.next());
61  	}
62  	
63  	@Test
64  	public void singleSymol() {
65  		final SimpleScannerInput input = new SimpleScannerInput("[]{}+:;=>*");
66  		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
67  		final Scanner scanner = ModelDslScanner.create();
68  		scanner.scan(input, output);
69  		assertEquals(true, output.hasNext());
70  		assertEquals(SquareBracketOpenToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
71  		assertEquals(SquareBracketCloseToken.create(Position.create("", 0 + 1, 1 + 1, 1)), output.next());
72  		assertEquals(CurlyBracketOpenToken.create(Position.create("", 0 + 1, 2 + 1, 2)), output.next());
73  		assertEquals(CurlyBracketCloseToken.create(Position.create("", 0 + 1, 3 + 1, 3)), output.next());
74  		assertEquals(PlusSymbolToken.create(Position.create("", 0 + 1, 4 + 1, 4)), output.next());
75  		assertEquals(ColonToken.create(Position.create("", 0 + 1, 5 + 1, 5)), output.next());
76  		assertEquals(SemicolonToken.create(Position.create("", 0 + 1, 6 + 1, 6)), output.next());
77  		assertEquals(EqualToken.create(Position.create("", 0 + 1, 7 + 1, 7)), output.next());
78  		assertEquals(GreaterSymbolToken.create(Position.create("", 0 + 1, 8 + 1, 8)), output.next());
79  		assertEquals(AsteriskToken.create(Position.create("", 0 + 1, 9 + 1, 9)), output.next());
80  		assertEquals(EndToken.create(Position.create("", 0 + 1, 10 + 1, 10)), output.peek());
81  		output.removeFirst();
82  		assertEquals(false, output.hasNext());
83  	}
84  	
85  	@Test
86  	public void keywords() {
87  		final SimpleScannerInput input = new SimpleScannerInput("abstract class group prior");
88  		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
89  		final Scanner scanner = ModelDslScanner.create();
90  		scanner.scan(input, output);
91  		assertEquals(AbstractToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
92  		assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 8 + 1, 8)), output.next());
93  		assertEquals(ClassToken.create(Position.create("", 0 + 1, 9 + 1, 9)), output.next());
94  		assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 14 + 1, 14)), output.next());
95  		assertEquals(GroupToken.create(Position.create("", 0 + 1, 15 + 1, 15)), output.next());
96  		assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 20 + 1, 20)), output.next());
97  		assertEquals(PriorToken.create(Position.create("", 0 + 1, 21 + 1, 21)), output.next());
98  		assertEquals(EndToken.create(Position.create("", 0 + 1, 26 + 1, 26)), output.next());
99  	}
100 	
101 	@Test
102 	public void identifiers() {
103 		final SimpleScannerInput input =
104 				new SimpleScannerInput(
105 						"Bond007 what_is_the_answer_to_life_the_universe_and_everything  ThreadsafeButton_Channel SimpleNodeTester");
106 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
107 		final Scanner scanner = ModelDslScanner.create();
108 		scanner.scan(input, output);
109 		assertEquals(IdentifierToken.create("Bond007", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
110 		assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 7 + 1, 7)), output.next());
111 		assertEquals(
112 				IdentifierToken.create(
113 						"what_is_the_answer_to_life_the_universe_and_everything",
114 						Position.create("", 0 + 1, 8 + 1, 8)),
115 				output.next());
116 		assertEquals(WhitespaceToken.create("  ", Position.create("", 0 + 1, 62 + 1, 62)), output.next());
117 		assertEquals(
118 				IdentifierToken.create("ThreadsafeButton_Channel", Position.create("", 0 + 1, 64 + 1, 64)),
119 				output.next());
120 		assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 88 + 1, 88)), output.next());
121 		assertEquals(IdentifierToken.create("SimpleNodeTester", Position.create("", 0 + 1, 89 + 1, 89)), output.next());
122 		assertEquals(EndToken.create(Position.create("", 0 + 1, 105 + 1, 105)), output.next());
123 	}
124 	
125 	@Test
126 	public void classInGroup() {
127 		final SimpleScannerInput input = new SimpleScannerInput("ichbingruppe:group=[IchBinEINEkLASSE:class={}]");
128 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
129 		final Scanner scanner = ModelDslScanner.create();
130 		scanner.scan(input, output);
131 		assertEquals(IdentifierToken.create("ichbingruppe", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
132 		assertEquals(ColonToken.create(Position.create("", 0 + 1, 12 + 1, 12)), output.next());
133 		assertEquals(GroupToken.create(Position.create("", 0 + 1, 13 + 1, 13)), output.next());
134 		assertEquals(EqualToken.create(Position.create("", 0 + 1, 18 + 1, 18)), output.next());
135 		assertEquals(SquareBracketOpenToken.create(Position.create("", 0 + 1, 19 + 1, 19)), output.next());
136 		assertEquals(IdentifierToken.create("IchBinEINEkLASSE", Position.create("", 0 + 1, 20 + 1, 20)), output.next());
137 		assertEquals(ColonToken.create(Position.create("", 0 + 1, 36 + 1, 36)), output.next());
138 		assertEquals(ClassToken.create(Position.create("", 0 + 1, 37 + 1, 37)), output.next());
139 		assertEquals(EqualToken.create(Position.create("", 0 + 1, 42 + 1, 42)), output.next());
140 		assertEquals(CurlyBracketOpenToken.create(Position.create("", 0 + 1, 43 + 1, 43)), output.next());
141 		assertEquals(CurlyBracketCloseToken.create(Position.create("", 0 + 1, 44 + 1, 44)), output.next());
142 		assertEquals(SquareBracketCloseToken.create(Position.create("", 0 + 1, 45 + 1, 45)), output.next());
143 		assertEquals(EndToken.create(Position.create("", 0 + 1, 46 + 1, 46)), output.next());
144 	}
145 	
146 	@Test
147 	public void testInvalidChar() {
148 		final SimpleScannerInput input = new SimpleScannerInput("^ichbingruppe:group=[IchBinEINEkLASSE:class^={}]");
149 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
150 		final Scanner scanner = ModelDslScanner.create();
151 		scanner.scan(input, output);
152 		assertEquals(InvalidToken.create('^', Position.create("", 0 + 1, 0 + 1, 0)), output.next());
153 		assertEquals(IdentifierToken.create("ichbingruppe", Position.create("", 0 + 1, 1 + 1, 1)), output.next());
154 		assertEquals(ColonToken.create(Position.create("", 0 + 1, 13 + 1, 13)), output.next());
155 		assertEquals(GroupToken.create(Position.create("", 0 + 1, 14 + 1, 14)), output.next());
156 		assertEquals(EqualToken.create(Position.create("", 0 + 1, 19 + 1, 19)), output.next());
157 		assertEquals(SquareBracketOpenToken.create(Position.create("", 0 + 1, 20 + 1, 20)), output.next());
158 		assertEquals(IdentifierToken.create("IchBinEINEkLASSE", Position.create("", 0 + 1, 21 + 1, 21)), output.next());
159 		assertEquals(ColonToken.create(Position.create("", 0 + 1, 37 + 1, 37)), output.next());
160 		assertEquals(ClassToken.create(Position.create("", 0 + 1, 38 + 1, 38)), output.next());
161 		assertEquals(InvalidToken.create('^', Position.create("", 0 + 1, 43 + 1, 43)), output.next());
162 		assertEquals(EqualToken.create(Position.create("", 0 + 1, 44 + 1, 44)), output.next());
163 		assertEquals(CurlyBracketOpenToken.create(Position.create("", 0 + 1, 45 + 1, 45)), output.next());
164 		assertEquals(CurlyBracketCloseToken.create(Position.create("", 0 + 1, 46 + 1, 46)), output.next());
165 		assertEquals(SquareBracketCloseToken.create(Position.create("", 0 + 1, 47 + 1, 47)), output.next());
166 		assertEquals(EndToken.create(Position.create("", 0 + 1, 48 + 1, 48)), output.next());
167 	}
168 	
169 	@Test
170 	public void testClassNames() {
171 		final SimpleScannerInput input = new SimpleScannerInput("IchBinEINEkLASSE:class={};className:class={};");
172 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
173 		final Scanner scanner = ModelDslScanner.create();
174 		scanner.scan(input, output);
175 		assertEquals(IdentifierToken.create("IchBinEINEkLASSE", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
176 		assertTrue(output.next() instanceof ColonToken);
177 		assertTrue(output.next() instanceof ClassToken);
178 		assertTrue(output.next() instanceof EqualToken);
179 		assertTrue(output.next() instanceof CurlyBracketOpenToken);
180 		assertTrue(output.next() instanceof CurlyBracketCloseToken);
181 		assertTrue(output.next() instanceof SemicolonToken);
182 		assertEquals(IdentifierToken.create("className", Position.create("", 0 + 1, 26 + 1, 26)), output.next());
183 		assertTrue(output.next() instanceof ColonToken);
184 		assertTrue(output.next() instanceof ClassToken);
185 		assertTrue(output.next() instanceof EqualToken);
186 		assertTrue(output.next() instanceof CurlyBracketOpenToken);
187 		assertTrue(output.next() instanceof CurlyBracketCloseToken);
188 		assertTrue(output.next() instanceof SemicolonToken);
189 		assertEquals(EndToken.create(Position.create("", 0 + 1, 45 + 1, 45)), output.next());
190 	}
191 	
192 	@Test
193 	public void invalid() {
194 		final SimpleScannerInput input = new SimpleScannerInput("abstract  &hallo");
195 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
196 		final Scanner scanner = ModelDslScanner.create();
197 		scanner.scan(input, output);
198 		assertEquals(AbstractToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
199 		assertEquals(WhitespaceToken.create("  ", Position.create("", 0 + 1, 8 + 1, 8)), output.next());
200 		assertEquals(InvalidToken.create("&", Position.create("", 0 + 1, 10 + 1, 10)), output.next());
201 		assertEquals(IdentifierToken.create("hallo", Position.create("", 0 + 1, 11 + 1, 11)), output.next());
202 		assertEquals(EndToken.create(Position.create("", 0 + 1, 16 + 1, 16)), output.next());
203 	}
204 	
205 	@Test
206 	public void testComment1() {
207 		final SimpleScannerInput input = new SimpleScannerInput("#Ich bin ein Kommentar#");
208 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
209 		final Scanner scanner = ModelDslScanner.create();
210 		scanner.scan(input, output);
211 		assertEquals(
212 				CommentTextToken.create("#Ich bin ein Kommentar#", Position.create("", 0 + 1, 0 + 1, 0)),
213 				output.next());
214 		assertEquals(EndToken.create(Position.create("", 0 + 1, 23 + 1, 23)), output.next());
215 	}
216 	
217 	@Test
218 	public void testComment2() {
219 		final SimpleScannerInput input = new SimpleScannerInput("class#Ich bin ein Kommentar#;");
220 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
221 		final Scanner scanner = ModelDslScanner.create();
222 		scanner.scan(input, output);
223 		assertTrue(output.next() instanceof ClassToken);
224 		assertEquals(
225 				CommentTextToken.create("#Ich bin ein Kommentar#", Position.create("", 0 + 1, 5 + 1, 5)),
226 				output.next());
227 		assertTrue(output.next() instanceof SemicolonToken);
228 		assertEquals(EndToken.create(Position.create("", 0 + 1, 29 + 1, 29)), output.next());
229 	}
230 	
231 	@Test
232 	public void testComment3() {
233 		final SimpleScannerInput input = new SimpleScannerInput("#Ich bin ein Kommentar;");
234 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
235 		final Scanner scanner = ModelDslScanner.create();
236 		scanner.scan(input, output);
237 		assertEquals(
238 				CommentTextToken.create("#Ich bin ein Kommentar;#", Position.create("", 0 + 1, 0 + 1, 0)),
239 				output.next());
240 		assertEquals(EndToken.create(Position.create("", 0 + 1, 23 + 1, 23)), output.next());
241 		
242 	}
243 	
244 	@Test
245 	public void testComment4() {
246 		final SimpleScannerInput input = new SimpleScannerInput("#auto:class={};");
247 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
248 		final Scanner scanner = ModelDslScanner.create();
249 		scanner.scan(input, output);
250 		assertEquals(CommentTextToken.create("#auto:class={};#", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
251 		assertEquals(EndToken.create(Position.create("", 0 + 1, 15 + 1, 15)), output.next());
252 		
253 	}
254 	
255 	@Test
256 	public void testComment5() {
257 		final SimpleScannerInput input = new SimpleScannerInput("#######");
258 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
259 		final Scanner scanner = ModelDslScanner.create();
260 		scanner.scan(input, output);
261 		assertEquals(CommentTextToken.create("##", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
262 		assertEquals(CommentTextToken.create("##", Position.create("", 0 + 1, 2 + 1, 2)), output.next());
263 		assertEquals(CommentTextToken.create("##", Position.create("", 0 + 1, 4 + 1, 4)), output.next());
264 		assertEquals(CommentTextToken.create("##", Position.create("", 0 + 1, 6 + 1, 6)), output.next());
265 		assertEquals(EndToken.create(Position.create("", 0 + 1, 7 + 1, 7)), output.next());
266 		
267 	}
268 	
269 	@Test
270 	public void testComment6() {
271 		final SimpleScannerInput input = new SimpleScannerInput("/*Kommentar;*/");
272 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
273 		final Scanner scanner = ModelDslScanner.create();
274 		scanner.scan(input, output);
275 		assertEquals(InvalidToken.create("/", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
276 		assertEquals(AsteriskToken.create(Position.create("", 0 + 1, 1 + 1, 1)), output.next());
277 		assertEquals(IdentifierToken.create("Kommentar", Position.create("", 0 + 1, 2 + 1, 2)), output.next());
278 		assertEquals(SemicolonToken.create(Position.create("", 0 + 1, 11 + 1, 11)), output.next());
279 		assertEquals(AsteriskToken.create(Position.create("", 0 + 1, 12 + 1, 12)), output.next());
280 		assertEquals(InvalidToken.create("/", Position.create("", 0 + 1, 13 + 1, 13)), output.next());
281 		assertEquals(EndToken.create(Position.create("", 0 + 1, 14 + 1, 14)), output.next());
282 		
283 	}
284 	
285 	@Test
286 	public void testCommentEmpty() {
287 		final SimpleScannerInput input = new SimpleScannerInput("##");
288 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
289 		final Scanner scanner = ModelDslScanner.create();
290 		scanner.scan(input, output);
291 		assertEquals(CommentTextToken.create("##", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
292 		assertEquals(EndToken.create(Position.create("", 0 + 1, 2 + 1, 2)), output.next());
293 	}
294 	
295 	@Test
296 	public void testWhitespace() {
297 		final SimpleScannerInput input = new SimpleScannerInput("abstract hallo class");
298 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
299 		final Scanner scanner = ModelDslScanner.create();
300 		scanner.scan(input, output);
301 		assertEquals(AbstractToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
302 		assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 8 + 1, 8)), output.next());
303 		assertEquals(IdentifierToken.create("hallo", Position.create("", 0 + 1, 9 + 1, 9)), output.next());
304 		assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 14 + 1, 14)), output.next());
305 		assertEquals(ClassToken.create(Position.create("", 0 + 1, 15 + 1, 15)), output.next());
306 		assertEquals(EndToken.create(Position.create("", 0 + 1, 20 + 1, 20)), output.next());
307 	}
308 	
309 	@Test
310 	public void testWhitespace2() {
311 		final SimpleScannerInput input = new SimpleScannerInput("  abstract    hallo     class");
312 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
313 		final Scanner scanner = ModelDslScanner.create();
314 		scanner.scan(input, output);
315 		assertEquals(WhitespaceToken.create("  ", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
316 		assertEquals(AbstractToken.create(Position.create("", 0 + 1, 2 + 1, 2)), output.next());
317 		assertEquals(WhitespaceToken.create("    ", Position.create("", 0 + 1, 10 + 1, 10)), output.next());
318 		assertEquals(IdentifierToken.create("hallo", Position.create("", 0 + 1, 14 + 1, 14)), output.next());
319 		assertEquals(WhitespaceToken.create("     ", Position.create("", 0 + 1, 19 + 1, 19)), output.next());
320 		assertEquals(ClassToken.create(Position.create("", 0 + 1, 24 + 1, 24)), output.next());
321 		assertEquals(EndToken.create(Position.create("", 0 + 1, 29 + 1, 29)), output.next());
322 	}
323 	
324 	@Test
325 	public void testTab() {
326 		final SimpleScannerInput input = new SimpleScannerInput(" abstract	hallo	class");
327 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
328 		final Scanner scanner = ModelDslScanner.create();
329 		scanner.scan(input, output);
330 		assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
331 		assertEquals(AbstractToken.create(Position.create("", 0 + 1, 1 + 1, 1)), output.next());
332 		assertEquals(WhitespaceToken.create("	", Position.create("", 0 + 1, 9 + 1, 9)), output.next());
333 		assertEquals(IdentifierToken.create("hallo", Position.create("", 0 + 1, 10 + 1, 10)), output.next());
334 		assertEquals(WhitespaceToken.create("	", Position.create("", 0 + 1, 15 + 1, 15)), output.next());
335 		assertEquals(ClassToken.create(Position.create("", 0 + 1, 16 + 1, 16)), output.next());
336 		assertEquals(EndToken.create(Position.create("", 0 + 1, 21 + 1, 21)), output.next());
337 	}
338 	
339 	@Test
340 	public void testInvalidCharacterInKeyword() {
341 		final SimpleScannerInput input = new SimpleScannerInput("cla&%ss");
342 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
343 		final Scanner scanner = ModelDslScanner.create();
344 		scanner.scan(input, output);
345 		assertEquals(IdentifierToken.create("cla", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
346 		assertEquals(InvalidToken.create("&%", Position.create("", 0 + 1, 3 + 1, 3)), output.next());
347 		assertEquals(IdentifierToken.create("ss", Position.create("", 0 + 1, 5 + 1, 5)), output.next());
348 		assertEquals(EndToken.create(Position.create("", 0 + 1, 7 + 1, 7)), output.next());
349 	}
350 	
351 	@Test
352 	public void testExclamationCharacterInKeyword() {
353 		final SimpleScannerInput input = new SimpleScannerInput("cla!ss");
354 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
355 		final Scanner scanner = ModelDslScanner.create();
356 		scanner.scan(input, output);
357 		assertEquals(IdentifierToken.create("cla", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
358 		assertTrue(output.next() instanceof ExclamationToken);
359 		assertEquals(IdentifierToken.create("ss", Position.create("", 0 + 1, 4 + 1, 4)), output.next());
360 		assertEquals(EndToken.create(Position.create("", 0 + 1, 6 + 1, 6)), output.next());
361 	}
362 	
363 	@Test
364 	public void testCommentCharacterInKeyword() {
365 		final SimpleScannerInput input = new SimpleScannerInput("cla#this is a class#ss");
366 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
367 		final Scanner scanner = ModelDslScanner.create();
368 		scanner.scan(input, output);
369 		assertEquals(IdentifierToken.create("cla", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
370 		assertEquals(CommentTextToken.create("#this is a class#", Position.create("", 0 + 1, 3 + 1, 3)), output.next());
371 		assertEquals(IdentifierToken.create("ss", Position.create("", 0 + 1, 20 + 1, 20)), output.next());
372 		assertEquals(EndToken.create(Position.create("", 0 + 1, 22 + 1, 22)), output.next());
373 	}
374 	
375 	@Test
376 	public void testWhitespaceInKeyword() {
377 		final SimpleScannerInput input = new SimpleScannerInput("cla   ss");
378 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
379 		final Scanner scanner = ModelDslScanner.create();
380 		scanner.scan(input, output);
381 		assertEquals(IdentifierToken.create("cla", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
382 		assertEquals(WhitespaceToken.create("   ", Position.create("", 0 + 1, 3 + 1, 3)), output.next());
383 		assertEquals(IdentifierToken.create("ss", Position.create("", 0 + 1, 6 + 1, 6)), output.next());
384 		assertEquals(EndToken.create(Position.create("", 0 + 1, 8 + 1, 8)), output.next());
385 	}
386 	
387 	@Test
388 	public void testExceptionAsKeyword() {
389 		final SimpleScannerInput input = new SimpleScannerInput("exception");
390 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
391 		final Scanner scanner = ModelDslScanner.create();
392 		scanner.scan(input, output);
393 		assertEquals(ExceptionToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
394 		assertEquals(EndToken.create(Position.create("", 0 + 1, 9 + 1, 9)), output.next());
395 	}
396 	
397 	@Test
398 	public void testServiceAsKeyword() {
399 		final SimpleScannerInput input = new SimpleScannerInput("service");
400 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
401 		final Scanner scanner = ModelDslScanner.create();
402 		scanner.scan(input, output);
403 		assertEquals(ServiceToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
404 		assertEquals(EndToken.create(Position.create("", 0 + 1, 7 + 1, 7)), output.next());
405 	}
406 	
407 	@Test
408 	public void testTransientAsKeyword() {
409 		final SimpleScannerInput input = new SimpleScannerInput("transient");
410 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
411 		final Scanner scanner = ModelDslScanner.create();
412 		scanner.scan(input, output);
413 		assertEquals(TransientToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
414 		assertEquals(EndToken.create(Position.create("", 0 + 1, 9 + 1, 9)), output.next());
415 	}
416 	
417 	@Test
418 	public void testFindableAsKeyword() {
419 		final SimpleScannerInput input = new SimpleScannerInput("findable");
420 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
421 		final Scanner scanner = ModelDslScanner.create();
422 		scanner.scan(input, output);
423 		assertEquals(FindableToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
424 		assertEquals(EndToken.create(Position.create("", 0 + 1, 8 + 1, 8)), output.next());
425 	}
426 	
427 	@Test
428 	public void testArrowAsToken() {
429 		final SimpleScannerInput input = new SimpleScannerInput("->");
430 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
431 		final Scanner scanner = ModelDslScanner.create();
432 		scanner.scan(input, output);
433 		assertEquals(ArrowToken.create(Position.create("", 1, 1, 0)), output.next());
434 		assertEquals(EndToken.create(Position.create("", 1, 3, 2)), output.next());
435 	}
436 	
437 	@Test
438 	public void testDoubleSquareBracketOpenAsToken() {
439 		final SimpleScannerInput input = new SimpleScannerInput("[[");
440 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
441 		final Scanner scanner = ModelDslScanner.create();
442 		scanner.scan(input, output);
443 		assertEquals(DoubleSquareBracketOpenToken.create(Position.create("", 1, 1, 0)), output.next());
444 		assertEquals(EndToken.create(Position.create("", 0 + 1, 2 + 1, 2)), output.next());
445 	}
446 	
447 	@Test
448 	public void testDoubleSquareBracketCloseAsToken() {
449 		final SimpleScannerInput input = new SimpleScannerInput("]]");
450 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
451 		final Scanner scanner = ModelDslScanner.create();
452 		scanner.scan(input, output);
453 		assertEquals(DoubleSquareBracketCloseToken.create(Position.create("", 1, 1, 0)), output.next());
454 		assertEquals(EndToken.create(Position.create("", 1, 3, 2)), output.next());
455 	}
456 	
457 	@Test
458 	public void testTripleSquareBracketCloseAsToken() {
459 		final SimpleScannerInput input = new SimpleScannerInput("] ]]");
460 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
461 		final Scanner scanner = ModelDslScanner.create();
462 		scanner.scan(input, output);
463 		assertEquals(SquareBracketCloseToken.create(Position.create("", 1, 1, 0)), output.next());
464 		assertEquals(WhitespaceToken.create(" ", Position.create("", 1, 2, 1)), output.next());
465 		assertEquals(DoubleSquareBracketCloseToken.create(Position.create("", 1, 3, 2)), output.next());
466 		assertEquals(EndToken.create(Position.create("", 1, 5, 4)), output.next());
467 	}
468 	
469 	@Test
470 	public void testAsteriskAsToken() {
471 		final SimpleScannerInput input = new SimpleScannerInput("*");
472 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
473 		final Scanner scanner = ModelDslScanner.create();
474 		scanner.scan(input, output);
475 		assertEquals(AsteriskToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
476 		assertEquals(EndToken.create(Position.create("", 0 + 1, 1 + 1, 1)), output.next());
477 	}
478 	
479 	@Test
480 	public void testMutableAsToken() {
481 		final SimpleScannerInput input = new SimpleScannerInput("mutable");
482 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
483 		final Scanner scanner = ModelDslScanner.create();
484 		scanner.scan(input, output);
485 		assertEquals(MutableToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
486 		assertEquals(EndToken.create(Position.create("", 0 + 1, 7 + 1, 7)), output.next());
487 	}
488 	
489 	@Test
490 	public void testSymmetricAsToken() {
491 		final SimpleScannerInput input = new SimpleScannerInput("symmetric");
492 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
493 		final Scanner scanner = ModelDslScanner.create();
494 		scanner.scan(input, output);
495 		assertEquals(SymmetricToken.create(Position.create("", 0 + 1, 0 + 1, 0)), output.next());
496 		assertEquals(EndToken.create(Position.create("", 0 + 1, 9 + 1, 9)), output.next());
497 	}
498 	
499 	@Test
500 	public void testWhitespaceInIdentifier() {
501 		final SimpleScannerInput input = new SimpleScannerInput("Hannover Hbf:class");
502 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
503 		final Scanner scanner = ModelDslScanner.create();
504 		scanner.scan(input, output);
505 		assertEquals(IdentifierToken.create("Hannover", Position.create("", 0 + 1, 0 + 1, 0)), output.next());
506 		assertEquals(WhitespaceToken.create(" ", Position.create("", 0 + 1, 8 + 1, 8)), output.next());
507 		assertEquals(IdentifierToken.create("Hbf", Position.create("", 0 + 1, 9 + 1, 9)), output.next());
508 		assertEquals(ColonToken.create(Position.create("", 0 + 1, 12 + 1, 12)), output.next());
509 		assertEquals(ClassToken.create(Position.create("", 0 + 1, 13 + 1, 13)), output.next());
510 		assertEquals(EndToken.create(Position.create("", 0 + 1, 18 + 1, 18)), output.next());
511 	}
512 	
513 	/**
514 	 * Input: MyGroup:group=[MyClass:class={MyOperation:[[()->[String->Integer]!!]];};];.
515 	 */
516 	@Test
517 	public void testExclamationToken() {
518 		final SimpleScannerInput input =
519 				new SimpleScannerInput("MyGroup:group=[MyClass:class={MyOperation:[[()->[String->Integer]!]];};];");
520 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
521 		final Scanner scanner = ModelDslScanner.create();
522 		scanner.scan(input, output);
523 		
524 		final TokenStream stream = SimpleTokenStream.create();
525 		
526 		final IdentifierToken myGroupToken = IdentifierToken.create("MyGroup", Position.create("", 1, 1, 0));
527 		final IdentifierToken myClassToken = IdentifierToken.create("MyClass", Position.create("", 1, 16, 15));
528 		final IdentifierToken myOperationToken = IdentifierToken.create("MyOperation", Position.create("", 1, 31, 30));
529 		
530 		stream.add(myGroupToken);
531 		stream.add(ColonToken.create(Position.create("", 1, 8, 7)));
532 		stream.add(GroupToken.create(Position.create("", 1, 9, 8)));
533 		stream.add(EqualToken.create(Position.create("", 1, 14, 13)));
534 		stream.add(SquareBracketOpenToken.create(Position.create("", 1, 15, 14)));
535 		stream.add(myClassToken);
536 		stream.add(ColonToken.create(Position.create("", 1, 23, 22)));
537 		stream.add(ClassToken.create(Position.create("", 1, 24, 23)));
538 		stream.add(EqualToken.create(Position.create("", 1, 29, 28)));
539 		stream.add(CurlyBracketOpenToken.create(Position.create("", 1, 30, 29)));
540 		stream.add(myOperationToken);
541 		stream.add(ColonToken.create(Position.create("", 1, 42, 41)));
542 		stream.add(DoubleSquareBracketOpenToken.create(Position.create("", 1, 43, 42)));
543 		stream.add(BracketOpenToken.create(Position.create("", 1, 45, 44)));
544 		stream.add(BracketCloseToken.create(Position.create("", 1, 46, 45)));
545 		stream.add(ArrowToken.create(Position.create("", 1, 47, 46)));
546 		stream.add(SquareBracketOpenToken.create(Position.create("", 1, 49, 48)));
547 		stream.add(IdentifierToken.create("String", Position.create("", 1, 50, 49)));
548 		stream.add(ArrowToken.create(Position.create("", 1, 56, 55)));
549 		stream.add(IdentifierToken.create("Integer", Position.create("", 1, 58, 57)));
550 		stream.add(SquareBracketCloseToken.create(Position.create("", 1, 65, 64)));
551 		stream.add(ExclamationToken.create(Position.create("", 1, 66, 65)));
552 		stream.add(DoubleSquareBracketCloseToken.create(Position.create("", 1, 67, 66)));
553 		stream.add(SemicolonToken.create(Position.create("", 1, 69, 68)));
554 		stream.add(CurlyBracketCloseToken.create(Position.create("", 1, 70, 69)));
555 		stream.add(SemicolonToken.create(Position.create("", 1, 71, 70)));
556 		stream.add(SquareBracketCloseToken.create(Position.create("", 1, 72, 71)));
557 		stream.add(SemicolonToken.create(Position.create("", 1, 73, 72)));
558 		stream.add(EndToken.create(Position.create("", 1, 74, 73)));
559 		
560 		this.assertEqualsTokenStream(stream, output);
561 	}
562 	
563 	/**
564 	 * Input: MyGroup:group=[MyClass:class={MyOperation:[[()->{TestException}!]];};TestException:exception={};];.
565 	 */
566 	@Test
567 	public void testExclamationToken2() {
568 		final SimpleScannerInput input =
569 				new SimpleScannerInput(
570 						"MyGroup:group=[MyClass:class={MyOperation:[[()->{TestException}!]];};TestException:exception={};];");
571 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
572 		final Scanner scanner = ModelDslScanner.create();
573 		scanner.scan(input, output);
574 		
575 		final TokenStream stream = SimpleTokenStream.create();
576 		
577 		final IdentifierToken myGroupToken = IdentifierToken.create("MyGroup", Position.create("", 1, 1, 0));
578 		final IdentifierToken myClassToken = IdentifierToken.create("MyClass", Position.create("", 1, 16, 15));
579 		final IdentifierToken myOperationToken = IdentifierToken.create("MyOperation", Position.create("", 1, 31, 30));
580 		final IdentifierToken testExceptionReferenceToken =
581 				IdentifierToken.create("TestException", Position.create("", 1, 50, 49));
582 		final IdentifierToken testExceptionToken =
583 				IdentifierToken.create("TestException", Position.create("", 1, 70, 69));
584 		final CurlyBracketOpenToken sumTypeStart = CurlyBracketOpenToken.create(Position.create("", 1, 49, 48));
585 		
586 		stream.add(myGroupToken);
587 		stream.add(ColonToken.create(Position.create("", 1, 8, 7)));
588 		stream.add(GroupToken.create(Position.create("", 1, 9, 8)));
589 		stream.add(EqualToken.create(Position.create("", 1, 14, 13)));
590 		stream.add(SquareBracketOpenToken.create(Position.create("", 1, 15, 14)));
591 		stream.add(myClassToken);
592 		stream.add(ColonToken.create(Position.create("", 1, 23, 22)));
593 		stream.add(ClassToken.create(Position.create("", 1, 24, 23)));
594 		stream.add(EqualToken.create(Position.create("", 1, 29, 28)));
595 		stream.add(CurlyBracketOpenToken.create(Position.create("", 1, 30, 29)));
596 		stream.add(myOperationToken);
597 		stream.add(ColonToken.create(Position.create("", 1, 42, 41)));
598 		stream.add(DoubleSquareBracketOpenToken.create(Position.create("", 1, 43, 42)));
599 		stream.add(BracketOpenToken.create(Position.create("", 1, 45, 44)));
600 		stream.add(BracketCloseToken.create(Position.create("", 1, 46, 45)));
601 		stream.add(ArrowToken.create(Position.create("", 1, 47, 46)));
602 		stream.add(sumTypeStart);
603 		stream.add(testExceptionReferenceToken);
604 		stream.add(CurlyBracketCloseToken.create(Position.create("", 1, 63, 62)));
605 		stream.add(ExclamationToken.create(Position.create("", 1, 64, 63)));
606 		stream.add(DoubleSquareBracketCloseToken.create(Position.create("", 1, 65, 64)));
607 		stream.add(SemicolonToken.create(Position.create("", 1, 67, 66)));
608 		stream.add(CurlyBracketCloseToken.create(Position.create("", 1, 68, 67)));
609 		stream.add(SemicolonToken.create(Position.create("", 1, 69, 68)));
610 		stream.add(testExceptionToken);
611 		stream.add(ColonToken.create(Position.create("", 1, 83, 82)));
612 		stream.add(ExceptionToken.create(Position.create("", 1, 84, 83)));
613 		stream.add(EqualToken.create(Position.create("", 1, 93, 92)));
614 		stream.add(CurlyBracketOpenToken.create(Position.create("", 1, 94, 93)));
615 		stream.add(CurlyBracketCloseToken.create(Position.create("", 1, 95, 94)));
616 		stream.add(SemicolonToken.create(Position.create("", 1, 96, 95)));
617 		stream.add(SquareBracketCloseToken.create(Position.create("", 1, 97, 96)));
618 		stream.add(SemicolonToken.create(Position.create("", 1, 98, 97)));
619 		stream.add(EndToken.create(Position.create("", 1, 99, 98)));
620 		
621 		this.assertEqualsTokenStream(stream, output);
622 	}
623 	
624 	/**
625 	 * Input: MyGroup:group!!!!{};];.
626 	 */
627 	@Test
628 	public void testExclamationToken3() {
629 		final SimpleScannerInput input = new SimpleScannerInput("MyGroup:group!!!!{};];");
630 		final VerboseTokenStream output = new VerboseTokenStream(SimpleTokenStream.create());
631 		final Scanner scanner = ModelDslScanner.create();
632 		scanner.scan(input, output);
633 		
634 		final TokenStream stream = SimpleTokenStream.create();
635 		
636 		final IdentifierToken myGroupToken = IdentifierToken.create("MyGroup", Position.create("", 1, 1, 0));
637 		
638 		stream.add(myGroupToken);
639 		stream.add(ColonToken.create(Position.create("", 1, 8, 7)));
640 		stream.add(GroupToken.create(Position.create("", 1, 9, 8)));
641 		stream.add(ExclamationToken.create(Position.create("", 1, 14, 13)));
642 		stream.add(ExclamationToken.create(Position.create("", 1, 15, 14)));
643 		stream.add(ExclamationToken.create(Position.create("", 1, 16, 15)));
644 		stream.add(ExclamationToken.create(Position.create("", 1, 17, 16)));
645 		stream.add(CurlyBracketOpenToken.create(Position.create("", 1, 18, 17)));
646 		stream.add(CurlyBracketCloseToken.create(Position.create("", 1, 19, 18)));
647 		stream.add(SemicolonToken.create(Position.create("", 1, 20, 19)));
648 		stream.add(SquareBracketCloseToken.create(Position.create("", 1, 21, 20)));
649 		stream.add(SemicolonToken.create(Position.create("", 1, 22, 21)));
650 		stream.add(EndToken.create(Position.create("", 1, 23, 22)));
651 		
652 		this.assertEqualsTokenStream(stream, output);
653 	}
654 	
655 	/**
656 	 * Check if two TokenStream contains the same tokens in the same order.
657 	 * 
658 	 * @param expected
659 	 *            the expected tokenstream
660 	 * @param actual
661 	 *            the actual tokenstream
662 	 */
663 	private void assertEqualsTokenStream(final TokenStream expected, final TokenStream actual) {
664 		
665 		while (expected.hasNext()) {
666 			assertTrue(actual.hasNext());
667 			assertEquals(expected.next(), actual.next());
668 		}
669 		
670 		assertTrue(!actual.hasNext());
671 	}
672 }