ctf/org.eclipse.tracecompass.ctf.parser.tests/src/org/eclipse/tracecompass/ctf/parser/tests/CtfLexerTest.java

   1 /*******************************************************************************
   2  * Copyright (c) 2013, 2014 Etienne Bergeron
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Etienne Bergeron - Initial API and implementation
  11  *******************************************************************************/
  12
  13 package org.eclipse.tracecompass.ctf.parser.tests;
  14
  15 import static org.junit.Assert.fail;
  16
  17 import java.util.LinkedList;
  18 import java.util.List;
  19
  20 import org.antlr.runtime.ANTLRStringStream;
  21 import org.antlr.runtime.CharStream;
  22 import org.antlr.runtime.Token;
  23 import org.eclipse.tracecompass.ctf.parser.CTFLexer;
  24 import org.junit.Ignore;
  25 import org.junit.Test;
  26
  27 /**
  28  * This test validates the CTF-Lexer implementation.
  29  *
  30  * The test splits a string into tokens with the compiled lexer and
  31  * validates the sequences of tokens produced by comparing their type
  32  * and content.
  33  *
  34  * @author Etienne Bergeron
  35  */
  36 public class CtfLexerTest {
  37
  38     // ------------------------------------------------------------------------
  39     // Attributes
  40     // ------------------------------------------------------------------------
  41
  42     private final List<Token> tokens = new LinkedList<>();
  43
  44     // ------------------------------------------------------------------------
  45     // Operations
  46     // ------------------------------------------------------------------------
  47
  48     private void tokenize(String content) {
  49         CharStream cs = new ANTLRStringStream(content);
  50         CTFLexer lexer = new CTFLexer(cs);
  51
  52         tokens.clear();
  53         for (;;) {
  54           Token token = lexer.nextToken();
  55           if (token == Token.EOF_TOKEN) {
  56             return;
  57           }
  58           tokens.add(token);
  59         }
  60     }
  61
  62     private void checkToken(int type, String content) {
  63         Token token = tokens.remove(0);
  64         if (token.getType() != type) {
  65             fail("Invalid type [value " + token.getType()
  66                     + " but expect " + type + "]."
  67                     + " Fail to tokenize:" + content);
  68         } else if (token.getText().compareTo(content) != 0) {
  69             fail("Invalid content [value " + token.getText()
  70                     + " but expect " + content + "].");
  71         }
  72     }
  73
  74     private void checkSingle(int type, String content) {
  75        tokenize(content);
  76        checkToken(type, content);
  77     }
  78
  79     // ------------------------------------------------------------------------
  80     // Test cases
  81     // ------------------------------------------------------------------------
  82
  83     /**
  84      *  Validate the parsing of keywords
  85      */
  86     @Test
  87     public void testKeywords() {
  88         checkSingle(CTFLexer.ALIGNTOK, "align");
  89         checkSingle(CTFLexer.CONSTTOK, "const");
  90         checkSingle(CTFLexer.CHARTOK, "char");
  91         checkSingle(CTFLexer.DOUBLETOK, "double");
  92         checkSingle(CTFLexer.ENUMTOK, "enum");
  93         checkSingle(CTFLexer.EVENTTOK, "event");
  94         checkSingle(CTFLexer.FLOATINGPOINTTOK, "floating_point");
  95         checkSingle(CTFLexer.FLOATTOK, "float");
  96         checkSingle(CTFLexer.INTEGERTOK, "integer");
  97         checkSingle(CTFLexer.INTTOK, "int");
  98         checkSingle(CTFLexer.LONGTOK, "long");
  99         checkSingle(CTFLexer.SHORTTOK, "short");
 100         checkSingle(CTFLexer.SIGNEDTOK, "signed");
 101         checkSingle(CTFLexer.STREAMTOK, "stream");
 102         checkSingle(CTFLexer.STRINGTOK, "string");
 103         checkSingle(CTFLexer.STRUCTTOK, "struct");
 104         checkSingle(CTFLexer.TRACETOK, "trace");
 105         checkSingle(CTFLexer.TYPEALIASTOK, "typealias");
 106         checkSingle(CTFLexer.TYPEDEFTOK, "typedef");
 107         checkSingle(CTFLexer.UNSIGNEDTOK, "unsigned");
 108         checkSingle(CTFLexer.VARIANTTOK, "variant");
 109         checkSingle(CTFLexer.VOIDTOK, "void");
 110         checkSingle(CTFLexer.BOOLTOK, "_Bool");
 111         checkSingle(CTFLexer.COMPLEXTOK, "_Complex");
 112         checkSingle(CTFLexer.IMAGINARYTOK, "_Imaginary");
 113         checkSingle(CTFLexer.ENVTOK, "env");
 114         checkSingle(CTFLexer.CLOCKTOK, "clock");
 115         checkSingle(CTFLexer.CALLSITETOK, "callsite");
 116         checkSingle(CTFLexer.NANNUMBERTOK, "NaN");
 117         checkSingle(CTFLexer.INFINITYTOK,  "+inf");
 118         checkSingle(CTFLexer.NINFINITYTOK, "-inf");
 119     }
 120
 121     /**
 122      *  Validate the parsing of symbols
 123      */
 124     @Test
 125     public void testSymbols() {
 126         tokenize(" , : ... ");
 127         checkToken(CTFLexer.WS, " ");
 128         checkToken(CTFLexer.SEPARATOR, ",");
 129         checkToken(CTFLexer.WS, " ");
 130         checkToken(CTFLexer.COLON, ":");
 131         checkToken(CTFLexer.WS, " ");
 132         checkToken(CTFLexer.ELIPSES, "...");
 133         checkToken(CTFLexer.WS, " ");
 134
 135         tokenize(" = := = ");
 136         checkToken(CTFLexer.WS, " ");
 137         checkToken(CTFLexer.ASSIGNMENT, "=");
 138         checkToken(CTFLexer.WS, " ");
 139         checkToken(CTFLexer.TYPE_ASSIGNMENT, ":=");
 140         checkToken(CTFLexer.WS, " ");
 141
 142         tokenize(" <<>> ");
 143         checkToken(CTFLexer.WS, " ");
 144         checkToken(CTFLexer.LT, "<");
 145         checkToken(CTFLexer.LT, "<");
 146         checkToken(CTFLexer.GT, ">");
 147         checkToken(CTFLexer.GT, ">");
 148         checkToken(CTFLexer.WS, " ");
 149
 150         tokenize(" ({[]}) ");
 151         checkToken(CTFLexer.WS, " ");
 152         checkToken(CTFLexer.LPAREN, "(");
 153         checkToken(CTFLexer.LCURL, "{");
 154         checkToken(CTFLexer.OPENBRAC, "[");
 155         checkToken(CTFLexer.CLOSEBRAC, "]");
 156         checkToken(CTFLexer.RCURL, "}");
 157         checkToken(CTFLexer.RPAREN, ")");
 158         checkToken(CTFLexer.WS, " ");
 159
 160         tokenize(";;");
 161         checkToken(CTFLexer.TERM, ";");
 162         checkToken(CTFLexer.TERM, ";");
 163
 164         tokenize(" ++ -- ");
 165         checkToken(CTFLexer.WS, " ");
 166         checkToken(CTFLexer.SIGN, "+");
 167         checkToken(CTFLexer.SIGN, "+");
 168         checkToken(CTFLexer.WS, " ");
 169         checkToken(CTFLexer.SIGN, "-");
 170         checkToken(CTFLexer.SIGN, "-");
 171         checkToken(CTFLexer.WS, " ");
 172
 173         tokenize("-> .*.");
 174         checkToken(CTFLexer.ARROW, "->");
 175         checkToken(CTFLexer.WS, " ");
 176         checkToken(CTFLexer.DOT, ".");
 177         checkToken(CTFLexer.POINTER, "*");
 178         checkToken(CTFLexer.DOT, ".");
 179     }
 180
 181     /**
 182      *  Validate the parsing of literals
 183      */
 184     @Test
 185     public void testLiterals() {
 186         tokenize("01 02 010");
 187         checkToken(CTFLexer.OCTAL_LITERAL, "01");
 188         checkToken(CTFLexer.WS, " ");
 189         checkToken(CTFLexer.OCTAL_LITERAL, "02");
 190         checkToken(CTFLexer.WS, " ");
 191         checkToken(CTFLexer.OCTAL_LITERAL, "010");
 192
 193         tokenize("1 2 10 1024 ");
 194         checkToken(CTFLexer.DECIMAL_LITERAL, "1");
 195         checkToken(CTFLexer.WS, " ");
 196         checkToken(CTFLexer.DECIMAL_LITERAL, "2");
 197         checkToken(CTFLexer.WS, " ");
 198         checkToken(CTFLexer.DECIMAL_LITERAL, "10");
 199         checkToken(CTFLexer.WS, " ");
 200         checkToken(CTFLexer.DECIMAL_LITERAL, "1024");
 201         checkToken(CTFLexer.WS, " ");
 202
 203         tokenize("0x01 0x02 0x0F0");
 204         checkToken(CTFLexer.HEX_LITERAL, "0x01");
 205         checkToken(CTFLexer.WS, " ");
 206         checkToken(CTFLexer.HEX_LITERAL, "0x02");
 207         checkToken(CTFLexer.WS, " ");
 208         checkToken(CTFLexer.HEX_LITERAL, "0x0F0");
 209     }
 210
 211     /**
 212      *  Validate the parsing of literals with hexa prefix
 213      */
 214     @Test
 215     public void testLiteralPrefixes() {
 216         checkSingle(CTFLexer.HEX_LITERAL, "0x1");
 217         checkSingle(CTFLexer.HEX_LITERAL, "0X1");
 218     }
 219
 220     /**
 221      *  Validate the parsing of literals with type suffix
 222      */
 223     @Test
 224     public void testLiteralSuffixes() {
 225         checkSingle(CTFLexer.DECIMAL_LITERAL, "0l");
 226         checkSingle(CTFLexer.DECIMAL_LITERAL, "0L");
 227         checkSingle(CTFLexer.DECIMAL_LITERAL, "0ll");
 228         checkSingle(CTFLexer.DECIMAL_LITERAL, "0LL");
 229         checkSingle(CTFLexer.DECIMAL_LITERAL, "0ul");
 230         checkSingle(CTFLexer.DECIMAL_LITERAL, "0uL");
 231         checkSingle(CTFLexer.DECIMAL_LITERAL, "0ull");
 232         checkSingle(CTFLexer.DECIMAL_LITERAL, "0uLL");
 233         checkSingle(CTFLexer.DECIMAL_LITERAL, "0Ul");
 234         checkSingle(CTFLexer.DECIMAL_LITERAL, "0UL");
 235         checkSingle(CTFLexer.DECIMAL_LITERAL, "0Ull");
 236         checkSingle(CTFLexer.DECIMAL_LITERAL, "0ULL");
 237     }
 238
 239     /**
 240      *  Validate the accepted characters in literals.
 241      */
 242     @Test
 243     public void testLiteralDigits() {
 244         checkSingle(CTFLexer.OCTAL_LITERAL, "001234567");
 245
 246         checkSingle(CTFLexer.DECIMAL_LITERAL, "123456");
 247         checkSingle(CTFLexer.DECIMAL_LITERAL, "987654");
 248
 249         checkSingle(CTFLexer.HEX_LITERAL, "0x012345");
 250         checkSingle(CTFLexer.HEX_LITERAL, "0x678990");
 251         checkSingle(CTFLexer.HEX_LITERAL, "0xABCDEF");
 252         checkSingle(CTFLexer.HEX_LITERAL, "0xabcdef");
 253     }
 254
 255     /**
 256      *  Validate zero literal to be the right token.
 257      */
 258     @Test
 259     public void testLiteralZero() {
 260         checkSingle(CTFLexer.OCTAL_LITERAL, "00");
 261         checkSingle(CTFLexer.DECIMAL_LITERAL, "0");
 262         checkSingle(CTFLexer.HEX_LITERAL, "0x0");
 263     }
 264
 265     /**
 266      *  Validate character literals
 267      */
 268     @Test
 269     public void testCharLiteral() {
 270         checkSingle(CTFLexer.CHARACTER_LITERAL, "'x'");
 271         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\''");
 272         checkSingle(CTFLexer.CHARACTER_LITERAL, "' '");
 273         checkSingle(CTFLexer.CHARACTER_LITERAL, "L'1'");
 274     }
 275
 276     /**
 277      *  Validate escaped character literals
 278      */
 279     @Test
 280     public void testEscapeCharLiteral() {
 281         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\a'");
 282         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\b'");
 283         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\f'");
 284         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\n'");
 285         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\r'");
 286         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\t'");
 287         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\v'");
 288         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\''");
 289         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\\"'");
 290         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\\\'");
 291
 292         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\001'");
 293         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\01'");
 294         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\1'");
 295
 296         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\x1A'");
 297         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\x1a'");
 298         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\xa'");
 299         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\x0'");
 300
 301         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\uABCD'");
 302         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\u0123'");
 303         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\u012345678'");
 304         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\uFEDCBA987'");
 305     }
 306
 307     /**
 308      *  Validate string literals
 309      */
 310     @Test
 311     public void testStringLiteral() {
 312         checkSingle(CTFLexer.STRING_LITERAL, "\"\"");
 313         checkSingle(CTFLexer.STRING_LITERAL, "\"x\"");
 314         checkSingle(CTFLexer.STRING_LITERAL, "\"\\\"\"");
 315         checkSingle(CTFLexer.STRING_LITERAL, "\" \"");
 316         checkSingle(CTFLexer.STRING_LITERAL, "L\"1\"");
 317
 318         checkSingle(CTFLexer.STRING_LITERAL, "\"This is \\n a multiline\\r\\n\"");
 319         checkSingle(CTFLexer.STRING_LITERAL, "L\"This is \\n a multiline\\r\\n\"");
 320     }
 321
 322     /**
 323      *  Validate string literals with escape sequence
 324      */
 325     @Test
 326     public void testEscapeStringLiteral() {
 327         checkSingle(CTFLexer.STRING_LITERAL, "\"\\a\"");
 328         checkSingle(CTFLexer.STRING_LITERAL, "\"\\b\"");
 329         checkSingle(CTFLexer.STRING_LITERAL, "\"\\f\"");
 330         checkSingle(CTFLexer.STRING_LITERAL, "\"\\n\"");
 331         checkSingle(CTFLexer.STRING_LITERAL, "\"\\r\"");
 332         checkSingle(CTFLexer.STRING_LITERAL, "\"\\t\"");
 333         checkSingle(CTFLexer.STRING_LITERAL, "\"\\v\"");
 334         checkSingle(CTFLexer.STRING_LITERAL, "\"\\'\"");
 335         checkSingle(CTFLexer.STRING_LITERAL, "\"\\\"\"");
 336         checkSingle(CTFLexer.STRING_LITERAL, "\"\\\\\"");
 337
 338         checkSingle(CTFLexer.STRING_LITERAL, "\"\001\"");
 339         checkSingle(CTFLexer.STRING_LITERAL, "\"\01\"");
 340         checkSingle(CTFLexer.STRING_LITERAL, "\"\1\"");
 341
 342         checkSingle(CTFLexer.STRING_LITERAL, "\"\\x1A\"");
 343         checkSingle(CTFLexer.STRING_LITERAL, "\"\\x1a\"");
 344         checkSingle(CTFLexer.STRING_LITERAL, "\"\\xa\"");
 345         checkSingle(CTFLexer.STRING_LITERAL, "\"\\x0\"");
 346
 347         checkSingle(CTFLexer.STRING_LITERAL, "\"\uABCD\"");
 348         checkSingle(CTFLexer.STRING_LITERAL, "\"\u0123\"");
 349         checkSingle(CTFLexer.STRING_LITERAL, "\"\u012345678\"");
 350         checkSingle(CTFLexer.STRING_LITERAL, "\"\uFEDCBA987\"");
 351     }
 352
 353     /**
 354      *  Validate spaces parsing
 355      */
 356     @Test
 357     public void testWhitespaces() {
 358         tokenize("  \r\t\n\u000C ");
 359         checkToken(CTFLexer.WS, " ");
 360         checkToken(CTFLexer.WS, " ");
 361         checkToken(CTFLexer.WS, "\r");
 362         checkToken(CTFLexer.WS, "\t");
 363         checkToken(CTFLexer.WS, "\n");
 364         checkToken(CTFLexer.WS, "\u000C");
 365         checkToken(CTFLexer.WS, " ");
 366     }
 367
 368     /**
 369      *  Validate comments parsing
 370      */
 371     @Test
 372     public void testComment() {
 373         tokenize(" /* test */ ");
 374         checkToken(CTFLexer.WS, " ");
 375         checkToken(CTFLexer.COMMENT, "/* test */");
 376         checkToken(CTFLexer.WS, " ");
 377     }
 378
 379     /**
 380      *  Validate complex nested comments parsing
 381      */
 382     @Test
 383     public void testNestedComment() {
 384         tokenize(" /* /* */ ");
 385         checkToken(CTFLexer.WS, " ");
 386         checkToken(CTFLexer.COMMENT, "/* /* */");
 387         checkToken(CTFLexer.WS, " ");
 388
 389         tokenize(" /* /* * ** / */ ");
 390         checkToken(CTFLexer.WS, " ");
 391         checkToken(CTFLexer.COMMENT, "/* /* * ** / */");
 392         checkToken(CTFLexer.WS, " ");
 393     }
 394
 395     /**
 396      *  Validate multi-lines comments
 397      */
 398     @Test
 399     public void testMultiLineComment() {
 400         tokenize(" /*\ntest\n*/ ");
 401         checkToken(CTFLexer.WS, " ");
 402         checkToken(CTFLexer.COMMENT, "/*\ntest\n*/");
 403         checkToken(CTFLexer.WS, " ");
 404     }
 405
 406     /**
 407      *  Validate single line comments
 408      */
 409     @Test
 410     public void testLineComment() {
 411         tokenize(" // asdad\r\n ");
 412         checkToken(CTFLexer.WS, " ");
 413         checkToken(CTFLexer.LINE_COMMENT, "// asdad\r\n");
 414         checkToken(CTFLexer.WS, " ");
 415     }
 416
 417     /**
 418      *  Validate incomplete comments parsing
 419      */
 420     @Ignore("Lexer must be fixed first")
 421     @Test
 422     public void testLineCommentWithEOF() {
 423         tokenize("//");
 424         checkToken(CTFLexer.LINE_COMMENT, "//");
 425     }
 426
 427     /**
 428      *  Validate parsing of mixed kind of comments
 429      */
 430     @Test
 431     public void testMixedComment() {
 432         tokenize(" // /*\n");
 433         checkToken(CTFLexer.WS, " ");
 434         checkToken(CTFLexer.LINE_COMMENT, "// /*\n");
 435
 436         tokenize(" /*\n//\n*/ ");
 437         checkToken(CTFLexer.WS, " ");
 438         checkToken(CTFLexer.COMMENT, "/*\n//\n*/");
 439         checkToken(CTFLexer.WS, " ");
 440     }
 441
 442     /**
 443      *  Validate parsing identifiers
 444      */
 445     @Test
 446     public void testIdentifier() {
 447         tokenize("_ a a1 B ");
 448         checkToken(CTFLexer.IDENTIFIER, "_");
 449         checkToken(CTFLexer.WS, " ");
 450         checkToken(CTFLexer.IDENTIFIER, "a");
 451         checkToken(CTFLexer.WS, " ");
 452         checkToken(CTFLexer.IDENTIFIER, "a1");
 453         checkToken(CTFLexer.WS, " ");
 454         checkToken(CTFLexer.IDENTIFIER, "B");
 455         checkToken(CTFLexer.WS, " ");
 456     }
 457
 458     /**
 459      *  Validate accepted characters within an identifier
 460      */
 461     @Test
 462     public void testIdentifierLetters() {
 463         checkSingle(CTFLexer.IDENTIFIER, "ABCDEFGHI");
 464         checkSingle(CTFLexer.IDENTIFIER, "JKLMNOPQR");
 465         checkSingle(CTFLexer.IDENTIFIER, "STUVWXYZ");
 466         checkSingle(CTFLexer.IDENTIFIER, "abcdefghi");
 467         checkSingle(CTFLexer.IDENTIFIER, "jklmnopqr");
 468         checkSingle(CTFLexer.IDENTIFIER, "stuvwxyz");
 469         checkSingle(CTFLexer.IDENTIFIER, "_0123456789");
 470     }
 471 }