ctf/org.eclipse.tracecompass.ctf.parser.tests/src/org/eclipse/tracecompass/ctf/parser/tests/CtfLexerTest.java

   1 /*******************************************************************************
   2  * Copyright (c) 2013, 2014 Etienne Bergeron
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Etienne Bergeron - Initial API and implementation
  11  *******************************************************************************/
  12
  13 package org.eclipse.tracecompass.ctf.parser.tests;
  14
  15 import static org.junit.Assert.fail;
  16
  17 import java.util.LinkedList;
  18 import java.util.List;
  19
  20 import org.antlr.runtime.ANTLRStringStream;
  21 import org.antlr.runtime.CharStream;
  22 import org.antlr.runtime.Token;
  23 import org.eclipse.tracecompass.ctf.parser.CTFLexer;
  24 import org.junit.Ignore;
  25 import org.junit.Test;
  26
  27 /**
  28  * This test validates the CTF-Lexer implementation.
  29  *
  30  * The test splits a string into tokens with the compiled lexer and
  31  * validates the sequences of tokens produced by comparing their type
  32  * and content.
  33  *
  34  * @author Etienne Bergeron
  35  */
  36 public class CtfLexerTest {
  37
  38     // ------------------------------------------------------------------------
  39     // Attributes
  40     // ------------------------------------------------------------------------
  41
  42     private final List<Token> tokens = new LinkedList<>();
  43
  44     // ------------------------------------------------------------------------
  45     // Operations
  46     // ------------------------------------------------------------------------
  47
  48     private void tokenize(String content) {
  49         CharStream cs = new ANTLRStringStream(content);
  50         CTFLexer lexer = new CTFLexer(cs);
  51
  52         tokens.clear();
  53         for (;;) {
  54           Token token = lexer.nextToken();
  55           if (token == Token.EOF_TOKEN) {
  56             return;
  57           }
  58           tokens.add(token);
  59         }
  60     }
  61
  62     private void checkToken(int type, String content) {
  63         Token token = tokens.remove(0);
  64         if (token.getType() != type) {
  65             fail("Invalid type [value " + token.getType()
  66                     + " but expect " + type + "]."
  67                     + " Fail to tokenize:" + content);
  68         } else if (token.getText().compareTo(content) != 0) {
  69             fail("Invalid content [value " + token.getText()
  70                     + " but expect " + content + "].");
  71         }
  72     }
  73
  74     private void checkSingle(int type, String content) {
  75        tokenize(content);
  76        checkToken(type, content);
  77     }
  78
  79     // ------------------------------------------------------------------------
  80     // Test cases
  81     // ------------------------------------------------------------------------
  82
  83     /**
  84      *  Validate the parsing of keywords
  85      */
  86     @Test
  87     public void testKeywords() {
  88         checkSingle(CTFLexer.ALIGNTOK, "align");
  89         checkSingle(CTFLexer.CONSTTOK, "const");
  90         checkSingle(CTFLexer.CHARTOK, "char");
  91         checkSingle(CTFLexer.DOUBLETOK, "double");
  92         checkSingle(CTFLexer.ENUMTOK, "enum");
  93         checkSingle(CTFLexer.EVENTTOK, "event");
  94         checkSingle(CTFLexer.FLOATINGPOINTTOK, "floating_point");
  95         checkSingle(CTFLexer.FLOATTOK, "float");
  96         checkSingle(CTFLexer.INTEGERTOK, "integer");
  97         checkSingle(CTFLexer.INTTOK, "int");
  98         checkSingle(CTFLexer.LONGTOK, "long");
  99         checkSingle(CTFLexer.SHORTTOK, "short");
 100         checkSingle(CTFLexer.SIGNEDTOK, "signed");
 101         checkSingle(CTFLexer.STREAMTOK, "stream");
 102         checkSingle(CTFLexer.STRINGTOK, "string");
 103         checkSingle(CTFLexer.STRUCTTOK, "struct");
 104         checkSingle(CTFLexer.TRACETOK, "trace");
 105         checkSingle(CTFLexer.TYPEALIASTOK, "typealias");
 106         checkSingle(CTFLexer.TYPEDEFTOK, "typedef");
 107         checkSingle(CTFLexer.UNSIGNEDTOK, "unsigned");
 108         checkSingle(CTFLexer.VARIANTTOK, "variant");
 109         checkSingle(CTFLexer.VOIDTOK, "void");
 110         checkSingle(CTFLexer.BOOLTOK, "_Bool");
 111         checkSingle(CTFLexer.COMPLEXTOK, "_Complex");
 112         checkSingle(CTFLexer.IMAGINARYTOK, "_Imaginary");
 113         checkSingle(CTFLexer.ENVTOK, "env");
 114         checkSingle(CTFLexer.CLOCKTOK, "clock");
 115         checkSingle(CTFLexer.NANNUMBERTOK, "NaN");
 116         checkSingle(CTFLexer.INFINITYTOK,  "+inf");
 117         checkSingle(CTFLexer.NINFINITYTOK, "-inf");
 118     }
 119
 120     /**
 121      *  Validate the parsing of symbols
 122      */
 123     @Test
 124     public void testSymbols() {
 125         tokenize(" , : ... ");
 126         checkToken(CTFLexer.WS, " ");
 127         checkToken(CTFLexer.SEPARATOR, ",");
 128         checkToken(CTFLexer.WS, " ");
 129         checkToken(CTFLexer.COLON, ":");
 130         checkToken(CTFLexer.WS, " ");
 131         checkToken(CTFLexer.ELIPSES, "...");
 132         checkToken(CTFLexer.WS, " ");
 133
 134         tokenize(" = := = ");
 135         checkToken(CTFLexer.WS, " ");
 136         checkToken(CTFLexer.ASSIGNMENT, "=");
 137         checkToken(CTFLexer.WS, " ");
 138         checkToken(CTFLexer.TYPE_ASSIGNMENT, ":=");
 139         checkToken(CTFLexer.WS, " ");
 140
 141         tokenize(" <<>> ");
 142         checkToken(CTFLexer.WS, " ");
 143         checkToken(CTFLexer.LT, "<");
 144         checkToken(CTFLexer.LT, "<");
 145         checkToken(CTFLexer.GT, ">");
 146         checkToken(CTFLexer.GT, ">");
 147         checkToken(CTFLexer.WS, " ");
 148
 149         tokenize(" ({[]}) ");
 150         checkToken(CTFLexer.WS, " ");
 151         checkToken(CTFLexer.LPAREN, "(");
 152         checkToken(CTFLexer.LCURL, "{");
 153         checkToken(CTFLexer.OPENBRAC, "[");
 154         checkToken(CTFLexer.CLOSEBRAC, "]");
 155         checkToken(CTFLexer.RCURL, "}");
 156         checkToken(CTFLexer.RPAREN, ")");
 157         checkToken(CTFLexer.WS, " ");
 158
 159         tokenize(";;");
 160         checkToken(CTFLexer.TERM, ";");
 161         checkToken(CTFLexer.TERM, ";");
 162
 163         tokenize(" ++ -- ");
 164         checkToken(CTFLexer.WS, " ");
 165         checkToken(CTFLexer.SIGN, "+");
 166         checkToken(CTFLexer.SIGN, "+");
 167         checkToken(CTFLexer.WS, " ");
 168         checkToken(CTFLexer.SIGN, "-");
 169         checkToken(CTFLexer.SIGN, "-");
 170         checkToken(CTFLexer.WS, " ");
 171
 172         tokenize("-> .*.");
 173         checkToken(CTFLexer.ARROW, "->");
 174         checkToken(CTFLexer.WS, " ");
 175         checkToken(CTFLexer.DOT, ".");
 176         checkToken(CTFLexer.POINTER, "*");
 177         checkToken(CTFLexer.DOT, ".");
 178     }
 179
 180     /**
 181      *  Validate the parsing of literals
 182      */
 183     @Test
 184     public void testLiterals() {
 185         tokenize("01 02 010");
 186         checkToken(CTFLexer.OCTAL_LITERAL, "01");
 187         checkToken(CTFLexer.WS, " ");
 188         checkToken(CTFLexer.OCTAL_LITERAL, "02");
 189         checkToken(CTFLexer.WS, " ");
 190         checkToken(CTFLexer.OCTAL_LITERAL, "010");
 191
 192         tokenize("1 2 10 1024 ");
 193         checkToken(CTFLexer.DECIMAL_LITERAL, "1");
 194         checkToken(CTFLexer.WS, " ");
 195         checkToken(CTFLexer.DECIMAL_LITERAL, "2");
 196         checkToken(CTFLexer.WS, " ");
 197         checkToken(CTFLexer.DECIMAL_LITERAL, "10");
 198         checkToken(CTFLexer.WS, " ");
 199         checkToken(CTFLexer.DECIMAL_LITERAL, "1024");
 200         checkToken(CTFLexer.WS, " ");
 201
 202         tokenize("0x01 0x02 0x0F0");
 203         checkToken(CTFLexer.HEX_LITERAL, "0x01");
 204         checkToken(CTFLexer.WS, " ");
 205         checkToken(CTFLexer.HEX_LITERAL, "0x02");
 206         checkToken(CTFLexer.WS, " ");
 207         checkToken(CTFLexer.HEX_LITERAL, "0x0F0");
 208     }
 209
 210     /**
 211      *  Validate the parsing of literals with hexa prefix
 212      */
 213     @Test
 214     public void testLiteralPrefixes() {
 215         checkSingle(CTFLexer.HEX_LITERAL, "0x1");
 216         checkSingle(CTFLexer.HEX_LITERAL, "0X1");
 217     }
 218
 219     /**
 220      *  Validate the parsing of literals with type suffix
 221      */
 222     @Test
 223     public void testLiteralSuffixes() {
 224         checkSingle(CTFLexer.DECIMAL_LITERAL, "0l");
 225         checkSingle(CTFLexer.DECIMAL_LITERAL, "0L");
 226         checkSingle(CTFLexer.DECIMAL_LITERAL, "0ll");
 227         checkSingle(CTFLexer.DECIMAL_LITERAL, "0LL");
 228         checkSingle(CTFLexer.DECIMAL_LITERAL, "0ul");
 229         checkSingle(CTFLexer.DECIMAL_LITERAL, "0uL");
 230         checkSingle(CTFLexer.DECIMAL_LITERAL, "0ull");
 231         checkSingle(CTFLexer.DECIMAL_LITERAL, "0uLL");
 232         checkSingle(CTFLexer.DECIMAL_LITERAL, "0Ul");
 233         checkSingle(CTFLexer.DECIMAL_LITERAL, "0UL");
 234         checkSingle(CTFLexer.DECIMAL_LITERAL, "0Ull");
 235         checkSingle(CTFLexer.DECIMAL_LITERAL, "0ULL");
 236     }
 237
 238     /**
 239      *  Validate the accepted characters in literals.
 240      */
 241     @Test
 242     public void testLiteralDigits() {
 243         checkSingle(CTFLexer.OCTAL_LITERAL, "001234567");
 244
 245         checkSingle(CTFLexer.DECIMAL_LITERAL, "123456");
 246         checkSingle(CTFLexer.DECIMAL_LITERAL, "987654");
 247
 248         checkSingle(CTFLexer.HEX_LITERAL, "0x012345");
 249         checkSingle(CTFLexer.HEX_LITERAL, "0x678990");
 250         checkSingle(CTFLexer.HEX_LITERAL, "0xABCDEF");
 251         checkSingle(CTFLexer.HEX_LITERAL, "0xabcdef");
 252     }
 253
 254     /**
 255      *  Validate zero literal to be the right token.
 256      */
 257     @Test
 258     public void testLiteralZero() {
 259         checkSingle(CTFLexer.OCTAL_LITERAL, "00");
 260         checkSingle(CTFLexer.DECIMAL_LITERAL, "0");
 261         checkSingle(CTFLexer.HEX_LITERAL, "0x0");
 262     }
 263
 264     /**
 265      *  Validate character literals
 266      */
 267     @Test
 268     public void testCharLiteral() {
 269         checkSingle(CTFLexer.CHARACTER_LITERAL, "'x'");
 270         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\''");
 271         checkSingle(CTFLexer.CHARACTER_LITERAL, "' '");
 272         checkSingle(CTFLexer.CHARACTER_LITERAL, "L'1'");
 273     }
 274
 275     /**
 276      *  Validate escaped character literals
 277      */
 278     @Test
 279     public void testEscapeCharLiteral() {
 280         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\a'");
 281         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\b'");
 282         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\f'");
 283         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\n'");
 284         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\r'");
 285         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\t'");
 286         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\v'");
 287         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\''");
 288         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\\"'");
 289         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\\\'");
 290
 291         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\001'");
 292         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\01'");
 293         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\1'");
 294
 295         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\x1A'");
 296         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\x1a'");
 297         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\xa'");
 298         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\\x0'");
 299
 300         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\uABCD'");
 301         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\u0123'");
 302         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\u012345678'");
 303         checkSingle(CTFLexer.CHARACTER_LITERAL, "'\uFEDCBA987'");
 304     }
 305
 306     /**
 307      *  Validate string literals
 308      */
 309     @Test
 310     public void testStringLiteral() {
 311         checkSingle(CTFLexer.STRING_LITERAL, "\"\"");
 312         checkSingle(CTFLexer.STRING_LITERAL, "\"x\"");
 313         checkSingle(CTFLexer.STRING_LITERAL, "\"\\\"\"");
 314         checkSingle(CTFLexer.STRING_LITERAL, "\" \"");
 315         checkSingle(CTFLexer.STRING_LITERAL, "L\"1\"");
 316
 317         checkSingle(CTFLexer.STRING_LITERAL, "\"This is \\n a multiline\\r\\n\"");
 318         checkSingle(CTFLexer.STRING_LITERAL, "L\"This is \\n a multiline\\r\\n\"");
 319     }
 320
 321     /**
 322      *  Validate string literals with escape sequence
 323      */
 324     @Test
 325     public void testEscapeStringLiteral() {
 326         checkSingle(CTFLexer.STRING_LITERAL, "\"\\a\"");
 327         checkSingle(CTFLexer.STRING_LITERAL, "\"\\b\"");
 328         checkSingle(CTFLexer.STRING_LITERAL, "\"\\f\"");
 329         checkSingle(CTFLexer.STRING_LITERAL, "\"\\n\"");
 330         checkSingle(CTFLexer.STRING_LITERAL, "\"\\r\"");
 331         checkSingle(CTFLexer.STRING_LITERAL, "\"\\t\"");
 332         checkSingle(CTFLexer.STRING_LITERAL, "\"\\v\"");
 333         checkSingle(CTFLexer.STRING_LITERAL, "\"\\'\"");
 334         checkSingle(CTFLexer.STRING_LITERAL, "\"\\\"\"");
 335         checkSingle(CTFLexer.STRING_LITERAL, "\"\\\\\"");
 336
 337         checkSingle(CTFLexer.STRING_LITERAL, "\"\001\"");
 338         checkSingle(CTFLexer.STRING_LITERAL, "\"\01\"");
 339         checkSingle(CTFLexer.STRING_LITERAL, "\"\1\"");
 340
 341         checkSingle(CTFLexer.STRING_LITERAL, "\"\\x1A\"");
 342         checkSingle(CTFLexer.STRING_LITERAL, "\"\\x1a\"");
 343         checkSingle(CTFLexer.STRING_LITERAL, "\"\\xa\"");
 344         checkSingle(CTFLexer.STRING_LITERAL, "\"\\x0\"");
 345
 346         checkSingle(CTFLexer.STRING_LITERAL, "\"\uABCD\"");
 347         checkSingle(CTFLexer.STRING_LITERAL, "\"\u0123\"");
 348         checkSingle(CTFLexer.STRING_LITERAL, "\"\u012345678\"");
 349         checkSingle(CTFLexer.STRING_LITERAL, "\"\uFEDCBA987\"");
 350     }
 351
 352     /**
 353      *  Validate spaces parsing
 354      */
 355     @Test
 356     public void testWhitespaces() {
 357         tokenize("  \r\t\n\u000C ");
 358         checkToken(CTFLexer.WS, " ");
 359         checkToken(CTFLexer.WS, " ");
 360         checkToken(CTFLexer.WS, "\r");
 361         checkToken(CTFLexer.WS, "\t");
 362         checkToken(CTFLexer.WS, "\n");
 363         checkToken(CTFLexer.WS, "\u000C");
 364         checkToken(CTFLexer.WS, " ");
 365     }
 366
 367     /**
 368      *  Validate comments parsing
 369      */
 370     @Test
 371     public void testComment() {
 372         tokenize(" /* test */ ");
 373         checkToken(CTFLexer.WS, " ");
 374         checkToken(CTFLexer.COMMENT, "/* test */");
 375         checkToken(CTFLexer.WS, " ");
 376     }
 377
 378     /**
 379      *  Validate complex nested comments parsing
 380      */
 381     @Test
 382     public void testNestedComment() {
 383         tokenize(" /* /* */ ");
 384         checkToken(CTFLexer.WS, " ");
 385         checkToken(CTFLexer.COMMENT, "/* /* */");
 386         checkToken(CTFLexer.WS, " ");
 387
 388         tokenize(" /* /* * ** / */ ");
 389         checkToken(CTFLexer.WS, " ");
 390         checkToken(CTFLexer.COMMENT, "/* /* * ** / */");
 391         checkToken(CTFLexer.WS, " ");
 392     }
 393
 394     /**
 395      *  Validate multi-lines comments
 396      */
 397     @Test
 398     public void testMultiLineComment() {
 399         tokenize(" /*\ntest\n*/ ");
 400         checkToken(CTFLexer.WS, " ");
 401         checkToken(CTFLexer.COMMENT, "/*\ntest\n*/");
 402         checkToken(CTFLexer.WS, " ");
 403     }
 404
 405     /**
 406      *  Validate single line comments
 407      */
 408     @Test
 409     public void testLineComment() {
 410         tokenize(" // asdad\r\n ");
 411         checkToken(CTFLexer.WS, " ");
 412         checkToken(CTFLexer.LINE_COMMENT, "// asdad\r\n");
 413         checkToken(CTFLexer.WS, " ");
 414     }
 415
 416     /**
 417      *  Validate incomplete comments parsing
 418      */
 419     @Ignore("Lexer must be fixed first")
 420     @Test
 421     public void testLineCommentWithEOF() {
 422         tokenize("//");
 423         checkToken(CTFLexer.LINE_COMMENT, "//");
 424     }
 425
 426     /**
 427      *  Validate parsing of mixed kind of comments
 428      */
 429     @Test
 430     public void testMixedComment() {
 431         tokenize(" // /*\n");
 432         checkToken(CTFLexer.WS, " ");
 433         checkToken(CTFLexer.LINE_COMMENT, "// /*\n");
 434
 435         tokenize(" /*\n//\n*/ ");
 436         checkToken(CTFLexer.WS, " ");
 437         checkToken(CTFLexer.COMMENT, "/*\n//\n*/");
 438         checkToken(CTFLexer.WS, " ");
 439     }
 440
 441     /**
 442      *  Validate parsing identifiers
 443      */
 444     @Test
 445     public void testIdentifier() {
 446         tokenize("_ a a1 B ");
 447         checkToken(CTFLexer.IDENTIFIER, "_");
 448         checkToken(CTFLexer.WS, " ");
 449         checkToken(CTFLexer.IDENTIFIER, "a");
 450         checkToken(CTFLexer.WS, " ");
 451         checkToken(CTFLexer.IDENTIFIER, "a1");
 452         checkToken(CTFLexer.WS, " ");
 453         checkToken(CTFLexer.IDENTIFIER, "B");
 454         checkToken(CTFLexer.WS, " ");
 455     }
 456
 457     /**
 458      *  Validate accepted characters within an identifier
 459      */
 460     @Test
 461     public void testIdentifierLetters() {
 462         checkSingle(CTFLexer.IDENTIFIER, "ABCDEFGHI");
 463         checkSingle(CTFLexer.IDENTIFIER, "JKLMNOPQR");
 464         checkSingle(CTFLexer.IDENTIFIER, "STUVWXYZ");
 465         checkSingle(CTFLexer.IDENTIFIER, "abcdefghi");
 466         checkSingle(CTFLexer.IDENTIFIER, "jklmnopqr");
 467         checkSingle(CTFLexer.IDENTIFIER, "stuvwxyz");
 468         checkSingle(CTFLexer.IDENTIFIER, "_0123456789");
 469     }
 470 }