compiler2/ttcn3/compiler.l

   1 /******************************************************************************
   2  * Copyright (c) 2000-2016 Ericsson Telecom AB
   3  * All rights reserved. This program and the accompanying materials
   4  * are made available under the terms of the Eclipse Public License v1.0
   5  * which accompanies this distribution, and is available at
   6  * http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Contributors:
   9  *   Baji, Laszlo
  10  *   Balasko, Jeno
  11  *   Baranyi, Botond
  12  *   Cserveni, Akos
  13  *   Delic, Adam
  14  *   Feher, Csaba
  15  *   Forstner, Matyas
  16  *   Kovacs, Ferenc
  17  *   Kremer, Peter
  18  *   Raduly, Csaba
  19  *   Szabados, Kristof
  20  *   Szabo, Bence Janos
  21  *   Szabo, Janos Zoltan – initial implementation
  22  *   Zalanyi, Balazs Andor
  23  *
  24  ******************************************************************************/
  25 %option noyywrap
  26 %option never-interactive
  27 %option nounput
  28 %{
  29
  30 /* Tokenizer for TTCN-3 Core Language */
  31
  32 #include "../../common/dbgnew.hh"
  33 #include "compiler.h"
  34 #include "../string.hh"
  35 #include "../CompilerError.hh"
  36 #include "../Real.hh"
  37 #include "../Value.hh"
  38 #include "AST_ttcn3.hh"
  39 #include "Ttcnstuff.hh" // at least for PortTypeBody::PortOperationMode_t
  40 #include "Statement.hh" // for Statement::statementtype_t
  41 #include "Attributes.hh"
  42
  43 #include <string.h>
  44 #include <ctype.h>
  45 #include <openssl/md5.h>
  46
  47 namespace Common {
  48   class IndexedValue;
  49   class Location;
  50 }
  51
  52 namespace Ttcn {
  53   class ParamRedirect;
  54   class Statement;
  55   class AltGuard;
  56   class IfClause;
  57   class IfClauses;
  58   class NamedTemplate;
  59   class NamedTemplates;
  60   class IndexedTemplate;
  61   class IndexedTemplates;
  62   class Templates;
  63   class CompTypeRefList;
  64 }
  65
  66 using namespace Common;
  67 using namespace Ttcn;
  68
  69 #include "compiler.tab.hh"
  70
  71 #define yylval ttcn3_lval
  72 #define yylloc ttcn3_lloc
  73
  74 /* global variable indicating the location of the returned token to bison */
  75 extern YYLTYPE yylloc;
  76
  77 extern bool is_erroneous_parsed;
  78
  79 /* always points to the first character of the regexp to be recognized */
  80 static int current_line, current_column;
  81
  82 /* when reporting an error in linemarker or preprocessor
  83  * directive the real file name and line number is needed */
  84 static const char* real_infile;
  85 /* real_lineno = current_line + real_lineno_offset */
  86 static int real_lineno_offset;
  87
  88 static bool dot_flag = false;
  89 /* type of the backup token (that was found after a dot) */
  90 static int backup_token;
  91 /* semantic value of the backup token */
  92 static YYSTYPE backup_lval;
  93 /* location of the backup token */
  94 static YYLTYPE backup_lloc;
  95
  96 static MD5_CTX md5_ctx;
  97
  98 static void fill_location()
  99 {
 100   yylloc.first_line = current_line;
 101   yylloc.first_column = current_column;
 102   current_column += yyleng;
 103   yylloc.last_line = current_line;
 104   yylloc.last_column = current_column;
 105 }
 106
 107 static void update_md5()
 108 {
 109   MD5_Update(&md5_ctx, yytext, yyleng);
 110   MD5_Update(&md5_ctx, " ", 1);
 111 }
 112
 113 #define RETURN_SAVED_DOT \
 114   do { \
 115     yytext[0] = '\0'; \
 116     yylloc.first_line = dot_line; \
 117     yylloc.first_column = dot_column; \
 118     yylloc.last_line = dot_line; \
 119     yylloc.last_column = dot_column + 1; \
 120     return '.'; \
 121   } while (0)
 122
 123 /* return macro for simple tokens without semantic value */
 124 #define RETURN(ret_val) \
 125   do { \
 126     update_md5(); \
 127     fill_location(); \
 128     if (dot_flag) { \
 129       backup_token = ret_val; \
 130       backup_lloc = yylloc; \
 131       RETURN_SAVED_DOT; \
 132     } else return ret_val; \
 133   } while (0)
 134
 135 /* same as RETURN(ret_val) macro but without location update,
 136  * usually a return after an error */
 137 #define RETURN_NOLOCUPD(ret_val) \
 138   do { \
 139     update_md5(); \
 140     if (dot_flag) { \
 141       backup_token = ret_val; \
 142       backup_lloc = yylloc; \
 143       RETURN_SAVED_DOT; \
 144     } else return ret_val; \
 145   } while (0)
 146
 147 /* return macro for simple tokens with semantic value */
 148 #define RETURN_LVAL(ret_val) \
 149   do { \
 150     update_md5(); \
 151     fill_location(); \
 152     if (dot_flag) { \
 153       backup_token = ret_val; \
 154       backup_lval = yylval; \
 155       backup_lloc = yylloc; \
 156       RETURN_SAVED_DOT; \
 157     } else return ret_val; \
 158   } while (0)
 159
 160 /* return macro for special tokens that are glued together with previous dot */
 161 #define RETURN_DOT(ret_val) \
 162   do { \
 163     update_md5(); \
 164     if (dot_flag) { \
 165       dot_flag = false; \
 166       yylloc.first_line = dot_line; \
 167       yylloc.first_column = dot_column; \
 168       current_column += yyleng; \
 169       yylloc.last_line = current_line; \
 170       yylloc.last_column = current_column; \
 171       return Dot##ret_val; \
 172     } else { \
 173       fill_location(); \
 174       return ret_val; \
 175     } \
 176   } while (0)
 177
 178 extern string *parse_charstring_value(const char *str, const Location& loc);
 179
 180 %}
 181
 182 NUMBER 0|([1-9][0-9]*)
 183
 184 FLOAT ({NUMBER}\.[0-9]+)|({NUMBER}(\.[0-9]+)?[Ee][+-]?{NUMBER})
 185
 186 IDENTIFIER [A-Za-z][A-Za-z0-9_]*
 187
 188 LINECOMMENT "//"[^\r\n]*
 189
 190 WHITESPACE [ \t\v\f]
 191
 192 NEWLINE \r|\n|\r\n
 193
 194 LINEMARKER {NUMBER}{WHITESPACE}+\"([^\\\"\r\n]|\\[^\r\n])*\"
 195
 196 UID [uU][+]?[0-9A-Fa-f]{1,8}
 197
 198 TITAN "$#&&&(#TITANERRONEOUS$#&&^#% "
 199
 200 %x SC_blockcomment SC_cstring
 201 %x SC_binstring SC_binstring_bad
 202 %s SC_charkeyword
 203
 204 %%
 205   /* local variables of yylex() */
 206   int start_line = 0, start_column = 0; /**< used by block comments and
 207                                              string literals */
 208   int dot_line = 0, dot_column = 0; /**< location of the previous '.' token */
 209   /* variables used when processing binary strings */
 210   expstring_t binstr = NULL; /**< the string itself */
 211   bool valid_bit = false, /**< binstr is valid bitstring */
 212     valid_oct = false, /**< binstr is valid octetstring */
 213     half_oct = false, /**< binstr is not a valid octetstr but a valid hexstr */
 214     contains_match = false, /**< binstr contains matching symbol */
 215     contains_ws = false; /**< binstr contains whitespace characters */
 216
 217   if (dot_flag) {
 218     if (backup_token == '.') {
 219       /* Two dots were found in the previous call: the first one was returned,
 220        * the second one is now in the backup. Let's assume that we have just
 221        * found the second one. */
 222       dot_line = backup_lloc.first_line;
 223       dot_column = backup_lloc.first_column;
 224     } else {
 225       /* Return the token and its semantic value that was backed up after the
 226        * last token (which was a dot). */
 227       dot_flag = false;
 228       yylval = backup_lval;
 229       yylloc = backup_lloc;
 230       return backup_token;
 231     }
 232   }
 233
 234
 235 {TITAN} {
 236   // hack: to avoid the erroneous parsing reporting a syntax error where it
 237   //       also lists TTCN3ModuleKeyword as a possible nonterminal to use
 238   if (is_erroneous_parsed) {
 239     RETURN(TitanErroneousHackKeyword);
 240   } else {
 241     Location loc(infile, current_line, current_column, current_line,
 242                  current_column + yyleng);
 243     loc.error("Unexpected `%s'.", yytext);
 244   }
 245 }
 246
 247         /* Eat up comments and whitespaces */
 248
 249 "/*" {
 250   start_line = current_line;
 251   start_column = current_column;
 252   current_column += 2;
 253   BEGIN(SC_blockcomment);
 254 }
 255
 256 <SC_blockcomment> /* -------- SC_blockcomment scope -------------- */
 257 {
 258
 259   "*/" {
 260     current_column += 2;
 261     BEGIN(INITIAL);
 262   }
 263
 264   {NEWLINE} {
 265     current_line++;
 266     current_column = 0;
 267   }
 268
 269   . current_column++;
 270
 271 } /* SC_blockcomment */
 272
 273 {LINECOMMENT}?{NEWLINE} {
 274   current_line++;
 275   current_column = 0;
 276 }
 277
 278 {LINECOMMENT} {
 279   current_column += yyleng;
 280 }
 281
 282 {WHITESPACE}+ current_column += yyleng;
 283
 284         /* C preprocessor line markers */
 285
 286 ^{WHITESPACE}*"#"({WHITESPACE}*"line")?{WHITESPACE}+{LINEMARKER}[^\r\n]*{NEWLINE} {
 287   bool error_flag = false;
 288   int real_lineno = current_line + real_lineno_offset;
 289   /* skipping the leading whitespaces */
 290   int marker_begin = 0;
 291   while (yytext[marker_begin] != '#') marker_begin++;
 292   /* skipping the trailing whitespaces and newline */
 293   int marker_end = yyleng - 1;
 294   while (yytext[marker_end] == '\r' || yytext[marker_end] == '\n' ||
 295          yytext[marker_end] == ' ' || yytext[marker_end] == '\t') marker_end--;
 296   marker_end++;
 297   Location loc(real_infile, real_lineno, current_column + marker_begin,
 298     real_lineno, current_column + marker_end);
 299   Error_Context cntxt(&loc, "In preprocessor line marker");
 300   /* parsing out the line number */
 301   int lineno_begin = marker_begin + 1;
 302   while (!isdigit((unsigned char)yytext[lineno_begin])) lineno_begin++;
 303   int lineno_end = lineno_begin + 1;
 304   while (isdigit((int)yytext[lineno_end])) lineno_end++;
 305   errno = 0;
 306   int new_lineno = strtol(yytext + lineno_begin, NULL, 10);
 307   if (errno != 0) {
 308     Location lineno_loc(real_infile, real_lineno, current_column + lineno_begin,
 309       real_lineno, current_column + lineno_end);
 310     string lineno_str(lineno_end - lineno_begin, yytext + lineno_begin);
 311     lineno_loc.error("Line number `%s' is too large for being represented in "
 312       "memory: %s", lineno_str.c_str(), strerror(errno));
 313     error_flag = true;
 314   }
 315   /* parsing out the file name */
 316   int filename_begin = lineno_end + 1;
 317   while (yytext[filename_begin] != '"') filename_begin++;
 318   filename_begin++;
 319   int filename_end = filename_begin;
 320   while (yytext[filename_end] != '"') {
 321     if (yytext[filename_end] == '\\') filename_end += 2;
 322     else filename_end++;
 323   }
 324   Location filename_loc(real_infile, real_lineno, current_column +
 325     filename_begin - 1, real_lineno, current_column + filename_end + 1);
 326   string filename_str(filename_end - filename_begin, yytext + filename_begin);
 327   string *parsed_filename = parse_charstring_value(filename_str.c_str(),
 328     filename_loc);
 329   if (!parsed_filename) error_flag = true;
 330   /* updating the line/column counters */
 331   if (error_flag) {
 332     /* the line marker is erroneous, use the real line numbers */
 333     infile = real_infile;
 334     current_line = real_lineno + 1;
 335     real_lineno_offset = 0;
 336   } else {
 337     /* set the given line number */
 338     infile = Location::add_source_file_name(*parsed_filename);
 339     current_line = new_lineno;
 340     real_lineno_offset = real_lineno + 1 - new_lineno;
 341   }
 342   current_column = 0;
 343   delete parsed_filename;
 344 }
 345
 346 ^{WHITESPACE}*"#"[^\r\n]* {
 347   int real_lineno = current_line + real_lineno_offset;
 348   /* skip the leading and trailing whitespaces */
 349   int marker_begin = 0;
 350   while (yytext[marker_begin] != '#') marker_begin++;
 351   int marker_end = yyleng - 1;
 352   while (yytext[marker_end] == ' ' || yytext[marker_end] == '\t') marker_end--;
 353   marker_end++;
 354   Location loc(real_infile, real_lineno, current_column + marker_begin,
 355     real_lineno, current_column + marker_end);
 356   loc.error("Invalid/unsupported preprocessor directive or line marker: `%s'",
 357     string(marker_end - marker_begin, yytext + marker_begin).c_str());
 358   current_column += yyleng;
 359 }
 360
 361         /* Keywords */
 362
 363 action          RETURN(ActionKeyword);
 364 activate        RETURN(ActivateKeyword);
 365 address         RETURN(AddressKeyword);
 366 alive           RETURN_DOT(AliveKeyword);
 367 all             RETURN(AllKeyword);
 368 alt             RETURN(AltKeyword);
 369 altstep         RETURN(AltstepKeyword);
 370 and             RETURN(AndKeyword);
 371 and4b           RETURN(And4bKeyword);
 372 any             RETURN(AnyKeyword);
 373 anytype         RETURN(AnyTypeKeyword);
 374 apply           RETURN_DOT(ApplyKeyword);
 375 bitstring       RETURN(BitStringKeyword);
 376 boolean         RETURN(BooleanKeyword);
 377 break           RETURN(BreakKeyword);
 378 call            RETURN_DOT(CallOpKeyword);
 379 case            RETURN(CaseKeyword);
 380 catch           RETURN_DOT(CatchOpKeyword);
 381 char            { BEGIN(SC_charkeyword); RETURN(CharKeyword); }
 382 charstring      RETURN(CharStringKeyword);
 383 checkstate      RETURN_DOT(CheckStateKeyword);
 384 check           RETURN_DOT(CheckOpKeyword);
 385 clear           RETURN_DOT(ClearOpKeyword);
 386 complement      RETURN(ComplementKeyword);
 387 component       RETURN(ComponentKeyword);
 388 connect         RETURN(ConnectKeyword);
 389 const           RETURN(ConstKeyword);
 390 continue        RETURN(ContinueKeyword);
 391 control         RETURN(ControlKeyword);
 392 create          RETURN_DOT(CreateKeyword);
 393 decmatch        RETURN(DecodedMatchKeyword);
 394 deactivate      RETURN(DeactivateKeyword);
 395 default         RETURN(DefaultKeyword);
 396 derefers        RETURN(DerefersKeyword);
 397 disconnect      RETURN(DisconnectKeyword);
 398 display         RETURN(DisplayKeyword);
 399 do              RETURN(DoKeyword);
 400 done            RETURN_DOT(DoneKeyword);
 401 else            RETURN(ElseKeyword);
 402 encode          RETURN(EncodeKeyword);
 403 enumerated      RETURN(EnumKeyword);
 404 error           RETURN(ErrorKeyword);
 405 except          RETURN(ExceptKeyword);
 406 exception       RETURN(ExceptionKeyword);
 407 execute         RETURN(ExecuteKeyword);
 408 extends         RETURN(ExtendsKeyword);
 409 extension       RETURN(ExtensionKeyword);
 410 external        RETURN(ExtKeyword);
 411 fail            RETURN(FailKeyword);
 412 false           RETURN(FalseKeyword);
 413 float           RETURN(FloatKeyword);
 414 for             RETURN(ForKeyword);
 415 friend          RETURN(FriendKeyword);
 416 from            RETURN(FromKeyword);
 417 function        RETURN(FunctionKeyword);
 418 getcall         RETURN_DOT(GetCallOpKeyword);
 419 getreply        RETURN_DOT(GetReplyOpKeyword);
 420 getverdict      RETURN(GetVerdictKeyword);
 421 goto            RETURN(GotoKeyword);
 422 group           RETURN(GroupKeyword);
 423 halt            RETURN_DOT(HaltKeyword);
 424 hexstring       RETURN(HexStringKeyword);
 425 if              RETURN(IfKeyword);
 426 ifpresent       RETURN(IfPresentKeyword);
 427 import          RETURN(ImportKeyword);
 428 in              RETURN(InParKeyword);
 429 inconc          RETURN(InconcKeyword);
 430 infinity        RETURN(InfinityKeyword);
 431 inout           RETURN(InOutParKeyword);
 432 integer         RETURN(IntegerKeyword);
 433 interleave      RETURN(InterleavedKeyword);
 434 kill            RETURN_DOT(KillKeyword);
 435 killed          RETURN_DOT(KilledKeyword);
 436 label           RETURN(LabelKeyword);
 437 language        RETURN(LanguageKeyword);
 438 length          RETURN(LengthKeyword);
 439 log             RETURN(LogKeyword);
 440 map             RETURN(MapKeyword);
 441 match           RETURN(MatchKeyword);
 442 message         RETURN(MessageKeyword);
 443 mixed           RETURN(MixedKeyword);
 444 mod             RETURN(ModKeyword);
 445 modifies        RETURN(ModifiesKeyword);
 446 module          RETURN(TTCN3ModuleKeyword);
 447 modulepar       RETURN(ModuleParKeyword);
 448 mtc             RETURN(MTCKeyword);
 449 noblock         RETURN(NoBlockKeyword);
 450 none            RETURN(NoneKeyword);
 451 not             RETURN(NotKeyword);
 452 not_a_number  RETURN(NaNKeyword);
 453 not4b           RETURN(Not4bKeyword);
 454 nowait          RETURN(NowaitKeyword);
 455 null            RETURN(NullKeyword);
 456 objid           RETURN(ObjectIdentifierKeyword);
 457 octetstring     RETURN(OctetStringKeyword);
 458 of              RETURN(OfKeyword);
 459 omit            RETURN(OmitKeyword);
 460 on              RETURN(OnKeyword);
 461 optional        RETURN(OptionalKeyword);
 462 or              RETURN(OrKeyword);
 463 or4b            RETURN(Or4bKeyword);
 464 out             RETURN(OutParKeyword);
 465 override        RETURN(OverrideKeyword);
 466 param           RETURN(ParamKeyword);
 467 pass            RETURN(PassKeyword);
 468 pattern         RETURN(PatternKeyword);
 469 permutation     RETURN(PermutationKeyword);
 470 port            RETURN(PortKeyword);
 471 present         RETURN(PresentKeyword);
 472 private         RETURN(PrivateKeyword);
 473 procedure       RETURN(ProcedureKeyword);
 474 public          RETURN(PublicKeyword);
 475 raise           RETURN_DOT(RaiseKeyword);
 476 read            RETURN_DOT(ReadKeyword);
 477 receive         RETURN_DOT(ReceiveOpKeyword);
 478 record          RETURN(RecordKeyword);
 479 recursive       RETURN(RecursiveKeyword);
 480 refers          RETURN(RefersKeyword);
 481 rem             RETURN(RemKeyword);
 482 repeat          RETURN(RepeatKeyword);
 483 reply           RETURN_DOT(ReplyKeyword);
 484 return          RETURN(ReturnKeyword);
 485 running         RETURN_DOT(RunningKeyword);
 486 runs            RETURN(RunsKeyword);
 487 select          RETURN(SelectKeyword);
 488 self            RETURN(SelfKeyword);
 489 send            RETURN_DOT(SendOpKeyword);
 490 sender          RETURN(SenderKeyword);
 491 set             RETURN(SetKeyword);
 492 setverdict      RETURN(SetVerdictKeyword);
 493 signature       RETURN(SignatureKeyword);
 494 start           RETURN_DOT(StartKeyword);
 495 stop            RETURN_DOT(StopKeyword);
 496 subset          RETURN(SubsetKeyword);
 497 superset        RETURN(SupersetKeyword);
 498 system          RETURN(SystemKeyword);
 499 template        RETURN(TemplateKeyword);
 500 testcase        RETURN(TestcaseKeyword);
 501 timeout         RETURN_DOT(TimeoutKeyword);
 502 timer           RETURN(TimerKeyword);
 503 to              RETURN(ToKeyword);
 504 trigger         RETURN_DOT(TriggerOpKeyword);
 505 true            RETURN(TrueKeyword);
 506 type            RETURN(TypeDefKeyword);
 507 union           RETURN(UnionKeyword);
 508 universal       RETURN(UniversalKeyword);
 509 unmap           RETURN(UnmapKeyword);
 510 value           RETURN(ValueKeyword);
 511 valueof         RETURN(ValueofKeyword);
 512 var             RETURN(VarKeyword);
 513 variant         RETURN(VariantKeyword);
 514 verdicttype     RETURN(VerdictTypeKeyword);
 515 while           RETURN(WhileKeyword);
 516 with            RETURN(WithKeyword);
 517 xor             RETURN(XorKeyword);
 518 xor4b           RETURN(Xor4bKeyword);
 519
 520   /* modifier keywords */
 521
 522 "@nocase"        RETURN(NocaseKeyword);
 523 "@lazy"          RETURN(LazyKeyword);
 524 "@decoded"       RETURN(DecodedKeyword);
 525 "@deterministic" RETURN(DeterministicKeyword);
 526 "@fuzzy"         RETURN(FuzzyKeyword);
 527 "@index"         RETURN(IndexKeyword);
 528
 529   /* special TITAN specific keywords */
 530
 531 "@try"    RETURN(TitanSpecificTryKeyword);
 532 "@catch"  RETURN(TitanSpecificCatchKeyword);
 533 "@profiler" RETURN(TitanSpecificProfilerKeyword);
 534
 535
 536         /* Predefined function identifiers */
 537
 538 bit2hex         RETURN(bit2hexKeyword);
 539 bit2int         RETURN(bit2intKeyword);
 540 bit2oct         RETURN(bit2octKeyword);
 541 bit2str         RETURN(bit2strKeyword);
 542 char2int        RETURN(char2intKeyword);
 543 char2oct        RETURN(char2octKeyword);
 544 decomp          RETURN(decompKeyword);
 545 float2int       RETURN(float2intKeyword);
 546 float2str       RETURN(float2strKeyword);
 547 hex2bit         RETURN(hex2bitKeyword);
 548 hex2int         RETURN(hex2intKeyword);
 549 hex2oct         RETURN(hex2octKeyword);
 550 hex2str         RETURN(hex2strKeyword);
 551 int2bit         RETURN(int2bitKeyword);
 552 int2char        RETURN(int2charKeyword);
 553 int2enum  RETURN(int2enumKeyword);
 554 int2float       RETURN(int2floatKeyword);
 555 int2hex         RETURN(int2hexKeyword);
 556 int2oct         RETURN(int2octKeyword);
 557 int2str         RETURN(int2strKeyword);
 558 int2unichar     RETURN(int2unicharKeyword);
 559 isvalue         RETURN(isvalueKeyword);
 560 isbound         RETURN(isboundKeyword);
 561 ischosen        RETURN(ischosenKeyword);
 562 ispresent       RETURN(ispresentKeyword);
 563 lengthof        RETURN(lengthofKeyword);
 564 oct2bit         RETURN(oct2bitKeyword);
 565 oct2char        RETURN(oct2charKeyword);
 566 oct2hex         RETURN(oct2hexKeyword);
 567 oct2int         RETURN(oct2intKeyword);
 568 oct2str         RETURN(oct2strKeyword);
 569 regexp          RETURN(regexpKeyword);
 570 replace         RETURN(replaceKeyword);
 571 rnd                     RETURN(rndKeyword);
 572 sizeof          RETURN(sizeofKeyword);
 573 str2bit         RETURN(str2bitKeyword);
 574 str2float       RETURN(str2floatKeyword);
 575 str2hex         RETURN(str2hexKeyword);
 576 str2int         RETURN(str2intKeyword);
 577 str2oct         RETURN(str2octKeyword);
 578 substr          RETURN(substrKeyword);
 579 unichar2int     RETURN(unichar2intKeyword);
 580 unichar2char    RETURN(unichar2charKeyword);
 581 log2str         RETURN(log2strKeyword);
 582 enum2int        RETURN(enum2intKeyword);
 583 encvalue        RETURN(encvalueKeyword);
 584 decvalue        RETURN(decvalueKeyword);
 585 testcasename    RETURN(testcasenameKeyword);
 586 ttcn2string RETURN(ttcn2stringKeyword);
 587 string2ttcn RETURN(string2ttcnKeyword);
 588 unichar2oct     RETURN(unichar2octKeyword);
 589 oct2unichar     RETURN(oct2unicharKeyword);
 590 remove_bom      RETURN(remove_bomKeyWord);
 591 get_stringencoding      RETURN(get_stringencodingKeyWord);
 592 encode_base64   RETURN(encode_base64KeyWord);
 593 decode_base64   RETURN(decode_base64KeyWord);
 594 encvalue_unichar RETURN(encvalue_unicharKeyWord);
 595 decvalue_unichar RETURN(decvalue_unicharKeyWord);
 596 any2unistr      RETURN(any2unistrKeyWord);
 597 hostid          RETURN(hostidKeyWord);
 598
 599         /* Values */
 600
 601 {NUMBER} {
 602   Location loc(infile, current_line, current_column, current_line,
 603     current_column + yyleng);
 604   yylval.int_val = new int_val_t(yytext, loc);
 605   RETURN_LVAL(Number);
 606 }
 607
 608 {FLOAT} {
 609   Location loc(infile, current_line, current_column, current_line,
 610     current_column + yyleng);
 611   yylval.float_val = string2Real(yytext, loc);
 612   RETURN_LVAL(FloatValue);
 613 }
 614
 615 NULL RETURN(NullValue);
 616
 617 "'" {
 618   binstr=memptystr();
 619   valid_bit=true;
 620   valid_oct=true;
 621   half_oct=false;
 622   contains_match=false;
 623   contains_ws=false;
 624   start_line = current_line;
 625   start_column = current_column;
 626   current_column++;
 627   MD5_Update(&md5_ctx, yytext, yyleng);
 628   BEGIN(SC_binstring);
 629 }
 630
 631 \" {
 632   yylval.str = memptystr();
 633   start_line = current_line;
 634   start_column = current_column;
 635   current_column++;
 636   MD5_Update(&md5_ctx, yytext, yyleng);
 637   BEGIN(SC_cstring);
 638 }
 639
 640 <SC_charkeyword>
 641 {
 642   {UID} {
 643     yylval.str = mcopystrn(yytext, yyleng);
 644     RETURN_LVAL(Cstring);
 645   }
 646
 647   [,] { RETURN(*yytext); }
 648
 649   [)] { BEGIN(INITIAL); RETURN(*yytext); }
 650 }
 651
 652 <SC_binstring> /* -------- SC_binstring scope -------------- */
 653 {
 654
 655   {WHITESPACE}+ {
 656     contains_ws = true;
 657     current_column += yyleng;
 658   }
 659
 660   {WHITESPACE}*{NEWLINE} {
 661     contains_ws = true;
 662     current_line++;
 663     current_column = 0;
 664   }
 665
 666   [01] {
 667     binstr = mputc(binstr, yytext[0]);
 668     half_oct = !half_oct;
 669     current_column++;
 670   }
 671
 672   [2-9A-F] {
 673     binstr = mputc(binstr, yytext[0]);
 674     valid_bit = false;
 675     half_oct = !half_oct;
 676     current_column++;
 677   }
 678
 679   [a-f] {
 680     binstr = mputc(binstr, yytext[0] - 'a' + 'A');
 681     valid_bit = false;
 682     half_oct = !half_oct;
 683     current_column++;
 684   }
 685
 686   "?"|"*" {
 687     binstr = mputc(binstr, yytext[0]);
 688     contains_match = true;
 689     if (half_oct) valid_oct = false;
 690     current_column++;
 691   }
 692
 693   "'"[bBhHoO] {
 694     yylloc.first_line = start_line;
 695     yylloc.first_column = start_column;
 696     yylloc.last_line = current_line;
 697     yylloc.last_column = current_column + 2;
 698     Location loc(infile, yylloc);
 699     int ret_val = TOK_errval;
 700     switch (yytext[1]) {
 701     case 'b': {
 702       Location loc2(infile, current_line, current_column + 1, current_line,
 703         current_column + 2);
 704       loc2.warning("The last character of a bitstring literal should be "
 705         "`B' instead of `b'");
 706       /* no break */ }
 707     case 'B':
 708       if (valid_bit) {
 709         if (contains_ws) loc.warning("Bitstring %s contains whitespace and/or "
 710           "newline character(s)", contains_match ? "match" : "value");
 711         ret_val = contains_match ? BitStringMatch : Bstring;
 712         yylval.string_val = new string(binstr);
 713       } else loc.error("Bitstring value contains invalid character");
 714       break;
 715     case 'h': {
 716       Location loc2(infile, current_line, current_column + 1, current_line,
 717         current_column + 2);
 718       loc2.warning("The last character of a hexstring literal should be "
 719         "`H' instead of `h'");
 720       /* no break */ }
 721     case 'H':
 722       if (contains_ws) loc.warning("Hexstring %s contains whitespace and/or "
 723         "newline character(s)", contains_match ? "match" : "value");
 724       ret_val = contains_match ? HexStringMatch : Hstring;
 725       yylval.string_val = new string(binstr);
 726       break;
 727     case 'o': {
 728       Location loc2(infile, current_line, current_column + 1, current_line,
 729         current_column + 2);
 730       loc2.warning("The last character of an octetstring literal should be "
 731         "`O' instead of `o'");
 732       /* no break */ }
 733     case 'O':
 734       if (valid_oct && !half_oct) {
 735         if (contains_ws) loc.warning("Octetstring %s contains whitespace "
 736           "and/or newline character(s)", contains_match ? "match" : "value");
 737         ret_val = contains_match ? OctetStringMatch : Ostring;
 738         yylval.string_val = new string(binstr);
 739       } else if (contains_match) {
 740         loc.error("Octetstring match contains half octet(s)");
 741       } else {
 742         loc.error("Octetstring value contains odd number of hexadecimal "
 743           "digits");
 744       }
 745     }
 746     MD5_Update(&md5_ctx, binstr, strlen(binstr));
 747     Free(binstr);
 748     update_md5();
 749     BEGIN(INITIAL);
 750     current_column += 2;
 751     if (dot_flag) {
 752       backup_token = ret_val;
 753       backup_lval = yylval;
 754       backup_lloc = yylloc;
 755       RETURN_SAVED_DOT;
 756     } else return ret_val;
 757   }
 758
 759   "'" {
 760     yylloc.first_line = start_line;
 761     yylloc.first_column = start_column;
 762     current_column++;
 763     yylloc.last_line = current_line;
 764     yylloc.last_column = current_column;
 765     Location loc(infile, yylloc);
 766     loc.error("Invalid binary string literal. Expecting `B', `H' or `O' after "
 767       "the closing `''");
 768     MD5_Update(&md5_ctx, binstr, strlen(binstr));
 769     Free(binstr);
 770     BEGIN(INITIAL);
 771     RETURN_NOLOCUPD(TOK_errval);
 772   }
 773
 774   . {
 775     Location loc(infile, current_line, current_column, current_line,
 776       current_column + 1);
 777     int c = (unsigned char)yytext[0];
 778     loc.error("Invalid character `%c' (0x%02X) in binary string",
 779       isprint(c) ? c : '?', c);
 780     MD5_Update(&md5_ctx, binstr, strlen(binstr));
 781     Free(binstr);
 782     MD5_Update(&md5_ctx, yytext, 1);
 783     current_column++;
 784     BEGIN(SC_binstring_bad);
 785   }
 786
 787 } /* SC_binstring scope */
 788
 789 <SC_binstring_bad> /* -------- SC_binstring_bad scope -------------- */
 790 {
 791
 792   {WHITESPACE}+ current_column += yyleng;
 793
 794   {WHITESPACE}*{NEWLINE} {
 795     current_line++;
 796     current_column = 0;
 797   }
 798
 799   "'"[bBhHoO]? {
 800     current_column += yyleng;
 801     yylloc.first_line = start_line;
 802     yylloc.first_column = start_column;
 803     yylloc.last_line = current_line;
 804     yylloc.last_column = current_column;
 805     BEGIN(INITIAL);
 806     RETURN_NOLOCUPD(TOK_errval);
 807   }
 808
 809   . {
 810     MD5_Update(&md5_ctx, yytext, yyleng);
 811     current_column++;
 812   }
 813
 814 } /* SC_binstring_bad scope */
 815
 816 <SC_cstring> /* -------- SC_cstring scope -------------- */
 817 {
 818
 819   \\?{NEWLINE} { /* newline possibly preceded by backslash */
 820     yylval.str = mputstr(yylval.str, yytext);
 821     current_line++;
 822     current_column = 0;
 823   }
 824
 825   \"\"|\\. { /* two doublequotes or any backslash-escaped char */
 826     yylval.str = mputstr(yylval.str, yytext);
 827     current_column += 2;
 828     /* Note that both get added ("external representation").
 829      * parse_charstring_value() in charstring_la.l is responsible
 830      * for transforming the string to "internal representation" */
 831   }
 832
 833   \" {
 834     current_column++;
 835     yylloc.first_line = start_line;
 836     yylloc.first_column = start_column;
 837     yylloc.last_line = current_line;
 838     yylloc.last_column = current_column;
 839     MD5_Update(&md5_ctx, yylval.str, strlen(yylval.str));
 840     update_md5();
 841     BEGIN(INITIAL);
 842     if (dot_flag) {
 843       backup_token = Cstring;
 844       backup_lval = yylval;
 845       backup_lloc = yylloc;
 846       RETURN_SAVED_DOT;
 847     } else return Cstring;
 848   }
 849
 850   . {
 851     yylval.str = mputc(yylval.str, yytext[0]);
 852     current_column++;
 853   }
 854
 855 } /* SC_cstring scope */
 856
 857         /* Macros */
 858
 859 "%moduleId" {
 860   yylval.macrotype = Value::MACRO_MODULEID;
 861   RETURN_LVAL(MacroValue);
 862 }
 863 "%fileName" {
 864   yylval.macrotype = Value::MACRO_FILENAME;
 865   RETURN_LVAL(MacroValue);
 866 }
 867 "%lineNumber" {
 868   yylval.macrotype = Value::MACRO_LINENUMBER;
 869   RETURN_LVAL(MacroValue);
 870 }
 871 "%definitionId" {
 872   yylval.macrotype = Value::MACRO_DEFINITIONID;
 873   RETURN_LVAL(MacroValue);
 874 }
 875 "%testcaseId" {
 876   yylval.macrotype = Value::MACRO_TESTCASEID;
 877   RETURN_LVAL(MacroValue);
 878 }
 879 "%"{IDENTIFIER} {
 880   fill_location();
 881   Location loc(infile, yylloc);
 882   loc.error("Invalid macro notation: `%s'", yytext);
 883   RETURN_NOLOCUPD(TOK_errval);
 884 }
 885
 886 "__MODULE__" {
 887   yylval.macrotype = Value::MACRO_MODULEID;
 888   RETURN_LVAL(MacroValue);
 889 }
 890 "__FILE__" {
 891   yylval.macrotype = Value::MACRO_FILEPATH;
 892   RETURN_LVAL(MacroValue);
 893 }
 894 "__BFILE__" {
 895   yylval.macrotype = Value::MACRO_BFILENAME;
 896   RETURN_LVAL(MacroValue);
 897 }
 898 "__LINE__" {
 899   yylval.macrotype = Value::MACRO_LINENUMBER_C;
 900   RETURN_LVAL(MacroValue);
 901 }
 902 "__SCOPE__" {
 903   yylval.macrotype = Value::MACRO_SCOPE;
 904   RETURN_LVAL(MacroValue);
 905 }
 906 "__TESTCASE__" {
 907   yylval.macrotype = Value::MACRO_TESTCASEID;
 908   RETURN_LVAL(MacroValue);
 909 }
 910 "__"{IDENTIFIER}"__" {
 911   fill_location();
 912   Location loc(infile, yylloc);
 913   loc.error("Invalid macro notation: `%s'", yytext);
 914   RETURN_NOLOCUPD(TOK_errval);
 915 }
 916
 917         /* Multi-character operators */
 918
 919 ":="            RETURN(AssignmentChar);
 920 "\.\."          RETURN(DotDot);
 921 "->"            RETURN(PortRedirectSymbol);
 922 "=="            RETURN(EQ);
 923 "!="            RETURN(NE);
 924 ">="            RETURN(GE);
 925 "<="            RETURN(LE);
 926 "<<"            RETURN(SL);
 927 ">>"            RETURN(SR);
 928 "<@"            RETURN(RL);
 929 "@>"            RETURN(_RR);
 930
 931 "++" |
 932 "--" {
 933   fill_location();
 934   Location loc(infile, yylloc);
 935   loc.error("Operator `%s' is reserved for future use", yytext);
 936 }
 937
 938         /* Invalid operators */
 939
 940 "::=" {
 941   fill_location();
 942   Location loc(infile, yylloc);
 943   loc.error("`::=' is not a valid assignment operator in TTCN-3. Did you mean "
 944     "`:='?");
 945   RETURN_NOLOCUPD(AssignmentChar);
 946 }
 947
 948 "=" {
 949   fill_location();
 950   Location loc(infile, yylloc);
 951   loc.error("A single `=' character cannot be used in TTCN-3. Did you mean "
 952     "the assignment sign `:=' or the equality operator `=='?");
 953   /* the former is more probable than the latter */
 954   RETURN_NOLOCUPD(AssignmentChar);
 955 }
 956
 957 "<>" {
 958   fill_location();
 959   Location loc(infile, yylloc);
 960   loc.error("`<>' is not a valid comparison operator in TTCN-3. Did you mean "
 961     "`!='?");
 962   RETURN_NOLOCUPD(NE);
 963 }
 964
 965         /* Identifiers */
 966
 967 {IDENTIFIER} {
 968   yylval.id = new Identifier(Identifier::ID_TTCN, string(yyleng, yytext));
 969   RETURN_LVAL(IDentifier);
 970 }
 971
 972         /* Single character tokens (brackets, operators, etc.) */
 973
 974 \. {
 975   update_md5();
 976   if (dot_flag) {
 977     /* store this dot in the backup */
 978     backup_token = '.';
 979     backup_lloc.first_line = current_line;
 980     backup_lloc.first_column = current_column;
 981     current_column++;
 982     backup_lloc.last_line = current_line;
 983     backup_lloc.last_column = current_column;
 984     /* return the dot that was found previously */
 985     RETURN_SAVED_DOT;
 986   } else {
 987     dot_flag = true;
 988     dot_line = current_line;
 989     dot_column = current_column;
 990     current_column++;
 991   }
 992 }
 993
 994 [()\[\]{}+\-\*/&:;,<>\?!]  RETURN(*yytext);
 995
 996         /* Invalid characters */
 997
 998 . {
 999   fill_location();
1000   Location loc(infile, yylloc);
1001   int c = (unsigned char)yytext[0];
1002   loc.error("Character `%c' (0x%02X) is not used in TTCN-3",
1003     isprint(c) ? c : '?', c);
1004 }
1005
1006         /* EOF rule */
1007
1008 <*><<EOF>> {
1009   if (YY_START != INITIAL) {
1010     Location loc(infile, start_line, start_column, current_line,
1011       current_column);
1012     switch (YY_START) {
1013     case SC_blockcomment:
1014       loc.error("Unterminated block comment");
1015       break;
1016     case SC_binstring:
1017       Free(binstr);
1018       /* no break */
1019     case SC_binstring_bad:
1020       loc.error("Unterminated binary string literal");
1021       break;
1022     case SC_cstring:
1023       Free(yylval.str);
1024       loc.error("Unterminated character string literal");
1025     }
1026     BEGIN(INITIAL);
1027   }
1028   if (dot_flag) {
1029     dot_flag = false;
1030     RETURN_SAVED_DOT;
1031   } else {
1032     yylloc.first_line = current_line;
1033     yylloc.first_column = current_column;
1034     yylloc.last_line = current_line;
1035     yylloc.last_column = current_column + 1;
1036     return EOF;
1037   }
1038 }
1039
1040 %%
1041
1042 void init_ttcn3_lex()
1043 {
1044   dot_flag = false;
1045   current_line = 1;
1046   current_column = 0;
1047   real_infile = infile;
1048   real_lineno_offset = 0;
1049   MD5_Init(&md5_ctx);
1050 }
1051
1052 void init_erroneous_lex(const char* p_infile, int p_line, int p_column)
1053 {
1054   infile = p_infile;
1055   current_line = p_line;
1056   current_column = p_column;
1057   real_infile = infile;
1058   real_lineno_offset = 0;
1059   dot_flag = false;
1060 }
1061
1062 void free_dot_flag_stuff()
1063 {
1064   if (dot_flag) {
1065     dot_flag = false;
1066     /* clean up the semantic value of the token that was backed up */
1067     switch (backup_token) {
1068     case IDentifier:
1069       delete backup_lval.id;
1070       break;
1071     case Bstring:
1072     case Hstring:
1073     case Ostring:
1074     case BitStringMatch:
1075     case HexStringMatch:
1076     case OctetStringMatch:
1077       delete backup_lval.string_val;
1078       break;
1079     case Cstring:
1080       Free(backup_lval.str);
1081     default:
1082       break;
1083     }
1084   }
1085 }
1086
1087 void free_ttcn3_lex()
1088 {
1089   free_dot_flag_stuff();
1090   fclose(ttcn3_in);
1091   ttcn3_lex_destroy();
1092 }
1093
1094 /* called from ttcn3_parse_file to finalize MD5 and add it to the module */
1095 void set_md5_checksum(Ttcn::Module *m)
1096 {
1097   unsigned char md5_sum[MD5_DIGEST_LENGTH];
1098   MD5_Final(md5_sum, &md5_ctx);
1099   m->set_checksum(sizeof(md5_sum), md5_sum);
1100 }