compiler2/ttcn3/compiler.l

   1 /******************************************************************************
   2  * Copyright (c) 2000-2016 Ericsson Telecom AB
   3  * All rights reserved. This program and the accompanying materials
   4  * are made available under the terms of the Eclipse Public License v1.0
   5  * which accompanies this distribution, and is available at
   6  * http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Contributors:
   9  *   Baji, Laszlo
  10  *   Balasko, Jeno
  11  *   Baranyi, Botond
  12  *   Cserveni, Akos
  13  *   Delic, Adam
  14  *   Feher, Csaba
  15  *   Forstner, Matyas
  16  *   Kovacs, Ferenc
  17  *   Kremer, Peter
  18  *   Raduly, Csaba
  19  *   Szabados, Kristof
  20  *   Szabo, Janos Zoltan – initial implementation
  21  *   Zalanyi, Balazs Andor
  22  *
  23  ******************************************************************************/
  24 %option noyywrap
  25 %option never-interactive
  26 %option nounput
  27 %{
  28
  29 /* Tokenizer for TTCN-3 Core Language */
  30
  31 #include "../../common/dbgnew.hh"
  32 #include "compiler.h"
  33 #include "../string.hh"
  34 #include "../CompilerError.hh"
  35 #include "../Real.hh"
  36 #include "../Value.hh"
  37 #include "AST_ttcn3.hh"
  38 #include "Ttcnstuff.hh" // at least for PortTypeBody::PortOperationMode_t
  39 #include "Statement.hh" // for Statement::statementtype_t
  40 #include "Attributes.hh"
  41
  42 #include <string.h>
  43 #include <ctype.h>
  44 #include <openssl/md5.h>
  45
  46 namespace Common {
  47   class IndexedValue;
  48   class Location;
  49 }
  50
  51 namespace Ttcn {
  52   class ParamRedirect;
  53   class Statement;
  54   class AltGuard;
  55   class IfClause;
  56   class IfClauses;
  57   class NamedTemplate;
  58   class NamedTemplates;
  59   class IndexedTemplate;
  60   class IndexedTemplates;
  61   class Templates;
  62   class CompTypeRefList;
  63 }
  64
  65 using namespace Common;
  66 using namespace Ttcn;
  67
  68 #include "compiler.tab.hh"
  69
  70 #define yylval ttcn3_lval
  71 #define yylloc ttcn3_lloc
  72
  73 /* global variable indicating the location of the returned token to bison */
  74 extern YYLTYPE yylloc;
  75
  76 extern bool is_erroneous_parsed;
  77
  78 /* always points to the first character of the regexp to be recognized */
  79 static int current_line, current_column;
  80
  81 /* when reporting an error in linemarker or preprocessor
  82  * directive the real file name and line number is needed */
  83 static const char* real_infile;
  84 /* real_lineno = current_line + real_lineno_offset */
  85 static int real_lineno_offset;
  86
  87 static bool dot_flag = false;
  88 /* type of the backup token (that was found after a dot) */
  89 static int backup_token;
  90 /* semantic value of the backup token */
  91 static YYSTYPE backup_lval;
  92 /* location of the backup token */
  93 static YYLTYPE backup_lloc;
  94
  95 static MD5_CTX md5_ctx;
  96
  97 static void fill_location()
  98 {
  99   yylloc.first_line = current_line;
 100   yylloc.first_column = current_column;
 101   current_column += yyleng;
 102   yylloc.last_line = current_line;
 103   yylloc.last_column = current_column;
 104 }
 105
 106 static void update_md5()
 107 {
 108   MD5_Update(&md5_ctx, yytext, yyleng);
 109   MD5_Update(&md5_ctx, " ", 1);
 110 }
 111
 112 #define RETURN_SAVED_DOT \
 113   do { \
 114     yytext[0] = '\0'; \
 115     yylloc.first_line = dot_line; \
 116     yylloc.first_column = dot_column; \
 117     yylloc.last_line = dot_line; \
 118     yylloc.last_column = dot_column + 1; \
 119     return '.'; \
 120   } while (0)
 121
 122 /* return macro for simple tokens without semantic value */
 123 #define RETURN(ret_val) \
 124   do { \
 125     update_md5(); \
 126     fill_location(); \
 127     if (dot_flag) { \
 128       backup_token = ret_val; \
 129       backup_lloc = yylloc; \
 130       RETURN_SAVED_DOT; \
 131     } else return ret_val; \
 132   } while (0)
 133
 134 /* same as RETURN(ret_val) macro but without location update,
 135  * usually a return after an error */
 136 #define RETURN_NOLOCUPD(ret_val) \
 137   do { \
 138     update_md5(); \
 139     if (dot_flag) { \
 140       backup_token = ret_val; \
 141       backup_lloc = yylloc; \
 142       RETURN_SAVED_DOT; \
 143     } else return ret_val; \
 144   } while (0)
 145
 146 /* return macro for simple tokens with semantic value */
 147 #define RETURN_LVAL(ret_val) \
 148   do { \
 149     update_md5(); \
 150     fill_location(); \
 151     if (dot_flag) { \
 152       backup_token = ret_val; \
 153       backup_lval = yylval; \
 154       backup_lloc = yylloc; \
 155       RETURN_SAVED_DOT; \
 156     } else return ret_val; \
 157   } while (0)
 158
 159 /* return macro for special tokens that are glued together with previous dot */
 160 #define RETURN_DOT(ret_val) \
 161   do { \
 162     update_md5(); \
 163     if (dot_flag) { \
 164       dot_flag = false; \
 165       yylloc.first_line = dot_line; \
 166       yylloc.first_column = dot_column; \
 167       current_column += yyleng; \
 168       yylloc.last_line = current_line; \
 169       yylloc.last_column = current_column; \
 170       return Dot##ret_val; \
 171     } else { \
 172       fill_location(); \
 173       return ret_val; \
 174     } \
 175   } while (0)
 176
 177 extern string *parse_charstring_value(const char *str, const Location& loc);
 178
 179 %}
 180
 181 NUMBER 0|([1-9][0-9]*)
 182
 183 FLOAT ({NUMBER}\.[0-9]+)|({NUMBER}(\.[0-9]+)?[Ee][+-]?{NUMBER})
 184
 185 IDENTIFIER [A-Za-z][A-Za-z0-9_]*
 186
 187 LINECOMMENT "//"[^\r\n]*
 188
 189 WHITESPACE [ \t\v\f]
 190
 191 NEWLINE \r|\n|\r\n
 192
 193 LINEMARKER {NUMBER}{WHITESPACE}+\"([^\\\"\r\n]|\\[^\r\n])*\"
 194
 195 TITAN "$#&&&(#TITANERRONEOUS$#&&^#% "
 196
 197 %x SC_blockcomment SC_cstring
 198 %x SC_binstring SC_binstring_bad
 199
 200 %%
 201   /* local variables of yylex() */
 202   int start_line = 0, start_column = 0; /**< used by block comments and
 203                                              string literals */
 204   int dot_line = 0, dot_column = 0; /**< location of the previous '.' token */
 205   /* variables used when processing binary strings */
 206   expstring_t binstr = NULL; /**< the string itself */
 207   bool valid_bit = false, /**< binstr is valid bitstring */
 208     valid_oct = false, /**< binstr is valid octetstring */
 209     half_oct = false, /**< binstr is not a valid octetstr but a valid hexstr */
 210     contains_match = false, /**< binstr contains matching symbol */
 211     contains_ws = false; /**< binstr contains whitespace characters */
 212
 213   if (dot_flag) {
 214     if (backup_token == '.') {
 215       /* Two dots were found in the previous call: the first one was returned,
 216        * the second one is now in the backup. Let's assume that we have just
 217        * found the second one. */
 218       dot_line = backup_lloc.first_line;
 219       dot_column = backup_lloc.first_column;
 220     } else {
 221       /* Return the token and its semantic value that was backed up after the
 222        * last token (which was a dot). */
 223       dot_flag = false;
 224       yylval = backup_lval;
 225       yylloc = backup_lloc;
 226       return backup_token;
 227     }
 228   }
 229
 230
 231 {TITAN} {
 232   // hack: to avoid the erroneous parsing reporting a syntax error where it
 233   //       also lists TTCN3ModuleKeyword as a possible nonterminal to use
 234   if (is_erroneous_parsed) {
 235     RETURN(TitanErroneousHackKeyword);
 236   } else {
 237     Location loc(infile, current_line, current_column, current_line,
 238                  current_column + yyleng);
 239     loc.error("Unexpected `%s'.", yytext);
 240   }
 241 }
 242
 243         /* Eat up comments and whitespaces */
 244
 245 "/*" {
 246   start_line = current_line;
 247   start_column = current_column;
 248   current_column += 2;
 249   BEGIN(SC_blockcomment);
 250 }
 251
 252 <SC_blockcomment> /* -------- SC_blockcomment scope -------------- */
 253 {
 254
 255   "*/" {
 256     current_column += 2;
 257     BEGIN(INITIAL);
 258   }
 259
 260   {NEWLINE} {
 261     current_line++;
 262     current_column = 0;
 263   }
 264
 265   . current_column++;
 266
 267 } /* SC_blockcomment */
 268
 269 {LINECOMMENT}?{NEWLINE} {
 270   current_line++;
 271   current_column = 0;
 272 }
 273
 274 {LINECOMMENT} {
 275   current_column += yyleng;
 276 }
 277
 278 {WHITESPACE}+ current_column += yyleng;
 279
 280         /* C preprocessor line markers */
 281
 282 ^{WHITESPACE}*"#"({WHITESPACE}*"line")?{WHITESPACE}+{LINEMARKER}[^\r\n]*{NEWLINE} {
 283   bool error_flag = false;
 284   int real_lineno = current_line + real_lineno_offset;
 285   /* skipping the leading whitespaces */
 286   int marker_begin = 0;
 287   while (yytext[marker_begin] != '#') marker_begin++;
 288   /* skipping the trailing whitespaces and newline */
 289   int marker_end = yyleng - 1;
 290   while (yytext[marker_end] == '\r' || yytext[marker_end] == '\n' ||
 291          yytext[marker_end] == ' ' || yytext[marker_end] == '\t') marker_end--;
 292   marker_end++;
 293   Location loc(real_infile, real_lineno, current_column + marker_begin,
 294     real_lineno, current_column + marker_end);
 295   Error_Context cntxt(&loc, "In preprocessor line marker");
 296   /* parsing out the line number */
 297   int lineno_begin = marker_begin + 1;
 298   while (!isdigit((unsigned char)yytext[lineno_begin])) lineno_begin++;
 299   int lineno_end = lineno_begin + 1;
 300   while (isdigit((int)yytext[lineno_end])) lineno_end++;
 301   errno = 0;
 302   int new_lineno = strtol(yytext + lineno_begin, NULL, 10);
 303   if (errno != 0) {
 304     Location lineno_loc(real_infile, real_lineno, current_column + lineno_begin,
 305       real_lineno, current_column + lineno_end);
 306     string lineno_str(lineno_end - lineno_begin, yytext + lineno_begin);
 307     lineno_loc.error("Line number `%s' is too large for being represented in "
 308       "memory: %s", lineno_str.c_str(), strerror(errno));
 309     error_flag = true;
 310   }
 311   /* parsing out the file name */
 312   int filename_begin = lineno_end + 1;
 313   while (yytext[filename_begin] != '"') filename_begin++;
 314   filename_begin++;
 315   int filename_end = filename_begin;
 316   while (yytext[filename_end] != '"') {
 317     if (yytext[filename_end] == '\\') filename_end += 2;
 318     else filename_end++;
 319   }
 320   Location filename_loc(real_infile, real_lineno, current_column +
 321     filename_begin - 1, real_lineno, current_column + filename_end + 1);
 322   string filename_str(filename_end - filename_begin, yytext + filename_begin);
 323   string *parsed_filename = parse_charstring_value(filename_str.c_str(),
 324     filename_loc);
 325   if (!parsed_filename) error_flag = true;
 326   /* updating the line/column counters */
 327   if (error_flag) {
 328     /* the line marker is erroneous, use the real line numbers */
 329     infile = real_infile;
 330     current_line = real_lineno + 1;
 331     real_lineno_offset = 0;
 332   } else {
 333     /* set the given line number */
 334     infile = Location::add_source_file_name(*parsed_filename);
 335     current_line = new_lineno;
 336     real_lineno_offset = real_lineno + 1 - new_lineno;
 337   }
 338   current_column = 0;
 339   delete parsed_filename;
 340 }
 341
 342 ^{WHITESPACE}*"#"[^\r\n]* {
 343   int real_lineno = current_line + real_lineno_offset;
 344   /* skip the leading and trailing whitespaces */
 345   int marker_begin = 0;
 346   while (yytext[marker_begin] != '#') marker_begin++;
 347   int marker_end = yyleng - 1;
 348   while (yytext[marker_end] == ' ' || yytext[marker_end] == '\t') marker_end--;
 349   marker_end++;
 350   Location loc(real_infile, real_lineno, current_column + marker_begin,
 351     real_lineno, current_column + marker_end);
 352   loc.error("Invalid/unsupported preprocessor directive or line marker: `%s'",
 353     string(marker_end - marker_begin, yytext + marker_begin).c_str());
 354   current_column += yyleng;
 355 }
 356
 357         /* Keywords */
 358
 359 action          RETURN(ActionKeyword);
 360 activate        RETURN(ActivateKeyword);
 361 address         RETURN(AddressKeyword);
 362 alive           RETURN_DOT(AliveKeyword);
 363 all             RETURN(AllKeyword);
 364 alt             RETURN(AltKeyword);
 365 altstep         RETURN(AltstepKeyword);
 366 and             RETURN(AndKeyword);
 367 and4b           RETURN(And4bKeyword);
 368 any             RETURN(AnyKeyword);
 369 anytype         RETURN(AnyTypeKeyword);
 370 apply           RETURN_DOT(ApplyKeyword);
 371 bitstring       RETURN(BitStringKeyword);
 372 boolean         RETURN(BooleanKeyword);
 373 break           RETURN(BreakKeyword);
 374 call            RETURN_DOT(CallOpKeyword);
 375 case            RETURN(CaseKeyword);
 376 catch           RETURN_DOT(CatchOpKeyword);
 377 char            RETURN(CharKeyword);
 378 charstring      RETURN(CharStringKeyword);
 379 check           RETURN_DOT(CheckOpKeyword);
 380 clear           RETURN_DOT(ClearOpKeyword);
 381 complement      RETURN(ComplementKeyword);
 382 component       RETURN(ComponentKeyword);
 383 connect         RETURN(ConnectKeyword);
 384 const           RETURN(ConstKeyword);
 385 continue        RETURN(ContinueKeyword);
 386 control         RETURN(ControlKeyword);
 387 create          RETURN_DOT(CreateKeyword);
 388 deactivate      RETURN(DeactivateKeyword);
 389 default         RETURN(DefaultKeyword);
 390 derefers        RETURN(DerefersKeyword);
 391 disconnect      RETURN(DisconnectKeyword);
 392 display         RETURN(DisplayKeyword);
 393 do              RETURN(DoKeyword);
 394 done            RETURN_DOT(DoneKeyword);
 395 else            RETURN(ElseKeyword);
 396 encode          RETURN(EncodeKeyword);
 397 enumerated      RETURN(EnumKeyword);
 398 error           RETURN(ErrorKeyword);
 399 except          RETURN(ExceptKeyword);
 400 exception       RETURN(ExceptionKeyword);
 401 execute         RETURN(ExecuteKeyword);
 402 extends         RETURN(ExtendsKeyword);
 403 extension       RETURN(ExtensionKeyword);
 404 external        RETURN(ExtKeyword);
 405 fail            RETURN(FailKeyword);
 406 false           RETURN(FalseKeyword);
 407 float           RETURN(FloatKeyword);
 408 for             RETURN(ForKeyword);
 409 friend          RETURN(FriendKeyword);
 410 from            RETURN(FromKeyword);
 411 function        RETURN(FunctionKeyword);
 412 getcall         RETURN_DOT(GetCallOpKeyword);
 413 getreply        RETURN_DOT(GetReplyOpKeyword);
 414 getverdict      RETURN(GetVerdictKeyword);
 415 goto            RETURN(GotoKeyword);
 416 group           RETURN(GroupKeyword);
 417 halt            RETURN_DOT(HaltKeyword);
 418 hexstring       RETURN(HexStringKeyword);
 419 if              RETURN(IfKeyword);
 420 ifpresent       RETURN(IfPresentKeyword);
 421 import          RETURN(ImportKeyword);
 422 in              RETURN(InParKeyword);
 423 inconc          RETURN(InconcKeyword);
 424 infinity        RETURN(InfinityKeyword);
 425 inout           RETURN(InOutParKeyword);
 426 integer         RETURN(IntegerKeyword);
 427 interleave      RETURN(InterleavedKeyword);
 428 kill            RETURN_DOT(KillKeyword);
 429 killed          RETURN_DOT(KilledKeyword);
 430 label           RETURN(LabelKeyword);
 431 language        RETURN(LanguageKeyword);
 432 length          RETURN(LengthKeyword);
 433 log             RETURN(LogKeyword);
 434 map             RETURN(MapKeyword);
 435 match           RETURN(MatchKeyword);
 436 message         RETURN(MessageKeyword);
 437 mixed           RETURN(MixedKeyword);
 438 mod             RETURN(ModKeyword);
 439 modifies        RETURN(ModifiesKeyword);
 440 module          RETURN(TTCN3ModuleKeyword);
 441 modulepar       RETURN(ModuleParKeyword);
 442 mtc             RETURN(MTCKeyword);
 443 noblock         RETURN(NoBlockKeyword);
 444 none            RETURN(NoneKeyword);
 445 not             RETURN(NotKeyword);
 446 not_a_number  RETURN(NaNKeyword);
 447 not4b           RETURN(Not4bKeyword);
 448 nowait          RETURN(NowaitKeyword);
 449 null            RETURN(NullKeyword);
 450 objid           RETURN(ObjectIdentifierKeyword);
 451 octetstring     RETURN(OctetStringKeyword);
 452 of              RETURN(OfKeyword);
 453 omit            RETURN(OmitKeyword);
 454 on              RETURN(OnKeyword);
 455 optional        RETURN(OptionalKeyword);
 456 or              RETURN(OrKeyword);
 457 or4b            RETURN(Or4bKeyword);
 458 out             RETURN(OutParKeyword);
 459 override        RETURN(OverrideKeyword);
 460 param           RETURN(ParamKeyword);
 461 pass            RETURN(PassKeyword);
 462 pattern         RETURN(PatternKeyword);
 463 permutation     RETURN(PermutationKeyword);
 464 port            RETURN(PortKeyword);
 465 present         RETURN(PresentKeyword);
 466 private         RETURN(PrivateKeyword);
 467 procedure       RETURN(ProcedureKeyword);
 468 public          RETURN(PublicKeyword);
 469 raise           RETURN_DOT(RaiseKeyword);
 470 read            RETURN_DOT(ReadKeyword);
 471 receive         RETURN_DOT(ReceiveOpKeyword);
 472 record          RETURN(RecordKeyword);
 473 recursive       RETURN(RecursiveKeyword);
 474 refers          RETURN(RefersKeyword);
 475 rem             RETURN(RemKeyword);
 476 repeat          RETURN(RepeatKeyword);
 477 reply           RETURN_DOT(ReplyKeyword);
 478 return          RETURN(ReturnKeyword);
 479 running         RETURN_DOT(RunningKeyword);
 480 runs            RETURN(RunsKeyword);
 481 select          RETURN(SelectKeyword);
 482 self            RETURN(SelfKeyword);
 483 send            RETURN_DOT(SendOpKeyword);
 484 sender          RETURN(SenderKeyword);
 485 set             RETURN(SetKeyword);
 486 setverdict      RETURN(SetVerdictKeyword);
 487 signature       RETURN(SignatureKeyword);
 488 start           RETURN_DOT(StartKeyword);
 489 stop            RETURN_DOT(StopKeyword);
 490 subset          RETURN(SubsetKeyword);
 491 superset        RETURN(SupersetKeyword);
 492 system          RETURN(SystemKeyword);
 493 template        RETURN(TemplateKeyword);
 494 testcase        RETURN(TestcaseKeyword);
 495 timeout         RETURN_DOT(TimeoutKeyword);
 496 timer           RETURN(TimerKeyword);
 497 to              RETURN(ToKeyword);
 498 trigger         RETURN_DOT(TriggerOpKeyword);
 499 true            RETURN(TrueKeyword);
 500 type            RETURN(TypeDefKeyword);
 501 union           RETURN(UnionKeyword);
 502 universal       RETURN(UniversalKeyword);
 503 unmap           RETURN(UnmapKeyword);
 504 value           RETURN(ValueKeyword);
 505 valueof         RETURN(ValueofKeyword);
 506 var             RETURN(VarKeyword);
 507 variant         RETURN(VariantKeyword);
 508 verdicttype     RETURN(VerdictTypeKeyword);
 509 while           RETURN(WhileKeyword);
 510 with            RETURN(WithKeyword);
 511 xor             RETURN(XorKeyword);
 512 xor4b           RETURN(Xor4bKeyword);
 513
 514   /* modifier keywords */
 515
 516 "@nocase" RETURN(NocaseKeyword);
 517 "@lazy"   RETURN(LazyKeyword);
 518
 519   /* special TITAN specific keywords */
 520
 521 "@try"    RETURN(TitanSpecificTryKeyword);
 522 "@catch"  RETURN(TitanSpecificCatchKeyword);
 523 "@profiler" RETURN(TitanSpecificProfilerKeyword);
 524
 525
 526         /* Predefined function identifiers */
 527
 528 bit2hex         RETURN(bit2hexKeyword);
 529 bit2int         RETURN(bit2intKeyword);
 530 bit2oct         RETURN(bit2octKeyword);
 531 bit2str         RETURN(bit2strKeyword);
 532 char2int        RETURN(char2intKeyword);
 533 char2oct        RETURN(char2octKeyword);
 534 decomp          RETURN(decompKeyword);
 535 float2int       RETURN(float2intKeyword);
 536 float2str       RETURN(float2strKeyword);
 537 hex2bit         RETURN(hex2bitKeyword);
 538 hex2int         RETURN(hex2intKeyword);
 539 hex2oct         RETURN(hex2octKeyword);
 540 hex2str         RETURN(hex2strKeyword);
 541 int2bit         RETURN(int2bitKeyword);
 542 int2char        RETURN(int2charKeyword);
 543 int2enum  RETURN(int2enumKeyword);
 544 int2float       RETURN(int2floatKeyword);
 545 int2hex         RETURN(int2hexKeyword);
 546 int2oct         RETURN(int2octKeyword);
 547 int2str         RETURN(int2strKeyword);
 548 int2unichar     RETURN(int2unicharKeyword);
 549 isvalue         RETURN(isvalueKeyword);
 550 isbound         RETURN(isboundKeyword);
 551 ischosen        RETURN(ischosenKeyword);
 552 ispresent       RETURN(ispresentKeyword);
 553 lengthof        RETURN(lengthofKeyword);
 554 oct2bit         RETURN(oct2bitKeyword);
 555 oct2char        RETURN(oct2charKeyword);
 556 oct2hex         RETURN(oct2hexKeyword);
 557 oct2int         RETURN(oct2intKeyword);
 558 oct2str         RETURN(oct2strKeyword);
 559 regexp          RETURN(regexpKeyword);
 560 replace         RETURN(replaceKeyword);
 561 rnd                     RETURN(rndKeyword);
 562 sizeof          RETURN(sizeofKeyword);
 563 str2bit         RETURN(str2bitKeyword);
 564 str2float       RETURN(str2floatKeyword);
 565 str2hex         RETURN(str2hexKeyword);
 566 str2int         RETURN(str2intKeyword);
 567 str2oct         RETURN(str2octKeyword);
 568 substr          RETURN(substrKeyword);
 569 unichar2int     RETURN(unichar2intKeyword);
 570 unichar2char    RETURN(unichar2charKeyword);
 571 log2str         RETURN(log2strKeyword);
 572 enum2int        RETURN(enum2intKeyword);
 573 encvalue        RETURN(encvalueKeyword);
 574 decvalue        RETURN(decvalueKeyword);
 575 testcasename    RETURN(testcasenameKeyword);
 576 ttcn2string RETURN(ttcn2stringKeyword);
 577 string2ttcn RETURN(string2ttcnKeyword);
 578 unichar2oct     RETURN(unichar2octKeyword);
 579 oct2unichar     RETURN(oct2unicharKeyword);
 580 remove_bom      RETURN(remove_bomKeyWord);
 581 get_stringencoding      RETURN(get_stringencodingKeyWord);
 582 encode_base64   RETURN(encode_base64KeyWord);
 583 decode_base64   RETURN(decode_base64KeyWord);
 584
 585         /* Values */
 586
 587 {NUMBER} {
 588   Location loc(infile, current_line, current_column, current_line,
 589     current_column + yyleng);
 590   yylval.int_val = new int_val_t(yytext, loc);
 591   RETURN_LVAL(Number);
 592 }
 593
 594 {FLOAT} {
 595   Location loc(infile, current_line, current_column, current_line,
 596     current_column + yyleng);
 597   yylval.float_val = string2Real(yytext, loc);
 598   RETURN_LVAL(FloatValue);
 599 }
 600
 601 NULL RETURN(NullValue);
 602
 603 "'" {
 604   binstr=memptystr();
 605   valid_bit=true;
 606   valid_oct=true;
 607   half_oct=false;
 608   contains_match=false;
 609   contains_ws=false;
 610   start_line = current_line;
 611   start_column = current_column;
 612   current_column++;
 613   MD5_Update(&md5_ctx, yytext, yyleng);
 614   BEGIN(SC_binstring);
 615 }
 616
 617 \" {
 618   yylval.str = memptystr();
 619   start_line = current_line;
 620   start_column = current_column;
 621   current_column++;
 622   MD5_Update(&md5_ctx, yytext, yyleng);
 623   BEGIN(SC_cstring);
 624 }
 625
 626 <SC_binstring> /* -------- SC_binstring scope -------------- */
 627 {
 628
 629   {WHITESPACE}+ {
 630     contains_ws = true;
 631     current_column += yyleng;
 632   }
 633
 634   {WHITESPACE}*{NEWLINE} {
 635     contains_ws = true;
 636     current_line++;
 637     current_column = 0;
 638   }
 639
 640   [01] {
 641     binstr = mputc(binstr, yytext[0]);
 642     half_oct = !half_oct;
 643     current_column++;
 644   }
 645
 646   [2-9A-F] {
 647     binstr = mputc(binstr, yytext[0]);
 648     valid_bit = false;
 649     half_oct = !half_oct;
 650     current_column++;
 651   }
 652
 653   [a-f] {
 654     binstr = mputc(binstr, yytext[0] - 'a' + 'A');
 655     valid_bit = false;
 656     half_oct = !half_oct;
 657     current_column++;
 658   }
 659
 660   "?"|"*" {
 661     binstr = mputc(binstr, yytext[0]);
 662     contains_match = true;
 663     if (half_oct) valid_oct = false;
 664     current_column++;
 665   }
 666
 667   "'"[bBhHoO] {
 668     yylloc.first_line = start_line;
 669     yylloc.first_column = start_column;
 670     yylloc.last_line = current_line;
 671     yylloc.last_column = current_column + 2;
 672     Location loc(infile, yylloc);
 673     int ret_val = TOK_errval;
 674     switch (yytext[1]) {
 675     case 'b': {
 676       Location loc2(infile, current_line, current_column + 1, current_line,
 677         current_column + 2);
 678       loc2.warning("The last character of a bitstring literal should be "
 679         "`B' instead of `b'");
 680       /* no break */ }
 681     case 'B':
 682       if (valid_bit) {
 683         if (contains_ws) loc.warning("Bitstring %s contains whitespace and/or "
 684           "newline character(s)", contains_match ? "match" : "value");
 685         ret_val = contains_match ? BitStringMatch : Bstring;
 686         yylval.string_val = new string(binstr);
 687       } else loc.error("Bitstring value contains invalid character");
 688       break;
 689     case 'h': {
 690       Location loc2(infile, current_line, current_column + 1, current_line,
 691         current_column + 2);
 692       loc2.warning("The last character of a hexstring literal should be "
 693         "`H' instead of `h'");
 694       /* no break */ }
 695     case 'H':
 696       if (contains_ws) loc.warning("Hexstring %s contains whitespace and/or "
 697         "newline character(s)", contains_match ? "match" : "value");
 698       ret_val = contains_match ? HexStringMatch : Hstring;
 699       yylval.string_val = new string(binstr);
 700       break;
 701     case 'o': {
 702       Location loc2(infile, current_line, current_column + 1, current_line,
 703         current_column + 2);
 704       loc2.warning("The last character of an octetstring literal should be "
 705         "`O' instead of `o'");
 706       /* no break */ }
 707     case 'O':
 708       if (valid_oct && !half_oct) {
 709         if (contains_ws) loc.warning("Octetstring %s contains whitespace "
 710           "and/or newline character(s)", contains_match ? "match" : "value");
 711         ret_val = contains_match ? OctetStringMatch : Ostring;
 712         yylval.string_val = new string(binstr);
 713       } else if (contains_match) {
 714         loc.error("Octetstring match contains half octet(s)");
 715       } else {
 716         loc.error("Octetstring value contains odd number of hexadecimal "
 717           "digits");
 718       }
 719     }
 720     MD5_Update(&md5_ctx, binstr, strlen(binstr));
 721     Free(binstr);
 722     update_md5();
 723     BEGIN(INITIAL);
 724     current_column += 2;
 725     if (dot_flag) {
 726       backup_token = ret_val;
 727       backup_lval = yylval;
 728       backup_lloc = yylloc;
 729       RETURN_SAVED_DOT;
 730     } else return ret_val;
 731   }
 732
 733   "'" {
 734     yylloc.first_line = start_line;
 735     yylloc.first_column = start_column;
 736     current_column++;
 737     yylloc.last_line = current_line;
 738     yylloc.last_column = current_column;
 739     Location loc(infile, yylloc);
 740     loc.error("Invalid binary string literal. Expecting `B', `H' or `O' after "
 741       "the closing `''");
 742     MD5_Update(&md5_ctx, binstr, strlen(binstr));
 743     Free(binstr);
 744     BEGIN(INITIAL);
 745     RETURN_NOLOCUPD(TOK_errval);
 746   }
 747
 748   . {
 749     Location loc(infile, current_line, current_column, current_line,
 750       current_column + 1);
 751     int c = (unsigned char)yytext[0];
 752     loc.error("Invalid character `%c' (0x%02X) in binary string",
 753       isprint(c) ? c : '?', c);
 754     MD5_Update(&md5_ctx, binstr, strlen(binstr));
 755     Free(binstr);
 756     MD5_Update(&md5_ctx, yytext, 1);
 757     current_column++;
 758     BEGIN(SC_binstring_bad);
 759   }
 760
 761 } /* SC_binstring scope */
 762
 763 <SC_binstring_bad> /* -------- SC_binstring_bad scope -------------- */
 764 {
 765
 766   {WHITESPACE}+ current_column += yyleng;
 767
 768   {WHITESPACE}*{NEWLINE} {
 769     current_line++;
 770     current_column = 0;
 771   }
 772
 773   "'"[bBhHoO]? {
 774     current_column += yyleng;
 775     yylloc.first_line = start_line;
 776     yylloc.first_column = start_column;
 777     yylloc.last_line = current_line;
 778     yylloc.last_column = current_column;
 779     BEGIN(INITIAL);
 780     RETURN_NOLOCUPD(TOK_errval);
 781   }
 782
 783   . {
 784     MD5_Update(&md5_ctx, yytext, yyleng);
 785     current_column++;
 786   }
 787
 788 } /* SC_binstring_bad scope */
 789
 790 <SC_cstring> /* -------- SC_cstring scope -------------- */
 791 {
 792
 793   \\?{NEWLINE} { /* newline possibly preceded by backslash */
 794     yylval.str = mputstr(yylval.str, yytext);
 795     current_line++;
 796     current_column = 0;
 797   }
 798
 799   \"\"|\\. { /* two doublequotes or any backslash-escaped char */
 800     yylval.str = mputstr(yylval.str, yytext);
 801     current_column += 2;
 802     /* Note that both get added ("external representation").
 803      * parse_charstring_value() in charstring_la.l is responsible
 804      * for transforming the string to "internal representation" */
 805   }
 806
 807   \" {
 808     current_column++;
 809     yylloc.first_line = start_line;
 810     yylloc.first_column = start_column;
 811     yylloc.last_line = current_line;
 812     yylloc.last_column = current_column;
 813     MD5_Update(&md5_ctx, yylval.str, strlen(yylval.str));
 814     update_md5();
 815     BEGIN(INITIAL);
 816     if (dot_flag) {
 817       backup_token = Cstring;
 818       backup_lval = yylval;
 819       backup_lloc = yylloc;
 820       RETURN_SAVED_DOT;
 821     } else return Cstring;
 822   }
 823
 824   . {
 825     yylval.str = mputc(yylval.str, yytext[0]);
 826     current_column++;
 827   }
 828
 829 } /* SC_cstring scope */
 830
 831         /* Macros */
 832
 833 "%moduleId" {
 834   yylval.macrotype = Value::MACRO_MODULEID;
 835   RETURN_LVAL(MacroValue);
 836 }
 837 "%fileName" {
 838   yylval.macrotype = Value::MACRO_FILENAME;
 839   RETURN_LVAL(MacroValue);
 840 }
 841 "%lineNumber" {
 842   yylval.macrotype = Value::MACRO_LINENUMBER;
 843   RETURN_LVAL(MacroValue);
 844 }
 845 "%definitionId" {
 846   yylval.macrotype = Value::MACRO_DEFINITIONID;
 847   RETURN_LVAL(MacroValue);
 848 }
 849 "%testcaseId" {
 850   yylval.macrotype = Value::MACRO_TESTCASEID;
 851   RETURN_LVAL(MacroValue);
 852 }
 853 "%"{IDENTIFIER} {
 854   fill_location();
 855   Location loc(infile, yylloc);
 856   loc.error("Invalid macro notation: `%s'", yytext);
 857   RETURN_NOLOCUPD(TOK_errval);
 858 }
 859
 860 "__MODULE__" {
 861   yylval.macrotype = Value::MACRO_MODULEID;
 862   RETURN_LVAL(MacroValue);
 863 }
 864 "__FILE__" {
 865   yylval.macrotype = Value::MACRO_FILEPATH;
 866   RETURN_LVAL(MacroValue);
 867 }
 868 "__BFILE__" {
 869   yylval.macrotype = Value::MACRO_BFILENAME;
 870   RETURN_LVAL(MacroValue);
 871 }
 872 "__LINE__" {
 873   yylval.macrotype = Value::MACRO_LINENUMBER_C;
 874   RETURN_LVAL(MacroValue);
 875 }
 876 "__SCOPE__" {
 877   yylval.macrotype = Value::MACRO_SCOPE;
 878   RETURN_LVAL(MacroValue);
 879 }
 880 "__TESTCASE__" {
 881   yylval.macrotype = Value::MACRO_TESTCASEID;
 882   RETURN_LVAL(MacroValue);
 883 }
 884 "__"{IDENTIFIER}"__" {
 885   fill_location();
 886   Location loc(infile, yylloc);
 887   loc.error("Invalid macro notation: `%s'", yytext);
 888   RETURN_NOLOCUPD(TOK_errval);
 889 }
 890
 891         /* Multi-character operators */
 892
 893 ":="            RETURN(AssignmentChar);
 894 "\.\."          RETURN(DotDot);
 895 "->"            RETURN(PortRedirectSymbol);
 896 "=="            RETURN(EQ);
 897 "!="            RETURN(NE);
 898 ">="            RETURN(GE);
 899 "<="            RETURN(LE);
 900 "<<"            RETURN(SL);
 901 ">>"            RETURN(SR);
 902 "<@"            RETURN(RL);
 903 "@>"            RETURN(_RR);
 904
 905 "++" |
 906 "--" {
 907   fill_location();
 908   Location loc(infile, yylloc);
 909   loc.error("Operator `%s' is reserved for future use", yytext);
 910 }
 911
 912         /* Invalid operators */
 913
 914 "::=" {
 915   fill_location();
 916   Location loc(infile, yylloc);
 917   loc.error("`::=' is not a valid assignment operator in TTCN-3. Did you mean "
 918     "`:='?");
 919   RETURN_NOLOCUPD(AssignmentChar);
 920 }
 921
 922 "=" {
 923   fill_location();
 924   Location loc(infile, yylloc);
 925   loc.error("A single `=' character cannot be used in TTCN-3. Did you mean "
 926     "the assignment sign `:=' or the equality operator `=='?");
 927   /* the former is more probable than the latter */
 928   RETURN_NOLOCUPD(AssignmentChar);
 929 }
 930
 931 "<>" {
 932   fill_location();
 933   Location loc(infile, yylloc);
 934   loc.error("`<>' is not a valid comparison operator in TTCN-3. Did you mean "
 935     "`!='?");
 936   RETURN_NOLOCUPD(NE);
 937 }
 938
 939         /* Identifiers */
 940
 941 {IDENTIFIER} {
 942   yylval.id = new Identifier(Identifier::ID_TTCN, string(yyleng, yytext));
 943   RETURN_LVAL(IDentifier);
 944 }
 945
 946         /* Single character tokens (brackets, operators, etc.) */
 947
 948 \. {
 949   update_md5();
 950   if (dot_flag) {
 951     /* store this dot in the backup */
 952     backup_token = '.';
 953     backup_lloc.first_line = current_line;
 954     backup_lloc.first_column = current_column;
 955     current_column++;
 956     backup_lloc.last_line = current_line;
 957     backup_lloc.last_column = current_column;
 958     /* return the dot that was found previously */
 959     RETURN_SAVED_DOT;
 960   } else {
 961     dot_flag = true;
 962     dot_line = current_line;
 963     dot_column = current_column;
 964     current_column++;
 965   }
 966 }
 967
 968 [()\[\]{}+\-\*/&:;,<>\?!]  RETURN(*yytext);
 969
 970         /* Invalid characters */
 971
 972 . {
 973   fill_location();
 974   Location loc(infile, yylloc);
 975   int c = (unsigned char)yytext[0];
 976   loc.error("Character `%c' (0x%02X) is not used in TTCN-3",
 977     isprint(c) ? c : '?', c);
 978 }
 979
 980         /* EOF rule */
 981
 982 <*><<EOF>> {
 983   if (YY_START != INITIAL) {
 984     Location loc(infile, start_line, start_column, current_line,
 985       current_column);
 986     switch (YY_START) {
 987     case SC_blockcomment:
 988       loc.error("Unterminated block comment");
 989       break;
 990     case SC_binstring:
 991       Free(binstr);
 992       /* no break */
 993     case SC_binstring_bad:
 994       loc.error("Unterminated binary string literal");
 995       break;
 996     case SC_cstring:
 997       Free(yylval.str);
 998       loc.error("Unterminated character string literal");
 999     }
1000     BEGIN(INITIAL);
1001   }
1002   if (dot_flag) {
1003     dot_flag = false;
1004     RETURN_SAVED_DOT;
1005   } else {
1006     yylloc.first_line = current_line;
1007     yylloc.first_column = current_column;
1008     yylloc.last_line = current_line;
1009     yylloc.last_column = current_column + 1;
1010     return EOF;
1011   }
1012 }
1013
1014 %%
1015
1016 void init_ttcn3_lex()
1017 {
1018   dot_flag = false;
1019   current_line = 1;
1020   current_column = 0;
1021   real_infile = infile;
1022   real_lineno_offset = 0;
1023   MD5_Init(&md5_ctx);
1024 }
1025
1026 void init_erroneous_lex(const char* p_infile, int p_line, int p_column)
1027 {
1028   infile = p_infile;
1029   current_line = p_line;
1030   current_column = p_column;
1031   real_infile = infile;
1032   real_lineno_offset = 0;
1033   dot_flag = false;
1034 }
1035
1036 void free_dot_flag_stuff()
1037 {
1038   if (dot_flag) {
1039     dot_flag = false;
1040     /* clean up the semantic value of the token that was backed up */
1041     switch (backup_token) {
1042     case IDentifier:
1043       delete backup_lval.id;
1044       break;
1045     case Bstring:
1046     case Hstring:
1047     case Ostring:
1048     case BitStringMatch:
1049     case HexStringMatch:
1050     case OctetStringMatch:
1051       delete backup_lval.string_val;
1052       break;
1053     case Cstring:
1054       Free(backup_lval.str);
1055     default:
1056       break;
1057     }
1058   }
1059 }
1060
1061 void free_ttcn3_lex()
1062 {
1063   free_dot_flag_stuff();
1064   fclose(ttcn3_in);
1065   ttcn3_lex_destroy();
1066 }
1067
1068 /* called from ttcn3_parse_file to finalize MD5 and add it to the module */
1069 void set_md5_checksum(Ttcn::Module *m)
1070 {
1071   unsigned char md5_sum[MD5_DIGEST_LENGTH];
1072   MD5_Final(md5_sum, &md5_ctx);
1073   m->set_checksum(sizeof(md5_sum), md5_sum);
1074 }