compiler2/ttcn3/compiler.l

   1 /******************************************************************************
   2  * Copyright (c) 2000-2014 Ericsson Telecom AB
   3  * All rights reserved. This program and the accompanying materials
   4  * are made available under the terms of the Eclipse Public License v1.0
   5  * which accompanies this distribution, and is available at
   6  * http://www.eclipse.org/legal/epl-v10.html
   7  ******************************************************************************/
   8 %option noyywrap
   9 %option never-interactive
  10 %option nounput
  11 %{
  12
  13 /* Tokenizer for TTCN-3 Core Language */
  14
  15 #include "../../common/dbgnew.hh"
  16 #include "compiler.h"
  17 #include "../string.hh"
  18 #include "../CompilerError.hh"
  19 #include "../Real.hh"
  20 #include "../Value.hh"
  21 #include "AST_ttcn3.hh"
  22 #include "Ttcnstuff.hh" // at least for PortTypeBody::PortOperationMode_t
  23 #include "Statement.hh" // for Statement::statementtype_t
  24 #include "Attributes.hh"
  25
  26 #include <string.h>
  27 #include <ctype.h>
  28 #include <openssl/md5.h>
  29
  30 namespace Common {
  31   class IndexedValue;
  32   class Location;
  33 }
  34
  35 namespace Ttcn {
  36   class ParamRedirect;
  37   class Statement;
  38   class AltGuard;
  39   class IfClause;
  40   class IfClauses;
  41   class NamedTemplate;
  42   class NamedTemplates;
  43   class IndexedTemplate;
  44   class IndexedTemplates;
  45   class Templates;
  46   class CompTypeRefList;
  47 }
  48
  49 using namespace Common;
  50 using namespace Ttcn;
  51
  52 #include "compiler.tab.hh"
  53
  54 #define yylval ttcn3_lval
  55 #define yylloc ttcn3_lloc
  56
  57 /* global variable indicating the location of the returned token to bison */
  58 extern YYLTYPE yylloc;
  59
  60 extern bool is_erroneous_parsed;
  61
  62 /* always points to the first character of the regexp to be recognized */
  63 static int current_line, current_column;
  64
  65 /* when reporting an error in linemarker or preprocessor
  66  * directive the real file name and line number is needed */
  67 static const char* real_infile;
  68 /* real_lineno = current_line + real_lineno_offset */
  69 static int real_lineno_offset;
  70
  71 static bool dot_flag = false;
  72 /* type of the backup token (that was found after a dot) */
  73 static int backup_token;
  74 /* semantic value of the backup token */
  75 static YYSTYPE backup_lval;
  76 /* location of the backup token */
  77 static YYLTYPE backup_lloc;
  78
  79 static MD5_CTX md5_ctx;
  80
  81 static void fill_location()
  82 {
  83   yylloc.first_line = current_line;
  84   yylloc.first_column = current_column;
  85   current_column += yyleng;
  86   yylloc.last_line = current_line;
  87   yylloc.last_column = current_column;
  88 }
  89
  90 static void update_md5()
  91 {
  92   MD5_Update(&md5_ctx, yytext, yyleng);
  93   MD5_Update(&md5_ctx, " ", 1);
  94 }
  95
  96 #define RETURN_SAVED_DOT \
  97   do { \
  98     yytext[0] = '\0'; \
  99     yylloc.first_line = dot_line; \
 100     yylloc.first_column = dot_column; \
 101     yylloc.last_line = dot_line; \
 102     yylloc.last_column = dot_column + 1; \
 103     return '.'; \
 104   } while (0)
 105
 106 /* return macro for simple tokens without semantic value */
 107 #define RETURN(ret_val) \
 108   do { \
 109     update_md5(); \
 110     fill_location(); \
 111     if (dot_flag) { \
 112       backup_token = ret_val; \
 113       backup_lloc = yylloc; \
 114       RETURN_SAVED_DOT; \
 115     } else return ret_val; \
 116   } while (0)
 117
 118 /* same as RETURN(ret_val) macro but without location update,
 119  * usually a return after an error */
 120 #define RETURN_NOLOCUPD(ret_val) \
 121   do { \
 122     update_md5(); \
 123     if (dot_flag) { \
 124       backup_token = ret_val; \
 125       backup_lloc = yylloc; \
 126       RETURN_SAVED_DOT; \
 127     } else return ret_val; \
 128   } while (0)
 129
 130 /* return macro for simple tokens with semantic value */
 131 #define RETURN_LVAL(ret_val) \
 132   do { \
 133     update_md5(); \
 134     fill_location(); \
 135     if (dot_flag) { \
 136       backup_token = ret_val; \
 137       backup_lval = yylval; \
 138       backup_lloc = yylloc; \
 139       RETURN_SAVED_DOT; \
 140     } else return ret_val; \
 141   } while (0)
 142
 143 /* return macro for special tokens that are glued together with previous dot */
 144 #define RETURN_DOT(ret_val) \
 145   do { \
 146     update_md5(); \
 147     if (dot_flag) { \
 148       dot_flag = false; \
 149       yylloc.first_line = dot_line; \
 150       yylloc.first_column = dot_column; \
 151       current_column += yyleng; \
 152       yylloc.last_line = current_line; \
 153       yylloc.last_column = current_column; \
 154       return Dot##ret_val; \
 155     } else { \
 156       fill_location(); \
 157       return ret_val; \
 158     } \
 159   } while (0)
 160
 161 extern string *parse_charstring_value(const char *str, const Location& loc);
 162
 163 %}
 164
 165 NUMBER 0|([1-9][0-9]*)
 166
 167 FLOAT ({NUMBER}\.[0-9]+)|({NUMBER}(\.[0-9]+)?[Ee][+-]?{NUMBER})
 168
 169 IDENTIFIER [A-Za-z][A-Za-z0-9_]*
 170
 171 LINECOMMENT "//"[^\r\n]*
 172
 173 WHITESPACE [ \t\v\f]
 174
 175 NEWLINE \r|\n|\r\n
 176
 177 LINEMARKER {NUMBER}{WHITESPACE}+\"([^\\\"\r\n]|\\[^\r\n])*\"
 178
 179 TITAN "$#&&&(#TITANERRONEOUS$#&&^#% "
 180
 181 %x SC_blockcomment SC_cstring
 182 %x SC_binstring SC_binstring_bad
 183
 184 %%
 185   /* local variables of yylex() */
 186   int start_line = 0, start_column = 0; /**< used by block comments and
 187                                              string literals */
 188   int dot_line = 0, dot_column = 0; /**< location of the previous '.' token */
 189   /* variables used when processing binary strings */
 190   expstring_t binstr = NULL; /**< the string itself */
 191   bool valid_bit = false, /**< binstr is valid bitstring */
 192     valid_oct = false, /**< binstr is valid octetstring */
 193     half_oct = false, /**< binstr is not a valid octetstr but a valid hexstr */
 194     contains_match = false, /**< binstr contains matching symbol */
 195     contains_ws = false; /**< binstr contains whitespace characters */
 196
 197   if (dot_flag) {
 198     if (backup_token == '.') {
 199       /* Two dots were found in the previous call: the first one was returned,
 200        * the second one is now in the backup. Let's assume that we have just
 201        * found the second one. */
 202       dot_line = backup_lloc.first_line;
 203       dot_column = backup_lloc.first_column;
 204     } else {
 205       /* Return the token and its semantic value that was backed up after the
 206        * last token (which was a dot). */
 207       dot_flag = false;
 208       yylval = backup_lval;
 209       yylloc = backup_lloc;
 210       return backup_token;
 211     }
 212   }
 213
 214
 215 {TITAN} {
 216   // hack: to avoid the erroneous parsing reporting a syntax error where it
 217   //       also lists TTCN3ModuleKeyword as a possible nonterminal to use
 218   if (is_erroneous_parsed) {
 219     RETURN(TitanErroneousHackKeyword);
 220   } else {
 221     Location loc(infile, current_line, current_column, current_line,
 222                  current_column + yyleng);
 223     loc.error("Unexpected `%s'.", yytext);
 224   }
 225 }
 226
 227         /* Eat up comments and whitespaces */
 228
 229 "/*" {
 230   start_line = current_line;
 231   start_column = current_column;
 232   current_column += 2;
 233   BEGIN(SC_blockcomment);
 234 }
 235
 236 <SC_blockcomment> /* -------- SC_blockcomment scope -------------- */
 237 {
 238
 239   "*/" {
 240     current_column += 2;
 241     BEGIN(INITIAL);
 242   }
 243
 244   {NEWLINE} {
 245     current_line++;
 246     current_column = 0;
 247   }
 248
 249   . current_column++;
 250
 251 } /* SC_blockcomment */
 252
 253 {LINECOMMENT}?{NEWLINE} {
 254   current_line++;
 255   current_column = 0;
 256 }
 257
 258 {LINECOMMENT} {
 259   Location loc(infile, current_line, current_column, current_line,
 260     current_column + yyleng);
 261   loc.error("Unterminated line comment (missing newline character at the end "
 262     "of file)");
 263   current_column += yyleng;
 264 }
 265
 266 {WHITESPACE}+ current_column += yyleng;
 267
 268         /* C preprocessor line markers */
 269
 270 ^{WHITESPACE}*"#"({WHITESPACE}*"line")?{WHITESPACE}+{LINEMARKER}[^\r\n]*{NEWLINE} {
 271   bool error_flag = false;
 272   int real_lineno = current_line + real_lineno_offset;
 273   /* skipping the leading whitespaces */
 274   int marker_begin = 0;
 275   while (yytext[marker_begin] != '#') marker_begin++;
 276   /* skipping the trailing whitespaces and newline */
 277   int marker_end = yyleng - 1;
 278   while (yytext[marker_end] == '\r' || yytext[marker_end] == '\n' ||
 279          yytext[marker_end] == ' ' || yytext[marker_end] == '\t') marker_end--;
 280   marker_end++;
 281   Location loc(real_infile, real_lineno, current_column + marker_begin,
 282     real_lineno, current_column + marker_end);
 283   Error_Context cntxt(&loc, "In preprocessor line marker");
 284   /* parsing out the line number */
 285   int lineno_begin = marker_begin + 1;
 286   while (!isdigit((unsigned char)yytext[lineno_begin])) lineno_begin++;
 287   int lineno_end = lineno_begin + 1;
 288   while (isdigit((int)yytext[lineno_end])) lineno_end++;
 289   errno = 0;
 290   int new_lineno = strtol(yytext + lineno_begin, NULL, 10);
 291   if (errno != 0) {
 292     Location lineno_loc(real_infile, real_lineno, current_column + lineno_begin,
 293       real_lineno, current_column + lineno_end);
 294     string lineno_str(lineno_end - lineno_begin, yytext + lineno_begin);
 295     lineno_loc.error("Line number `%s' is too large for being represented in "
 296       "memory: %s", lineno_str.c_str(), strerror(errno));
 297     error_flag = true;
 298   }
 299   /* parsing out the file name */
 300   int filename_begin = lineno_end + 1;
 301   while (yytext[filename_begin] != '"') filename_begin++;
 302   filename_begin++;
 303   int filename_end = filename_begin;
 304   while (yytext[filename_end] != '"') {
 305     if (yytext[filename_end] == '\\') filename_end += 2;
 306     else filename_end++;
 307   }
 308   Location filename_loc(real_infile, real_lineno, current_column +
 309     filename_begin - 1, real_lineno, current_column + filename_end + 1);
 310   string filename_str(filename_end - filename_begin, yytext + filename_begin);
 311   string *parsed_filename = parse_charstring_value(filename_str.c_str(),
 312     filename_loc);
 313   if (!parsed_filename) error_flag = true;
 314   /* updating the line/column counters */
 315   if (error_flag) {
 316     /* the line marker is erroneous, use the real line numbers */
 317     infile = real_infile;
 318     current_line = real_lineno + 1;
 319     real_lineno_offset = 0;
 320   } else {
 321     /* set the given line number */
 322     infile = Location::add_source_file_name(*parsed_filename);
 323     current_line = new_lineno;
 324     real_lineno_offset = real_lineno + 1 - new_lineno;
 325   }
 326   current_column = 0;
 327   delete parsed_filename;
 328 }
 329
 330 ^{WHITESPACE}*"#"[^\r\n]* {
 331   int real_lineno = current_line + real_lineno_offset;
 332   /* skip the leading and trailing whitespaces */
 333   int marker_begin = 0;
 334   while (yytext[marker_begin] != '#') marker_begin++;
 335   int marker_end = yyleng - 1;
 336   while (yytext[marker_end] == ' ' || yytext[marker_end] == '\t') marker_end--;
 337   marker_end++;
 338   Location loc(real_infile, real_lineno, current_column + marker_begin,
 339     real_lineno, current_column + marker_end);
 340   loc.error("Invalid/unsupported preprocessor directive or line marker: `%s'",
 341     string(marker_end - marker_begin, yytext + marker_begin).c_str());
 342   current_column += yyleng;
 343 }
 344
 345         /* Keywords */
 346
 347 action          RETURN(ActionKeyword);
 348 activate        RETURN(ActivateKeyword);
 349 address         RETURN(AddressKeyword);
 350 alive           RETURN_DOT(AliveKeyword);
 351 all             RETURN(AllKeyword);
 352 alt             RETURN(AltKeyword);
 353 altstep         RETURN(AltstepKeyword);
 354 and             RETURN(AndKeyword);
 355 and4b           RETURN(And4bKeyword);
 356 any             RETURN(AnyKeyword);
 357 anytype         RETURN(AnyTypeKeyword);
 358 apply           RETURN_DOT(ApplyKeyword);
 359 bitstring       RETURN(BitStringKeyword);
 360 boolean         RETURN(BooleanKeyword);
 361 break           RETURN(BreakKeyword);
 362 call            RETURN_DOT(CallOpKeyword);
 363 case            RETURN(CaseKeyword);
 364 catch           RETURN_DOT(CatchOpKeyword);
 365 char            RETURN(CharKeyword);
 366 charstring      RETURN(CharStringKeyword);
 367 check           RETURN_DOT(CheckOpKeyword);
 368 clear           RETURN_DOT(ClearOpKeyword);
 369 complement      RETURN(ComplementKeyword);
 370 component       RETURN(ComponentKeyword);
 371 connect         RETURN(ConnectKeyword);
 372 const           RETURN(ConstKeyword);
 373 continue        RETURN(ContinueKeyword);
 374 control         RETURN(ControlKeyword);
 375 create          RETURN_DOT(CreateKeyword);
 376 deactivate      RETURN(DeactivateKeyword);
 377 default         RETURN(DefaultKeyword);
 378 derefers        RETURN(DerefersKeyword);
 379 disconnect      RETURN(DisconnectKeyword);
 380 display         RETURN(DisplayKeyword);
 381 do              RETURN(DoKeyword);
 382 done            RETURN_DOT(DoneKeyword);
 383 else            RETURN(ElseKeyword);
 384 encode          RETURN(EncodeKeyword);
 385 enumerated      RETURN(EnumKeyword);
 386 error           RETURN(ErrorKeyword);
 387 except          RETURN(ExceptKeyword);
 388 exception       RETURN(ExceptionKeyword);
 389 execute         RETURN(ExecuteKeyword);
 390 extends         RETURN(ExtendsKeyword);
 391 extension       RETURN(ExtensionKeyword);
 392 external        RETURN(ExtKeyword);
 393 fail            RETURN(FailKeyword);
 394 false           RETURN(FalseKeyword);
 395 float           RETURN(FloatKeyword);
 396 for             RETURN(ForKeyword);
 397 friend          RETURN(FriendKeyword);
 398 from            RETURN(FromKeyword);
 399 function        RETURN(FunctionKeyword);
 400 getcall         RETURN_DOT(GetCallOpKeyword);
 401 getreply        RETURN_DOT(GetReplyOpKeyword);
 402 getverdict      RETURN(GetVerdictKeyword);
 403 goto            RETURN(GotoKeyword);
 404 group           RETURN(GroupKeyword);
 405 halt            RETURN_DOT(HaltKeyword);
 406 hexstring       RETURN(HexStringKeyword);
 407 if              RETURN(IfKeyword);
 408 ifpresent       RETURN(IfPresentKeyword);
 409 import          RETURN(ImportKeyword);
 410 in              RETURN(InParKeyword);
 411 inconc          RETURN(InconcKeyword);
 412 infinity        RETURN(InfinityKeyword);
 413 inout           RETURN(InOutParKeyword);
 414 integer         RETURN(IntegerKeyword);
 415 interleave      RETURN(InterleavedKeyword);
 416 kill            RETURN_DOT(KillKeyword);
 417 killed          RETURN_DOT(KilledKeyword);
 418 label           RETURN(LabelKeyword);
 419 language        RETURN(LanguageKeyword);
 420 length          RETURN(LengthKeyword);
 421 log             RETURN(LogKeyword);
 422 map             RETURN(MapKeyword);
 423 match           RETURN(MatchKeyword);
 424 message         RETURN(MessageKeyword);
 425 mixed           RETURN(MixedKeyword);
 426 mod             RETURN(ModKeyword);
 427 modifies        RETURN(ModifiesKeyword);
 428 module          RETURN(TTCN3ModuleKeyword);
 429 modulepar       RETURN(ModuleParKeyword);
 430 mtc             RETURN(MTCKeyword);
 431 noblock         RETURN(NoBlockKeyword);
 432 none            RETURN(NoneKeyword);
 433 not             RETURN(NotKeyword);
 434 not_a_number  RETURN(NaNKeyword);
 435 not4b           RETURN(Not4bKeyword);
 436 nowait          RETURN(NowaitKeyword);
 437 null            RETURN(NullKeyword);
 438 objid           RETURN(ObjectIdentifierKeyword);
 439 octetstring     RETURN(OctetStringKeyword);
 440 of              RETURN(OfKeyword);
 441 omit            RETURN(OmitKeyword);
 442 on              RETURN(OnKeyword);
 443 optional        RETURN(OptionalKeyword);
 444 or              RETURN(OrKeyword);
 445 or4b            RETURN(Or4bKeyword);
 446 out             RETURN(OutParKeyword);
 447 override        RETURN(OverrideKeyword);
 448 param           RETURN(ParamKeyword);
 449 pass            RETURN(PassKeyword);
 450 pattern         RETURN(PatternKeyword);
 451 permutation     RETURN(PermutationKeyword);
 452 port            RETURN(PortKeyword);
 453 present         RETURN(PresentKeyword);
 454 private         RETURN(PrivateKeyword);
 455 procedure       RETURN(ProcedureKeyword);
 456 public          RETURN(PublicKeyword);
 457 raise           RETURN_DOT(RaiseKeyword);
 458 read            RETURN_DOT(ReadKeyword);
 459 receive         RETURN_DOT(ReceiveOpKeyword);
 460 record          RETURN(RecordKeyword);
 461 recursive       RETURN(RecursiveKeyword);
 462 refers          RETURN(RefersKeyword);
 463 rem             RETURN(RemKeyword);
 464 repeat          RETURN(RepeatKeyword);
 465 reply           RETURN_DOT(ReplyKeyword);
 466 return          RETURN(ReturnKeyword);
 467 running         RETURN_DOT(RunningKeyword);
 468 runs            RETURN(RunsKeyword);
 469 select          RETURN(SelectKeyword);
 470 self            RETURN(SelfKeyword);
 471 send            RETURN_DOT(SendOpKeyword);
 472 sender          RETURN(SenderKeyword);
 473 set             RETURN(SetKeyword);
 474 setverdict      RETURN(SetVerdictKeyword);
 475 signature       RETURN(SignatureKeyword);
 476 start           RETURN_DOT(StartKeyword);
 477 stop            RETURN_DOT(StopKeyword);
 478 subset          RETURN(SubsetKeyword);
 479 superset        RETURN(SupersetKeyword);
 480 system          RETURN(SystemKeyword);
 481 template        RETURN(TemplateKeyword);
 482 testcase        RETURN(TestcaseKeyword);
 483 timeout         RETURN_DOT(TimeoutKeyword);
 484 timer           RETURN(TimerKeyword);
 485 to              RETURN(ToKeyword);
 486 trigger         RETURN_DOT(TriggerOpKeyword);
 487 true            RETURN(TrueKeyword);
 488 type            RETURN(TypeDefKeyword);
 489 union           RETURN(UnionKeyword);
 490 universal       RETURN(UniversalKeyword);
 491 unmap           RETURN(UnmapKeyword);
 492 value           RETURN(ValueKeyword);
 493 valueof         RETURN(ValueofKeyword);
 494 var             RETURN(VarKeyword);
 495 variant         RETURN(VariantKeyword);
 496 verdicttype     RETURN(VerdictTypeKeyword);
 497 while           RETURN(WhileKeyword);
 498 with            RETURN(WithKeyword);
 499 xor             RETURN(XorKeyword);
 500 xor4b           RETURN(Xor4bKeyword);
 501
 502   /* special TITAN specific keywords */
 503
 504 "@try"    RETURN(TitanSpecificTryKeyword);
 505 "@catch"  RETURN(TitanSpecificCatchKeyword);
 506 "@lazy"   RETURN(TitanSpecificLazyKeyword);
 507 "@profiler" RETURN(TitanSpecificProfilerKeyword);
 508
 509         /* Predefined function identifiers */
 510
 511 bit2hex         RETURN(bit2hexKeyword);
 512 bit2int         RETURN(bit2intKeyword);
 513 bit2oct         RETURN(bit2octKeyword);
 514 bit2str         RETURN(bit2strKeyword);
 515 char2int        RETURN(char2intKeyword);
 516 char2oct        RETURN(char2octKeyword);
 517 decomp          RETURN(decompKeyword);
 518 float2int       RETURN(float2intKeyword);
 519 float2str       RETURN(float2strKeyword);
 520 hex2bit         RETURN(hex2bitKeyword);
 521 hex2int         RETURN(hex2intKeyword);
 522 hex2oct         RETURN(hex2octKeyword);
 523 hex2str         RETURN(hex2strKeyword);
 524 int2bit         RETURN(int2bitKeyword);
 525 int2char        RETURN(int2charKeyword);
 526 int2float       RETURN(int2floatKeyword);
 527 int2hex         RETURN(int2hexKeyword);
 528 int2oct         RETURN(int2octKeyword);
 529 int2str         RETURN(int2strKeyword);
 530 int2unichar     RETURN(int2unicharKeyword);
 531 isvalue         RETURN(isvalueKeyword);
 532 isbound         RETURN(isboundKeyword);
 533 ischosen        RETURN(ischosenKeyword);
 534 ispresent       RETURN(ispresentKeyword);
 535 lengthof        RETURN(lengthofKeyword);
 536 oct2bit         RETURN(oct2bitKeyword);
 537 oct2char        RETURN(oct2charKeyword);
 538 oct2hex         RETURN(oct2hexKeyword);
 539 oct2int         RETURN(oct2intKeyword);
 540 oct2str         RETURN(oct2strKeyword);
 541 regexp          RETURN(regexpKeyword);
 542 replace         RETURN(replaceKeyword);
 543 rnd                     RETURN(rndKeyword);
 544 sizeof          RETURN(sizeofKeyword);
 545 str2bit         RETURN(str2bitKeyword);
 546 str2float       RETURN(str2floatKeyword);
 547 str2hex         RETURN(str2hexKeyword);
 548 str2int         RETURN(str2intKeyword);
 549 str2oct         RETURN(str2octKeyword);
 550 substr          RETURN(substrKeyword);
 551 unichar2int     RETURN(unichar2intKeyword);
 552 unichar2char    RETURN(unichar2charKeyword);
 553 log2str         RETURN(log2strKeyword);
 554 enum2int        RETURN(enum2intKeyword);
 555 encvalue        RETURN(encvalueKeyword);
 556 decvalue        RETURN(decvalueKeyword);
 557 testcasename    RETURN(testcasenameKeyword);
 558 ttcn2string RETURN(ttcn2stringKeyword);
 559 string2ttcn RETURN(string2ttcnKeyword);
 560 unichar2oct     RETURN(unichar2octKeyword);
 561 oct2unichar     RETURN(oct2unicharKeyword);
 562 remove_bom      RETURN(remove_bomKeyWord);
 563 get_stringencoding      RETURN(get_stringencodingKeyWord);
 564 encode_base64   RETURN(encode_base64KeyWord);
 565 decode_base64   RETURN(decode_base64KeyWord);
 566
 567         /* Values */
 568
 569 {NUMBER} {
 570   Location loc(infile, current_line, current_column, current_line,
 571     current_column + yyleng);
 572   yylval.int_val = new int_val_t(yytext, loc);
 573   RETURN_LVAL(Number);
 574 }
 575
 576 {FLOAT} {
 577   Location loc(infile, current_line, current_column, current_line,
 578     current_column + yyleng);
 579   yylval.float_val = string2Real(yytext, loc);
 580   RETURN_LVAL(FloatValue);
 581 }
 582
 583 NULL RETURN(NullValue);
 584
 585 "'" {
 586   binstr=memptystr();
 587   valid_bit=true;
 588   valid_oct=true;
 589   half_oct=false;
 590   contains_match=false;
 591   contains_ws=false;
 592   start_line = current_line;
 593   start_column = current_column;
 594   current_column++;
 595   MD5_Update(&md5_ctx, yytext, yyleng);
 596   BEGIN(SC_binstring);
 597 }
 598
 599 \" {
 600   yylval.str = memptystr();
 601   start_line = current_line;
 602   start_column = current_column;
 603   current_column++;
 604   MD5_Update(&md5_ctx, yytext, yyleng);
 605   BEGIN(SC_cstring);
 606 }
 607
 608 <SC_binstring> /* -------- SC_binstring scope -------------- */
 609 {
 610
 611   {WHITESPACE}+ {
 612     contains_ws = true;
 613     current_column += yyleng;
 614   }
 615
 616   {WHITESPACE}*{NEWLINE} {
 617     contains_ws = true;
 618     current_line++;
 619     current_column = 0;
 620   }
 621
 622   [01] {
 623     binstr = mputc(binstr, yytext[0]);
 624     half_oct = !half_oct;
 625     current_column++;
 626   }
 627
 628   [2-9A-F] {
 629     binstr = mputc(binstr, yytext[0]);
 630     valid_bit = false;
 631     half_oct = !half_oct;
 632     current_column++;
 633   }
 634
 635   [a-f] {
 636     binstr = mputc(binstr, yytext[0] - 'a' + 'A');
 637     valid_bit = false;
 638     half_oct = !half_oct;
 639     current_column++;
 640   }
 641
 642   "?"|"*" {
 643     binstr = mputc(binstr, yytext[0]);
 644     contains_match = true;
 645     if (half_oct) valid_oct = false;
 646     current_column++;
 647   }
 648
 649   "'"[bBhHoO] {
 650     yylloc.first_line = start_line;
 651     yylloc.first_column = start_column;
 652     yylloc.last_line = current_line;
 653     yylloc.last_column = current_column + 2;
 654     Location loc(infile, yylloc);
 655     int ret_val = TOK_errval;
 656     switch (yytext[1]) {
 657     case 'b': {
 658       Location loc2(infile, current_line, current_column + 1, current_line,
 659         current_column + 2);
 660       loc2.warning("The last character of a bitstring literal should be "
 661         "`B' instead of `b'");
 662       /* no break */ }
 663     case 'B':
 664       if (valid_bit) {
 665         if (contains_ws) loc.warning("Bitstring %s contains whitespace and/or "
 666           "newline character(s)", contains_match ? "match" : "value");
 667         ret_val = contains_match ? BitStringMatch : Bstring;
 668         yylval.string_val = new string(binstr);
 669       } else loc.error("Bitstring value contains invalid character");
 670       break;
 671     case 'h': {
 672       Location loc2(infile, current_line, current_column + 1, current_line,
 673         current_column + 2);
 674       loc2.warning("The last character of a hexstring literal should be "
 675         "`H' instead of `h'");
 676       /* no break */ }
 677     case 'H':
 678       if (contains_ws) loc.warning("Hexstring %s contains whitespace and/or "
 679         "newline character(s)", contains_match ? "match" : "value");
 680       ret_val = contains_match ? HexStringMatch : Hstring;
 681       yylval.string_val = new string(binstr);
 682       break;
 683     case 'o': {
 684       Location loc2(infile, current_line, current_column + 1, current_line,
 685         current_column + 2);
 686       loc2.warning("The last character of an octetstring literal should be "
 687         "`O' instead of `o'");
 688       /* no break */ }
 689     case 'O':
 690       if (valid_oct && !half_oct) {
 691         if (contains_ws) loc.warning("Octetstring %s contains whitespace "
 692           "and/or newline character(s)", contains_match ? "match" : "value");
 693         ret_val = contains_match ? OctetStringMatch : Ostring;
 694         yylval.string_val = new string(binstr);
 695       } else if (contains_match) {
 696         loc.error("Octetstring match contains half octet(s)");
 697       } else {
 698         loc.error("Octetstring value contains odd number of hexadecimal "
 699           "digits");
 700       }
 701     }
 702     MD5_Update(&md5_ctx, binstr, strlen(binstr));
 703     Free(binstr);
 704     update_md5();
 705     BEGIN(INITIAL);
 706     current_column += 2;
 707     if (dot_flag) {
 708       backup_token = ret_val;
 709       backup_lval = yylval;
 710       backup_lloc = yylloc;
 711       RETURN_SAVED_DOT;
 712     } else return ret_val;
 713   }
 714
 715   "'" {
 716     yylloc.first_line = start_line;
 717     yylloc.first_column = start_column;
 718     current_column++;
 719     yylloc.last_line = current_line;
 720     yylloc.last_column = current_column;
 721     Location loc(infile, yylloc);
 722     loc.error("Invalid binary string literal. Expecting `B', `H' or `O' after "
 723       "the closing `''");
 724     MD5_Update(&md5_ctx, binstr, strlen(binstr));
 725     Free(binstr);
 726     BEGIN(INITIAL);
 727     RETURN_NOLOCUPD(TOK_errval);
 728   }
 729
 730   . {
 731     Location loc(infile, current_line, current_column, current_line,
 732       current_column + 1);
 733     int c = (unsigned char)yytext[0];
 734     loc.error("Invalid character `%c' (0x%02X) in binary string",
 735       isprint(c) ? c : '?', c);
 736     MD5_Update(&md5_ctx, binstr, strlen(binstr));
 737     Free(binstr);
 738     MD5_Update(&md5_ctx, yytext, 1);
 739     current_column++;
 740     BEGIN(SC_binstring_bad);
 741   }
 742
 743 } /* SC_binstring scope */
 744
 745 <SC_binstring_bad> /* -------- SC_binstring_bad scope -------------- */
 746 {
 747
 748   {WHITESPACE}+ current_column += yyleng;
 749
 750   {WHITESPACE}*{NEWLINE} {
 751     current_line++;
 752     current_column = 0;
 753   }
 754
 755   "'"[bBhHoO]? {
 756     current_column += yyleng;
 757     yylloc.first_line = start_line;
 758     yylloc.first_column = start_column;
 759     yylloc.last_line = current_line;
 760     yylloc.last_column = current_column;
 761     BEGIN(INITIAL);
 762     RETURN_NOLOCUPD(TOK_errval);
 763   }
 764
 765   . {
 766     MD5_Update(&md5_ctx, yytext, yyleng);
 767     current_column++;
 768   }
 769
 770 } /* SC_binstring_bad scope */
 771
 772 <SC_cstring> /* -------- SC_cstring scope -------------- */
 773 {
 774
 775   \\?{NEWLINE} { /* newline possibly preceded by backslash */
 776     yylval.str = mputstr(yylval.str, yytext);
 777     current_line++;
 778     current_column = 0;
 779   }
 780
 781   \"\"|\\. { /* two doublequotes or any backslash-escaped char */
 782     yylval.str = mputstr(yylval.str, yytext);
 783     current_column += 2;
 784     /* Note that both get added ("external representation").
 785      * parse_charstring_value() in charstring_la.l is responsible
 786      * for transforming the string to "internal representation" */
 787   }
 788
 789   \" {
 790     current_column++;
 791     yylloc.first_line = start_line;
 792     yylloc.first_column = start_column;
 793     yylloc.last_line = current_line;
 794     yylloc.last_column = current_column;
 795     MD5_Update(&md5_ctx, yylval.str, strlen(yylval.str));
 796     update_md5();
 797     BEGIN(INITIAL);
 798     if (dot_flag) {
 799       backup_token = Cstring;
 800       backup_lval = yylval;
 801       backup_lloc = yylloc;
 802       RETURN_SAVED_DOT;
 803     } else return Cstring;
 804   }
 805
 806   . {
 807     yylval.str = mputc(yylval.str, yytext[0]);
 808     current_column++;
 809   }
 810
 811 } /* SC_cstring scope */
 812
 813         /* Macros */
 814
 815 "%moduleId" {
 816   yylval.macrotype = Value::MACRO_MODULEID;
 817   RETURN_LVAL(MacroValue);
 818 }
 819 "%fileName" {
 820   yylval.macrotype = Value::MACRO_FILENAME;
 821   RETURN_LVAL(MacroValue);
 822 }
 823 "%lineNumber" {
 824   yylval.macrotype = Value::MACRO_LINENUMBER;
 825   RETURN_LVAL(MacroValue);
 826 }
 827 "%definitionId" {
 828   yylval.macrotype = Value::MACRO_DEFINITIONID;
 829   RETURN_LVAL(MacroValue);
 830 }
 831 "%testcaseId" {
 832   yylval.macrotype = Value::MACRO_TESTCASEID;
 833   RETURN_LVAL(MacroValue);
 834 }
 835 "%"{IDENTIFIER} {
 836   fill_location();
 837   Location loc(infile, yylloc);
 838   loc.error("Invalid macro notation: `%s'", yytext);
 839   RETURN_NOLOCUPD(TOK_errval);
 840 }
 841
 842 "__MODULE__" {
 843   yylval.macrotype = Value::MACRO_MODULEID;
 844   RETURN_LVAL(MacroValue);
 845 }
 846 "__FILE__" {
 847   yylval.macrotype = Value::MACRO_FILEPATH;
 848   RETURN_LVAL(MacroValue);
 849 }
 850 "__BFILE__" {
 851   yylval.macrotype = Value::MACRO_BFILENAME;
 852   RETURN_LVAL(MacroValue);
 853 }
 854 "__LINE__" {
 855   yylval.macrotype = Value::MACRO_LINENUMBER_C;
 856   RETURN_LVAL(MacroValue);
 857 }
 858 "__SCOPE__" {
 859   yylval.macrotype = Value::MACRO_SCOPE;
 860   RETURN_LVAL(MacroValue);
 861 }
 862 "__TESTCASE__" {
 863   yylval.macrotype = Value::MACRO_TESTCASEID;
 864   RETURN_LVAL(MacroValue);
 865 }
 866 "__"{IDENTIFIER}"__" {
 867   fill_location();
 868   Location loc(infile, yylloc);
 869   loc.error("Invalid macro notation: `%s'", yytext);
 870   RETURN_NOLOCUPD(TOK_errval);
 871 }
 872
 873         /* Multi-character operators */
 874
 875 ":="            RETURN(AssignmentChar);
 876 "\.\."          RETURN(DotDot);
 877 "->"            RETURN(PortRedirectSymbol);
 878 "=="            RETURN(EQ);
 879 "!="            RETURN(NE);
 880 ">="            RETURN(GE);
 881 "<="            RETURN(LE);
 882 "<<"            RETURN(SL);
 883 ">>"            RETURN(SR);
 884 "<@"            RETURN(RL);
 885 "@>"            RETURN(_RR);
 886
 887 "++" |
 888 "--" {
 889   fill_location();
 890   Location loc(infile, yylloc);
 891   loc.error("Operator `%s' is reserved for future use", yytext);
 892 }
 893
 894         /* Invalid operators */
 895
 896 "::=" {
 897   fill_location();
 898   Location loc(infile, yylloc);
 899   loc.error("`::=' is not a valid assignment operator in TTCN-3. Did you mean "
 900     "`:='?");
 901   RETURN_NOLOCUPD(AssignmentChar);
 902 }
 903
 904 "=" {
 905   fill_location();
 906   Location loc(infile, yylloc);
 907   loc.error("A single `=' character cannot be used in TTCN-3. Did you mean "
 908     "the assignment sign `:=' or the equality operator `=='?");
 909   /* the former is more probable than the latter */
 910   RETURN_NOLOCUPD(AssignmentChar);
 911 }
 912
 913 "<>" {
 914   fill_location();
 915   Location loc(infile, yylloc);
 916   loc.error("`<>' is not a valid comparison operator in TTCN-3. Did you mean "
 917     "`!='?");
 918   RETURN_NOLOCUPD(NE);
 919 }
 920
 921         /* Identifiers */
 922
 923 {IDENTIFIER} {
 924   yylval.id = new Identifier(Identifier::ID_TTCN, string(yyleng, yytext));
 925   RETURN_LVAL(IDentifier);
 926 }
 927
 928         /* Single character tokens (brackets, operators, etc.) */
 929
 930 \. {
 931   update_md5();
 932   if (dot_flag) {
 933     /* store this dot in the backup */
 934     backup_token = '.';
 935     backup_lloc.first_line = current_line;
 936     backup_lloc.first_column = current_column;
 937     current_column++;
 938     backup_lloc.last_line = current_line;
 939     backup_lloc.last_column = current_column;
 940     /* return the dot that was found previously */
 941     RETURN_SAVED_DOT;
 942   } else {
 943     dot_flag = true;
 944     dot_line = current_line;
 945     dot_column = current_column;
 946     current_column++;
 947   }
 948 }
 949
 950 [()\[\]{}+\-\*/&:;,<>\?!]  RETURN(*yytext);
 951
 952         /* Invalid characters */
 953
 954 . {
 955   fill_location();
 956   Location loc(infile, yylloc);
 957   int c = (unsigned char)yytext[0];
 958   loc.error("Character `%c' (0x%02X) is not used in TTCN-3",
 959     isprint(c) ? c : '?', c);
 960 }
 961
 962         /* EOF rule */
 963
 964 <*><<EOF>> {
 965   if (YY_START != INITIAL) {
 966     Location loc(infile, start_line, start_column, current_line,
 967       current_column);
 968     switch (YY_START) {
 969     case SC_blockcomment:
 970       loc.error("Unterminated block comment");
 971       break;
 972     case SC_binstring:
 973       Free(binstr);
 974       /* no break */
 975     case SC_binstring_bad:
 976       loc.error("Unterminated binary string literal");
 977       break;
 978     case SC_cstring:
 979       Free(yylval.str);
 980       loc.error("Unterminated character string literal");
 981     }
 982     BEGIN(INITIAL);
 983   }
 984   if (dot_flag) {
 985     dot_flag = false;
 986     RETURN_SAVED_DOT;
 987   } else {
 988     yylloc.first_line = current_line;
 989     yylloc.first_column = current_column;
 990     yylloc.last_line = current_line;
 991     yylloc.last_column = current_column + 1;
 992     return EOF;
 993   }
 994 }
 995
 996 %%
 997
 998 void init_ttcn3_lex()
 999 {
1000   dot_flag = false;
1001   current_line = 1;
1002   current_column = 0;
1003   real_infile = infile;
1004   real_lineno_offset = 0;
1005   MD5_Init(&md5_ctx);
1006 }
1007
1008 void init_erroneous_lex(const char* p_infile, int p_line, int p_column)
1009 {
1010   infile = p_infile;
1011   current_line = p_line;
1012   current_column = p_column;
1013   real_infile = infile;
1014   real_lineno_offset = 0;
1015   dot_flag = false;
1016 }
1017
1018 void free_dot_flag_stuff()
1019 {
1020   if (dot_flag) {
1021     dot_flag = false;
1022     /* clean up the semantic value of the token that was backed up */
1023     switch (backup_token) {
1024     case IDentifier:
1025       delete backup_lval.id;
1026       break;
1027     case Bstring:
1028     case Hstring:
1029     case Ostring:
1030     case BitStringMatch:
1031     case HexStringMatch:
1032     case OctetStringMatch:
1033       delete backup_lval.string_val;
1034       break;
1035     case Cstring:
1036       Free(backup_lval.str);
1037     default:
1038       break;
1039     }
1040   }
1041 }
1042
1043 void free_ttcn3_lex()
1044 {
1045   free_dot_flag_stuff();
1046   fclose(ttcn3_in);
1047   ttcn3_lex_destroy();
1048 }
1049
1050 /* called from ttcn3_parse_file to finalize MD5 and add it to the module */
1051 void set_md5_checksum(Ttcn::Module *m)
1052 {
1053   unsigned char md5_sum[MD5_DIGEST_LENGTH];
1054   MD5_Final(md5_sum, &md5_ctx);
1055   m->set_checksum(sizeof(md5_sum), md5_sum);
1056 }