regression_test/logger/emergency_logging/EmergencyLogTest.sh

   1 #!/bin/bash
   2 ###############################################################################
   3 # Copyright (c) 2000-2016 Ericsson Telecom AB
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Eclipse Public License v1.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.eclipse.org/legal/epl-v10.html
   8 #
   9 # Contributors:
  10 #   Balasko, Jeno
  11 #   Baranyi, Botond
  12 #
  13 ###############################################################################
  14
  15 MYEXE="EmergencyLogTest"
  16 LOGDIR="logs"
  17 SUCCESS=0
  18 LIST_OF_FAILED="failed_testcases.txt"
  19 DEBUG="true"
  20
  21 #####################################
  22 #init:
  23 # sets TTCN3_DIR, if is needed
  24 # makes executable
  25 #####################################
  26 init() {
  27   if [ "$TTCN3_DIR" == "" ]
  28   then
  29      echo "TTCN3_DIR should be set"
  30      exit
  31   fi
  32
  33   make
  34   chmod +x ./$MYEXE
  35   success=0
  36   failed=0
  37   echo "Failed tests:" > ${LIST_OF_FAILED}
  38   rm -f logs/*
  39   #TODO: set cut off length according to TimeStampFormat
  40 }
  41
  42 #####################################
  43 # debug
  44 # $1 : log file
  45 #####################################
  46 debug(){
  47   if [ "$DEBUG" == "true" ] ; then
  48    echo "$1"
  49   fi
  50 }
  51
  52 #####################################
  53 # modify_logfile
  54 # $1 : log file
  55 #####################################
  56 modify_logfile() {
  57   #remove timestamps, data different for run by run
  58   debug "Modifying file $1"
  59   cmd='
  60 s/\ (No such file or directory)//g
  61 s/^EXECUTOR_RUNTIME - MTC was created. Process id: [0-9]*\./EXECUTOR_RUNTIME - MTC was created. Process id:/g
  62 s/rocess id: [0-9][0-9]*/rocess id: X/g
  63 s/I\/O: [0-9][0-9]*/I\/O: Y/g
  64 s/switches: [0-9][0-9]*/switches: S/g
  65 s/block output operations: [0-9]*/block output operations: B/g
  66 s/block input operations: [0-9]*/block input operations: I/g
  67 s/system time: [0-9]*\.[0-9]*/system time: SYS/g
  68 s/user time: [0-9]*\.[0-9]*/user time: T/g
  69 s/seed [0-9][0-9]*[.][0-9]*/seed /g
  70 s/returned [0-9][0-9]*[.][0-9]*/returned R/g
  71 s/srand48.[\-]*[0-9]*.\./srand X/g
  72 s/t1: [0-9]e-[0-9]* s/t1: T s/g
  73 s/t1: [0-9e.-]* s/t1: T s/g
  74 s/t: [0-9e.-]* s/t: T s/g
  75 s/Mytime: [0-9.]*e-[0-9]* s/Mytime: T s/g
  76 s/Mytime: [0-9.]* s/Mytime: T s/g
  77 s/reference [0-9]* finished/reference R finished/g
  78 s/started on [a-zA-Z0-9._-]*. Version: .*/started on X. Version: V/g
  79 s/^[0-9.:]* //g
  80 s/[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*/a\.b\.c\.d/g
  81 s/The address of MC was set to [a-zA-Z0-9._-]*/The address of MC was set to X/g
  82 s/resident set size: [0-9]*/resident set size: S/g
  83 s/Action: Finish:[0-9][0-9]*[.][0-9]*/Action: Finish: F/g
  84 s/Action: Elapsed time:[0-9][0-9]*[.][0-9]*/Action: Elapsed time:E/g
  85 s/Read timer t: [0-9e.-]* s/Read timer t: T s/g
  86 s/Read timer t1: [0-9e.-]* s/Read timer t1: T s/g
  87 s/Action: Start:[0-9e.-]* s/Action: Start: S s/g
  88 s/^PARALLEL_PTC.*//g
  89 s/UNIX pathname .*$/UNIX pathname /g
  90 s/^EXECUTOR_COMPONENT.*$//g
  91 s/^PORTEVENT_MQUEUE.*$//g
  92 s/^$//g
  93 '
  94   sed -e "$cmd" < "$1" > "${LOGDIR}/tmp"
  95
  96   sed -e "/^$/d" < "${LOGDIR}/tmp" > "$1"
  97   #mv "${LOGDIR}/tmp" "$1"
  98 }
  99
 100 #####################################
 101 # run_and_modify_logfile
 102 # $1 : cfg file
 103 # $2 : cfg file base without extension
 104 #####################################
 105 run_and_modify_logfile() {
 106   echo $TTCN3_DIR/bin/ttcn3_start $MYEXE $1
 107   $TTCN3_DIR/bin/ttcn3_start $MYEXE $1
 108   cp $LOGDIR/${MYEXE}-mtc.log $LOGDIR/${2}-mtc.log
 109   cp $LOGDIR/${MYEXE}-hc.log $LOGDIR/${2}-hc.log
 110
 111   #remove timestamp, diff data:
 112   modify_logfile "${LOGDIR}/${2}-mtc.log"
 113   modify_logfile "${LOGDIR}/${2}-hc.log"
 114
 115   #emergency log handling
 116   if [ -e "$LOGDIR/${MYEXE}-mtc.log_emergency" ]
 117   then
 118    debug "Emergency log file name: $LOGDIR/${2}-mtc.log_emergency"
 119    cp "$LOGDIR/${MYEXE}-mtc.log_emergency" "$LOGDIR/${2}-mtc.log_emergency"
 120    modify_logfile "$LOGDIR/${2}-mtc.log_emergency"
 121   fi
 122
 123   i=3
 124   while [ -e "$LOGDIR/${MYEXE}-$i.log" ]
 125   do
 126        cp "$LOGDIR/${MYEXE}-$i.log" "$LOGDIR/${2}-$i.log"
 127        modify_logfile "$LOGDIR/${2}-$i.log"
 128        let "i = $i + 1"
 129   done
 130 }
 131 #####################################
 132 # create EmergencyLogCommentedOut (ELCO) file from the original log file
 133 # $1:original config file
 134 # output "${1}_ELCO.cfg"
 135 #####################################
 136 create_ELCO_config() {
 137   elco_cfg=`basename $1 ".cfg"`
 138   elco_cfg="${elco_cfg}_ELCO.cfg"
 139   cp $1 "${elco_cfg}"
 140   cmd='
 141 s/^\*\.Emergency/#\*\.Emergency/g'
 142   sed -e "$cmd" < "${elco_cfg}" > tmp
 143   mv tmp "${elco_cfg}"
 144   debug "OLD  cfg name: $1"
 145   debug "ELCO cfg name: ${elco_cfg}"
 146 }
 147 #####################################
 148 # compare_with_ELCO
 149 # $1: componenent id (mtc, hc, 3, 4 etc)
 150 #####################################
 151 compare_with_ELCO() {
 152  diff -u "$LOGDIR/${elco_cfg_base}-$1.log" "$LOGDIR/${orig_cfg_base}-$1.log"  > "${LOGDIR}/diff_${orig_cfg_base}_$1.log"
 153  if [ "$?" -eq $SUCCESS ]
 154  then
 155    debug ">>>tc $orig_cfg_base $1 part success<<<"
 156    let "success = $success + 1"
 157  else
 158    debug ">>>tc $orig_cfg_base $1 part failed<<<"
 159    echo "tc $orig_cfg_base $1 part failed" >> ${LIST_OF_FAILED}
 160    let "failed = $failed + 1"
 161  fi
 162  debug "success: ${success} failed: ${failed}"
 163 }
 164
 165 #####################################
 166 # compare_with_expected
 167 # $1: componenent id (mtc, hc, 3, 4 etc)
 168 # $2: expected log
 169 #####################################
 170 compare_with_expected() {
 171  diff -u "$2" "${LOGDIR}/${orig_cfg_base}-$1.log" > "${LOGDIR}/diff_${orig_cfg_base}_$1.log"
 172  if [ "$?" -eq $SUCCESS ]
 173  then
 174    debug ">>>tc $orig_cfg_base $1 part success<<<"
 175    let "success = $success + 1"
 176  else
 177    debug ">>>tc $orig_cfg_base $1 part failed<<<"
 178    echo "tc $orig_cfg_base $1 part failed" >> ${LIST_OF_FAILED}
 179    let "failed = $failed + 1"
 180  fi
 181  debug "success: ${success} failed: ${failed}"
 182 }
 183
 184 #####################################
 185 # compare_with_expected_emergency
 186 # $1: componenent id (mtc, hc, 3, 4 etc)
 187 # $2: expected log
 188 #####################################
 189 compare_with_expected_emergency() {
 190  #debug "==>diff -u $4 ${LOGDIR}/${orig_cfg_base}-$1.log_emergency  > ${LOGDIR}/diff_${orig_cfg_base}_$1.log_emergency"
 191  diff -u "$2" "${LOGDIR}/${orig_cfg_base}-$1.log_emergency"  > "${LOGDIR}/diff_${orig_cfg_base}_$1.log_emergency"
 192  if [ "$?" -eq $SUCCESS ]
 193  then
 194    debug ">>>tc $orig_cfg_base $1 part success<<<"
 195    let "success = $success + 1"
 196  else
 197    debug ">>>tc $orig_cfg_base $1 part failed<<<"
 198    echo "tc $orig_cfg_base $1 emergency part failed" >> ${LIST_OF_FAILED}
 199    let "failed = $failed + 1"
 200  fi
 201  debug "success: ${success} failed: ${failed}"
 202 }
 203 #####################################
 204 # run_and_compare_log_files
 205 # $1 first config file
 206 # $2 2nd config file
 207 # $3 output file (diff_)
 208 #####################################
 209 run_and_compare_log_files() {
 210  #running with orig cfg:
 211  rm  ${LOGDIR}/${MYEXE}*.log
 212  orig_cfg_base=`basename $1 ".cfg"`
 213  run_and_modify_logfile "$1" "$orig_cfg_base"
 214
 215  if [ "$2" == "" ]; then
 216    #running with modified cfg:
 217    create_ELCO_config "$1"
 218  else
 219    elco_cfg=$2
 220  fi
 221
 222  elco_cfg_base=`basename "$elco_cfg" ".cfg"`
 223  debug "ELCO cfg name: ${elco_cfg}"
 224  run_and_modify_logfile "${elco_cfg}"  "$elco_cfg_base"
 225
 226  #diff:
 227  compare_with_ELCO "mtc"
 228  compare_with_ELCO "hc"
 229
 230  i=3
 231  while [ -e "$LOGDIR/${elco_cfg_base}-$i.log" -a -e "$LOGDIR/${orig_cfg_base}-$i.log" ]
 232  do
 233    compare_with_ELCO "$i"
 234    let "i = $i + 1"
 235  done
 236
 237 }
 238 #####################################
 239 # run_and_compare_log_file_with_expected
 240 # $1 first config file
 241 # $2 expected mtc log file
 242 # $3 expected hc log file
 243 # $4 expected emergency log file
 244 #####################################
 245 run_and_compare_log_file_with_expected() {
 246   rm -f ${LOGDIR}/${MYEXE}*.log*
 247   #running with orig cfg:
 248   orig_cfg_base=`basename $1 ".cfg"`
 249   run_and_modify_logfile "$1" "$orig_cfg_base"
 250
 251
 252   debug "cfg:  $1"
 253   debug "mtclog:  $2"
 254   debug "hclog: $3"
 255   #diff mtc with the expected:
 256   if [ -e "$2" ]; then
 257      modify_logfile "$2"
 258      compare_with_expected "mtc" "$2"
 259   else
 260     debug ">>>tc $orig_cfg_base mtc part failed, not existing expected log file $2<<<"
 261     echo "tc $orig_cfg_base mtc part failed" >> ${LIST_OF_FAILED}
 262     let "failed = $failed + 1"
 263   fi
 264
 265   #diff:
 266
 267   if [ "$3" != "" ]; then
 268    if [ -e "$3" ]; then
 269       modify_logfile "$3"
 270       compare_with_expected "hc" "$3"
 271    else
 272      debug ">>>tc $orig_cfg_base mtc part failed, not existing expected log file $3<<<"
 273      echo "tc $orig_cfg_base mtc part failed" >> ${LIST_OF_FAILED}
 274      let "failed = $failed + 1"
 275    fi
 276   fi
 277
 278   #mtc emergency log kezeles
 279   if [ "$4" != "" ]; then
 280    if [ -e "$4" ]; then
 281       modify_logfile "$4"
 282       compare_with_expected_emergency "mtc" "$4"
 283    else
 284      debug ">>>tc $orig_cfg_base mtc part failed, not existing expected log file $4<<<"
 285      echo "tc $orig_cfg_base mtc part failed, not existing expected log file $4" >> ${LIST_OF_FAILED}
 286      let "failed = $failed + 1"
 287    fi
 288   fi
 289 }
 290
 291 #####################################
 292 # evaluate
 293 #####################################
 294 evaluate() {
 295   echo "Summary: success: ${success}, failed: ${failed} "
 296   if [ $failed -eq 0 ]
 297   then
 298    echo "Overall verdict: success"
 299    exit_code=0
 300   else
 301    echo "Overall verdict: failed"
 302    cat ${LIST_OF_FAILED} |
 303    while read  line
 304    do
 305     echo $line
 306    done
 307    exit_code=1
 308   fi
 309
 310   exit $exit_code
 311 }
 312
 313 #### MAIN ###
 314
 315 # see the files "logs/diff*.log" !
 316
 317 init
 318 #======= Buffer All ========
 319 run_and_compare_log_files EL_BufferAll_1.cfg  #compatibility test with minimal coverage (compare logs with/without EL setting if EL-event does not happen
 320 run_and_compare_log_files EL_BufferAll_2.cfg  #compatibility test with coverage "tc_parallel_portconn" (compare logs with/without EL setting if EL-event does not happen) - NOT STABILE, hc FAILED
 321 run_and_compare_log_files EL_BufferAll_3.cfg EL_BufferAll_3_NOEL.cfg #the same log expected with two different log files: EL and NOEL, EL-event occurs in the first case - PASSED
 322 run_and_compare_log_files EL_BufferAll_4.cfg  #compatibility test with "tc_timer" (compare logs with/without EL setting if EL-event does not happen) - hc FAILED
 323 run_and_compare_log_files EL_BufferAll_5.cfg  #compatibility test "with Titan_LogTest.tc_encdec" - NOT STABILE, mtc PASSED, hc PASSED
 324 run_and_compare_log_files EL_BufferAll_6.cfg  #compatibility test with "tc_function_rnd and tc_encdec" - mtc PASSED, hc PASSED
 325 run_and_compare_log_file_with_expected "EL_BufferAll_7.cfg"  "EL_BufferAll_7_mtc_expected.log"  # - mtc PASSED
 326 run_and_compare_log_file_with_expected "EL_BufferAll_7A.cfg"  "EL_BufferAll_7A_mtc_expected.log"  # - mtc PASSED
 327 run_and_compare_log_files EL_BufferAll_8.cfg   #compatibility test with MAXIMAL coverage - NOT STABILE; mtc PASSED hc FAILED (1 line order problem,
 328 run_and_compare_log_file_with_expected "EL_BufferAll_9.cfg"  "EL_BufferAll_9_mtc_expected.log"  # - More EL-event can be logged after each other - mtc FAILED ( 1 line missing), hc FAILED
 329 run_and_compare_log_file_with_expected "EL_BufferAll_10.cfg"  "EL_BufferAll_10_mtc_expected.log"  # - More EL-event with big buffer - mtc PASSED
 330 run_and_compare_log_file_with_expected "EL_BufferAll_11.cfg"  "EL_BufferAll_11_mtc_expected.log"  # - More EL-event with big buffer - mtc PASSED
 331 run_and_compare_log_file_with_expected "EL_BufferAll_12.cfg"  "EL_BufferAll_12_mtc_expected.log"  # - More EL-event with very small buffer - (buffer size=2) - mtc FAILED, one line missing
 332 run_and_compare_log_file_with_expected "EL_BufferAll_13.cfg"  "EL_BufferAll_13_mtc_expected.log"  # - More EL-event with very small buffer - (buffer size=3) - mtc FAILED, one line missing
 333 #======= Buffer Masked ========
 334 run_and_compare_log_files EL_BufferMasked_1.cfg  #compatibility test with minimal coverage (compare logs with/without EL setting if EL-event does not happen
 335 run_and_compare_log_files EL_BufferMasked_2.cfg  #compatibility test with coverage "tc_parallel_portconn" (compare logs with/without EL setting if EL-event does not happen) - NOT STABILE
 336 run_and_compare_log_files EL_BufferMasked_4.cfg  #compatibility test with "tc_timer" (compare logs with/without EL setting if EL-event does not happen) - PASSED
 337 run_and_compare_log_files EL_BufferMasked_5.cfg  #compatibility test "with Titan_LogTest.tc_encdec" - NOT STABILE
 338 run_and_compare_log_files EL_BufferMasked_3.cfg EL_BufferMasked_3_NOEL.cfg #the same log expected with two different log files: EL and NOEL, EL-event occurs in the first case - PASSED
 339 run_and_compare_log_files EL_BufferMasked_6.cfg  #compatibility test with "tc_function_rnd and tc_encdec" - PASSED
 340 run_and_compare_log_file_with_expected "EL_BufferMasked_7.cfg"  "EL_BufferMasked_7_mtc_expected.log" "" "EL_BufferMasked_7_mtc_expected.log_emergency" # - PASSED
 341 run_and_compare_log_files EL_BufferMasked_8.cfg  #compatibility test with MAXIMAL coverage - NOT STABILE
 342 run_and_compare_log_file_with_expected "EL_BufferMasked_9.cfg"  "EL_BufferMasked_9_mtc_expected.log" ""  "EL_BufferMasked_9_mtc_expected.log_emergency" # - More EL-event can be logged after each other - PASSED
 343 run_and_compare_log_file_with_expected "EL_BufferMasked_10.cfg"  "EL_BufferMasked_10_mtc_expected.log" ""  "EL_BufferMasked_10_mtc_expected.log_emergency" # - More EL-event with big buffer - PASSED
 344 run_and_compare_log_file_with_expected "EL_BufferMasked_11.cfg"  "EL_BufferMasked_11_mtc_expected.log"  ""  "EL_BufferMasked_11_mtc_expected.log_emergency" # - More EL-event with big buffer - PASSED
 345 run_and_compare_log_file_with_expected "EL_BufferMasked_12.cfg"  "EL_BufferMasked_12_mtc_expected.log"  "" "EL_BufferMasked_12_mtc_expected.log_emergency" # - More EL-event with very small buffer - (buffer size=2) - mtc.log_emergency FAILED (wrong order)
 346 run_and_compare_log_file_with_expected "EL_BufferMasked_13.cfg"  "EL_BufferMasked_13_mtc_expected.log"  "" "EL_BufferMasked_13_mtc_expected.log_emergency" # - More EL-event with very small buffer - (buffer size=3) - mtc.log_emergency FAILED (wrong order)
 347 #Stat: 63 success/ 0 failed
 348 evaluate