etc/autotest/product_handler.py

   1 ###############################################################################
   2 # Copyright (c) 2000-2014 Ericsson Telecom AB
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Eclipse Public License v1.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.eclipse.org/legal/epl-v10.html
   7 ###############################################################################
   8 import os, re
   9 import titan_publisher, utils
  10
  11 MAKEFILE_PATCH = 'makefile_patch.sh'
  12
  13 class product_handler:
  14   """ It is assumed that the VOB is reachable (e.g. through an SSHFS mounted
  15       directory) on the master machine.  The source package is copied to all
  16       slaves with SCP.  It is assumed that the logger is always set.
  17
  18       Saving stdout and stderr of product actions is optional.  During
  19       publishing these files should be linked or copied to somewhere under
  20       `public_html'.
  21   """
  22   def __init__(self, logger = None, config = None):
  23     self._logger = logger
  24     self._config = config
  25
  26   def config_products(self, target_path):
  27     """ Localize first in the following order: `test', `demo'.  If neither of
  28         these directories are found the `src' will be copied, it's a must
  29         have.  If there's a Makefile or .prj in `test', `demo', the list of
  30         files will be gathered from there.  Otherwise the contents of these
  31         directories excluding the Makefile will be copied to the target
  32         directory.  A new Makefile will be generated for the copied files.
  33         Configspec 129 is used.  It's assumed that the ClearCase access is
  34         working and all files are accessible through SSHFS etc.  The `test'
  35         directory always has higher priority than `demo'.  However, `demo' is
  36         the one released to the customer and not `test', but we're using the
  37         TCC_Common VOB anyway.
  38
  39         We generate single mode Makefiles here for simplicity.  We can grep
  40         through the sources for a `create' call, but it's so ugly.  It is
  41         assumed that the files enumerated in Makefiles or .prj files are all
  42         relative to the current directory.  The source and target paths are
  43         class parameters.  It is assumed that the Makefile and the project
  44         file are all in the `test' directory.  The Makefile will be ignored if
  45         there's a project file in the same directory.
  46
  47         The distribution of the VOB package is the job of the master.
  48         Configuring the products with `ttcn3_makefilegen' is the job of the
  49         slaves.
  50
  51         Returns:
  52           0 if everything went fine and the VOB package is ready for
  53           distribution.  1 otherwise.
  54     """
  55     utils.run_cmd('rm -rf ' + target_path)
  56     for kind, products in self._config.products.iteritems():
  57       for product in products:
  58         product_name = product['name'].strip()
  59         local_src_path = os.path.join(os.path.join(self._config.common['vob'], kind), product_name)
  60         src_dir = os.path.join(local_src_path, 'src')
  61         test_dir = os.path.join(local_src_path, 'test')
  62         demo_dir = os.path.join(local_src_path, 'demo')
  63         if not os.path.isdir(src_dir):
  64           self._logger.error('Missing `src\' directory for product `%s\' ' \
  65                              'in %s, skipping product' % (product_name, local_src_path))
  66           continue
  67         else:
  68           dirs_to_copy = []
  69           files_to_copy = []
  70           if os.path.isdir(test_dir):
  71             dirs_to_copy.append(test_dir)
  72           elif os.path.isdir(demo_dir):
  73             dirs_to_copy.append(demo_dir)
  74           else:
  75             # No `demo' or `test'.  The `src' is copied only if the other
  76             # directories are missing.  There can be junk files as well.  The
  77             # Makefile patch script must have a fixed name
  78             # `makefile_patch.sh'.
  79             dirs_to_copy.append(src_dir)
  80             self._logger.debug('Product `%s\' in %s doesn\'t have the `demo\' or `test\' directories'
  81                                % (product_name, local_src_path))
  82           product_target_path = os.path.join(os.path.join(target_path, kind), product_name)
  83           utils.run_cmd('mkdir -p ' + product_target_path)
  84           has_prj_file = False
  85           for dir in dirs_to_copy:
  86             for dir_path, dir_names, file_names in os.walk(dir):
  87               if not has_prj_file and \
  88                 len([file_name for file_name in file_names \
  89                      if file_name.endswith('.prj')]) > 0: has_prj_file = True
  90               for file_name in file_names:
  91                 if not has_prj_file:  # Trust the project file if we have one.
  92                   files_to_copy.append(os.path.join(dir_path, file_name))
  93                 if (file_name == 'Makefile' and not has_prj_file) or file_name.endswith('.prj'):
  94                   (makefile_patch, extracted_files) = \
  95                     self.extract_files(dir_path, os.path.join(dir_path, file_name))
  96                   files_to_copy.extend(extracted_files)
  97                   if makefile_patch:
  98                     utils.run_cmd('cp -Lf %s %s' \
  99                       % (makefile_patch, os.path.join(product_target_path, MAKEFILE_PATCH)))
 100           utils.run_cmd('cp -Lf %s %s' % (' '.join(files_to_copy), product_target_path))
 101           utils.run_cmd('chmod 644 * ; chmod 755 *.sh', product_target_path)
 102           self._logger.debug('Product `%s\' was configured successfully ' \
 103                              'with %d files' % (product_name, len(files_to_copy)))
 104     return 0
 105
 106   def build_products(self, proddir, logdir, config, rt2 = False):
 107     """ Build the products provided in the list.  Simple `compiler -s' etc.
 108         commands are executed from the directories of the products.  The
 109         actions must be synchronized with the product configuration files.
 110         The stderr and stdout of actions is captured here, but it's optional.
 111
 112         Arguments:
 113           The directory of the products, the actual build configuration and
 114           runtime.
 115
 116         Returns:
 117           The build results in the following format:
 118
 119           results['kind1'] = [
 120             {'name1':{'action1':(1, o1, e1), 'action2':-1}},
 121             {'name2':{'action1':(1, o1, e1), 'action2':-1}},
 122             {'name3':-1}
 123           ]
 124
 125           The standard output and error channels are returned for each action
 126           with the return value.  The return value is usually the exit status
 127           of `make' or the `compiler'.  If the element is a simple integer
 128           value the action was disabled for the current product.  The output
 129           of this function is intended to be used by the presentation layer.
 130     """
 131     results = {}
 132     if not os.path.isdir(logdir):
 133       utils.run_cmd('mkdir -p %s' % logdir)
 134     for kind, products in self._config.products.iteritems():
 135       results[kind] = []
 136       for product in products:
 137         product_name = product['name'].strip()
 138         product_dir = os.path.join(proddir, os.path.join(kind, product_name))
 139         if not os.path.isdir(product_dir):
 140           # No `src' was found for the product.  Maybe a list would be better
 141           # instead.
 142           results[kind].append({product_name:-1})
 143           continue
 144         info = {product_name:{}}
 145         for product_key in product.iterkeys():
 146           files = ' '.join(filter(lambda file: file.endswith('.ttcn') \
 147                                   or file.endswith('.asn'), \
 148                                   os.listdir(product_dir)))
 149           cmd = None
 150           if product_key == 'semantic':
 151             if product[product_key]:
 152               cmd = '%s/bin/compiler -s %s %s' % (config['installdir'], rt2 and '-R' or '', files)
 153             else:
 154               info[product_name][product_key] = -1
 155               continue
 156           elif product_key == 'translate':
 157             if product[product_key]:
 158               cmd = '%s/bin/compiler %s %s' % (config['installdir'], rt2 and '-R' or '', files)
 159             else:
 160               info[product_name][product_key] = -1
 161               continue
 162           elif product_key == 'compile' or product_key == 'run':
 163             if product[product_key]:
 164               utils.run_cmd('cd %s && %s/bin/ttcn3_makefilegen ' \
 165                             '-fp %s *' % (product_dir, config['installdir'], rt2 and '-R' or ''))
 166               if os.path.isfile(os.path.join(product_dir, MAKEFILE_PATCH)):
 167                 self._logger.debug('Patching Makefile of product `%s\' for the %s runtime'
 168                                    % (product_name, rt2 and 'function-test' or 'load-test'))
 169                 utils.run_cmd('cd %s && mv Makefile Makefile.bak' % product_dir)
 170                 utils.run_cmd('cd %s && ./%s Makefile.bak Makefile' % (product_dir, MAKEFILE_PATCH))
 171               cmd = 'make clean ; make dep ; make -j4 ; %s' % (product_key == 'run' and 'make run' or '')
 172             else:
 173               info[product_name][product_key] = -1
 174               continue
 175           else:
 176             # Skip `name' or other things.
 177             continue
 178           (retval, stdout, stderr) = utils.run_cmd(cmd, product_dir, 900)
 179           prod_stdout = os.path.join(logdir, '%s_%s_%s_stdout.%s' \
 180             % (kind, product_name, product_key, rt2 and 'rt2' or 'rt1'))
 181           prod_stderr = os.path.join(logdir, '%s_%s_%s_stderr.%s' \
 182             % (kind, product_name, product_key, rt2 and 'rt2' or 'rt1'))
 183           output_files = (prod_stdout, prod_stderr)
 184           try:
 185             out_file = open(prod_stdout, 'wt')
 186             err_file = open(prod_stderr, 'wt')
 187             if 'vobtest_logs' not in config or config['vobtest_logs']:
 188               out_file.write(' '.join(stdout))
 189               err_file.write(' '.join(stderr))
 190             out_file.close()
 191             err_file.close()
 192           except IOError, (errno, strerror):
 193             self._logger.error('Error while dumping product results: %s (%s)' \
 194                                % (strerror, errno))
 195           info[product_name][product_key] = (retval, output_files, stdout, stderr)
 196         results[kind].append(info)
 197     return results
 198
 199   def extract_files(self, path, filename):
 200     """ Extract the files need to be copied all around from a given Makefile
 201         or .prj file.  It handles wrapped lines (i.e. '\') in Makefiles.  """
 202
 203     # Interesting patterns in Makefiles and .prj files.  Tuples are faster
 204     # than lists, use them for storing constants.
 205     prj_matches = ( \
 206       '<Module>\s*(.+)\s*</Module>', \
 207       '<TestPort>\s*(.+)\s*</TestPort>', \
 208       '<Config>\s*(.+)\s*</Config>', \
 209       '<Other>\s*(.+)\s*</Other>', \
 210       '<Other_Source>\s*(.+)\s*</Other_Source>', \
 211       '<File path="\s*(.+)\s*"', \
 212       '<File_Group path="\s*(.+)\s*"' )
 213
 214     makefile_matches = ( \
 215       '^TTCN3_MODULES =\s*(.+)', \
 216       '^ASN1_MODULES =\s*(.+)', \
 217       '^USER_SOURCES =\s*(.+)', \
 218       '^USER_HEADERS =\s*(.+)', \
 219       '^OTHER_FILES =\s*(.+)' )
 220
 221     try:
 222       file = open(filename, 'rt')
 223     except IOError:
 224       self._logger.error('File `%s\' cannot be opened for reading' % filename)
 225       return (None, [])
 226
 227     files = []
 228     makefile_patch = None
 229     if re.search('.*[Mm]akefile$', filename):
 230       multi_line = False
 231       for line in file:
 232         line = line.strip()
 233         if multi_line:
 234           files.extend(map(lambda f: os.path.join(path, f), line.split()))
 235           multi_line = line.endswith('\\')
 236           if multi_line:
 237             files.pop()
 238         else:
 239           for line_match in makefile_matches:
 240             matched = re.search(line_match, line)
 241             if matched:
 242               files.extend(map(lambda f: os.path.join(path, f),
 243                                matched.group(1).split()))
 244               multi_line = line.endswith('\\')
 245               if multi_line:
 246                 files.pop()
 247     elif re.search('.*\.prj$', filename) or re.search('.*\.grp', filename):
 248       files_to_exclude = []
 249       for line in file:
 250         # Only basic support for Makefile patching, since it doesn't have a
 251         # bright future in its current form...
 252         matched = re.search('<ScriptFile_AfterMake>\s*(.+)\s*</ScriptFile_AfterMake>', line)
 253         if matched:
 254           makefile_patch = os.path.join(path, matched.group(1))
 255           continue
 256         matched = re.search('<UnUsed_List>\s*(.+)\s*</UnUsed_List>', line)
 257         if matched:
 258           files_to_exclude.extend(matched.group(1).split(','))
 259           continue
 260         for line_match in prj_matches:
 261           matched = re.search(line_match, line)
 262           if matched and matched.group(1) not in files_to_exclude:
 263             file_to_append = os.path.join(path, matched.group(1))
 264             files_to_append = []
 265             if file_to_append != filename and file_to_append.endswith('.grp'):
 266               self._logger.debug('Group file `%s\' found' % file_to_append)
 267               last_slash = file_to_append.rfind('/')
 268               if last_slash != -1:
 269                 grp_dir = file_to_append[:last_slash]
 270                 if path.startswith('/'):
 271                   grp_dir = os.path.join(path, grp_dir)
 272                 (not_used, files_to_append) = self.extract_files(grp_dir, file_to_append)
 273               else:
 274                 self._logger.warning('Skipping contents of `%s\', check ' \
 275                                      'this file by hand' % file_to_append)
 276             files.append(file_to_append)
 277             files.extend(files_to_append)
 278             break
 279     else:
 280       self._logger.error('Unsupported project description file: %s\n' % filename)
 281     file.close()
 282     return (makefile_patch, files)