1 ###############################################################################
2 # Copyright (c) 2000-2014 Ericsson Telecom AB
3 # All rights reserved. This program and the accompanying materials
4 # are made available under the terms of the Eclipse Public License v1.0
5 # which accompanies this distribution, and is available at
6 # http://www.eclipse.org/legal/epl-v10.html
7 ###############################################################################
10 import logging
, optparse
, os
, re
, sys
, time
, fnmatch
11 import socket
, traceback
, types
12 # Never import everything! E.g. enumerate() can be redefined somewhere.
13 # Check out: http://www.phidgets.com/phorum/viewtopic.php?f=26&t=3315.
14 import product_handler
, titan_builder_cfg
, titan_publisher
, utils
, threading
16 LOG_FILENAME
= './titan_builder.log'
17 TRACE_FILENAME
= './titan_builder.err-log'
19 vobtest_lock
= threading
.Lock()
22 """ Class to process the semi-configuration file and provide easy access to
23 the configuration data read. This very same configuration file will be
26 Or simply include the parts of the current shell script setting all
27 environment variables?
29 def __init__(self
, logger
):
31 self
.products
= titan_builder_cfg
.products
32 self
.recipients
= titan_builder_cfg
.recipients
33 self
.configs
= titan_builder_cfg
.configs
34 self
.slaves
= titan_builder_cfg
.slaves
35 self
.common
= titan_builder_cfg
.common
37 self
.validate_config_data()
40 """ For debugging purposes only. """
41 results
= (str(self
.configs
), str(self
.products
), str(self
.recipients
), \
43 return '\n'.join(results
)
45 def is_used_config(self
, config
):
46 """ Check if the given build configuration is used by any of the slaves.
47 It is assumed that the configuration file is already validated.
50 config: The name of the build configuration.
52 for slave
in self
.slaves
:
53 if config
in self
.slaves
[slave
]['configs']:
55 # The build configuration will be skipped from the build process.
58 def validate_config_data(self
):
59 """ We have only one configuration file. The wrong addresses are filtered
60 out automatically. Add more checks. Rewrite `installdir' in case of
63 self
.recipients
= dict([(key
, self
.recipients
[key
]) \
64 for key
in self
.recipients \
65 if re
.match('^<[\w\-\.]+@(\w[\w\-]+\.)+\w+>$', self
.recipients
[key
])])
66 # elif current_section == 'slaves':
67 # row_data = [data.strip() for data in line.split()]
68 # if len(row_data) != 4:
69 # elif not re.match('^\w+\s*(\d{1,3}.){3}\d{1,3}\s*\w+\s*[\w/]+$', line):
70 # else: # 100% correct data all over.
71 # self.slaves[row_data[0]] = row_data[1:]
72 for config_name
, config_data
in self
.configs
.iteritems():
73 if 'foa' in config_data
and config_data
['foa'] and (not 'foadir' in config_data
or len(config_data
['foadir']) == 0):
74 config_data
['foadir'] = config_data
['installdir'] # The final build directory, it'll be linked.
75 config_data
['installdir'] = "%s/temporary_foa_builds/TTCNv3-%s" % ('/'.join(config_data
['foadir'].split('/')[0:-1]), utils
.get_time(True))
77 class MasterThread(threading
.Thread
):
78 def __init__(self
, titan_builder
, config
, config_name
, slave_list
, log_dir
, build_dir
, tests
):
79 threading
.Thread
.__init
__(self
)
80 self
.titan_builder
= titan_builder
82 self
.config_name
= config_name
83 self
.slave_list
= slave_list
84 self
.log_dir
= log_dir
85 self
.build_dir
= build_dir
89 self
.slave_list
.extend(self
.titan_builder
.master(self
.config
, self
.config_name
, self
.log_dir
, self
.build_dir
, self
.tests
))
91 class RegtestThread(threading
.Thread
):
92 def __init__(self
, titan_builder
, config
, slave_name
):
93 threading
.Thread
.__init
__(self
)
94 self
.titan_builder
= titan_builder
96 self
.slave_name
= slave_name
99 self
.titan_builder
.pass_regtest(self
.config
, self
.slave_name
)
101 class FunctestThread(threading
.Thread
):
102 def __init__(self
, titan_builder
, config
, slave_name
):
103 threading
.Thread
.__init
__(self
)
104 self
.titan_builder
= titan_builder
106 self
.slave_name
= slave_name
109 self
.titan_builder
.pass_functest(self
.config
, self
.slave_name
)
111 class PerftestThread(threading
.Thread
):
112 def __init__(self
, titan_builder
, config
, slave_name
):
113 threading
.Thread
.__init
__(self
)
114 self
.titan_builder
= titan_builder
116 self
.slave_name
= slave_name
119 self
.titan_builder
.pass_perftest(self
.config
, self
.slave_name
)
121 class EclipseThread(threading
.Thread
):
122 def __init__(self
, titan_builder
, config
, slave_name
):
123 threading
.Thread
.__init
__(self
)
124 self
.titan_builder
= titan_builder
126 self
.slave_name
= slave_name
129 self
.titan_builder
.pass_eclipse(self
.config
, self
.slave_name
)
131 class VobtestThread(threading
.Thread
):
132 def __init__(self
, titan_builder
, config
, slave_name
):
133 threading
.Thread
.__init
__(self
)
134 self
.titan_builder
= titan_builder
136 self
.slave_name
= slave_name
139 self
.titan_builder
.pass_vobtest(self
.config
, self
.slave_name
)
144 self
.logger
= self
.create_logger()
146 self
.config
= self
.create_config()
147 self
.publisher
= None
148 self
.publisher
= self
.create_publisher()
150 def create_logger(self
):
153 logger
= logging
.getLogger('titan_logger')
154 logger
.setLevel(logging
.DEBUG
)
155 formatter
= logging
.Formatter('%(asctime)s - %(levelname)s - %(message)s')
156 handler
= logging
.FileHandler(LOG_FILENAME
)
157 handler
.setFormatter(formatter
)
158 logger
.addHandler(handler
)
159 sth
= logging
.StreamHandler()
160 sth
.setLevel(logging
.DEBUG
)
161 logger
.addHandler(sth
)
162 return logger
# Just like in singleton.
164 def create_config(self
):
165 """ Create the configuration file handler class. If it's already created
166 the existing one will be returned. The configuration cannot be
167 changed during operation. It cannot be reloaded etc.
171 return config_handler(self
.logger
)
173 def create_publisher(self
):
175 return self
.publisher
176 return titan_publisher
.titan_publisher(self
.logger
, self
.config
)
178 def remove_dups(self
, list = []):
179 """ Remove duplicates from a list.
183 [tmp_list
.append(elem
) for elem
in list if not elem
in tmp_list
]
186 def build(self
, config
, slave_name
, reset
, set_addressees
, tests
, path
):
187 """ Build the specified build configurations. The configurations are
188 built sequentially. For the slaves a single build configuration should
189 be specified in the command line. The slave will abort build execution
190 if there are more build configurations specified. It's a limitation
191 and should be relaxed later.
194 config: The list of build configurations specified in the command line.
195 slave_name: The name of the slave if `--slave-mode' is on.
196 reset: Reset statistics.
197 set_addressees: List of recipients.
198 tests: Tests to run for all configurations.
201 Nothing. It's the main driver of the whole build.
205 self
.logger
.warning('Running all available build configurations from ' \
206 'the configuration file...')
207 config_list
.extend(self
.config
.configs
.keys())
208 elif not re
.match('^\w+(,\w+)*$', config
):
209 self
.logger
.error('Invalid build configuration list: `%s\'' % config
)
211 config
= self
.remove_dups(config
.split(','))
212 for config_elem
in config
:
213 if not config_elem
in self
.config
.configs
.keys():
214 self
.logger
.error('Build configuration `%s\' not found' \
217 config_list
.append(config_elem
)
218 if not len(config_list
) > 0:
219 self
.logger
.error('No valid build configurations were found, ' \
223 self
.config
.recipients
= {}
224 addressees
= set_addressees
.split(',')
225 for addressee
in addressees
:
226 self
.config
.recipients
[' '.join(addressee
.split(' ')[:-1])] = addressee
.split(' ')[-1];
228 everything_started_here
= utils
.get_time()
229 utils
.run_cmd('/bin/rm -rf %s %s && mkdir -p %s %s' \
230 % (self
.config
.common
['builddir'], self
.config
.common
['logdir'], self
.config
.common
['builddir'], self
.config
.common
['logdir']), None, 1800, self
.logger
)
233 for config_name
in config_list
:
234 if not self
.config
.is_used_config(config_name
):
235 self
.logger
.warning('Skipping unused build configuration: `%s\'' \
238 # Create the slave and configuration specific log directory. If the
239 # logs haven't arrived yet from the given slave, that slave should
240 # be considered lost.
241 build_dir
= os
.path
.join(self
.config
.common
['builddir'], config_name
)
242 log_dir
= os
.path
.join(self
.config
.common
['logdir'], config_name
)
243 utils
.run_cmd('/bin/rm -rf %s %s && mkdir -p %s %s' % (build_dir
, log_dir
, build_dir
, log_dir
), None, 1800, self
.logger
)
244 master_thread
= MasterThread(self
, self
.config
.configs
[config_name
], config_name
, slave_list
, log_dir
, build_dir
, tests
)
245 master_thread
.start()
246 master_threads
.append((config_name
, master_thread
))
247 for config_name
, master_thread
in master_threads
:
249 self
.logger
.debug('Master thread for `%s\' joined successfully' % config_name
)
250 everything_ended_here
= utils
.get_time()
251 self
.gather_all_stuff_together_and_present_to_the_public( \
252 everything_started_here
, everything_ended_here
, slave_list
, reset
)
254 # Run the tests on the given slave of each assigned build configuration.
255 # It may cause problems if several configurations are run one after
256 # another, but otherwise it's not possible assign multiple build
257 # configurations at all.
258 for config_name
in config_list
:
259 self
.logger
.debug('Hello, from a slave `%s\' running build ' \
260 'configuration `%s\'' \
261 % (slave_name
, config_name
))
262 if tests
and len(tests
) > 0:
263 self
.config
.configs
[config_name
]['functest'] = tests
.find('f') != -1
264 self
.config
.configs
[config_name
]['perftest'] = tests
.find('p') != -1
265 self
.config
.configs
[config_name
]['regtest'] = tests
.find('r') != -1
266 self
.slave(self
.config
.configs
[config_name
], config_name
, slave_name
)
268 def get_titan(self
, config
, config_name
, log_dir
, build_dir
):
269 """ Get the TITAN sources from the CVS repository. It can do checkouts by
270 tag and date only. If the version string is omitted HEAD will be
271 used. The checkout will be made into the build directory. The output
272 is not handled by the output handler yet.
275 The build configuration to get TITAN sources for. Log/build
279 0 on success. 1 if the checkout failed for some reason. It's most
280 probably a timeout, since the parameters are validated and the
281 existence of `cvs' is required. So, it's safe to abort the build if
284 command_line
= 'cd %s && cvs get TTCNv3' % build_dir
285 if re
.match('^v\d\-\d\-pl\d$', config
['version']):
286 command_line
= 'cd %s && cvs co -r%s TTCNv3' \
287 % (build_dir
, config
['version'])
288 elif re
.match('2\d{7}', config
['version']):
289 command_line
= 'cd %s && cvs co -D%s TTCNv3' \
290 % (build_dir
, config
['version'])
291 command_line
+= ' 1>%s/cvs-%s.stdout 2>%s/cvs-%s.stderr' \
292 % (log_dir
, config_name
, log_dir
, config_name
)
293 self
.logger
.debug('CVS checkout starting for config `%s\'', config_name
)
294 (retval
, stdout
, stderr
) = utils
.run_cmd(command_line
, None, 10800)
296 self
.logger
.error('The CVS checkout failed with command: `%s\', exit status: `%d\', stdout: `%s\', stderr: `%s\'' \
297 % (command_line
, retval
, stdout
, stderr
))
298 return 1 # `retval' is not handled yet.
299 self
.logger
.debug('CVS checkout finished for config `%s\'', config_name
)
302 def master(self
, config
, config_name
, log_dir
, build_dir
, tests
):
303 """ Prepare the packages for the slaves. The regression tests and
304 function tests are part of TITAN, hence the preparations regarding
305 those tests are done together with TITAN. It seems to make sense.
306 Delete only the `TTCNv3' directory when switching between build
307 configurations. It's advised to use a different global build
308 directory for the master and slaves.
311 The current build configuration and its name.
314 for slave_name
in self
.config
.slaves
:
315 slave
= self
.config
.slaves
[slave_name
]
316 if not config_name
in slave
['configs']:
318 slave_url
= '%s@%s' % (slave
['user'], slave
['ip'])
319 # Need more robust IP address checking. It doesn't work on my Debian
320 # laptop. It can return simply `127.0.0.1' and fool this check
322 is_localhost
= socket
.gethostbyname(socket
.gethostname()) == slave
['ip']
323 if self
.pass_prepare_titan(config
, config_name
, slave_name
, log_dir
, build_dir
):
324 continue # Configuration for a given slave is failed.
325 # The slave list is needed for the last pass.
326 slave_list
.append((slave_name
, config_name
, is_localhost
))
328 self
.logger
.debug('Removing old build `%s\' and log `%s\' ' \
329 'directories for slave `%s\' and build configuration `%s\'' \
330 % (config
['builddir'], config
['logdir'], slave_name
, config_name
))
331 if is_localhost
: # Cleanup first.
332 utils
.run_cmd('/bin/rm -rf %s %s && mkdir -p %s %s' \
333 % (config
['builddir'], config
['logdir'],
334 config
['builddir'], config
['logdir']), None, 1800, self
.logger
)
336 utils
.run_cmd('ssh %s \'/bin/rm -rf %s %s && mkdir -p %s %s\'' \
337 % (slave_url
, config
['builddir'], config
['logdir'],
338 config
['builddir'], config
['logdir']), None, 1800, self
.logger
)
340 if config
['perftest']:
341 self
.logger
.debug('Copying performance tests for slave `%s\'' % slave_name
)
342 self
.pass_prepare_perftest(config
, config_name
, slave
, slave_name
, slave_url
, \
344 if config
['vobtest']:
345 self
.logger
.debug('Copying VOB product tests for slave `%s\'' % slave_name
)
346 self
.pass_prepare_vobtest(config
, config_name
, slave
, slave_name
, slave_url
, \
349 if is_localhost
: # Optimize local builds.
350 self
.logger
.debug('It\'s a local build for slave `%s\' and build ' \
351 'configuration `%s\', working locally' \
352 % (slave_name
, config_name
))
353 utils
.run_cmd('cp %s/TTCNv3-%s.tar.bz2 %s' % \
354 (build_dir
, config_name
, config
['builddir']), None, 1800)
355 utils
.run_cmd('cp ./*.py ./*.sh %s' % config
['builddir'])
356 utils
.run_cmd('cd %s && %s/titan_builder.sh -s %s -c %s %s' \
357 % (config
['builddir'], config
['builddir'], \
358 slave_name
, config_name
, ((tests
and len(tests
) > 0) and ('-t %s' % tests
) or '')), None, 21600)
359 utils
.run_cmd('cp -r %s/%s/* %s' \
360 % (config
['logdir'], slave_name
, log_dir
))
362 self
.logger
.debug('It\'s a remote build for slave `%s\' and ' \
363 'build configuration `%s\', doing remote build' \
364 % (slave_name
, config_name
))
365 (retval
, stdout
, stderr
) = \
366 utils
.run_cmd('scp %s/TTCNv3-%s.tar.bz2 %s:%s' \
367 % (build_dir
, config_name
, slave_url
,
368 config
['builddir']), None, 1800)
370 self
.logger
.debug('The TITAN package is ready and distributed ' \
371 'for slave `%s\'' % slave_name
)
373 self
.logger
.error('Unable to distribute the TITAN package for ' \
374 'slave `%s\', it will be skipped from build ' \
375 'configuration `%s\'' % (slave_name
, config_name
))
377 utils
.run_cmd('scp ./*.py %s:%s' % (slave_url
, config
['builddir']))
378 utils
.run_cmd('scp ./*.sh %s:%s' % (slave_url
, config
['builddir']))
379 utils
.run_cmd('ssh %s \'cd %s && %s/titan_builder.sh -s %s -c ' \
380 '%s %s\'' % (slave_url
, config
['builddir'], \
381 config
['builddir'], slave_name
, config_name
, ((tests
and len(tests
) > 0) and ('-t %s' % tests
) or '') ), None, 21600)
382 utils
.run_cmd('scp -r %s:%s/%s/* %s' \
383 % (slave_url
, config
['logdir'], slave_name
, log_dir
))
387 def gather_all_stuff_together_and_present_to_the_public(self
, build_start
, \
388 build_end
, slave_list
, reset
):
389 """ Collect and process all logs. Only the CVS logs are coming from the
390 master. If the CSV output is not arrived from a slave, then the slave
391 will be considered lost.
393 build_root
= utils
.get_time(True)
394 html_root
= os
.path
.join(self
.config
.common
['htmldir'], build_root
)
395 utils
.run_cmd('mkdir -p %s' % html_root
, None, 1800, self
.logger
)
396 utils
.run_cmd('cd %s && /bin/rm -f latest && ln -s %s latest' % (self
.config
.common
['htmldir'], build_root
))
397 utils
.run_cmd('cp -r %s/* %s' % (self
.config
.common
['logdir'], html_root
))
398 email_file
= '%s/report.txt' % html_root
399 self
.publisher
.publish_csv2email(build_start
, build_end
, email_file
, \
400 slave_list
, build_root
, self
.config
.configs
, reset
)
401 self
.publisher
.publish_html(build_root
)
402 utils
.send_email(self
.logger
, self
.config
.recipients
, email_file
)
404 def pass_prepare_titan(self
, config
, config_name
, slave_name
, log_dir
, build_dir
):
405 """ Get TITAN from the CVS and configure it for the actual slave. Then
406 TITAN archive is created. The archive is not copied to the actual
407 slave, because this function can be a showstopper for the whole build
408 process for the actual slave.
411 The build configuration and its name, the actual slave's name,
412 log/build directories.
415 0 if everything went fine. 1 is returned when e.g. the CVS was
416 unreachable or the TITAN configuration failed for some reason.
417 Returning 1 should stop the build process for the actual slave.
419 if self
.get_titan(config
, config_name
, log_dir
, build_dir
):
420 self
.logger
.error('The CVS checkout failed for slave `%s\' and ' \
421 'build configuration `%s\'' \
422 % (slave_name
, config_name
))
424 if self
.config_titan(config
, build_dir
):
425 self
.logger
.error('Configuring TITAN failed for slave `%s\' ' \
426 'and build configuration `%s\'' \
427 % (slave_name
, config_name
))
429 utils
.run_cmd('cd %s && tar cf TTCNv3-%s.tar ./TTCNv3' \
430 % (build_dir
, config_name
), None, 1800)
431 utils
.run_cmd('cd %s && bzip2 TTCNv3-%s.tar' \
432 % (build_dir
, config_name
), None, 1800)
433 utils
.run_cmd('/bin/rm -rf %s/TTCNv3' % build_dir
, None, 1800)
436 def pass_prepare_perftest(self
, config
, config_name
, slave
, slave_name
, slave_url
, \
438 """ Copy the performance test package to the actual slave. It's a simple
439 archive. Its location is defined in the configuration file.
442 The build configuration and its name with the actual slave and its
443 name. Nothing is returned.
445 if os
.path
.isfile(config
['perftestdir']):
447 utils
.run_cmd('cp -f %s %s' % (config
['perftestdir'], \
450 (retval
, stdout
, stderr
) = utils
.run_cmd('scp %s %s:%s' \
451 % (config
['perftestdir'], slave_url
, config
['builddir']))
453 self
.logger
.error('Unable to copy performance test package ' \
454 'to slave `%s\'' % slave_name
)
456 self
.logger
.error('The performance test package cannot be found at ' \
457 '`%s\'' % config
['perftestdir'])
459 def pass_prepare_vobtest(self
, config
, config_name
, slave
, slave_name
, slave_url
, \
461 """ Collect and configure the VOB products. The products will be
462 collected only if there's no file matching the `vobtest-*.tar.bz2'
463 pattern in the build directory. The resulting archive is copied to
464 the given slave only if it's a remote slave. The errors are reported
465 to the local error log. The URL of the slave is calculated locally.
468 The build configuration and its name with the actual slave and its
469 name. Nothing is returned.
471 vobtest_lock
.acquire()
472 really_collect_products
= len([file for file in os
.listdir(self
.config
.common
['builddir']) \
473 if fnmatch
.fnmatch(file, 'vobtest\-.*\.tar\.bz2')]) == 0
474 if really_collect_products
:
475 handler
= product_handler
.product_handler(self
.logger
, self
.config
)
476 if handler
.config_products('%s/vobtest' % self
.config
.common
['builddir']):
477 self
.logger
.error('Configuring VOB products failed for slave: ' \
478 '`%s\' and build configuration: `%s\'' \
479 % (slave_name
, config_name
))
481 utils
.run_cmd('cd %s && tar cf vobtest-%s.tar ./vobtest' \
482 % (self
.config
.common
['builddir'], \
483 time
.strftime('%Y%m%d')), None, 1800)
484 utils
.run_cmd('cd %s && bzip2 vobtest-*.tar' \
485 % self
.config
.common
['builddir'], None, 1800)
486 utils
.run_cmd('/bin/rm -rf %s/vobtest %s/vobtest-*.tar' \
487 % (self
.config
.common
['builddir'], \
488 self
.config
.common
['builddir']), None, 1800)
490 self
.logger
.debug('VOB products don\'t need to be configured again')
491 vobtest_lock
.release()
493 (retval
, stdout
, stderr
) = \
494 utils
.run_cmd('scp %s/vobtest-*.tar.bz2 %s:%s' \
495 % (self
.config
.common
['builddir'], slave_url
, \
498 self
.logger
.error('Copying the VOB package to slave: ' \
499 '`%s\' failed for build configuration: `%s\'' \
500 % (slave_name
, config_name
))
502 utils
.run_cmd('cp %s/vobtest-*.tar.bz2 %s' \
503 % (self
.config
.common
['builddir'], config
['builddir']))
505 def slave(self
, config
, config_name
, slave_name
):
506 """ Run the build passes sequentially. If the TITAN build fails, the
507 remaining passes are skipped. Log everything. All the results will
508 be written in all supported formats. It should be configurable.
510 self
.logger
.debug('Setting environment variables from `pass_setenv()\'')
511 self
.pass_setenv(config
, slave_name
)
512 self
.logger
.debug('Building TITAN from `pass_titan()\'')
513 stamp_old
= utils
.get_time()
514 if not self
.pass_titan(config
, config_name
, slave_name
):
516 if config
['regtest']:
517 regtest_thread
= RegtestThread(self
, config
, slave_name
)
518 regtest_thread
.start()
519 test_threads
.append(('regression tests', regtest_thread
))
520 self
.logger
.debug('Running regression tests from `pass_regtest()\'')
521 if config
['functest']:
522 functest_thread
= FunctestThread(self
, config
, slave_name
)
523 functest_thread
.start()
524 test_threads
.append(('function tests', functest_thread
))
525 self
.logger
.debug('Running function tests from `pass_functest()\'')
526 if config
['perftest']:
527 perftest_thread
= PerftestThread(self
, config
, slave_name
)
528 perftest_thread
.start()
529 test_threads
.append(('performance tests', perftest_thread
))
530 self
.logger
.debug('Running performance tests from `pass_perftest()\'')
531 if 'eclipse' in config
and config
['eclipse']:
532 eclipse_thread
= EclipseThread(self
, config
, slave_name
)
533 eclipse_thread
.start()
534 test_threads
.append(('eclipse tests', eclipse_thread
))
535 self
.logger
.debug('Running Eclipse build from `pass_eclipse()\'')
536 if config
['vobtest']:
537 vobtest_thread
= VobtestThread(self
, config
, slave_name
)
538 vobtest_thread
.start()
539 test_threads
.append(('VOB product tests', vobtest_thread
))
540 self
.logger
.debug('Running VOB product tests from `pass_vobtest()\'')
541 for test_thread_name
, test_thread
in test_threads
:
543 self
.logger
.debug('Thread for `%s\' joined successfully' % test_thread_name
)
544 self
.publisher
.dump_csv(stamp_old
, utils
.get_time(), config
, config_name
, slave_name
)
545 self
.publisher
.dump_txt(stamp_old
, utils
.get_time(), config
, config_name
, slave_name
)
546 self
.publisher
.dump_html(stamp_old
, utils
.get_time(), config
, config_name
, slave_name
)
547 self
.logger
.debug('Finalizing build from using `pass_slave_postprocess()\'')
548 self
.pass_slave_postprocess(config
, config_name
, slave_name
)
550 def pass_slave_postprocess(self
, config
, config_name
, slave_name
):
551 """ Archive stuff and make everything available for the master. The
552 master will copy all necessary stuff. The build directory is
553 available until the next build. Do the cleanup here. The installation
554 directory is never removed.
557 The current build configuration.
559 utils
.run_cmd('cd %s && tar cf TTCNv3-%s-bin.tar ./TTCNv3' \
560 % (config
['builddir'], config_name
), None, 1800)
561 utils
.run_cmd('bzip2 %s/TTCNv3-%s-bin.tar' \
562 % (config
['builddir'], config_name
), None, 1800)
563 utils
.run_cmd('/bin/rm -rf %s/TTCNv3' % config
['builddir'], None, 1800)
564 utils
.run_cmd('/bin/rm -f %s/TTCNv3-%s.tar.bz2' \
565 % (config
['builddir'], config_name
))
566 utils
.run_cmd('cd %s && /bin/rm -f *.py *.pyc *.sh' % config
['builddir'])
567 utils
.run_cmd('mv -f %s %s %s/%s' % (LOG_FILENAME
, TRACE_FILENAME
, \
568 config
['logdir'], slave_name
))
570 def pass_titan(self
, config
, config_name
, slave_name
):
571 """ Build pass for TITAN itself. It is assumed that the master have
572 already copied the TITAN package to the build directory. It's the
573 only requirement here. If the installation fails the TITAN build is
574 considered as a failure. Only the `make install' is taken into
578 The current build configuration of the slave and its name.
581 1 on error, e.g. if the TITAN package is not present. 0 if the
582 TITAN package was found and the full build completed successfully.
584 stamp_begin
= utils
.get_time()
585 utils
.run_cmd('mkdir -p %s/%s' % (config
['logdir'], slave_name
), None, 1800, self
.logger
)
586 utils
.run_cmd('bunzip2 %s/TTCNv3-%s.tar.bz2' \
587 % (config
['builddir'], config_name
), None, 1800, self
.logger
)
588 utils
.run_cmd('cd %s && tar xf TTCNv3-%s.tar && bzip2 %s/TTCNv3-*.tar' \
589 % (config
['builddir'], config_name
, config
['builddir']), \
590 None, 1800, self
.logger
)
591 if not os
.path
.isdir('%s/TTCNv3' % config
['builddir']):
592 self
.logger
.error('The `%s/TTCNv3\' directory is not found' \
593 % config
['builddir'])
594 self
.publisher
.titan_out(config
, slave_name
, \
595 (stamp_begin
, utils
.get_time(), None))
597 utils
.run_cmd('cd %s && find . -exec touch {} \;' % config
['builddir'], None, 1800)
598 (ret_val_dep
, stdout_dep
, stderr_dep
) = \
599 utils
.run_cmd('cd %s/TTCNv3 && make dep 2>&1' \
600 % config
['builddir'], None, 1800)
601 (ret_val_make
, stdout_make
, stderr_make
) = \
602 utils
.run_cmd('cd %s/TTCNv3 && make -j4 2>&1' \
603 % config
['builddir'], None, 1800)
604 (ret_val_install
, stdout_install
, stderr_install
) = \
605 utils
.run_cmd('cd %s/TTCNv3 && make install 2>&1' \
606 % config
['builddir'], None, 1800)
607 if ret_val_make
or ret_val_install
:
608 self
.logger
.error('TITAN build failed for slave `%s\', please check ' \
609 'the logs for further investigation, stopping slave ' \
611 output
= (stamp_begin
, utils
.get_time(), \
612 ((ret_val_dep
, stdout_dep
, stderr_dep
), \
613 (ret_val_make
, stdout_make
, stderr_make
), \
614 (ret_val_install
, stdout_install
, stderr_install
)))
615 self
.publisher
.titan_out(config
, slave_name
, output
)
616 if ret_val_dep
or ret_val_make
or ret_val_install
:
619 if 'foa' in config
and config
['foa'] and 'foadir' in config
and config
['foadir'] != config
['installdir']:
620 # The `installdir' must be removed by hand after a FOA period. Cannot
621 # be automated in a sane way. For FOA `installdir' shall be a unique
622 # directory. E.g. date based. Otherwise, the builds are always
624 self
.logger
.debug('Linking directories for FOA build to `%s\'' % config
['foadir'])
625 (ret_val_rm
, stdout_rm
, stderr_rm
) = utils
.run_cmd('/bin/rm -rvf %s' % config
['foadir'])
626 if ret_val_rm
: # Sometimes it doesn't work.
627 self
.logger
.error('Unable to remove `%s\': `%s\'' % (config
['foadir'], ''.join(stderr_rm
)))
628 utils
.run_cmd('ln -s %s %s' % (config
['installdir'], config
['foadir']))
631 def pass_setenv(self
, config
, slave_name
):
632 """ Set some environment variables needed to run the TITAN tests. Don't
633 use uppercase latters in directory names. The GCC is added as well.
634 Always check if an environment variable exists before reading it.
637 The current build configuration of the slave and its name.
639 path
= os
.environ
.get('PATH')
640 ld_library_path
= os
.environ
.get('LD_LIBRARY_PATH')
641 os
.environ
['PATH'] = '%s/bin:%s/bin:%s' % (config
['installdir'], config
['gccdir'], path
and path
or '')
642 os
.environ
['LD_LIBRARY_PATH'] = '%s/lib:%s/lib:%s' % (config
['installdir'], config
['gccdir'], ld_library_path
and ld_library_path
or '')
643 os
.environ
['TTCN3_DIR'] = config
['installdir']
644 os
.environ
['TTCN3_LICENSE_FILE'] = config
['license']
646 def pass_regtest_helper(self
, config
, slave_name
, runtime
):
647 """ Run the regression tests with `make' and then `make run'. The output
648 is sent to the publisher as well. At the end, `make clean' is done to
649 save some bytes. Don't use `tee', since its exit code will always be
650 0. Only `stdout' is used.
653 config: The current build configuration.
654 slave_name: The name of the slave.
655 runtime: 0 for the load-test run-time, 1 for the function-test
658 utils
.run_cmd('cd %s/TTCNv3/regression_test && make distclean' \
659 % config
['builddir'], None, 1800)
660 (ret_val_make
, stdout_make
, stderr_make
) = \
661 utils
.run_cmd('cd %s/TTCNv3/regression_test && %s make 2>&1' \
662 % (config
['builddir'], runtime
and 'RT2=1' or ''), None, 3600)
664 self
.logger
.error('The regression test failed to build for the ' \
665 '`%s\' runtime' % (runtime
and 'function-test' or 'load-test'))
666 (ret_val_run
, stdout_run
, stderr_run
) = \
667 utils
.run_cmd('cd %s/TTCNv3/regression_test && %s make run 2>&1' \
668 % (config
['builddir'], runtime
and 'RT2=1' or ''), None, 1800)
671 for index
, line
in globals()['__builtins__'].enumerate(stdout_run
):
672 matched_line
= re
.search('Verdict stat.*pass \((\d+)\..*', line
)
673 if matched_line
and int(matched_line
.group(1)) != 100:
674 if all_fine
and not failed_lines
:
675 failed_lines
.append('\nThe failed tests were the following:\n\n')
676 if not re
.search('TverdictOper', stdout_run
[index
- 1]):
678 failed_lines
.append(stdout_run
[index
- 1])
679 failed_lines
.append(line
)
680 stdout_run
.extend(failed_lines
)
681 if ret_val_run
or not all_fine
:
682 self
.logger
.error('The regression test failed to run for the ' \
683 '`%s\' runtime' % (runtime
and 'function-test' or 'load-test'))
685 utils
.run_cmd('cd %s/TTCNv3/regression_test && make clean' \
686 % config
['builddir'], None, 1800)
687 return ((ret_val_make
, stdout_make
, stderr_make
), \
688 (ret_val_run
, stdout_run
, stderr_run
))
690 def pass_regtest(self
, config
, slave_name
):
691 """ Build and run the regression tests and publish the results. The
692 `pass_regtest_helper()' does the dirty job.
695 config: The current build configuration.
696 slave_name: The name of the slave.
699 stamp_begin
= utils
.get_time()
700 rt1_results
= self
.pass_regtest_helper(config
, slave_name
, 0)
701 output
['rt1'] = (stamp_begin
, utils
.get_time(), rt1_results
)
703 stamp_begin
= utils
.get_time()
704 rt2_results
= self
.pass_regtest_helper(config
, slave_name
, 1)
705 output
['rt2'] = (stamp_begin
, utils
.get_time(), rt2_results
)
706 self
.publisher
.regtest_out(config
, slave_name
, output
)
708 def pass_eclipse(self
, config
, slave_name
):
709 """ Build Eclipse plugins and publish them to an update site.
712 config: The current build configuration.
713 slave_name: The name of the slave.
716 stamp_begin
= utils
.get_time()
717 results
= utils
.run_cmd('cd %s/TTCNv3/eclipse/automatic_build && ant -d -l mylog.log -f build_main.xml updatesite.experimental 2>&1' \
718 % config
['builddir'], None, 1800)
719 log_dir
= os
.path
.join(config
['logdir'], slave_name
)
720 utils
.run_cmd('cp %s/TTCNv3/eclipse/automatic_build/mylog.log %s/eclipse-mylog.log' \
721 % (config
['builddir'], log_dir
))
722 output
= (stamp_begin
, utils
.get_time(), os
.path
.join(log_dir
, 'eclipse-mylog.log'), results
)
723 self
.publisher
.eclipse_out(config
, slave_name
, output
)
725 def pass_perftest_helper(self
, config
, slave_name
, runtime
):
726 """ Build the performance test and run it for some predefined CPS values.
727 These CPS values should come from the build configurations instead.
728 Obviously, if the build fails all test runs are skipped. It handles
729 its own tarball as well. It's unpacked at the beginning and removed
730 at the end. The results are also published.
733 The actual build configuration and the name of the slave. The
734 function returns nothing.
737 utils
.run_cmd('cd %s/perftest && ttcn3_makefilegen -e titansim %s ' \
738 '*.ttcnpp *.ttcnin *.ttcn *.cc *.cfg' \
739 % (config
['builddir'], (runtime
and '-fpgR' or '-fpg')))
740 # Avoid infinite recursion.
741 utils
.run_cmd('sed \'s/^-include $(DEPFILES)$//\' Makefile >Makefile-tmp && mv Makefile-tmp Makefile',
742 os
.path
.join(config
['builddir'], 'perftest'))
743 utils
.run_cmd('cd %s/perftest && make clean' % config
['builddir'])
744 (ret_val_dep
, stdout_dep
, stderr_dep
) = \
745 utils
.run_cmd('cd %s/perftest && find . -exec touch {} \; && make %s dep 2>&1' \
746 % (config
['builddir'], (runtime
and 'RT2=1' or '')), \
748 (ret_val_make
, stdout_make
, stderr_make
) = \
749 utils
.run_cmd('cd %s/perftest && make %s 2>&1' \
750 % (config
['builddir'], (runtime
and 'RT2=1' or '')), \
752 perftest_out
['dep'] = (ret_val_dep
, stdout_dep
, stderr_dep
)
753 perftest_out
['make'] = (ret_val_make
, stdout_make
, stderr_make
)
754 perftest_out
['run'] = []
756 cps_min
= config
.get('cpsmin', 1000)
757 cps_max
= config
.get('cpsmax', 2000)
758 cps_diff
= abs(cps_max
- cps_min
) / 5
759 for cps
in range(cps_min
, cps_max
+ cps_diff
, cps_diff
):
760 # These numbers should be platform dependent. Lower on slow
761 # machines and high on fast machines.
762 (ret_val_run
, stdout_run
, stderr_run
) = \
763 utils
.run_cmd('cd %s/perftest && cpp -DTSP_CPS_CPP=%d.0 config.cfg >config.cfg-tmp && ' \
764 'ttcn3_start ./titansim ./config.cfg-tmp 2>&1' \
765 % (config
['builddir'], cps
), None, 900)
766 for line
in stdout_run
:
767 matched_line
= re
.search('Verdict stat.*pass \((\d+)\..*', line
)
768 if matched_line
and int(matched_line
.group(1)) != 100:
769 self
.logger
.error('Performance test failed to run for `%d\' CPSs' % cps
)
771 perftest_out
['run'].append((cps
, (ret_val_run
, stdout_run
, stderr_run
)))
773 self
.logger
.error('Performance test compilation failed for the ' \
774 '`%s\' runtime' % (runtime
and 'function-test' or 'load-test'))
777 def pass_perftest(self
, config
, slave_name
):
778 """ Build and run the performance tests and publish the results. The
779 `pass_perftest_helper()' does the dirty job.
782 config: The current build configuration.
783 slave_name: The name of the slave.
785 utils
.run_cmd('bunzip2 perftest-*.tar.bz2', config
['builddir'], 1800)
786 utils
.run_cmd('tar xf ./perftest-*.tar && bzip2 ./perftest-*.tar', \
787 config
['builddir'], 1800)
788 if not os
.path
.isdir('%s/perftest' % config
['builddir']):
789 self
.logger
.error('The performance test is not available at ' \
790 '`%s/perftest\'' % config
['builddir'])
793 stamp_begin
= utils
.get_time()
794 rt1_results
= self
.pass_perftest_helper(config
, slave_name
, 0)
795 output
['rt1'] = (stamp_begin
, utils
.get_time(), rt1_results
)
797 stamp_begin
= utils
.get_time()
798 rt2_results
= self
.pass_perftest_helper(config
, slave_name
, 1)
799 output
['rt2'] = (stamp_begin
, utils
.get_time(), rt2_results
)
800 self
.publisher
.perftest_out(config
, slave_name
, output
)
801 utils
.run_cmd('/bin/rm -rf %s/perftest*' % config
['builddir'], None, 1800)
803 def pass_functest_helper(self
, config
, slave_name
, runtime
):
804 function_test_prefix
= '%s/TTCNv3/function_test' % config
['builddir']
805 function_test_prefixes
= ('%s/BER_EncDec' % function_test_prefix
, \
806 '%s/Config_Parser' % function_test_prefix
, \
807 '%s/RAW_EncDec' % function_test_prefix
, \
808 '%s/Semantic_Analyser' % function_test_prefix
, \
809 '%s/Text_EncDec' % function_test_prefix
, \
810 '%s/Semantic_Analyser/float' % function_test_prefix
, \
811 '%s/Semantic_Analyser/import_of_iports' % function_test_prefix
, \
812 '%s/Semantic_Analyser/options' % function_test_prefix
, \
813 '%s/Semantic_Analyser/ver' % function_test_prefix
, \
814 '%s/Semantic_Analyser/xer' % function_test_prefix
)
815 log_dir
= os
.path
.join(config
['logdir'], slave_name
)
817 stamp_old
= utils
.get_time()
818 for function_test
in function_test_prefixes
:
819 utils
.run_cmd('ln -s %s %s' % (config
['perl'], function_test
))
820 function_test_name
= function_test
.split('/')[-1]
821 ber_or_raw_or_text
= not (function_test_name
== 'Config_Parser' or function_test_name
== 'Semantic_Analyser')
822 utils
.run_cmd('cd %s && %s ./%s %s 2>&1 | tee %s/functest-%s.%s' \
823 % (function_test
, (runtime
and 'RT2=1' or ''),
824 (os
.path
.isfile('%s/run_test_all' % function_test
) \
825 and 'run_test_all' or 'run_test'), \
826 ((runtime
and not ber_or_raw_or_text
) and '-rt2' or ''), \
827 log_dir
, function_test_name
, \
828 (runtime
and 'rt2' or 'rt1')), None, 3600)
829 error_target
= os
.path
.join(log_dir
, 'functest-%s-error.%s' % (function_test_name
, (runtime
and 'rt2' or 'rt1')))
830 if ber_or_raw_or_text
:
831 utils
.run_cmd('cp %s/%s_TD.script_error %s' \
832 % (function_test
, function_test_name
, error_target
))
833 utils
.run_cmd('cp %s/%s_TD.fast_script_error %s' \
834 % (function_test
, function_test_name
, error_target
))
835 functest_out
[function_test_name
] = \
836 ('%s/functest-%s.%s' % (log_dir
, function_test_name
, (runtime
and 'rt2' or 'rt1')), \
837 (ber_or_raw_or_text
and error_target
or ''))
840 def pass_functest(self
, config
, slave_name
):
841 """ Build pass to build and run the function tests. The
842 `pass_functest_helper()' does the direty job.
845 stamp_begin
= utils
.get_time()
846 rt1_results
= self
.pass_functest_helper(config
, slave_name
, 0)
847 output
['rt1'] = (stamp_begin
, utils
.get_time(), rt1_results
)
849 stamp_begin
= utils
.get_time()
850 rt2_results
= self
.pass_functest_helper(config
, slave_name
, 1)
851 output
['rt2'] = (stamp_begin
, utils
.get_time(), rt2_results
)
852 self
.publisher
.functest_out(config
, slave_name
, output
)
854 def pass_vobtest(self
, config
, slave_name
):
855 """ Build pass for the VOB products. Currently, the VOB products are
856 compiled only due to the lack of usable tests written for them. The
857 output is stored here by the publisher. The normal runtime should
858 always be the first, it's a restriction of the publisher.
861 The actual build configuration and its name.
863 utils
.run_cmd('bunzip2 %s/vobtest-*.tar.bz2' \
864 % config
['builddir'], None, 1800)
865 utils
.run_cmd('cd %s && tar xf ./vobtest-*.tar && bzip2 ./vobtest-*.tar' \
866 % config
['builddir'], None, 1800)
867 if not os
.path
.isdir('%s/vobtest' % config
['builddir']):
868 self
.logger
.error('The products are not available at `%s/vobtest\'' \
869 % config
['builddir'])
870 self
.publisher
.vobtest_out(utils
.get_time(), utils
.get_time(), {})
873 stamp_begin
= utils
.get_time()
874 handler
= product_handler
.product_handler(self
.logger
, self
.config
)
875 log_dir
= '%s/%s/products' % (config
['logdir'], slave_name
)
876 results
= handler
.build_products('%s/vobtest' % config
['builddir'], \
877 log_dir
, config
, False)
878 output
['rt1'] = (stamp_begin
, utils
.get_time(), results
)
880 stamp_begin
= utils
.get_time()
881 results
= handler
.build_products('%s/vobtest' \
882 % config
['builddir'], log_dir
, config
, True)
883 output
['rt2'] = (stamp_begin
, utils
.get_time(), results
)
884 self
.publisher
.vobtest_out(config
, slave_name
, output
)
885 utils
.run_cmd('/bin/rm -rf %s/vobtest*' % config
['builddir'], None, 1800)
887 def config_titan(self
, config
, build_dir
):
888 """ Modify TITAN configuration files to create a platform-specific source
889 package. The original files are always preserved in an `*.orig' file.
890 `sed' would be shorter, but this way everything is under control.
891 Improve file handling. It is assumed, that there's a `TTCNv3' directory
892 in the build directory.
895 config: The build configuration we're configuring for.
898 If everything went fine 0 is returned. Otherwise 1 is returned and
899 the error messages will be logged. The screen always stays intact.
901 if not os
.path
.isdir('%s/TTCNv3' % build_dir
):
902 self
.logger
.error('The `%s/TTCNv3\' directory is not found' % build_dir
)
903 return 1 # It's a fatal error, no way out.
904 # Prepare all function tests. Add links to the `perl' interpreter and
905 # modify some some Makefiles containing the platform string.
906 if config
['functest']:
907 function_test_prefix
= '%s/TTCNv3/function_test' % build_dir
908 for function_test
in ('%s/BER_EncDec' % function_test_prefix
, \
909 '%s/Config_Parser' % function_test_prefix
, \
910 '%s/RAW_EncDec' % function_test_prefix
, \
911 '%s/Semantic_Analyser' % function_test_prefix
, \
912 '%s/Text_EncDec' % function_test_prefix
):
913 if os
.path
.isdir(function_test
):
914 if function_test
.endswith('BER_EncDec') or \
915 function_test
.endswith('RAW_EncDec') or \
916 function_test
.endswith('Text_EncDec'):
917 utils
.run_cmd('mv %s/Makefile %s/Makefile.orig' \
918 % (function_test
, function_test
))
919 berrawtext_makefile
= open('%s/Makefile.orig' % function_test
, 'rt')
920 berrawtext_makefile_new
= open('%s/Makefile' % function_test
, 'wt')
921 for line
in berrawtext_makefile
:
922 if re
.match('^PLATFORM\s*:?=\s*\w+$', line
): # Platform
923 # autodetect later. It is hard-coded into the build
925 berrawtext_makefile_new
.write('PLATFORM = %s\n' % config
['platform'])
926 elif re
.match('^CXX\s*:?=\s*.*$', line
) and ('cxx' in config
and len(config
['cxx']) > 0):
927 berrawtext_makefile_new
.write('CXX = %s\n' % config
['cxx'])
929 berrawtext_makefile_new
.write(line
)
930 if function_test
.endswith('BER_EncDec'):
931 utils
.run_cmd('mv %s/run_test %s/run_test.orig' \
932 % (function_test
, function_test
))
933 utils
.run_cmd('cat %s/run_test.orig | ' \
934 'sed s/TD.script/TD.fast_script/ >%s/run_test ' \
935 '&& chmod 755 %s/run_test' \
936 % (function_test
, function_test
, function_test
)) # Make it fast.
938 self
.logger
.warning('Function test directory `%s\' is not found'
940 # Add `-lncurses' for all `LINUX' targets. It's not always needed, hence
941 # platform autodetect won't help in this.
942 if config
['platform'] == 'LINUX':
943 mctr_makefile_name
= '%s/TTCNv3/mctr2/mctr/Makefile' % build_dir
944 if os
.path
.isfile(mctr_makefile_name
):
945 utils
.run_cmd('mv %s %s.orig' % (mctr_makefile_name
, mctr_makefile_name
))
946 mctr_makefile
= open('%s.orig' % mctr_makefile_name
, 'rt')
947 mctr_makefile_new
= open(mctr_makefile_name
, 'wt')
948 for line
in mctr_makefile
:
949 if re
.match('^LINUX_CLI_LIBS\s*:?=\s*$', line
):
950 mctr_makefile_new
.write('LINUX_CLI_LIBS := -lncurses\n')
952 mctr_makefile_new
.write(line
)
953 mctr_makefile
.close()
954 mctr_makefile_new
.close()
956 self
.logger
.warning('The `%s\' is not found' % mctr_makefile_name
)
957 # Prepare the main configuration file.
958 makefile_cfg_name
= '%s/TTCNv3/Makefile.cfg' % build_dir
959 if os
.path
.isfile(makefile_cfg_name
):
960 utils
.run_cmd('mv %s %s.orig' % (makefile_cfg_name
, makefile_cfg_name
))
961 makefile_cfg
= open('%s.orig' % makefile_cfg_name
, 'rt')
962 makefile_cfg_new
= open(makefile_cfg_name
, 'wt')
963 for line
in makefile_cfg
:
964 if re
.match('^TTCN3_DIR\s*:?=\s*.*$', line
):
965 # Use the environment.
967 elif re
.match('^DEBUG\s*:?=\s*.*$', line
):
968 makefile_cfg_new
.write('DEBUG := %s\n' % (config
['debug'] and 'yes' or 'no'))
969 elif re
.match('^# PLATFORM\s*:?=\s*.*$', line
) and len(config
['platform']) > 0:
970 # Automatic platform detection doesn't seem to work very well, so the
971 # platform should always be set explicitly.
972 makefile_cfg_new
.write('PLATFORM := %s\n' % config
['platform'])
973 elif re
.match('^JNI\s*:?=\s*.*$', line
):
974 # It's the so called `and-or' trick from http://diveintopython.org/
975 # power_of_introspection/and_or.html.
976 makefile_cfg_new
.write('JNI := %s\n' % (config
['jni'] and 'yes' or 'no'))
977 elif re
.match('^GUI\s*:?=\s*.*$', line
):
978 makefile_cfg_new
.write('GUI := %s\n' % (config
['gui'] and 'yes' or 'no'))
979 elif re
.match('^FLEX\s*:?=\s*.*$', line
) and len(config
['flex']) > 0:
980 makefile_cfg_new
.write('FLEX := %s\n' % config
['flex'])
981 elif re
.match('^BISON\s*:?=\s*.*$', line
) and len(config
['bison']) > 0:
982 makefile_cfg_new
.write('BISON := %s\n' % config
['bison'])
983 elif re
.match('^CC\s*:?=\s*.*$', line
) and len(config
['gccdir']) > 0:
984 makefile_cfg_new
.write('CC := %s/bin/%s\n' % (config
['gccdir'], (('cc' in config
and len(config
['cc']) > 0) and config
['cc'] or 'gcc')))
985 elif re
.match('^CXX\s*:?=\s*.*$', line
) and len(config
['gccdir']) > 0:
986 makefile_cfg_new
.write('CXX := %s/bin/%s\n' % (config
['gccdir'], (('cxx' in config
and len(config
['cxx']) > 0) and config
['cxx'] or 'g++')))
987 elif re
.match('^JDKDIR\s*:?=\s*.*$', line
) and len(config
['jdkdir']) > 0:
988 makefile_cfg_new
.write('JDKDIR := %s\n' % config
['jdkdir'])
989 elif re
.match('^QTDIR\s*:?=\s*.*$', line
) and len(config
['qtdir']) > 0:
990 makefile_cfg_new
.write('QTDIR = %s\n' % config
['qtdir'])
991 elif re
.match('^XMLDIR\s*:?=\s*.*$', line
) and len(config
['xmldir']) > 0:
992 makefile_cfg_new
.write('XMLDIR = %s\n' % config
['xmldir'])
993 elif re
.match('^OPENSSL_DIR\s*:?=\s*.*$', line
) and len(config
['openssldir']) > 0:
994 makefile_cfg_new
.write('OPENSSL_DIR = %s\n' % config
['openssldir'])
995 elif re
.match('^LDFLAGS\s*:?=\s*.*$', line
) and len(config
['ldflags']) > 0:
996 makefile_cfg_new
.write('LDFLAGS = %s\n' % config
['ldflags'])
997 elif re
.match('^COMPILERFLAGS\s*:?=\s*.*$', line
) and len(config
['compilerflags']) > 0:
998 makefile_cfg_new
.write('COMPILERFLAGS = %s\n' % config
['compilerflags'])
1000 makefile_cfg_new
.write(line
)
1001 makefile_cfg
.close()
1002 makefile_cfg_new
.close()
1004 self
.logger
.error('The `%s\' is not found, it seems to be a fake ' \
1005 'installation' % makefile_cfg_name
)
1006 return 1 # It's essential, exit immediately.
1007 if config
['regtest']:
1008 regtest_makefile_name
= '%s/TTCNv3/regression_test/Makefile.regression' % build_dir
1009 if os
.path
.isfile(regtest_makefile_name
):
1010 utils
.run_cmd('mv %s %s.orig' \
1011 % (regtest_makefile_name
, regtest_makefile_name
))
1012 regtest_makefile
= open('%s.orig' % regtest_makefile_name
, 'rt')
1013 regtest_makefile_new
= open(regtest_makefile_name
, 'wt')
1014 for line
in regtest_makefile
:
1015 if re
.match('^TTCN3_DIR\s*:?=\s*.*$', line
):
1016 # Use the environment.
1018 elif re
.match('^CC\s*:?=\s*.*$', line
) and len(config
['gccdir']) > 0:
1019 regtest_makefile_new
.write('CC := %s/bin/%s\n' % (config
['gccdir'], (('cc' in config
and len(config
['cc']) > 0) and config
['cc'] or 'gcc')))
1020 elif re
.match('^CXX\s*:?=\s*.*$', line
) and len(config
['gccdir']) > 0:
1021 regtest_makefile_new
.write('CXX := %s/bin/%s\n' % (config
['gccdir'], (('cxx' in config
and len(config
['cxx']) > 0) and config
['cxx'] or 'g++')))
1022 elif re
.match('^XMLDIR\s*:?=\s*.*$', line
) and len(config
['xmldir']) > 0:
1023 regtest_makefile_new
.write('XMLDIR = %s\n' % config
['xmldir'])
1025 regtest_makefile_new
.write(line
)
1026 regtest_makefile
.close()
1027 regtest_makefile_new
.close()
1029 self
.logger
.warning('Regression test configuration file `%s\' is ' \
1030 'not found' % regtest_makefile_name
)
1031 if 'xsdtests' in config
and not config
['xsdtests']:
1032 self
.logger
.warning('Disabling `xsd2ttcn\' tests to save some time')
1033 utils
.run_cmd('mv %s %s.orig' \
1034 % (regtest_makefile_name
.split('.')[0], \
1035 regtest_makefile_name
.split('.')[0]))
1036 utils
.run_cmd('cat %s.orig | sed s/\'xsdConverter\'/\'\'/ >%s' \
1037 % (regtest_makefile_name
.split('.')[0], \
1038 regtest_makefile_name
.split('.')[0]))
1039 self
.config_pdfs(config
, build_dir
)
1040 self
.logger
.debug('`%s/TTCNv3\' was configured and ready to build, all ' \
1041 'Makefiles were modified successfully' % build_dir
)
1042 return 0 # Allow the caller to catch errors. Use exceptions later
1043 # instead. Only `./TTCNv3' and `./TTCNv3/Makefile.cfg' is necessary for a
1044 # successful configuration. Other Makefiles can be missing.
1046 def config_pdfs(self
, config
, build_dir
):
1047 """ Optionally, copy .pdf files to the documentation directory or create
1048 fake .pdf files to make the installation successful. If the build
1049 configuration doesn't have the appropriate key nothing is done with
1050 .pdf files. If the directory of .pdf files doesn't exists the .pdf
1051 files will be faked.
1054 The actual build configuration.
1056 if 'pdfdir' in config
:
1057 if not os
.path
.isdir(config
['pdfdir']):
1058 self
.logger
.debug('Creating fake .pdf files in %s/TTCNv3/usrguide' % build_dir
)
1059 for file in os
.listdir('%s/TTCNv3/usrguide' % build_dir
):
1060 if file.endswith('.doc'):
1061 utils
.run_cmd('touch %s/TTCNv3/usrguide/%s.pdf' \
1062 % (build_dir
, file.split('.')[0]))
1064 self
.logger
.debug('Copying .pdf files from %s' % config
['pdfdir'])
1065 utils
.run_cmd('cp %s/*.pdf %s/TTCNv3/usrguide' % (config
['pdfdir'], build_dir
))
1067 self
.logger
.debug('The .pdf files are not in place, your ' \
1068 'installation will fail if you haven\'t fixed the ' \
1071 def dump_addressees(self
):
1072 for addressee
in self
.config
.recipients
:
1073 print('%s %s' % (addressee
, self
.config
.recipients
[addressee
]))
1075 def dump_configs(self
):
1076 configs
= self
.config
.configs
1077 slaves
= self
.config
.slaves
1078 for config_name
, config_data
in configs
.iteritems():
1080 for slave_name
, slave_data
in slaves
.iteritems():
1081 if config_name
in slave_data
['configs']:
1082 slave_list
.append(slave_name
)
1083 print('%s %s' % (config_name
, ', '.join(slave_list
)))
1085 def main(argv
= None):
1088 usage
= 'Usage: %prog [options]'
1089 version
= '%prog 0.0.5'
1090 parser
= optparse
.OptionParser(usage
= usage
, version
= version
)
1091 parser
.add_option('-a', '--addressees', action
= 'store_true', dest
= 'addressees', help = 'dump all addressees')
1092 parser
.add_option('-A', '--set-addressees', action
= 'store', type = 'string', dest
= 'set_addressees', help = 'set addressees from command line')
1093 parser
.add_option('-c', '--config-list', action
= 'store', type = 'string', dest
= 'config_list', help = 'list of build configurations')
1094 parser
.add_option('-d', '--dump', action
= 'store_true', dest
= 'dump', help = 'dump build configurations and the attached slaves', default
= False)
1095 parser
.add_option('-p', '--source-path', action
= 'store', type = 'string', dest
= 'source_path', help = 'instead of CVS use the given path')
1096 parser
.add_option('-r', '--reset', action
= 'store_true', dest
= 'reset', help = 'reset statistics', default
= False)
1097 parser
.add_option('-s', '--slave-mode', action
= 'store', type = 'string', dest
= 'slave_mode', help = 'enable slave mode', default
= None)
1098 parser
.add_option('-t', '--tests', action
= 'store', type = 'string', dest
= 'tests', help = 'tests to run')
1099 (options
, args
) = parser
.parse_args()
1100 # The slaves are always executing a specific build configuration.
1101 if not options
.config_list
and options
.slave_mode
:
1103 elif options
.addressees
:
1104 titan_builder().dump_addressees()
1106 titan_builder().dump_configs()
1108 builder
= titan_builder()
1109 builder
.build(options
.config_list
, options
.slave_mode
, options
.reset
, options
.set_addressees
, options
.tests
, options
.source_path
)
1112 if __name__
== '__main__':
1116 except SystemExit, e
:
1119 print('Exception caught, writing traceback info to log file `%s\'' \
1121 traceback
.print_exc(file = open(TRACE_FILENAME
, 'at'))
1122 sys
.exit(1) # Don't fall through.
This page took 0.069021 seconds and 5 git commands to generate.