lttng_ivc/utils/project.py

   1 # Copyright (c) 2017 Jonathan Rajotte-Julien <jonathan.rajotte-julien@efficios.com>
   2 #
   3 # Permission is hereby granted, free of charge, to any person obtaining a copy
   4 # of this software and associated documentation files (the "Software"), to deal
   5 # in the Software without restriction, including without limitation the rights
   6 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   7 # copies of the Software, and to permit persons to whom the Software is
   8 # furnished to do so, subject to the following conditions:
   9 #
  10 # The above copyright notice and this permission notice shall be included in all
  11 # copies or substantial portions of the Software.
  12 #
  13 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  16 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  17 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  18 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  19 # SOFTWARE.
  20
  21 import os
  22 import shutil
  23 import git
  24 import subprocess
  25 import logging
  26 import lttng_ivc.settings as Settings
  27
  28 from lttng_ivc.utils.utils import sha256_checksum
  29 from lttng_ivc.utils.utils import find_dir, find_file
  30
  31 _logger = logging.getLogger('project')
  32
  33 class Project(object):
  34
  35     def __init__(self, label, git_path, sha1, tmpdir):
  36         self.label = label
  37         self.git_path = git_path
  38         self.sha1 = sha1
  39
  40         """ Custom configure flags in the for of ['-x', 'arg']"""
  41         self.custom_configure_flags = []
  42         ccache = shutil.which("ccache")
  43         if ccache is not None:
  44             self.custom_configure_flags.append("CC={} gcc".format(ccache))
  45             self.custom_configure_flags.append("CXX={} g++".format(ccache))
  46         self.custom_configure_flags.append("CFLAGS=-g -O0".format(ccache))
  47
  48         """ A collection of Project dependencies """
  49         self.dependencies = {}
  50         # used for project cache and pickle validation
  51         self._immutable = False
  52         self._py_file_checksum = sha256_checksum(Settings.project_py_file_location)
  53
  54         # State
  55         self.isBuilt = False
  56         self.isConfigured = False
  57         self.isInstalled = False
  58         self.skip = False
  59
  60         self.basedir = tmpdir
  61         self.log_path = os.path.join(tmpdir, "log")
  62         self.source_path = os.path.join(tmpdir, "source")
  63         self.installation_path = os.path.join(tmpdir, "install")
  64
  65         if os.path.isdir(self.basedir):
  66             # Perform cleanup since it should not happen
  67             shutil.rmtree(self.basedir)
  68
  69         os.makedirs(self.log_path)
  70         os.makedirs(self.source_path)
  71         os.makedirs(self.installation_path)
  72
  73         self.special_env_variables = {}
  74
  75         # Init the repo for work
  76         self.checkout()
  77         self.bootstrap()
  78
  79     def add_special_env_variable(self, key, value):
  80         if key in self.special_env_variables:
  81             _logger.warning("{} Special var {} is already defined".format(
  82                                 self.label, key))
  83             raise Exception("Multiple definition of a special environment variable")
  84         self.special_env_variables[key] = value
  85
  86     def get_cppflags(self):
  87         cppflags = ["-I{}/include".format(self.installation_path)]
  88         for key, dep in self.dependencies.items():
  89             cppflags.append(dep.get_cppflags())
  90
  91         return " ".join(cppflags)
  92
  93     def get_ldflags(self):
  94         ldflags = ["-L{}/lib".format(self.installation_path)]
  95         for key, dep in self.dependencies.items():
  96             ldflags.append(dep.get_ldflags())
  97         return " ".join(ldflags)
  98
  99     def get_ld_library_path(self):
 100         library_path = ["{}/lib".format(self.installation_path)]
 101         for key, dep in self.dependencies.items():
 102             library_path.append(dep.get_ld_library_path())
 103         return ":".join(library_path)
 104
 105     def get_bin_path(self):
 106         bin_path = ["{}/bin".format(self.installation_path)]
 107         for key, dep in self.dependencies.items():
 108             bin_path.append(dep.get_bin_path())
 109         return ":".join(bin_path)
 110
 111     def get_env(self):
 112         """Modify environment to reflect dependency"""
 113         env_var = {"CPPFLAGS": (self.get_cppflags(), " "),
 114                    "LDFLAGS": (self.get_ldflags(), " "),
 115                    "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"),
 116                    }
 117
 118         env = os.environ.copy()
 119
 120         for var, value in self.special_env_variables.items():
 121             if var in env:
 122                 # Raise for now since no special cases is known
 123                 _logger.warning("{} Special var {} is already defined".format(
 124                                 self.label, var))
 125                 raise Exception("Multiple definition of a special environment variable")
 126             else:
 127                 env[var] = value
 128
 129         for key, dep in self.dependencies.items():
 130             # Extra space just in case
 131             for var, value in dep.special_env_variables.items():
 132                 if var in env:
 133                     # Raise for now since no special cases is known
 134                     _logger.warning("{} Special var {} is already defined".format(
 135                                 self.label, var))
 136                     raise Exception("Multiple definition of a special environment variable")
 137                 else:
 138                     env[var] = value
 139
 140         for var, (value, delimiter) in env_var.items():
 141             tmp = [value]
 142             if var in env:
 143                 tmp.append(env[var])
 144             env[var] = delimiter.join(tmp)
 145
 146         return env
 147
 148     def autobuild(self):
 149         """
 150         Perform the bootstrap, configuration, build and install the
 151         project. Build dependencies if not already built
 152         """
 153         if (self.isConfigured and self.isBuilt and self.isInstalled):
 154             return
 155
 156         if self._immutable:
 157             raise Exception("Object is immutable. Illegal autobuild")
 158
 159         for key, dep in self.dependencies.items():
 160             dep.autobuild()
 161
 162         if self.isConfigured ^ self.isBuilt ^ self.isInstalled:
 163             raise Exception("Project steps where manually triggered. Can't autobuild")
 164
 165         _logger.debug("{} Autobuild configure".format(self.label))
 166         try:
 167             self.configure()
 168         except subprocess.CalledProcessError as e:
 169             _logger.error("{} Configure failed. See {} for more details.".format(self.label, self.log_path))
 170             raise e
 171
 172         _logger.debug("{} Autobuild build".format(self.label))
 173         try:
 174             self.build()
 175         except subprocess.CalledProcessError as e:
 176             _logger.error("{} Build failed. See {} for more details.".format(self.label, self.log_path))
 177             raise e
 178
 179         _logger.debug("{} Autobuild install".format(self.label))
 180         try:
 181             self.install()
 182         except subprocess.CalledProcessError as e:
 183             _logger.error("{} Install failed. See {} for more details.".format(self.label, self.log_path))
 184             raise e
 185
 186     def checkout(self):
 187         if self._immutable:
 188             raise Exception("Object is immutable. Illegal checkout")
 189
 190         repo = git.Repo.clone_from(self.git_path, self.source_path)
 191         commit = repo.commit(self.sha1)
 192         repo.head.reference = commit
 193         assert repo.head.is_detached
 194         repo.head.reset(index=True, working_tree=True)
 195
 196     def bootstrap(self):
 197         """
 198         Bootstap the project. Raise subprocess.CalledProcessError on
 199         bootstrap error.
 200         """
 201         if self._immutable:
 202             raise Exception("Object is immutable. Illegal bootstrap")
 203
 204         out = os.path.join(self.log_path, "bootstrap.out")
 205         err = os.path.join(self.log_path, "bootstrap.err")
 206
 207         os.chdir(self.source_path)
 208         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 209             p = subprocess.run(['./bootstrap'], stdout=stdout, stderr=stderr)
 210         p.check_returncode()
 211         return p
 212
 213     def configure(self):
 214         """
 215         Configure the project.
 216         Raises subprocess.CalledProcessError on configure error
 217         """
 218         if self._immutable:
 219             raise Exception("Object is immutable. Illegal configure")
 220
 221         # Check that all our dependencies were actually installed
 222         for key, dep in self.dependencies.items():
 223             if not dep.isInstalled:
 224                 # TODO: Custom exception here Dependency Error
 225                 raise Exception("Dependency project flagged as not installed")
 226
 227         out = os.path.join(self.log_path, "configure.out")
 228         err = os.path.join(self.log_path, "configure.err")
 229
 230         os.chdir(self.source_path)
 231         args = ['./configure']
 232         prefix = '--prefix={}'.format(self.installation_path)
 233         args.append(prefix)
 234         args.extend(self.custom_configure_flags)
 235
 236         # TODO: log output and add INFO log point
 237         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 238             p = subprocess.run(args, env=self.get_env(), stdout=stdout,
 239                                stderr=stderr)
 240         p.check_returncode()
 241         self.isConfigured = True
 242         return p
 243
 244     def build(self):
 245         """
 246         Build the project. Raise subprocess.CalledProcessError on build
 247         error.
 248         """
 249         if self._immutable:
 250             raise Exception("Object is immutable. Illegal build")
 251
 252         out = os.path.join(self.log_path, "build.out")
 253         err = os.path.join(self.log_path, "build.err")
 254
 255         os.chdir(self.source_path)
 256         args = ['make']
 257         env = self.get_env()
 258
 259         # Number of usable cpu
 260         # https://docs.python.org/3/library/os.html#os.cpu_count
 261         num_cpu = str(len(os.sched_getaffinity(0)))
 262         args.append('-j')
 263         args.append(num_cpu)
 264         args.append('V=1')
 265
 266         # TODO: log output and add INFO log point with args
 267         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 268             p = subprocess.run(args, env=env, stdout=stdout,
 269                                stderr=stderr)
 270         p.check_returncode()
 271         self.isBuilt = True
 272         return p
 273
 274     def install(self):
 275         """
 276         Install the project. Raise subprocess.CalledProcessError on
 277         bootstrap error
 278         """
 279         if self._immutable:
 280             raise Exception("Object is immutable. Illegal install")
 281
 282         out = os.path.join(self.log_path, "install.out")
 283         err = os.path.join(self.log_path, "install.err")
 284
 285         os.chdir(self.source_path)
 286         args = ['make', 'install']
 287
 288         # TODO: log output and add INFO log point
 289         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 290             p = subprocess.run(args, env=self.get_env(), stdout=stdout,
 291                                stderr=stderr)
 292         p.check_returncode()
 293         self.isInstalled = True
 294         return p
 295
 296     def cleanup(self):
 297         if os.path.exists(self.source_path):
 298             shutil.rmtree(self.source_path)
 299         if os.path.exists(self.installation_path):
 300             shutil.rmtree(self.installation_path)
 301
 302
 303 class Lttng_modules(Project):
 304     def __init__(self, label, git_path, sha1, tmpdir):
 305         super(Lttng_modules, self).__init__(label=label, git_path=git_path,
 306                                             sha1=sha1, tmpdir=tmpdir)
 307         self.add_special_env_variable("MODPROBE_OPTIONS","-d {}".format(self.installation_path))
 308
 309     def bootstrap(self):
 310         pass
 311
 312     def configure(self):
 313         pass
 314
 315     def install(self):
 316         if self._immutable:
 317             raise Exception("Object is immutable. Illegal install")
 318         os.chdir(self.source_path)
 319         args = ['make', 'INSTALL_MOD_PATH={}'.format(self.installation_path),
 320                 'modules_install']
 321         p = subprocess.run(args, env=self.get_env(), stdout=subprocess.PIPE,
 322                            stderr=subprocess.PIPE)
 323         p.check_returncode()
 324
 325         # Perform a local depmod
 326         args = ['depmod', '-b', self.installation_path]
 327         p = subprocess.run(args, env=self.get_env())
 328         p.check_returncode()
 329         self.isInstalled = True
 330
 331     def autobuild(self):
 332         try:
 333             super(Lttng_modules, self).autobuild()
 334         except subprocess.CalledProcessError as e:
 335             self.skip = True
 336
 337
 338 class Lttng_ust(Project):
 339     def __init__(self, label, git_path, sha1, tmpdir):
 340         super(Lttng_ust, self).__init__(label=label, git_path=git_path,
 341                                         sha1=sha1, tmpdir=tmpdir)
 342         self.custom_configure_flags.extend(['--disable-man-pages'])
 343         self.custom_configure_flags.extend(['--enable-python-agent'])
 344         self.custom_configure_flags.extend(['--enable-java-agent-jul'])
 345
 346         jul_path = os.path.join(self.installation_path,
 347                 "share/java/liblttng-ust-agent.jar")
 348         classpath = ":".join([jul_path, '.'])
 349         self.add_special_env_variable("CLASSPATH", classpath)
 350
 351     def install(self):
 352         super(Lttng_ust, self).install()
 353         python_path = find_dir(self.installation_path, "lttngust")
 354         if python_path:
 355             # Fetch the parent of lttngust folder
 356             python_path = os.path.dirname(python_path)
 357             self.add_special_env_variable("PYTHONPATH", python_path)
 358
 359
 360
 361 class Lttng_tools(Project):
 362     def __init__(self, label, git_path, sha1, tmpdir):
 363         super(Lttng_tools, self).__init__(label=label, git_path=git_path,
 364                                         sha1=sha1, tmpdir=tmpdir)
 365         self.add_special_env_variable("LTTNG_SESSION_CONFIG_XSD_PATH",
 366                 os.path.join(self.installation_path, "share/xml/lttng/"))
 367
 368         # Find the mi xsd
 369         for xsd in Settings.mi_xsd_file_name:
 370             mi = find_file(self.source_path, xsd)
 371             if mi:
 372                 break
 373         if not mi:
 374             raise Exception("MI xsd not found")
 375         self.mi_xsd = mi
 376
 377
 378 class Babeltrace(Project):
 379     pass