lttng_ivc/utils/project.py

   1 import os
   2 import shutil
   3 import git
   4 import subprocess
   5 import logging
   6 import lttng_ivc.settings as Settings
   7
   8 from lttng_ivc.utils.utils import sha256_checksum
   9
  10 _logger = logging.getLogger('project')
  11
  12 class Project(object):
  13
  14     def __init__(self, label, git_path, sha1, tmpdir):
  15         self.label = label
  16         self.git_path = git_path
  17         self.sha1 = sha1
  18
  19         """ Custom configure flags in the for of ['-x', 'arg']"""
  20         self.custom_configure_flags = []
  21         ccache = shutil.which("ccache")
  22         if ccache is not None:
  23             self.custom_configure_flags.append("CC={} gcc".format(ccache))
  24             self.custom_configure_flags.append("CXX={} g++".format(ccache))
  25         self.custom_configure_flags.append("CFLAGS=-g -O0".format(ccache))
  26
  27         """ A collection of Project dependencies """
  28         self.dependencies = {}
  29         # used for project cache and pickle validation
  30         self._immutable = False
  31         self._py_file_checksum = sha256_checksum(Settings.project_py_file_location)
  32
  33         # State
  34         self.isBuilt = False
  35         self.isConfigured = False
  36         self.isInstalled = False
  37
  38         self.basedir = tmpdir
  39         self.log_path = os.path.join(tmpdir, "log")
  40         self.source_path = os.path.join(tmpdir, "source")
  41         self.installation_path = os.path.join(tmpdir, "install")
  42
  43         if os.path.isdir(self.basedir):
  44             # Perform cleanup since it should not happen
  45             shutil.rmtree(self.basedir)
  46
  47         os.makedirs(self.log_path)
  48         os.makedirs(self.source_path)
  49         os.makedirs(self.installation_path)
  50
  51         self.special_env_variables = {}
  52
  53         # Init the repo for work
  54         self.checkout()
  55         self.bootstrap()
  56
  57     def add_special_env_variable(self, key, value):
  58         if key in self.special_env_variables:
  59             _logger.warning("{} Special var {} is already defined".format(
  60                                 self.label, key))
  61             raise Exception("Multiple definition of a special environment variable")
  62         self.special_env_variables[key] = value
  63
  64     def get_cppflags(self):
  65         cppflags = ["-I{}/include".format(self.installation_path)]
  66         for key, dep in self.dependencies.items():
  67             cppflags.append(dep.get_cppflags())
  68
  69         return " ".join(cppflags)
  70
  71     def get_ldflags(self):
  72         ldflags = ["-L{}/lib".format(self.installation_path)]
  73         for key, dep in self.dependencies.items():
  74             ldflags.append(dep.get_ldflags())
  75         return " ".join(ldflags)
  76
  77     def get_ld_library_path(self):
  78         library_path = ["{}/lib".format(self.installation_path)]
  79         for key, dep in self.dependencies.items():
  80             library_path.append(dep.get_ld_library_path())
  81         return ":".join(library_path)
  82
  83     def get_bin_path(self):
  84         bin_path = ["{}/bin".format(self.installation_path)]
  85         for key, dep in self.dependencies.items():
  86             bin_path.append(dep.get_bin_path())
  87         return ":".join(bin_path)
  88
  89     def get_env(self):
  90         """Modify environment to reflect dependency"""
  91         env_var = {"CPPFLAGS": (self.get_cppflags(), " "),
  92                    "LDFLAGS": (self.get_ldflags(), " "),
  93                    "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"),
  94                    }
  95
  96         env = os.environ.copy()
  97
  98         for var, value in self.special_env_variables.items():
  99             if var in env:
 100                 # Raise for now since no special cases is known
 101                 _logger.warning("{} Special var {} is already defined".format(
 102                                 self.label, var))
 103                 raise Exception("Multiple definition of a special environment variable")
 104             else:
 105                 env[var] = value
 106
 107         for key, dep in self.dependencies.items():
 108             # Extra space just in case
 109             for var, value in dep.special_env_variables.items():
 110                 if var in env:
 111                     # Raise for now since no special cases is known
 112                     _logger.warning("{} Special var {} is already defined".format(
 113                                 self.label, var))
 114                     raise Exception("Multiple definition of a special environment variable")
 115                 else:
 116                     env[var] = value
 117
 118         for var, (value, delimiter) in env_var.items():
 119             tmp = [value]
 120             if var in env:
 121                 tmp.append(env[var])
 122             env[var] = delimiter.join(tmp)
 123
 124         return env
 125
 126     def autobuild(self):
 127         """
 128         Perform the bootstrap, configuration, build and install the
 129         project. Build dependencies if not already built
 130         """
 131         if (self.isConfigured and self.isBuilt and self.isInstalled):
 132             return
 133
 134         if self._immutable:
 135             raise Exception("Object is immutable. Illegal autobuild")
 136
 137         for key, dep in self.dependencies.items():
 138             dep.autobuild()
 139
 140         if self.isConfigured ^ self.isBuilt ^ self.isInstalled:
 141             raise Exception("Project steps where manually triggered. Can't autobuild")
 142
 143         _logger.debug("{} Autobuild configure".format(self.label))
 144         try:
 145             self.configure()
 146         except subprocess.CalledProcessError as e:
 147             _logger.error("{} Configure failed. See {} for more details.".format(self.label, self.log_path))
 148             raise e
 149
 150         _logger.debug("{} Autobuild build".format(self.label))
 151         try:
 152             self.build()
 153         except subprocess.CalledProcessError as e:
 154             _logger.error("{} Build failed. See {} for more details.".format(self.label, self.log_path))
 155             raise e
 156
 157         _logger.debug("{} Autobuild install".format(self.label))
 158         try:
 159             self.install()
 160         except subprocess.CalledProcessError as e:
 161             _logger.error("{} Install failed. See {} for more details.".format(self.label, self.log_path))
 162             raise e
 163
 164     def checkout(self):
 165         if self._immutable:
 166             raise Exception("Object is immutable. Illegal checkout")
 167
 168         repo = git.Repo.clone_from(self.git_path, self.source_path)
 169         commit = repo.commit(self.sha1)
 170         repo.head.reference = commit
 171         assert repo.head.is_detached
 172         repo.head.reset(index=True, working_tree=True)
 173
 174     def bootstrap(self):
 175         """
 176         Bootstap the project. Raise subprocess.CalledProcessError on
 177         bootstrap error.
 178         """
 179         if self._immutable:
 180             raise Exception("Object is immutable. Illegal bootstrap")
 181
 182         out = os.path.join(self.log_path, "bootstrap.out")
 183         err = os.path.join(self.log_path, "bootstrap.err")
 184
 185         os.chdir(self.source_path)
 186         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 187             p = subprocess.run(['./bootstrap'], stdout=stdout, stderr=stderr)
 188         p.check_returncode()
 189         return p
 190
 191     def configure(self):
 192         """
 193         Configure the project.
 194         Raises subprocess.CalledProcessError on configure error
 195         """
 196         if self._immutable:
 197             raise Exception("Object is immutable. Illegal configure")
 198
 199         # Check that all our dependencies were actually installed
 200         for key, dep in self.dependencies.items():
 201             if not dep.isInstalled:
 202                 # TODO: Custom exception here Dependency Error
 203                 raise Exception("Dependency project flagged as not installed")
 204
 205         out = os.path.join(self.log_path, "configure.out")
 206         err = os.path.join(self.log_path, "configure.err")
 207
 208         os.chdir(self.source_path)
 209         args = ['./configure']
 210         prefix = '--prefix={}'.format(self.installation_path)
 211         args.append(prefix)
 212         args.extend(self.custom_configure_flags)
 213
 214         # TODO: log output and add INFO log point
 215         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 216             p = subprocess.run(args, env=self.get_env(), stdout=stdout,
 217                                stderr=stderr)
 218         p.check_returncode()
 219         self.isConfigured = True
 220         return p
 221
 222     def build(self):
 223         """
 224         Build the project. Raise subprocess.CalledProcessError on build
 225         error.
 226         """
 227         if self._immutable:
 228             raise Exception("Object is immutable. Illegal build")
 229
 230         out = os.path.join(self.log_path, "build.out")
 231         err = os.path.join(self.log_path, "build.err")
 232
 233         os.chdir(self.source_path)
 234         args = ['make']
 235         env = self.get_env()
 236
 237         # Number of usable cpu
 238         # https://docs.python.org/3/library/os.html#os.cpu_count
 239         num_cpu = str(len(os.sched_getaffinity(0)))
 240         args.append('-j')
 241         args.append(num_cpu)
 242         args.append('V=1')
 243
 244         # TODO: log output and add INFO log point with args
 245         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 246             p = subprocess.run(args, env=env, stdout=stdout,
 247                                stderr=stderr)
 248         p.check_returncode()
 249         self.isBuilt = True
 250         return p
 251
 252     def install(self):
 253         """
 254         Install the project. Raise subprocess.CalledProcessError on
 255         bootstrap error
 256         """
 257         if self._immutable:
 258             raise Exception("Object is immutable. Illegal install")
 259
 260         out = os.path.join(self.log_path, "install.out")
 261         err = os.path.join(self.log_path, "install.err")
 262
 263         os.chdir(self.source_path)
 264         args = ['make', 'install']
 265
 266         # TODO: log output and add INFO log point
 267         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 268             p = subprocess.run(args, env=self.get_env(), stdout=stdout,
 269                                stderr=stderr)
 270         p.check_returncode()
 271         self.isInstalled = True
 272         return p
 273
 274     def cleanup(self):
 275         if os.path.exists(self.source_path):
 276             shutil.rmtree(self.source_path)
 277         if os.path.exists(self.installation_path):
 278             shutil.rmtree(self.installation_path)
 279
 280
 281 class Lttng_modules(Project):
 282     def __init__(self, label, git_path, sha1, tmpdir):
 283         super(Lttng_modules, self).__init__(label=label, git_path=git_path,
 284                                             sha1=sha1, tmpdir=tmpdir)
 285         self.add_special_env_variable("MODPROBE_OPTIONS","-d {}".format(self.installation_path))
 286
 287     def bootstrap(self):
 288         pass
 289
 290     def configure(self):
 291         pass
 292
 293     def install(self):
 294         if self._immutable:
 295             raise Exception("Object is immutable. Illegal install")
 296         os.chdir(self.source_path)
 297         args = ['make', 'INSTALL_MOD_PATH={}'.format(self.installation_path),
 298                 'modules_install']
 299         p = subprocess.run(args, env=self.get_env(), stdout=subprocess.PIPE,
 300                            stderr=subprocess.PIPE)
 301         p.check_returncode()
 302
 303         # Perform a local depmod
 304         args = ['depmod', '-b', self.installation_path]
 305         p = subprocess.run(args, env=self.get_env())
 306         p.check_returncode()
 307         self.isInstalled = True
 308
 309
 310 class Lttng_ust(Project):
 311     def __init__(self, label, git_path, sha1, tmpdir):
 312         super(Lttng_ust, self).__init__(label=label, git_path=git_path,
 313                                         sha1=sha1, tmpdir=tmpdir)
 314         self.custom_configure_flags.extend(['--disable-man-pages'])
 315
 316
 317 class Lttng_tools(Project):
 318     def __init__(self, label, git_path, sha1, tmpdir):
 319         super(Lttng_tools, self).__init__(label=label, git_path=git_path,
 320                                         sha1=sha1, tmpdir=tmpdir)
 321         self.add_special_env_variable("LTTNG_SESSION_CONFIG_XSD_PATH",
 322                 os.path.join(self.installation_path, "share/xml/lttng/"))
 323
 324
 325 class Babeltrace(Project):
 326     pass