lttng_ivc/utils/project.py

   1 import os
   2 import shutil
   3 import git
   4 import subprocess
   5 import logging
   6 import lttng_ivc.settings as Settings
   7
   8 from lttng_ivc.utils.utils import sha256_checksum
   9 from lttng_ivc.utils.utils import find_dir, find_file
  10
  11 _logger = logging.getLogger('project')
  12
  13 class Project(object):
  14
  15     def __init__(self, label, git_path, sha1, tmpdir):
  16         self.label = label
  17         self.git_path = git_path
  18         self.sha1 = sha1
  19
  20         """ Custom configure flags in the for of ['-x', 'arg']"""
  21         self.custom_configure_flags = []
  22         ccache = shutil.which("ccache")
  23         if ccache is not None:
  24             self.custom_configure_flags.append("CC={} gcc".format(ccache))
  25             self.custom_configure_flags.append("CXX={} g++".format(ccache))
  26         self.custom_configure_flags.append("CFLAGS=-g -O0".format(ccache))
  27
  28         """ A collection of Project dependencies """
  29         self.dependencies = {}
  30         # used for project cache and pickle validation
  31         self._immutable = False
  32         self._py_file_checksum = sha256_checksum(Settings.project_py_file_location)
  33
  34         # State
  35         self.isBuilt = False
  36         self.isConfigured = False
  37         self.isInstalled = False
  38         self.skip = False
  39
  40         self.basedir = tmpdir
  41         self.log_path = os.path.join(tmpdir, "log")
  42         self.source_path = os.path.join(tmpdir, "source")
  43         self.installation_path = os.path.join(tmpdir, "install")
  44
  45         if os.path.isdir(self.basedir):
  46             # Perform cleanup since it should not happen
  47             shutil.rmtree(self.basedir)
  48
  49         os.makedirs(self.log_path)
  50         os.makedirs(self.source_path)
  51         os.makedirs(self.installation_path)
  52
  53         self.special_env_variables = {}
  54
  55         # Init the repo for work
  56         self.checkout()
  57         self.bootstrap()
  58
  59     def add_special_env_variable(self, key, value):
  60         if key in self.special_env_variables:
  61             _logger.warning("{} Special var {} is already defined".format(
  62                                 self.label, key))
  63             raise Exception("Multiple definition of a special environment variable")
  64         self.special_env_variables[key] = value
  65
  66     def get_cppflags(self):
  67         cppflags = ["-I{}/include".format(self.installation_path)]
  68         for key, dep in self.dependencies.items():
  69             cppflags.append(dep.get_cppflags())
  70
  71         return " ".join(cppflags)
  72
  73     def get_ldflags(self):
  74         ldflags = ["-L{}/lib".format(self.installation_path)]
  75         for key, dep in self.dependencies.items():
  76             ldflags.append(dep.get_ldflags())
  77         return " ".join(ldflags)
  78
  79     def get_ld_library_path(self):
  80         library_path = ["{}/lib".format(self.installation_path)]
  81         for key, dep in self.dependencies.items():
  82             library_path.append(dep.get_ld_library_path())
  83         return ":".join(library_path)
  84
  85     def get_bin_path(self):
  86         bin_path = ["{}/bin".format(self.installation_path)]
  87         for key, dep in self.dependencies.items():
  88             bin_path.append(dep.get_bin_path())
  89         return ":".join(bin_path)
  90
  91     def get_env(self):
  92         """Modify environment to reflect dependency"""
  93         env_var = {"CPPFLAGS": (self.get_cppflags(), " "),
  94                    "LDFLAGS": (self.get_ldflags(), " "),
  95                    "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"),
  96                    }
  97
  98         env = os.environ.copy()
  99
 100         for var, value in self.special_env_variables.items():
 101             if var in env:
 102                 # Raise for now since no special cases is known
 103                 _logger.warning("{} Special var {} is already defined".format(
 104                                 self.label, var))
 105                 raise Exception("Multiple definition of a special environment variable")
 106             else:
 107                 env[var] = value
 108
 109         for key, dep in self.dependencies.items():
 110             # Extra space just in case
 111             for var, value in dep.special_env_variables.items():
 112                 if var in env:
 113                     # Raise for now since no special cases is known
 114                     _logger.warning("{} Special var {} is already defined".format(
 115                                 self.label, var))
 116                     raise Exception("Multiple definition of a special environment variable")
 117                 else:
 118                     env[var] = value
 119
 120         for var, (value, delimiter) in env_var.items():
 121             tmp = [value]
 122             if var in env:
 123                 tmp.append(env[var])
 124             env[var] = delimiter.join(tmp)
 125
 126         return env
 127
 128     def autobuild(self):
 129         """
 130         Perform the bootstrap, configuration, build and install the
 131         project. Build dependencies if not already built
 132         """
 133         if (self.isConfigured and self.isBuilt and self.isInstalled):
 134             return
 135
 136         if self._immutable:
 137             raise Exception("Object is immutable. Illegal autobuild")
 138
 139         for key, dep in self.dependencies.items():
 140             dep.autobuild()
 141
 142         if self.isConfigured ^ self.isBuilt ^ self.isInstalled:
 143             raise Exception("Project steps where manually triggered. Can't autobuild")
 144
 145         _logger.debug("{} Autobuild configure".format(self.label))
 146         try:
 147             self.configure()
 148         except subprocess.CalledProcessError as e:
 149             _logger.error("{} Configure failed. See {} for more details.".format(self.label, self.log_path))
 150             raise e
 151
 152         _logger.debug("{} Autobuild build".format(self.label))
 153         try:
 154             self.build()
 155         except subprocess.CalledProcessError as e:
 156             _logger.error("{} Build failed. See {} for more details.".format(self.label, self.log_path))
 157             raise e
 158
 159         _logger.debug("{} Autobuild install".format(self.label))
 160         try:
 161             self.install()
 162         except subprocess.CalledProcessError as e:
 163             _logger.error("{} Install failed. See {} for more details.".format(self.label, self.log_path))
 164             raise e
 165
 166     def checkout(self):
 167         if self._immutable:
 168             raise Exception("Object is immutable. Illegal checkout")
 169
 170         repo = git.Repo.clone_from(self.git_path, self.source_path)
 171         commit = repo.commit(self.sha1)
 172         repo.head.reference = commit
 173         assert repo.head.is_detached
 174         repo.head.reset(index=True, working_tree=True)
 175
 176     def bootstrap(self):
 177         """
 178         Bootstap the project. Raise subprocess.CalledProcessError on
 179         bootstrap error.
 180         """
 181         if self._immutable:
 182             raise Exception("Object is immutable. Illegal bootstrap")
 183
 184         out = os.path.join(self.log_path, "bootstrap.out")
 185         err = os.path.join(self.log_path, "bootstrap.err")
 186
 187         os.chdir(self.source_path)
 188         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 189             p = subprocess.run(['./bootstrap'], stdout=stdout, stderr=stderr)
 190         p.check_returncode()
 191         return p
 192
 193     def configure(self):
 194         """
 195         Configure the project.
 196         Raises subprocess.CalledProcessError on configure error
 197         """
 198         if self._immutable:
 199             raise Exception("Object is immutable. Illegal configure")
 200
 201         # Check that all our dependencies were actually installed
 202         for key, dep in self.dependencies.items():
 203             if not dep.isInstalled:
 204                 # TODO: Custom exception here Dependency Error
 205                 raise Exception("Dependency project flagged as not installed")
 206
 207         out = os.path.join(self.log_path, "configure.out")
 208         err = os.path.join(self.log_path, "configure.err")
 209
 210         os.chdir(self.source_path)
 211         args = ['./configure']
 212         prefix = '--prefix={}'.format(self.installation_path)
 213         args.append(prefix)
 214         args.extend(self.custom_configure_flags)
 215
 216         # TODO: log output and add INFO log point
 217         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 218             p = subprocess.run(args, env=self.get_env(), stdout=stdout,
 219                                stderr=stderr)
 220         p.check_returncode()
 221         self.isConfigured = True
 222         return p
 223
 224     def build(self):
 225         """
 226         Build the project. Raise subprocess.CalledProcessError on build
 227         error.
 228         """
 229         if self._immutable:
 230             raise Exception("Object is immutable. Illegal build")
 231
 232         out = os.path.join(self.log_path, "build.out")
 233         err = os.path.join(self.log_path, "build.err")
 234
 235         os.chdir(self.source_path)
 236         args = ['make']
 237         env = self.get_env()
 238
 239         # Number of usable cpu
 240         # https://docs.python.org/3/library/os.html#os.cpu_count
 241         num_cpu = str(len(os.sched_getaffinity(0)))
 242         args.append('-j')
 243         args.append(num_cpu)
 244         args.append('V=1')
 245
 246         # TODO: log output and add INFO log point with args
 247         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 248             p = subprocess.run(args, env=env, stdout=stdout,
 249                                stderr=stderr)
 250         p.check_returncode()
 251         self.isBuilt = True
 252         return p
 253
 254     def install(self):
 255         """
 256         Install the project. Raise subprocess.CalledProcessError on
 257         bootstrap error
 258         """
 259         if self._immutable:
 260             raise Exception("Object is immutable. Illegal install")
 261
 262         out = os.path.join(self.log_path, "install.out")
 263         err = os.path.join(self.log_path, "install.err")
 264
 265         os.chdir(self.source_path)
 266         args = ['make', 'install']
 267
 268         # TODO: log output and add INFO log point
 269         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 270             p = subprocess.run(args, env=self.get_env(), stdout=stdout,
 271                                stderr=stderr)
 272         p.check_returncode()
 273         self.isInstalled = True
 274         return p
 275
 276     def cleanup(self):
 277         if os.path.exists(self.source_path):
 278             shutil.rmtree(self.source_path)
 279         if os.path.exists(self.installation_path):
 280             shutil.rmtree(self.installation_path)
 281
 282
 283 class Lttng_modules(Project):
 284     def __init__(self, label, git_path, sha1, tmpdir):
 285         super(Lttng_modules, self).__init__(label=label, git_path=git_path,
 286                                             sha1=sha1, tmpdir=tmpdir)
 287         self.add_special_env_variable("MODPROBE_OPTIONS","-d {}".format(self.installation_path))
 288
 289     def bootstrap(self):
 290         pass
 291
 292     def configure(self):
 293         pass
 294
 295     def install(self):
 296         if self._immutable:
 297             raise Exception("Object is immutable. Illegal install")
 298         os.chdir(self.source_path)
 299         args = ['make', 'INSTALL_MOD_PATH={}'.format(self.installation_path),
 300                 'modules_install']
 301         p = subprocess.run(args, env=self.get_env(), stdout=subprocess.PIPE,
 302                            stderr=subprocess.PIPE)
 303         p.check_returncode()
 304
 305         # Perform a local depmod
 306         args = ['depmod', '-b', self.installation_path]
 307         p = subprocess.run(args, env=self.get_env())
 308         p.check_returncode()
 309         self.isInstalled = True
 310
 311     def autobuild(self):
 312         try:
 313             super(Lttng_modules, self).autobuild()
 314         except subprocess.CalledProcessError as e:
 315             self.skip = True
 316
 317
 318 class Lttng_ust(Project):
 319     def __init__(self, label, git_path, sha1, tmpdir):
 320         super(Lttng_ust, self).__init__(label=label, git_path=git_path,
 321                                         sha1=sha1, tmpdir=tmpdir)
 322         self.custom_configure_flags.extend(['--disable-man-pages'])
 323         self.custom_configure_flags.extend(['--enable-python-agent'])
 324         self.custom_configure_flags.extend(['--enable-java-agent-jul'])
 325
 326         jul_path = os.path.join(self.installation_path,
 327                 "share/java/liblttng-ust-agent.jar")
 328         classpath = ":".join([jul_path, Settings.log4j_class_path, '.'])
 329         self.add_special_env_variable("CLASSPATH", classpath)
 330
 331     def install(self):
 332         super(Lttng_ust, self).install()
 333         python_path = find_dir(self.installation_path, "lttngust")
 334         if python_path:
 335             # Fetch the parent of lttngust folder
 336             python_path = os.path.dirname(python_path)
 337             self.add_special_env_variable("PYTHONPATH", python_path)
 338
 339
 340
 341 class Lttng_tools(Project):
 342     def __init__(self, label, git_path, sha1, tmpdir):
 343         super(Lttng_tools, self).__init__(label=label, git_path=git_path,
 344                                         sha1=sha1, tmpdir=tmpdir)
 345         self.add_special_env_variable("LTTNG_SESSION_CONFIG_XSD_PATH",
 346                 os.path.join(self.installation_path, "share/xml/lttng/"))
 347
 348         # Find the mi xsd
 349         for xsd in Settings.mi_xsd_file_name:
 350             mi = find_file(self.source_path, xsd)
 351             if mi:
 352                 break
 353         if not mi:
 354             raise Exception("MI xsd not found")
 355         self.mi_xsd = mi
 356
 357
 358 class Babeltrace(Project):
 359     pass