lttng_ivc/utils/project.py

   1 import os
   2 import shutil
   3 import git
   4 import subprocess
   5 import logging
   6
   7 _logger = logging.getLogger('project')
   8
   9 class Project(object):
  10
  11     def __init__(self, label, git_path, sha1, tmpdir):
  12         self.label = label
  13         self.git_path = git_path
  14         self.sha1 = sha1
  15
  16         """ Custom configure flags in the for of ['-x', 'arg']"""
  17         self.custom_configure_flags = []
  18         ccache = shutil.which("ccache")
  19         if ccache is not None:
  20             self.custom_configure_flags.append("CC={} gcc".format(ccache))
  21             self.custom_configure_flags.append("CXX={} g++".format(ccache))
  22         self.custom_configure_flags.append("CFLAGS=-g -O0".format(ccache))
  23
  24         """ A collection of Project dependencies """
  25         self.dependencies = {}
  26         self._immutable = False
  27
  28         # State
  29         self.isBuilt = False
  30         self.isConfigured = False
  31         self.isInstalled = False
  32
  33         self.basedir = tmpdir
  34         self.log_path = os.path.join(tmpdir, "log")
  35         self.source_path = os.path.join(tmpdir, "source")
  36         self.installation_path = os.path.join(tmpdir, "install")
  37
  38         if os.path.isdir(self.basedir):
  39             # Perform cleanup since it should not happen
  40             shutil.rmtree(self.basedir)
  41
  42         os.makedirs(self.log_path)
  43         os.makedirs(self.source_path)
  44         os.makedirs(self.installation_path)
  45
  46         self.special_env_variables = {}
  47
  48         # Init the repo for work
  49         self.checkout()
  50         self.bootstrap()
  51
  52     def add_special_env_variable(self, key, value):
  53         if key in self.special_env_variables:
  54             _logger.warning("{} Special var {} is already defined".format(
  55                                 self.label, key))
  56             raise Exception("Multiple definition of a special environment variable")
  57         self.special_env_variables[key] = value
  58
  59     def get_cppflags(self):
  60         cppflags = ["-I{}/include".format(self.installation_path)]
  61         for key, dep in self.dependencies.items():
  62             cppflags.append(dep.get_cppflags())
  63
  64         return " ".join(cppflags)
  65
  66     def get_ldflags(self):
  67         ldflags = ["-L{}/lib".format(self.installation_path)]
  68         for key, dep in self.dependencies.items():
  69             ldflags.append(dep.get_ldflags())
  70         return " ".join(ldflags)
  71
  72     def get_ld_library_path(self):
  73         library_path = ["{}/lib".format(self.installation_path)]
  74         for key, dep in self.dependencies.items():
  75             library_path.append(dep.get_ld_library_path())
  76         return ":".join(library_path)
  77
  78     def get_bin_path(self):
  79         bin_path = ["{}/bin".format(self.installation_path)]
  80         for key, dep in self.dependencies.items():
  81             bin_path.append(dep.get_bin_path())
  82         return ":".join(bin_path)
  83
  84     def get_env(self):
  85         """Modify environment to reflect dependency"""
  86         env_var = {"CPPFLAGS": (self.get_cppflags(), " "),
  87                    "LDFLAGS": (self.get_ldflags(), " "),
  88                    "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"),
  89                    }
  90
  91         env = os.environ.copy()
  92
  93         for var, value in self.special_env_variables.items():
  94             if var in env:
  95                 # Raise for now since no special cases is known
  96                 _logger.warning("{} Special var {} is already defined".format(
  97                                 self.label, var))
  98                 raise Exception("Multiple definition of a special environment variable")
  99             else:
 100                 env[var] = value
 101
 102         for key, dep in self.dependencies.items():
 103             # Extra space just in case
 104             for var, value in dep.special_env_variables.items():
 105                 if var in env:
 106                     # Raise for now since no special cases is known
 107                     _logger.warning("{} Special var {} is already defined".format(
 108                                 self.label, var))
 109                     raise Exception("Multiple definition of a special environment variable")
 110                 else:
 111                     env[var] = value
 112
 113         for var, (value, delimiter) in env_var.items():
 114             tmp = [value]
 115             if var in env:
 116                 tmp.append(env[var])
 117             env[var] = delimiter.join(tmp)
 118
 119         return env
 120
 121     def autobuild(self):
 122         """
 123         Perform the bootstrap, configuration, build and install the
 124         project. Build dependencies if not already built
 125         """
 126         if (self.isConfigured and self.isBuilt and self.isInstalled):
 127             return
 128
 129         if self._immutable:
 130             raise Exception("Object is immutable. Illegal autobuild")
 131
 132         for key, dep in self.dependencies.items():
 133             dep.autobuild()
 134
 135         if self.isConfigured ^ self.isBuilt ^ self.isInstalled:
 136             raise Exception("Project steps where manually triggered. Can't autobuild")
 137
 138         _logger.debug("{} Autobuild configure".format(self.label))
 139         try:
 140             self.configure()
 141         except subprocess.CalledProcessError as e:
 142             _logger.error("{} Configure failed. See {} for more details.".format(self.label, self.log_path))
 143             raise e
 144
 145         _logger.debug("{} Autobuild build".format(self.label))
 146         try:
 147             self.build()
 148         except subprocess.CalledProcessError as e:
 149             _logger.error("{} Build failed. See {} for more details.".format(self.label, self.log_path))
 150             raise e
 151
 152         _logger.debug("{} Autobuild install".format(self.label))
 153         try:
 154             self.install()
 155         except subprocess.CalledProcessError as e:
 156             _logger.error("{} Install failed. See {} for more details.".format(self.label, self.log_path))
 157             raise e
 158
 159     def checkout(self):
 160         if self._immutable:
 161             raise Exception("Object is immutable. Illegal checkout")
 162
 163         repo = git.Repo.clone_from(self.git_path, self.source_path)
 164         commit = repo.commit(self.sha1)
 165         repo.head.reference = commit
 166         assert repo.head.is_detached
 167         repo.head.reset(index=True, working_tree=True)
 168
 169     def bootstrap(self):
 170         """
 171         Bootstap the project. Raise subprocess.CalledProcessError on
 172         bootstrap error.
 173         """
 174         if self._immutable:
 175             raise Exception("Object is immutable. Illegal bootstrap")
 176
 177         out = os.path.join(self.log_path, "bootstrap.out")
 178         err = os.path.join(self.log_path, "bootstrap.err")
 179
 180         os.chdir(self.source_path)
 181         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 182             p = subprocess.run(['./bootstrap'], stdout=stdout, stderr=stderr)
 183         p.check_returncode()
 184         return p
 185
 186     def configure(self):
 187         """
 188         Configure the project.
 189         Raises subprocess.CalledProcessError on configure error
 190         """
 191         if self._immutable:
 192             raise Exception("Object is immutable. Illegal configure")
 193
 194         # Check that all our dependencies were actually installed
 195         for key, dep in self.dependencies.items():
 196             if not dep.isInstalled:
 197                 # TODO: Custom exception here Dependency Error
 198                 raise Exception("Dependency project flagged as not installed")
 199
 200         out = os.path.join(self.log_path, "configure.out")
 201         err = os.path.join(self.log_path, "configure.err")
 202
 203         os.chdir(self.source_path)
 204         args = ['./configure']
 205         prefix = '--prefix={}'.format(self.installation_path)
 206         args.append(prefix)
 207         args.extend(self.custom_configure_flags)
 208
 209         # TODO: log output and add INFO log point
 210         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 211             p = subprocess.run(args, env=self.get_env(), stdout=stdout,
 212                                stderr=stderr)
 213         p.check_returncode()
 214         self.isConfigured = True
 215         return p
 216
 217     def build(self):
 218         """
 219         Build the project. Raise subprocess.CalledProcessError on build
 220         error.
 221         """
 222         if self._immutable:
 223             raise Exception("Object is immutable. Illegal build")
 224
 225         out = os.path.join(self.log_path, "build.out")
 226         err = os.path.join(self.log_path, "build.err")
 227
 228         os.chdir(self.source_path)
 229         args = ['make']
 230         env = self.get_env()
 231
 232         # Number of usable cpu
 233         # https://docs.python.org/3/library/os.html#os.cpu_count
 234         num_cpu = str(len(os.sched_getaffinity(0)))
 235         args.append('-j')
 236         args.append(num_cpu)
 237         args.append('V=1')
 238
 239         # TODO: log output and add INFO log point with args
 240         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 241             p = subprocess.run(args, env=env, stdout=stdout,
 242                                stderr=stderr)
 243         p.check_returncode()
 244         self.isBuilt = True
 245         return p
 246
 247     def install(self):
 248         """
 249         Install the project. Raise subprocess.CalledProcessError on
 250         bootstrap error
 251         """
 252         if self._immutable:
 253             raise Exception("Object is immutable. Illegal install")
 254
 255         out = os.path.join(self.log_path, "install.out")
 256         err = os.path.join(self.log_path, "install.err")
 257
 258         os.chdir(self.source_path)
 259         args = ['make', 'install']
 260
 261         # TODO: log output and add INFO log point
 262         with open(out, 'w') as stdout, open(err, 'w') as stderr:
 263             p = subprocess.run(args, env=self.get_env(), stdout=stdout,
 264                                stderr=stderr)
 265         p.check_returncode()
 266         self.isInstalled = True
 267         return p
 268
 269     def cleanup(self):
 270         if os.path.exists(self.source_path):
 271             shutil.rmtree(self.source_path)
 272         if os.path.exists(self.installation_path):
 273             shutil.rmtree(self.installation_path)
 274
 275
 276 class Lttng_modules(Project):
 277     def __init__(self, label, git_path, sha1, tmpdir):
 278         super(Lttng_modules, self).__init__(label=label, git_path=git_path,
 279                                             sha1=sha1, tmpdir=tmpdir)
 280         self.add_special_env_variable("MODPROBE_OPTIONS","-b {}".format(self.installation_path))
 281
 282     def bootstrap(self):
 283         pass
 284
 285     def configure(self):
 286         pass
 287
 288     def install(self):
 289         if self._immutable:
 290             raise Exception("Object is immutable. Illegal install")
 291         os.chdir(self.source_path)
 292         args = ['make', 'INSTALL_MOD_PATH={}'.format(self.installation_path),
 293                 'modules_install']
 294         p = subprocess.run(args, env=self.get_env(), stdout=subprocess.PIPE,
 295                            stderr=subprocess.PIPE)
 296         p.check_returncode()
 297
 298         # Perform a local depmod
 299         args = ['depmod', '-b', self.installation_path]
 300         p = subprocess.run(args, env=self.get_env())
 301         p.check_returncode()
 302         self.isInstalled = True
 303
 304
 305 class Lttng_ust(Project):
 306     def __init__(self, label, git_path, sha1, tmpdir):
 307         super(Lttng_ust, self).__init__(label=label, git_path=git_path,
 308                                         sha1=sha1, tmpdir=tmpdir)
 309         self.custom_configure_flags.extend(['--disable-man-pages'])
 310
 311
 312 class Lttng_tools(Project):
 313     def __init__(self, label, git_path, sha1, tmpdir):
 314         super(Lttng_tools, self).__init__(label=label, git_path=git_path,
 315                                         sha1=sha1, tmpdir=tmpdir)
 316         self.add_special_env_variable("LTTNG_SESSION_CONFIG_XSD_PATH",
 317                 os.path.join(self.installation_path, "share/xml/lttng/"))
 318
 319
 320 class Babeltrace(Project):
 321     pass