X-Git-Url: https://git.efficios.com/?a=blobdiff_plain;f=lttng_ivc%2Futils%2Fproject.py;h=5111819d432ed93c75d890bfff40d7c432c91bb9;hb=8e2c5f79e23839ec13866fc4827a1c402aab1687;hp=fb8c5c89f1ca1336f64edc8f2fc43085e3853dd7;hpb=de9b991ba1163882d83d4b60030014fedb8300cc;p=deliverable%2Flttng-ivc.git diff --git a/lttng_ivc/utils/project.py b/lttng_ivc/utils/project.py index fb8c5c8..5111819 100644 --- a/lttng_ivc/utils/project.py +++ b/lttng_ivc/utils/project.py @@ -1,9 +1,34 @@ +# Copyright (c) 2017 Jonathan Rajotte-Julien +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + import os import shutil import git import subprocess import logging +import lttng_ivc.settings as Settings + +from lttng_ivc.utils.utils import sha256_checksum +from lttng_ivc.utils.utils import find_dir, find_file +_logger = logging.getLogger('project') class Project(object): @@ -18,22 +43,32 @@ class Project(object): if ccache is not None: self.custom_configure_flags.append("CC={} gcc".format(ccache)) self.custom_configure_flags.append("CXX={} g++".format(ccache)) + self.custom_configure_flags.append("CFLAGS=-g -O0".format(ccache)) """ A collection of Project dependencies """ - self.dependencies = [] + self.dependencies = {} + # used for project cache and pickle validation + self._immutable = False + self._py_file_checksum = sha256_checksum(Settings.project_py_file_location) # State - self.isCheckedOut = False - self.isBootStrapped = False self.isBuilt = False self.isConfigured = False self.isInstalled = False + self.skip = False + + self.basedir = tmpdir + self.log_path = os.path.join(tmpdir, "log") + self.source_path = os.path.join(tmpdir, "source") + self.installation_path = os.path.join(tmpdir, "install") - self.source_path = tmpdir + "/source" - self.installation_path = tmpdir + "/install" + if os.path.isdir(self.basedir): + # Perform cleanup since it should not happen + shutil.rmtree(self.basedir) + + os.makedirs(self.log_path) os.makedirs(self.source_path) os.makedirs(self.installation_path) - self.logger = logging.getLogger('project.{}'.format(self.label)) self.special_env_variables = {} @@ -41,59 +76,73 @@ class Project(object): self.checkout() self.bootstrap() + def add_special_env_variable(self, key, value): + if key in self.special_env_variables: + _logger.warning("{} Special var {} is already defined".format( + self.label, key)) + raise Exception("Multiple definition of a special environment variable") + self.special_env_variables[key] = value + def get_cppflags(self): - return " -I{}/include".format(self.installation_path) + cppflags = ["-I{}/include".format(self.installation_path)] + for key, dep in self.dependencies.items(): + cppflags.append(dep.get_cppflags()) + + return " ".join(cppflags) def get_ldflags(self): - return " -L{}/lib".format(self.installation_path) + ldflags = ["-L{}/lib".format(self.installation_path)] + for key, dep in self.dependencies.items(): + ldflags.append(dep.get_ldflags()) + return " ".join(ldflags) def get_ld_library_path(self): - return "{}/lib".format(self.installation_path) + library_path = ["{}/lib".format(self.installation_path)] + for key, dep in self.dependencies.items(): + library_path.append(dep.get_ld_library_path()) + return ":".join(library_path) + + def get_bin_path(self): + bin_path = ["{}/bin".format(self.installation_path)] + for key, dep in self.dependencies.items(): + bin_path.append(dep.get_bin_path()) + return ":".join(bin_path) def get_env(self): """Modify environment to reflect dependency""" - cpp_flags = "" - ld_flags = "" - ld_library_path = "" + env_var = {"CPPFLAGS": (self.get_cppflags(), " "), + "LDFLAGS": (self.get_ldflags(), " "), + "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"), + } env = os.environ.copy() for var, value in self.special_env_variables.items(): if var in env: - # TODO: WARNING log point # Raise for now since no special cases is known - self.logger.warning("Special var % is already defined", var) + _logger.warning("{} Special var {} is already defined".format( + self.label, var)) raise Exception("Multiple definition of a special environment variable") else: env[var] = value - for dep in self.dependencies: + for key, dep in self.dependencies.items(): # Extra space just in case - cpp_flags += " {}".format(dep.get_cppflags()) - ld_flags += " {}".format(dep.get_ldflags()) - ld_library_path += "{}:".format(dep.get_ld_library_path()) for var, value in dep.special_env_variables.items(): if var in env: - # TODO: WARNING log point # Raise for now since no special cases is known - self.logger.warning("Special var % is already defined", var) + _logger.warning("{} Special var {} is already defined".format( + self.label, var)) raise Exception("Multiple definition of a special environment variable") else: env[var] = value - # TODO: INFO log point for each variable with project information - if cpp_flags: - if 'CPPFLAGS' in env: - cpp_flags = env['CPPFLAGS'] + cpp_flags - env['CPPFLAGS'] = cpp_flags - if ld_flags: - if 'LDFLAGS' in env: - ld_flags = env['LDFLAGS'] + ld_flags - env['LDFLAGS'] = ld_flags - if ld_library_path: - if 'LD_LIBRARY_PATH' in env: - ld_library_path = env['LD_LIBRARY_PATH'] + ":" + ld_library_path - env['LD_LIBRARY_PATH'] = ld_library_path + for var, (value, delimiter) in env_var.items(): + tmp = [value] + if var in env: + tmp.append(env[var]) + env[var] = delimiter.join(tmp) + return env def autobuild(self): @@ -101,17 +150,43 @@ class Project(object): Perform the bootstrap, configuration, build and install the project. Build dependencies if not already built """ - for dep in self.dependencies: + if (self.isConfigured and self.isBuilt and self.isInstalled): + return + + if self._immutable: + raise Exception("Object is immutable. Illegal autobuild") + + for key, dep in self.dependencies.items(): dep.autobuild() - if self.isCheckedOut ^ self.isBootStrapped ^ self.isBootStrapped ^ self.isBuilt ^ self.isConfigured ^ self.isInstalled: + if self.isConfigured ^ self.isBuilt ^ self.isInstalled: raise Exception("Project steps where manually triggered. Can't autobuild") - self.configure() - self.build() - self.install() + _logger.debug("{} Autobuild configure".format(self.label)) + try: + self.configure() + except subprocess.CalledProcessError as e: + _logger.error("{} Configure failed. See {} for more details.".format(self.label, self.log_path)) + raise e + + _logger.debug("{} Autobuild build".format(self.label)) + try: + self.build() + except subprocess.CalledProcessError as e: + _logger.error("{} Build failed. See {} for more details.".format(self.label, self.log_path)) + raise e + + _logger.debug("{} Autobuild install".format(self.label)) + try: + self.install() + except subprocess.CalledProcessError as e: + _logger.error("{} Install failed. See {} for more details.".format(self.label, self.log_path)) + raise e def checkout(self): + if self._immutable: + raise Exception("Object is immutable. Illegal checkout") + repo = git.Repo.clone_from(self.git_path, self.source_path) commit = repo.commit(self.sha1) repo.head.reference = commit @@ -123,9 +198,15 @@ class Project(object): Bootstap the project. Raise subprocess.CalledProcessError on bootstrap error. """ + if self._immutable: + raise Exception("Object is immutable. Illegal bootstrap") + + out = os.path.join(self.log_path, "bootstrap.out") + err = os.path.join(self.log_path, "bootstrap.err") + os.chdir(self.source_path) - p = subprocess.run(['./bootstrap'], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + with open(out, 'w') as stdout, open(err, 'w') as stderr: + p = subprocess.run(['./bootstrap'], stdout=stdout, stderr=stderr) p.check_returncode() return p @@ -134,12 +215,18 @@ class Project(object): Configure the project. Raises subprocess.CalledProcessError on configure error """ + if self._immutable: + raise Exception("Object is immutable. Illegal configure") + # Check that all our dependencies were actually installed - for dep in self.dependencies: + for key, dep in self.dependencies.items(): if not dep.isInstalled: # TODO: Custom exception here Dependency Error raise Exception("Dependency project flagged as not installed") + out = os.path.join(self.log_path, "configure.out") + err = os.path.join(self.log_path, "configure.err") + os.chdir(self.source_path) args = ['./configure'] prefix = '--prefix={}'.format(self.installation_path) @@ -147,8 +234,9 @@ class Project(object): args.extend(self.custom_configure_flags) # TODO: log output and add INFO log point - p = subprocess.run(args, env=self.get_env(), stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + with open(out, 'w') as stdout, open(err, 'w') as stderr: + p = subprocess.run(args, env=self.get_env(), stdout=stdout, + stderr=stderr) p.check_returncode() self.isConfigured = True return p @@ -158,20 +246,27 @@ class Project(object): Build the project. Raise subprocess.CalledProcessError on build error. """ + if self._immutable: + raise Exception("Object is immutable. Illegal build") + + out = os.path.join(self.log_path, "build.out") + err = os.path.join(self.log_path, "build.err") + os.chdir(self.source_path) args = ['make'] env = self.get_env() - env['CFLAGS'] = '-g -O0' # Number of usable cpu # https://docs.python.org/3/library/os.html#os.cpu_count num_cpu = str(len(os.sched_getaffinity(0))) args.append('-j') args.append(num_cpu) + args.append('V=1') # TODO: log output and add INFO log point with args - p = subprocess.run(args, env=env, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + with open(out, 'w') as stdout, open(err, 'w') as stderr: + p = subprocess.run(args, env=env, stdout=stdout, + stderr=stderr) p.check_returncode() self.isBuilt = True return p @@ -181,12 +276,19 @@ class Project(object): Install the project. Raise subprocess.CalledProcessError on bootstrap error """ + if self._immutable: + raise Exception("Object is immutable. Illegal install") + + out = os.path.join(self.log_path, "install.out") + err = os.path.join(self.log_path, "install.err") + os.chdir(self.source_path) args = ['make', 'install'] # TODO: log output and add INFO log point - p = subprocess.run(args, env=self.get_env(), stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + with open(out, 'w') as stdout, open(err, 'w') as stderr: + p = subprocess.run(args, env=self.get_env(), stdout=stdout, + stderr=stderr) p.check_returncode() self.isInstalled = True return p @@ -199,6 +301,11 @@ class Project(object): class Lttng_modules(Project): + def __init__(self, label, git_path, sha1, tmpdir): + super(Lttng_modules, self).__init__(label=label, git_path=git_path, + sha1=sha1, tmpdir=tmpdir) + self.add_special_env_variable("MODPROBE_OPTIONS","-d {}".format(self.installation_path)) + def bootstrap(self): pass @@ -206,6 +313,8 @@ class Lttng_modules(Project): pass def install(self): + if self._immutable: + raise Exception("Object is immutable. Illegal install") os.chdir(self.source_path) args = ['make', 'INSTALL_MOD_PATH={}'.format(self.installation_path), 'modules_install'] @@ -219,16 +328,51 @@ class Lttng_modules(Project): p.check_returncode() self.isInstalled = True + def autobuild(self): + try: + super(Lttng_modules, self).autobuild() + except subprocess.CalledProcessError as e: + self.skip = True + class Lttng_ust(Project): def __init__(self, label, git_path, sha1, tmpdir): super(Lttng_ust, self).__init__(label=label, git_path=git_path, sha1=sha1, tmpdir=tmpdir) self.custom_configure_flags.extend(['--disable-man-pages']) + self.custom_configure_flags.extend(['--enable-python-agent']) + self.custom_configure_flags.extend(['--enable-java-agent-jul']) + + jul_path = os.path.join(self.installation_path, + "share/java/liblttng-ust-agent.jar") + classpath = ":".join([jul_path, '.']) + self.add_special_env_variable("CLASSPATH", classpath) + + def install(self): + super(Lttng_ust, self).install() + python_path = find_dir(self.installation_path, "lttngust") + if python_path: + # Fetch the parent of lttngust folder + python_path = os.path.dirname(python_path) + self.add_special_env_variable("PYTHONPATH", python_path) + class Lttng_tools(Project): - pass + def __init__(self, label, git_path, sha1, tmpdir): + super(Lttng_tools, self).__init__(label=label, git_path=git_path, + sha1=sha1, tmpdir=tmpdir) + self.add_special_env_variable("LTTNG_SESSION_CONFIG_XSD_PATH", + os.path.join(self.installation_path, "share/xml/lttng/")) + + # Find the mi xsd + for xsd in Settings.mi_xsd_file_name: + mi = find_file(self.source_path, xsd) + if mi: + break + if not mi: + raise Exception("MI xsd not found") + self.mi_xsd = mi class Babeltrace(Project):