Fix: cleanup for classpath

[deliverable/lttng-ivc.git] / lttng_ivc / utils / project.py

diff --git a/lttng_ivc/utils/project.py b/lttng_ivc/utils/project.py
index fb8c5c89f1ca1336f64edc8f2fc43085e3853dd7..5111819d432ed93c75d890bfff40d7c432c91bb9 100644 (file)
--- a/lttng_ivc/utils/project.py
+++ b/lttng_ivc/utils/project.py
@@ -1,9 +1,34 @@
+# Copyright (c) 2017 Jonathan Rajotte-Julien <jonathan.rajotte-julien@efficios.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+

 import os
 import shutil
 import git
 import subprocess
 import logging
 import os
 import shutil
 import git
 import subprocess
 import logging

+import lttng_ivc.settings as Settings
+
+from lttng_ivc.utils.utils import sha256_checksum
+from lttng_ivc.utils.utils import find_dir, find_file

 
 

+_logger = logging.getLogger('project')

 
 class Project(object):
 
 
 class Project(object):
 


@@ -18,22 +43,32 @@ class Project(object):
         if ccache is not None:
             self.custom_configure_flags.append("CC={} gcc".format(ccache))
             self.custom_configure_flags.append("CXX={} g++".format(ccache))
         if ccache is not None:
             self.custom_configure_flags.append("CC={} gcc".format(ccache))
             self.custom_configure_flags.append("CXX={} g++".format(ccache))

+        self.custom_configure_flags.append("CFLAGS=-g -O0".format(ccache))

 
         """ A collection of Project dependencies """
 
         """ A collection of Project dependencies """

-        self.dependencies = []
+        self.dependencies = {}
+        # used for project cache and pickle validation
+        self._immutable = False
+        self._py_file_checksum = sha256_checksum(Settings.project_py_file_location)

 
         # State
 
         # State

-        self.isCheckedOut = False
-        self.isBootStrapped = False

         self.isBuilt = False
         self.isConfigured = False
         self.isInstalled = False
         self.isBuilt = False
         self.isConfigured = False
         self.isInstalled = False

+        self.skip = False
+
+        self.basedir = tmpdir
+        self.log_path = os.path.join(tmpdir, "log")
+        self.source_path = os.path.join(tmpdir, "source")
+        self.installation_path = os.path.join(tmpdir, "install")

 
 

-        self.source_path = tmpdir + "/source"
-        self.installation_path = tmpdir + "/install"
+        if os.path.isdir(self.basedir):
+            # Perform cleanup since it should not happen
+            shutil.rmtree(self.basedir)
+
+        os.makedirs(self.log_path)

         os.makedirs(self.source_path)
         os.makedirs(self.installation_path)
         os.makedirs(self.source_path)
         os.makedirs(self.installation_path)

-        self.logger = logging.getLogger('project.{}'.format(self.label))

 
         self.special_env_variables = {}
 
 
         self.special_env_variables = {}
 


@@ -41,59 +76,73 @@ class Project(object):
         self.checkout()
         self.bootstrap()
 
         self.checkout()
         self.bootstrap()
 

+    def add_special_env_variable(self, key, value):
+        if key in self.special_env_variables:
+            _logger.warning("{} Special var {} is already defined".format(
+                                self.label, key))
+            raise Exception("Multiple definition of a special environment variable")
+        self.special_env_variables[key] = value
+

     def get_cppflags(self):
     def get_cppflags(self):

-        return " -I{}/include".format(self.installation_path)
+        cppflags = ["-I{}/include".format(self.installation_path)]
+        for key, dep in self.dependencies.items():
+            cppflags.append(dep.get_cppflags())
+
+        return " ".join(cppflags)

 
     def get_ldflags(self):
 
     def get_ldflags(self):

-        return " -L{}/lib".format(self.installation_path)
+        ldflags = ["-L{}/lib".format(self.installation_path)]
+        for key, dep in self.dependencies.items():
+            ldflags.append(dep.get_ldflags())
+        return " ".join(ldflags)

 
     def get_ld_library_path(self):
 
     def get_ld_library_path(self):

-        return "{}/lib".format(self.installation_path)
+        library_path = ["{}/lib".format(self.installation_path)]
+        for key, dep in self.dependencies.items():
+            library_path.append(dep.get_ld_library_path())
+        return ":".join(library_path)
+
+    def get_bin_path(self):
+        bin_path = ["{}/bin".format(self.installation_path)]
+        for key, dep in self.dependencies.items():
+            bin_path.append(dep.get_bin_path())
+        return ":".join(bin_path)

 
     def get_env(self):
         """Modify environment to reflect dependency"""
 
     def get_env(self):
         """Modify environment to reflect dependency"""

-        cpp_flags = ""
-        ld_flags = ""
-        ld_library_path = ""
+        env_var = {"CPPFLAGS": (self.get_cppflags(), " "),
+                   "LDFLAGS": (self.get_ldflags(), " "),
+                   "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"),
+                   }

 
         env = os.environ.copy()
 
         for var, value in self.special_env_variables.items():
             if var in env:
 
         env = os.environ.copy()
 
         for var, value in self.special_env_variables.items():
             if var in env:

-                # TODO: WARNING log point

                 # Raise for now since no special cases is known
                 # Raise for now since no special cases is known

-                self.logger.warning("Special var % is already defined", var)
+                _logger.warning("{} Special var {} is already defined".format(
+                                self.label, var))

                 raise Exception("Multiple definition of a special environment variable")
             else:
                 env[var] = value
 
                 raise Exception("Multiple definition of a special environment variable")
             else:
                 env[var] = value
 

-        for dep in self.dependencies:
+        for key, dep in self.dependencies.items():

             # Extra space just in case
             # Extra space just in case

-            cpp_flags += " {}".format(dep.get_cppflags())
-            ld_flags += " {}".format(dep.get_ldflags())
-            ld_library_path += "{}:".format(dep.get_ld_library_path())

             for var, value in dep.special_env_variables.items():
                 if var in env:
             for var, value in dep.special_env_variables.items():
                 if var in env:

-                    # TODO: WARNING log point

                     # Raise for now since no special cases is known
                     # Raise for now since no special cases is known

-                    self.logger.warning("Special var % is already defined", var)
+                    _logger.warning("{} Special var {} is already defined".format(
+                                self.label, var))

                     raise Exception("Multiple definition of a special environment variable")
                 else:
                     env[var] = value
 
                     raise Exception("Multiple definition of a special environment variable")
                 else:
                     env[var] = value
 

-        # TODO: INFO log point for each variable with project information
-        if cpp_flags:
-            if 'CPPFLAGS' in env:
-                cpp_flags = env['CPPFLAGS'] + cpp_flags
-            env['CPPFLAGS'] = cpp_flags
-        if ld_flags:
-            if 'LDFLAGS' in env:
-                ld_flags = env['LDFLAGS'] + ld_flags
-            env['LDFLAGS'] = ld_flags
-        if ld_library_path:
-            if 'LD_LIBRARY_PATH' in env:
-                ld_library_path = env['LD_LIBRARY_PATH'] + ":" + ld_library_path
-            env['LD_LIBRARY_PATH'] = ld_library_path
+        for var, (value, delimiter) in env_var.items():
+            tmp = [value]
+            if var in env:
+                tmp.append(env[var])
+            env[var] = delimiter.join(tmp)
+

         return env
 
     def autobuild(self):
         return env
 
     def autobuild(self):


@@ -101,17 +150,43 @@ class Project(object):
         Perform the bootstrap, configuration, build and install the
         project. Build dependencies if not already built
         """
         Perform the bootstrap, configuration, build and install the
         project. Build dependencies if not already built
         """

-        for dep in self.dependencies:
+        if (self.isConfigured and self.isBuilt and self.isInstalled):
+            return
+
+        if self._immutable:
+            raise Exception("Object is immutable. Illegal autobuild")
+
+        for key, dep in self.dependencies.items():

             dep.autobuild()
 
             dep.autobuild()
 

-        if self.isCheckedOut ^ self.isBootStrapped ^ self.isBootStrapped ^ self.isBuilt ^ self.isConfigured ^ self.isInstalled:
+        if self.isConfigured ^ self.isBuilt ^ self.isInstalled:

             raise Exception("Project steps where manually triggered. Can't autobuild")
 
             raise Exception("Project steps where manually triggered. Can't autobuild")
 

-        self.configure()
-        self.build()
-        self.install()
+        _logger.debug("{} Autobuild configure".format(self.label))
+        try:
+            self.configure()
+        except subprocess.CalledProcessError as e:
+            _logger.error("{} Configure failed. See {} for more details.".format(self.label, self.log_path))
+            raise e
+
+        _logger.debug("{} Autobuild build".format(self.label))
+        try:
+            self.build()
+        except subprocess.CalledProcessError as e:
+            _logger.error("{} Build failed. See {} for more details.".format(self.label, self.log_path))
+            raise e
+
+        _logger.debug("{} Autobuild install".format(self.label))
+        try:
+            self.install()
+        except subprocess.CalledProcessError as e:
+            _logger.error("{} Install failed. See {} for more details.".format(self.label, self.log_path))
+            raise e

 
     def checkout(self):
 
     def checkout(self):

+        if self._immutable:
+            raise Exception("Object is immutable. Illegal checkout")
+

         repo = git.Repo.clone_from(self.git_path, self.source_path)
         commit = repo.commit(self.sha1)
         repo.head.reference = commit
         repo = git.Repo.clone_from(self.git_path, self.source_path)
         commit = repo.commit(self.sha1)
         repo.head.reference = commit


@@ -123,9 +198,15 @@ class Project(object):
         Bootstap the project. Raise subprocess.CalledProcessError on
         bootstrap error.
         """
         Bootstap the project. Raise subprocess.CalledProcessError on
         bootstrap error.
         """

+        if self._immutable:
+            raise Exception("Object is immutable. Illegal bootstrap")
+
+        out = os.path.join(self.log_path, "bootstrap.out")
+        err = os.path.join(self.log_path, "bootstrap.err")
+

         os.chdir(self.source_path)
         os.chdir(self.source_path)

-        p = subprocess.run(['./bootstrap'], stdout=subprocess.PIPE,
-                           stderr=subprocess.PIPE)
+        with open(out, 'w') as stdout, open(err, 'w') as stderr:
+            p = subprocess.run(['./bootstrap'], stdout=stdout, stderr=stderr)

         p.check_returncode()
         return p
 
         p.check_returncode()
         return p
 


@@ -134,12 +215,18 @@ class Project(object):
         Configure the project.
         Raises subprocess.CalledProcessError on configure error
         """
         Configure the project.
         Raises subprocess.CalledProcessError on configure error
         """

+        if self._immutable:
+            raise Exception("Object is immutable. Illegal configure")
+

         # Check that all our dependencies were actually installed
         # Check that all our dependencies were actually installed

-        for dep in self.dependencies:
+        for key, dep in self.dependencies.items():

             if not dep.isInstalled:
                 # TODO: Custom exception here Dependency Error
                 raise Exception("Dependency project flagged as not installed")
 
             if not dep.isInstalled:
                 # TODO: Custom exception here Dependency Error
                 raise Exception("Dependency project flagged as not installed")
 

+        out = os.path.join(self.log_path, "configure.out")
+        err = os.path.join(self.log_path, "configure.err")
+

         os.chdir(self.source_path)
         args = ['./configure']
         prefix = '--prefix={}'.format(self.installation_path)
         os.chdir(self.source_path)
         args = ['./configure']
         prefix = '--prefix={}'.format(self.installation_path)


@@ -147,8 +234,9 @@ class Project(object):
         args.extend(self.custom_configure_flags)
 
         # TODO: log output and add INFO log point
         args.extend(self.custom_configure_flags)
 
         # TODO: log output and add INFO log point

-        p = subprocess.run(args, env=self.get_env(), stdout=subprocess.PIPE,
-                           stderr=subprocess.PIPE)
+        with open(out, 'w') as stdout, open(err, 'w') as stderr:
+            p = subprocess.run(args, env=self.get_env(), stdout=stdout,
+                               stderr=stderr)

         p.check_returncode()
         self.isConfigured = True
         return p
         p.check_returncode()
         self.isConfigured = True
         return p


@@ -158,20 +246,27 @@ class Project(object):
         Build the project. Raise subprocess.CalledProcessError on build
         error.
         """
         Build the project. Raise subprocess.CalledProcessError on build
         error.
         """

+        if self._immutable:
+            raise Exception("Object is immutable. Illegal build")
+
+        out = os.path.join(self.log_path, "build.out")
+        err = os.path.join(self.log_path, "build.err")
+

         os.chdir(self.source_path)
         args = ['make']
         env = self.get_env()
         os.chdir(self.source_path)
         args = ['make']
         env = self.get_env()

-        env['CFLAGS'] = '-g -O0'

 
         # Number of usable cpu
         # https://docs.python.org/3/library/os.html#os.cpu_count
         num_cpu = str(len(os.sched_getaffinity(0)))
         args.append('-j')
         args.append(num_cpu)
 
         # Number of usable cpu
         # https://docs.python.org/3/library/os.html#os.cpu_count
         num_cpu = str(len(os.sched_getaffinity(0)))
         args.append('-j')
         args.append(num_cpu)

+        args.append('V=1')

 
         # TODO: log output and add INFO log point with args
 
         # TODO: log output and add INFO log point with args

-        p = subprocess.run(args, env=env, stdout=subprocess.PIPE,
-                           stderr=subprocess.PIPE)
+        with open(out, 'w') as stdout, open(err, 'w') as stderr:
+            p = subprocess.run(args, env=env, stdout=stdout,
+                               stderr=stderr)

         p.check_returncode()
         self.isBuilt = True
         return p
         p.check_returncode()
         self.isBuilt = True
         return p


@@ -181,12 +276,19 @@ class Project(object):
         Install the project. Raise subprocess.CalledProcessError on
         bootstrap error
         """
         Install the project. Raise subprocess.CalledProcessError on
         bootstrap error
         """

+        if self._immutable:
+            raise Exception("Object is immutable. Illegal install")
+
+        out = os.path.join(self.log_path, "install.out")
+        err = os.path.join(self.log_path, "install.err")
+

         os.chdir(self.source_path)
         args = ['make', 'install']
 
         # TODO: log output and add INFO log point
         os.chdir(self.source_path)
         args = ['make', 'install']
 
         # TODO: log output and add INFO log point

-        p = subprocess.run(args, env=self.get_env(), stdout=subprocess.PIPE,
-                           stderr=subprocess.PIPE)
+        with open(out, 'w') as stdout, open(err, 'w') as stderr:
+            p = subprocess.run(args, env=self.get_env(), stdout=stdout,
+                               stderr=stderr)

         p.check_returncode()
         self.isInstalled = True
         return p
         p.check_returncode()
         self.isInstalled = True
         return p


@@ -199,6 +301,11 @@ class Project(object):
 
 
 class Lttng_modules(Project):
 
 
 class Lttng_modules(Project):

+    def __init__(self, label, git_path, sha1, tmpdir):
+        super(Lttng_modules, self).__init__(label=label, git_path=git_path,
+                                            sha1=sha1, tmpdir=tmpdir)
+        self.add_special_env_variable("MODPROBE_OPTIONS","-d {}".format(self.installation_path))
+

     def bootstrap(self):
         pass
 
     def bootstrap(self):
         pass
 


@@ -206,6 +313,8 @@ class Lttng_modules(Project):
         pass
 
     def install(self):
         pass
 
     def install(self):

+        if self._immutable:
+            raise Exception("Object is immutable. Illegal install")

         os.chdir(self.source_path)
         args = ['make', 'INSTALL_MOD_PATH={}'.format(self.installation_path),
                 'modules_install']
         os.chdir(self.source_path)
         args = ['make', 'INSTALL_MOD_PATH={}'.format(self.installation_path),
                 'modules_install']


@@ -219,16 +328,51 @@ class Lttng_modules(Project):
         p.check_returncode()
         self.isInstalled = True
 
         p.check_returncode()
         self.isInstalled = True
 

+    def autobuild(self):
+        try:
+            super(Lttng_modules, self).autobuild()
+        except subprocess.CalledProcessError as e:
+            self.skip = True
+

 
 class Lttng_ust(Project):
     def __init__(self, label, git_path, sha1, tmpdir):
         super(Lttng_ust, self).__init__(label=label, git_path=git_path,
                                         sha1=sha1, tmpdir=tmpdir)
         self.custom_configure_flags.extend(['--disable-man-pages'])
 
 class Lttng_ust(Project):
     def __init__(self, label, git_path, sha1, tmpdir):
         super(Lttng_ust, self).__init__(label=label, git_path=git_path,
                                         sha1=sha1, tmpdir=tmpdir)
         self.custom_configure_flags.extend(['--disable-man-pages'])

+        self.custom_configure_flags.extend(['--enable-python-agent'])
+        self.custom_configure_flags.extend(['--enable-java-agent-jul'])
+
+        jul_path = os.path.join(self.installation_path,
+                "share/java/liblttng-ust-agent.jar")
+        classpath = ":".join([jul_path, '.'])
+        self.add_special_env_variable("CLASSPATH", classpath)
+
+    def install(self):
+        super(Lttng_ust, self).install()
+        python_path = find_dir(self.installation_path, "lttngust")
+        if python_path:
+            # Fetch the parent of lttngust folder
+            python_path = os.path.dirname(python_path)
+            self.add_special_env_variable("PYTHONPATH", python_path)
+

 
 
 class Lttng_tools(Project):
 
 
 class Lttng_tools(Project):

-    pass
+    def __init__(self, label, git_path, sha1, tmpdir):
+        super(Lttng_tools, self).__init__(label=label, git_path=git_path,
+                                        sha1=sha1, tmpdir=tmpdir)
+        self.add_special_env_variable("LTTNG_SESSION_CONFIG_XSD_PATH",
+                os.path.join(self.installation_path, "share/xml/lttng/"))
+
+        # Find the mi xsd
+        for xsd in Settings.mi_xsd_file_name:
+            mi = find_file(self.source_path, xsd)
+            if mi:
+                break
+        if not mi:
+            raise Exception("MI xsd not found")
+        self.mi_xsd = mi

 
 
 class Babeltrace(Project):
 
 
 class Babeltrace(Project):