lttng_ivc/utils/runtime.py

   1 import os
   2 import sys
   3 import shlex
   4 import subprocess
   5 import uuid
   6 import logging
   7 import shutil
   8 import contextlib
   9 import pprint
  10
  11 from tempfile import TemporaryDirectory
  12
  13 import lttng_ivc.settings as Settings
  14 _logger = logging.getLogger("Runtime")
  15
  16
  17 @contextlib.contextmanager
  18 def get_runtime(runtime_dir):
  19     runtime = Runtime(runtime_dir)
  20     try:
  21         yield runtime
  22     finally:
  23         runtime.close()
  24
  25
  26 class Runtime(object):
  27     def __init__(self, runtime_dir):
  28         """
  29         A dictionary of popen object eg. lttng-sessiond, relayd,
  30         anything really. Key is a uuid.
  31         """
  32         self.__subprocess = {}
  33         self.__stdout_stderr = {}
  34         self.__projects = []
  35
  36         self.__runtime_log = os.path.join(runtime_dir, "log")
  37         self.__runtime_log_sub = os.path.join(self.__runtime_log, "subprocess")
  38
  39         """
  40         Path of the copy of lttng_home folder after Runtime.close() is issued. This is
  41         to be used for post runtime analysis and mostly debugging on error.
  42         """
  43         self.__post_runtime_lttng_home_path = os.path.join(runtime_dir,
  44                 "lttng_home")
  45
  46         self._runtime_log_aggregation = os.path.join(self.__runtime_log, "runtime.log")
  47
  48         self._run_command_count = 0
  49
  50         self.special_env_variables = {"LTTNG_UST_DEBUG": "1",
  51                                       "LTTNG_APP_SOCKET_TIMEOUT": "-1",
  52                                       #"LTTNG_UST_REGISTER_TIMEOUT": "-1",
  53                                       "LTTNG_NETWORK_SOCKET_TIMEOUT": "-1"}
  54
  55         # Keep a reference on the object to keep it alive. It will close/clean on
  56         # exit.
  57         self.__lttng_home_dir = TemporaryDirectory(prefix=Settings.tmp_object_prefix)
  58         self.lttng_home = self.__lttng_home_dir.name
  59
  60         if len(self.lttng_home) > 88:
  61             raise Exception("TemporaryDirectory for lttng_home is to long. Use a short TMPDIR")
  62
  63         os.makedirs(self.__runtime_log)
  64         os.makedirs(self.__runtime_log_sub)
  65
  66     def add_project(self, project):
  67         self.__projects.append(project)
  68
  69     def remove_project(self, project):
  70         self.__projects.remove(project)
  71
  72     def subprocess_signal(self, subprocess_uuid, signal):
  73         self.__subproces[subprocess_uuid].send_signal(signal)
  74
  75     def subprocess_terminate(self, subprocess_uuid, timeout=60):
  76         process = self.__subprocess[subprocess_uuid]
  77         process.terminate()
  78         process.wait(timeout)
  79         stdout, stderr = self.__stdout_stderr[subprocess_uuid]
  80         stdout.close()
  81         stderr.close()
  82         return process
  83
  84     def subprocess_kill(self, subprocess_uuid):
  85         process = self.__subprocess[subprocess_uuid]
  86         process.kill()
  87         process.wait()
  88         stdout, stderr = self.__stdout_stderr[subprocess_uuid]
  89         stdout.close()
  90         stderr.close()
  91         return process
  92
  93     def get_subprocess_stdout_path(self, subprocess_uuid):
  94         stdout, stderr = self.__stdout_stderr[subprocess_uuid]
  95         return stdout.name
  96
  97     def get_subprocess_stderr_path(self, subprocess_uuid):
  98         stdout, stderr = self.__stdout_stderr[subprocess_uuid]
  99         return stderr.name
 100
 101     def spawn_subprocess(self, command_line, cwd=None):
 102         args = shlex.split(command_line)
 103         env = self.get_env()
 104
 105         if not os.path.isdir(self.lttng_home):
 106             raise Exception("lttng home does not exist")
 107
 108         tmp_id = uuid.uuid1()
 109         out_path = os.path.join(self.__runtime_log_sub, str(tmp_id) + ".out")
 110         err_path = os.path.join(self.__runtime_log_sub, str(tmp_id) + ".err")
 111
 112         stdout = open(out_path, 'w')
 113         stderr = open(err_path, 'w')
 114
 115         env_path = os.path.join(self.__runtime_log_sub, str(tmp_id) + ".env")
 116         with open(env_path, 'w') as env_out:
 117             pprint.pprint(env, stream=env_out)
 118
 119         p = subprocess.Popen(args, stdout=stdout, stderr=stderr, env=env, cwd=cwd)
 120         self.__subprocess[tmp_id] = p
 121         self.__stdout_stderr[tmp_id] = (stdout, stderr)
 122         _logger.debug("Spawned sub pid: {} args: {} stdout: {} stderr{}".format(p.pid, p.args, out_path, err_path))
 123         return tmp_id
 124
 125     def run(self, command_line, cwd=None, check_return=True, ld_preload="", classpath=""):
 126         """
 127         Run the command and return a tuple of a (CompletedProcess, stdout_path,
 128         stderr_path). The subprocess is already executed and returned. The
 129         callecaller is responsible for checking for errors.
 130         """
 131         args = shlex.split(command_line)
 132         env = self.get_env()
 133
 134         if ld_preload:
 135             env['LD_PRELOAD'] = ld_preload
 136         if classpath:
 137             env['CLASSPATH'] = classpath
 138
 139
 140         tmp_id = self._run_command_count
 141         self._run_command_count += 1
 142
 143         cmd_map = os.path.join(self.__runtime_log, "cmd.map")
 144         with open(cmd_map, 'a') as out:
 145             out.write("{}: {}\n".format(tmp_id, args))
 146
 147         out_path = os.path.join(self.__runtime_log, str(tmp_id) + ".out")
 148         err_path = os.path.join(self.__runtime_log, str(tmp_id) + ".err")
 149         stdout = open(out_path, "w")
 150         stderr = open(err_path, "w")
 151
 152         env_path = os.path.join(self.__runtime_log, str(tmp_id) + ".env")
 153         with open(env_path, 'w') as env_out:
 154             for key, value in env.items():
 155                 env_out.write('{}={}\n'.format(key, value))
 156
 157         cp = subprocess.run(args, stdout=stdout, stderr=stderr, env=env, cwd=cwd)
 158         _logger.debug("Command #{} args: {} stdout: {} stderr{}".format(tmp_id, cp.args, out_path, err_path))
 159
 160         # Add to the global log file. This can help a little. Leave the other
 161         # file available for per-run analysis
 162         with open(self._runtime_log_aggregation, "a") as log:
 163             with open(out_path, "r") as out:
 164                 log.write("Output for command #{} {}\n".format(tmp_id, command_line))
 165                 log.write("Start >>>>>>>>>>>>>>>>\n")
 166                 log.write(out.read())
 167                 log.write("End <<<<<<<<<<<<<<<<\n")
 168             with open(err_path, "r") as out:
 169                 log.write("Error for command #{} {}\n".format(tmp_id, command_line))
 170                 log.write("Start >>>>>>>>>>>>>>>>\n")
 171                 log.write(out.read())
 172                 log.write("End <<<<<<<<<<<<<<<<\n")
 173
 174         if check_return:
 175             cp.check_returncode()
 176
 177         return (cp, out_path, err_path)
 178
 179     def get_cppflags(self):
 180         cppflags = []
 181         for project in self.__projects:
 182             cppflags.append(project.get_cppflags())
 183         return " ".join(cppflags)
 184
 185     def get_ldflags(self):
 186         ldflags = []
 187         for project in self.__projects:
 188             ldflags.append(project.get_ldflags())
 189         return " ".join(ldflags)
 190
 191     def get_ld_library_path(self):
 192         library_path = []
 193         for project in self.__projects:
 194             library_path.append(project.get_ld_library_path())
 195         return ":".join(library_path)
 196
 197     def get_bin_path(self):
 198         path = []
 199         for project in self.__projects:
 200             path.append(project.get_bin_path())
 201         return ":".join(path)
 202
 203     def get_env(self):
 204         env = os.environ.copy()
 205
 206         env["LTTNG_HOME"] = self.lttng_home
 207
 208         env_fetch = {"CPPFLAGS": (self.get_cppflags(), " "),
 209                      "LDFLAGS": (self.get_ldflags(), " "),
 210                      "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"),
 211                      "PATH": (self.get_bin_path(), ":"),
 212                      }
 213         for key, (value, delimiter) in env_fetch.items():
 214             tmp_var = ""
 215             if key in env:
 216                 tmp_var = env[key]
 217             env[key] = delimiter.join([value, tmp_var])
 218
 219         for var, value in self.special_env_variables.items():
 220             if var in env:
 221                 # Raise for now since no special cases is known
 222                 _logger.warning("% Special var % is already defined",
 223                                 self.label, var)
 224                 raise Exception("Multiple definition of a special environment variable")
 225             else:
 226                 env[var] = value
 227
 228         for project in self.__projects:
 229             for var, value in project.special_env_variables.items():
 230                 if var in env:
 231                     # Raise for now since no special cases is known
 232                     _logger.warning("% Special var % is already defined",
 233                                     self.label, var)
 234                     raise Exception("Multiple definition of a special environment variable")
 235                 else:
 236                     env[var] = value
 237         return env
 238
 239     def load_test_module(self):
 240         # Base directory is provided by env
 241         self.run("modprobe lttng-test")
 242
 243     def unload_test_module(self, check_return=True):
 244         # Base directory is provided by env
 245         self.run("modprobe -r lttng-test", check_return=check_return)
 246
 247     def close(self):
 248         for key, subp in self.__subprocess.items():
 249             subp.terminate()
 250         for key, subp in self.__subprocess.items():
 251             # TODO move timeout to settings
 252             subp.wait(timeout=60)
 253         for key, (stdout, stderr) in self.__stdout_stderr.items():
 254             stdout.close()
 255             stderr.close()
 256
 257         # Always try to remove test module but do not perform check on return
 258         # value.
 259         self.unload_test_module(False)
 260
 261         # Copy the lttng_home used at runtime using hardlink to prevent useless
 262         # data duplication
 263         shutil.copytree(self.lttng_home, self.__post_runtime_lttng_home_path, copy_function=os.link)