lttng_ivc/utils/runtime.py

   1 # Copyright (c) 2017 Jonathan Rajotte-Julien <jonathan.rajotte-julien@efficios.com>
   2 #
   3 # Permission is hereby granted, free of charge, to any person obtaining a copy
   4 # of this software and associated documentation files (the "Software"), to deal
   5 # in the Software without restriction, including without limitation the rights
   6 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   7 # copies of the Software, and to permit persons to whom the Software is
   8 # furnished to do so, subject to the following conditions:
   9 #
  10 # The above copyright notice and this permission notice shall be included in all
  11 # copies or substantial portions of the Software.
  12 #
  13 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  16 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  17 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  18 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  19 # SOFTWARE.
  20
  21 import os
  22 import sys
  23 import shlex
  24 import subprocess
  25 import uuid
  26 import logging
  27 import shutil
  28 import contextlib
  29 import pprint
  30 import signal
  31
  32 from tempfile import TemporaryDirectory
  33
  34 import lttng_ivc.settings as Settings
  35 _logger = logging.getLogger("Runtime")
  36
  37
  38 @contextlib.contextmanager
  39 def get_runtime(runtime_dir):
  40     runtime = Runtime(runtime_dir)
  41     try:
  42         yield runtime
  43     finally:
  44         runtime.close()
  45
  46
  47 class Runtime(object):
  48     def __init__(self, runtime_dir):
  49         """
  50         A dictionary of popen object eg. lttng-sessiond, relayd,
  51         anything really. Key is a uuid.
  52         """
  53         self.__subprocess = {}
  54         self.__stdout_stderr = {}
  55         self.__projects = []
  56
  57         self.__runtime_log = os.path.join(runtime_dir, "log")
  58         self.__runtime_log_sub = os.path.join(self.__runtime_log, "subprocess")
  59
  60         """
  61         Path of the copy of lttng_home folder after Runtime.close() is issued. This is
  62         to be used for post runtime analysis and mostly debugging on error.
  63         """
  64         self.__post_runtime_lttng_home_path = os.path.join(runtime_dir,
  65                 "lttng_home")
  66
  67         self._runtime_log_aggregation = os.path.join(self.__runtime_log, "runtime.log")
  68
  69         self._run_command_count = 0
  70         self._is_test_modules_loaded = False
  71
  72         self.special_env_variables = {"LTTNG_UST_DEBUG": "1",
  73                                       "LTTNG_APP_SOCKET_TIMEOUT": "-1",
  74                                       #"LTTNG_UST_REGISTER_TIMEOUT": "-1",
  75                                       "LTTNG_NETWORK_SOCKET_TIMEOUT": "-1"}
  76
  77         # Keep a reference on the object to keep it alive. It will close/clean on
  78         # exit.
  79         self.__lttng_home_dir = TemporaryDirectory(prefix=Settings.tmp_object_prefix)
  80         self.lttng_home = self.__lttng_home_dir.name
  81
  82         if len(self.lttng_home) > 88:
  83             raise Exception("TemporaryDirectory for lttng_home is to long. Use a short TMPDIR")
  84
  85         os.makedirs(self.__runtime_log)
  86         os.makedirs(self.__runtime_log_sub)
  87
  88     def add_project(self, project):
  89         self.__projects.append(project)
  90
  91     def remove_project(self, project):
  92         self.__projects.remove(project)
  93
  94     def subprocess_signal(self, subprocess_uuid, signal):
  95         self.__subproces[subprocess_uuid].send_signal(signal)
  96
  97     def subprocess_terminate(self, subprocess_uuid, timeout=60, check_return=True):
  98         process = self.__subprocess[subprocess_uuid]
  99         process.terminate()
 100         try:
 101             process.wait(timeout)
 102         except subprocess.TimeoutExpired:
 103             # Force kill
 104             return self.subprocess_kill(subprocess_uuid)
 105         stdout, stderr = self.__stdout_stderr[subprocess_uuid]
 106         stdout.close()
 107         stderr.close()
 108         if check_return:
 109             if process.returncode != 0:
 110                 raise subprocess.CalledProcessError(process.returncode, process.args)
 111         return process
 112
 113     def subprocess_kill(self, subprocess_uuid):
 114         process = self.__subprocess[subprocess_uuid]
 115         process.kill()
 116         process.wait()
 117         stdout, stderr = self.__stdout_stderr[subprocess_uuid]
 118         stdout.close()
 119         stderr.close()
 120         return process
 121
 122     def subprocess_wait(self, subprocess_uuid, check_return=True):
 123         process = self.__subprocess[subprocess_uuid]
 124         process.wait()
 125         stdout, stderr = self.__stdout_stderr[subprocess_uuid]
 126         stdout.close()
 127         stderr.close()
 128         if check_return:
 129             if process.returncode != 0:
 130                 raise subprocess.CalledProcessError(process.returncode, process.args)
 131         return process
 132
 133     def get_subprocess_stdout_path(self, subprocess_uuid):
 134         stdout, stderr = self.__stdout_stderr[subprocess_uuid]
 135         return stdout.name
 136
 137     def get_subprocess_stderr_path(self, subprocess_uuid):
 138         stdout, stderr = self.__stdout_stderr[subprocess_uuid]
 139         return stderr.name
 140
 141     def spawn_subprocess(self, command_line, cwd=None):
 142         args = shlex.split(command_line)
 143         env = self.get_env()
 144
 145         if not os.path.isdir(self.lttng_home):
 146             raise Exception("lttng home does not exist")
 147
 148         tmp_id = uuid.uuid1()
 149         out_path = os.path.join(self.__runtime_log_sub, str(tmp_id) + ".out")
 150         err_path = os.path.join(self.__runtime_log_sub, str(tmp_id) + ".err")
 151
 152         stdout = open(out_path, 'w')
 153         stderr = open(err_path, 'w')
 154
 155         env_path = os.path.join(self.__runtime_log_sub, str(tmp_id) + ".env")
 156         with open(env_path, 'w') as env_out:
 157             pprint.pprint(env, stream=env_out)
 158
 159         p = subprocess.Popen(args, stdout=stdout, stderr=stderr, env=env, cwd=cwd)
 160         self.__subprocess[tmp_id] = p
 161         self.__stdout_stderr[tmp_id] = (stdout, stderr)
 162         _logger.debug("Spawned sub pid: {} args: {} stdout: {} stderr{}".format(p.pid, p.args, out_path, err_path))
 163         return tmp_id
 164
 165     def run(self, command_line, cwd=None, check_return=True, ld_preload="", classpath="", timeout=None):
 166         """
 167         Run the command and return a tuple of a (CompletedProcess, stdout_path,
 168         stderr_path). The subprocess is already executed and returned. The
 169         callecaller is responsible for checking for errors.
 170         """
 171         args = shlex.split(command_line)
 172         env = self.get_env()
 173
 174         if ld_preload:
 175             env['LD_PRELOAD'] = ld_preload
 176         if classpath:
 177             env['CLASSPATH'] = classpath
 178
 179
 180         tmp_id = self._run_command_count
 181         self._run_command_count += 1
 182
 183         cmd_map = os.path.join(self.__runtime_log, "cmd.map")
 184         with open(cmd_map, 'a') as out:
 185             out.write("{}: {}\n".format(tmp_id, args))
 186
 187         out_path = os.path.join(self.__runtime_log, str(tmp_id) + ".out")
 188         err_path = os.path.join(self.__runtime_log, str(tmp_id) + ".err")
 189         stdout = open(out_path, "w")
 190         stderr = open(err_path, "w")
 191
 192         env_path = os.path.join(self.__runtime_log, str(tmp_id) + ".env")
 193         with open(env_path, 'w') as env_out:
 194             for key, value in env.items():
 195                 env_out.write('{}={}\n'.format(key, value))
 196
 197         cp = subprocess.run(args, stdout=stdout, stderr=stderr, env=env,
 198                             cwd=cwd, timeout=timeout)
 199         _logger.debug("Command #{} args: {} stdout: {} stderr{}".format(tmp_id, cp.args, out_path, err_path))
 200
 201         # Add to the global log file. This can help a little. Leave the other
 202         # file available for per-run analysis
 203         with open(self._runtime_log_aggregation, "a") as log:
 204             with open(out_path, "r") as out:
 205                 log.write("Output for command #{} {}\n".format(tmp_id, command_line))
 206                 log.write("Start >>>>>>>>>>>>>>>>\n")
 207                 log.write(out.read())
 208                 log.write("End <<<<<<<<<<<<<<<<\n")
 209             with open(err_path, "r") as out:
 210                 log.write("Error for command #{} {}\n".format(tmp_id, command_line))
 211                 log.write("Start >>>>>>>>>>>>>>>>\n")
 212                 log.write(out.read())
 213                 log.write("End <<<<<<<<<<<<<<<<\n")
 214
 215         if check_return:
 216             cp.check_returncode()
 217
 218         return (cp, out_path, err_path)
 219
 220     def get_cppflags(self):
 221         cppflags = []
 222         for project in self.__projects:
 223             cppflags.append(project.get_cppflags())
 224         return " ".join(cppflags)
 225
 226     def get_ldflags(self):
 227         ldflags = []
 228         for project in self.__projects:
 229             ldflags.append(project.get_ldflags())
 230         return " ".join(ldflags)
 231
 232     def get_ld_library_path(self):
 233         library_path = []
 234         for project in self.__projects:
 235             library_path.append(project.get_ld_library_path())
 236         return ":".join(library_path)
 237
 238     def get_bin_path(self):
 239         path = []
 240         for project in self.__projects:
 241             path.append(project.get_bin_path())
 242         return ":".join(path)
 243
 244     def get_env(self):
 245         env = os.environ.copy()
 246
 247         env["LTTNG_HOME"] = self.lttng_home
 248
 249         env_fetch = {"CPPFLAGS": (self.get_cppflags(), " "),
 250                      "LDFLAGS": (self.get_ldflags(), " "),
 251                      "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"),
 252                      "PATH": (self.get_bin_path(), ":"),
 253                      }
 254         for key, (value, delimiter) in env_fetch.items():
 255             tmp_var = ""
 256             if key in env:
 257                 tmp_var = env[key]
 258             env[key] = delimiter.join([value, tmp_var])
 259
 260         for var, value in self.special_env_variables.items():
 261             if var in env:
 262                 # Raise for now since no special cases is known
 263                 _logger.warning("% Special var % is already defined",
 264                                 self.label, var)
 265                 raise Exception("Multiple definition of a special environment variable")
 266             else:
 267                 env[var] = value
 268
 269         for project in self.__projects:
 270             for var, value in project.special_env_variables.items():
 271                 if var in env:
 272                     # Raise for now since no special cases is known
 273                     _logger.warning("% Special var % is already defined",
 274                                     self.label, var)
 275                     raise Exception("Multiple definition of a special environment variable")
 276                 else:
 277                     env[var] = value
 278         return env
 279
 280     def load_test_module(self):
 281         # Base directory is provided by env
 282         self.run("modprobe lttng-test")
 283         self._is_test_modules_loaded = True
 284
 285     def unload_test_module(self, check_return=True):
 286         # Base directory is provided by env
 287         if self._is_test_modules_loaded:
 288             self.run("modprobe -r lttng-test", check_return=check_return)
 289
 290     def close(self):
 291         for key, subp in self.__subprocess.items():
 292             self.subprocess_kill(key)
 293
 294         # Always try to remove test module but do not perform check on return
 295         # value.
 296         self.unload_test_module(False)
 297
 298         # Copy the lttng_home used at runtime using hardlink to prevent useless
 299         # data duplication
 300         shutil.copytree(self.lttng_home, self.__post_runtime_lttng_home_path, copy_function=os.link)