X-Git-Url: http://git.efficios.com/?p=deliverable%2Flttng-ivc.git;a=blobdiff_plain;f=lttng_ivc%2Futils%2Fruntime.py;h=6c9f04c04e0deb77c3a8df035113b6d9fbf8b7b3;hp=1bd8cbf6dbe1abb1c790d8b62ff83b1f0036bbb9;hb=efdd48dbd95ae57db1b0f2dfe5af429feee2b06d;hpb=18aedaf96b66a383f46487b7203bc35fc2280900 diff --git a/lttng_ivc/utils/runtime.py b/lttng_ivc/utils/runtime.py index 1bd8cbf..6c9f04c 100644 --- a/lttng_ivc/utils/runtime.py +++ b/lttng_ivc/utils/runtime.py @@ -1,12 +1,49 @@ +# Copyright (c) 2017 Jonathan Rajotte-Julien +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + import os +import sys import shlex import subprocess import uuid import logging +import shutil +import contextlib +import pprint +import signal + +from tempfile import TemporaryDirectory +import lttng_ivc.settings as Settings _logger = logging.getLogger("Runtime") +@contextlib.contextmanager +def get_runtime(runtime_dir): + runtime = Runtime(runtime_dir) + try: + yield runtime + finally: + runtime.close() + + class Runtime(object): def __init__(self, runtime_dir): """ @@ -20,30 +57,58 @@ class Runtime(object): self.__runtime_log = os.path.join(runtime_dir, "log") self.__runtime_log_sub = os.path.join(self.__runtime_log, "subprocess") + """ + Path of the copy of lttng_home folder after Runtime.close() is issued. This is + to be used for post runtime analysis and mostly debugging on error. + """ + self.__post_runtime_lttng_home_path = os.path.join(runtime_dir, + "lttng_home") + self._runtime_log_aggregation = os.path.join(self.__runtime_log, "runtime.log") self._run_command_count = 0 + self._is_test_modules_loaded = False + + self.special_env_variables = {"LTTNG_UST_DEBUG": "1", + "LTTNG_APP_SOCKET_TIMEOUT": "-1", + #"LTTNG_UST_REGISTER_TIMEOUT": "-1", + "LTTNG_NETWORK_SOCKET_TIMEOUT": "-1"} + + # Keep a reference on the object to keep it alive. It will close/clean on + # exit. + self.__lttng_home_dir = TemporaryDirectory(prefix=Settings.tmp_object_prefix) + self.lttng_home = self.__lttng_home_dir.name - self.lttng_home = os.path.join(runtime_dir, "lttng_home") + if len(self.lttng_home) > 88: + raise Exception("TemporaryDirectory for lttng_home is to long. Use a short TMPDIR") - # TODO move exist_ok to false !!!! ONLY for testing - os.makedirs(self.__runtime_log, exist_ok=True) - os.makedirs(self.__runtime_log_sub, exist_ok=True) - os.makedirs(self.lttng_home, exist_ok=True) + os.makedirs(self.__runtime_log) + os.makedirs(self.__runtime_log_sub) def add_project(self, project): self.__projects.append(project) + def remove_project(self, project): + self.__projects.remove(project) + def subprocess_signal(self, subprocess_uuid, signal): self.__subproces[subprocess_uuid].send_signal(signal) - def subprocess_terminate(self, subprocess_uuid, timeout=60): + def subprocess_terminate(self, subprocess_uuid, timeout=60, check_return=True): process = self.__subprocess[subprocess_uuid] process.terminate() - process.wait(timeout) + try: + process.wait(timeout) + except subprocess.TimeoutExpired: + # Force kill + return self.subprocess_kill(subprocess_uuid) stdout, stderr = self.__stdout_stderr[subprocess_uuid] stdout.close() stderr.close() + if check_return: + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, process.args) + return process def subprocess_kill(self, subprocess_uuid): process = self.__subprocess[subprocess_uuid] @@ -52,6 +117,18 @@ class Runtime(object): stdout, stderr = self.__stdout_stderr[subprocess_uuid] stdout.close() stderr.close() + return process + + def subprocess_wait(self, subprocess_uuid, check_return=True): + process = self.__subprocess[subprocess_uuid] + process.wait() + stdout, stderr = self.__stdout_stderr[subprocess_uuid] + stdout.close() + stderr.close() + if check_return: + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, process.args) + return process def get_subprocess_stdout_path(self, subprocess_uuid): stdout, stderr = self.__stdout_stderr[subprocess_uuid] @@ -61,22 +138,31 @@ class Runtime(object): stdout, stderr = self.__stdout_stderr[subprocess_uuid] return stderr.name - def spawn_subprocess(self, command_line): + def spawn_subprocess(self, command_line, cwd=None): args = shlex.split(command_line) env = self.get_env() + if not os.path.isdir(self.lttng_home): + raise Exception("lttng home does not exist") + tmp_id = uuid.uuid1() out_path = os.path.join(self.__runtime_log_sub, str(tmp_id) + ".out") err_path = os.path.join(self.__runtime_log_sub, str(tmp_id) + ".err") - stdout = open(out_path, "w") - stderr = open(err_path, "w") - p = subprocess.Popen(args, stdout=stdout, stderr=stderr, env=env) + stdout = open(out_path, 'w') + stderr = open(err_path, 'w') + + env_path = os.path.join(self.__runtime_log_sub, str(tmp_id) + ".env") + with open(env_path, 'w') as env_out: + pprint.pprint(env, stream=env_out) + + p = subprocess.Popen(args, stdout=stdout, stderr=stderr, env=env, cwd=cwd) self.__subprocess[tmp_id] = p self.__stdout_stderr[tmp_id] = (stdout, stderr) _logger.debug("Spawned sub pid: {} args: {} stdout: {} stderr{}".format(p.pid, p.args, out_path, err_path)) + return tmp_id - def run(self, command_line): + def run(self, command_line, cwd=None, check_return=True, ld_preload="", classpath="", timeout=None): """ Run the command and return a tuple of a (CompletedProcess, stdout_path, stderr_path). The subprocess is already executed and returned. The @@ -85,38 +171,49 @@ class Runtime(object): args = shlex.split(command_line) env = self.get_env() + if ld_preload: + env['LD_PRELOAD'] = ld_preload + if classpath: + env['CLASSPATH'] = classpath + + tmp_id = self._run_command_count self._run_command_count += 1 + cmd_map = os.path.join(self.__runtime_log, "cmd.map") + with open(cmd_map, 'a') as out: + out.write("{}: {}\n".format(tmp_id, args)) + out_path = os.path.join(self.__runtime_log, str(tmp_id) + ".out") err_path = os.path.join(self.__runtime_log, str(tmp_id) + ".err") stdout = open(out_path, "w") stderr = open(err_path, "w") - stdout.write("Output for command #{} {}\n".format(tmp_id, command_line)) - stdout.write("Start >>>>>>>>>>>>>>>>\n") - stdout.flush() - - stderr.write("Output for command #{} {}\n".format(tmp_id, command_line)) - stderr.write("Start >>>>>>>>>>>>>>>>\n") - stderr.flush() + env_path = os.path.join(self.__runtime_log, str(tmp_id) + ".env") + with open(env_path, 'w') as env_out: + for key, value in env.items(): + env_out.write('{}={}\n'.format(key, value)) - cp = subprocess.run(args, stdout=stdout, stderr=stderr, env=env) + cp = subprocess.run(args, stdout=stdout, stderr=stderr, env=env, + cwd=cwd, timeout=timeout) _logger.debug("Command #{} args: {} stdout: {} stderr{}".format(tmp_id, cp.args, out_path, err_path)) - stdout.write("End <<<<<<<<<<<<<<<<\n") - stdout.close() - - stderr.write("End <<<<<<<<<<<<<<<<\n") - stderr.close() - # Add to the global log file. This can help a little. Leave the other # file available for per-run analysis with open(self._runtime_log_aggregation, "a") as log: with open(out_path, "r") as out: + log.write("Output for command #{} {}\n".format(tmp_id, command_line)) + log.write("Start >>>>>>>>>>>>>>>>\n") log.write(out.read()) + log.write("End <<<<<<<<<<<<<<<<\n") with open(err_path, "r") as out: + log.write("Error for command #{} {}\n".format(tmp_id, command_line)) + log.write("Start >>>>>>>>>>>>>>>>\n") log.write(out.read()) + log.write("End <<<<<<<<<<<<<<<<\n") + + if check_return: + cp.check_returncode() return (cp, out_path, err_path) @@ -136,7 +233,7 @@ class Runtime(object): library_path = [] for project in self.__projects: library_path.append(project.get_ld_library_path()) - return " ".join(library_path) + return ":".join(library_path) def get_bin_path(self): path = [] @@ -151,7 +248,7 @@ class Runtime(object): env_fetch = {"CPPFLAGS": (self.get_cppflags(), " "), "LDFLAGS": (self.get_ldflags(), " "), - "LD_LIRABRY_PATH": (self.get_ld_library_path(), ":"), + "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"), "PATH": (self.get_bin_path(), ":"), } for key, (value, delimiter) in env_fetch.items(): @@ -160,6 +257,15 @@ class Runtime(object): tmp_var = env[key] env[key] = delimiter.join([value, tmp_var]) + for var, value in self.special_env_variables.items(): + if var in env: + # Raise for now since no special cases is known + _logger.warning("% Special var % is already defined", + self.label, var) + raise Exception("Multiple definition of a special environment variable") + else: + env[var] = value + for project in self.__projects: for var, value in project.special_env_variables.items(): if var in env: @@ -171,17 +277,24 @@ class Runtime(object): env[var] = value return env + def load_test_module(self): + # Base directory is provided by env + self.run("modprobe lttng-test") + self._is_test_modules_loaded = True + + def unload_test_module(self, check_return=True): + # Base directory is provided by env + if self._is_test_modules_loaded: + self.run("modprobe -r lttng-test", check_return=check_return) + def close(self): for key, subp in self.__subprocess.items(): - subp.terminate() - for key, subp in self.__subprocess.items(): - try: - # TODO move timeout to settings - subp.wait(timeout=60) - except subprocess.TimeoutExpired as e: - # Force a little bit - subp.kill() - subp.wait() - for key, (stdout, stderr) in self.__stdout_stderr.items(): - stdout.close() - stderr.close() + self.subprocess_kill(key) + + # Always try to remove test module but do not perform check on return + # value. + self.unload_test_module(False) + + # Copy the lttng_home used at runtime using hardlink to prevent useless + # data duplication + shutil.copytree(self.lttng_home, self.__post_runtime_lttng_home_path, copy_function=os.link)