Commit | Line | Data |
---|---|---|
efdd48db JR |
1 | # Copyright (c) 2017 Jonathan Rajotte-Julien <jonathan.rajotte-julien@efficios.com> |
2 | # | |
3 | # Permission is hereby granted, free of charge, to any person obtaining a copy | |
4 | # of this software and associated documentation files (the "Software"), to deal | |
5 | # in the Software without restriction, including without limitation the rights | |
6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
7 | # copies of the Software, and to permit persons to whom the Software is | |
8 | # furnished to do so, subject to the following conditions: | |
9 | # | |
10 | # The above copyright notice and this permission notice shall be included in all | |
11 | # copies or substantial portions of the Software. | |
12 | # | |
13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |
16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
19 | # SOFTWARE. | |
20 | ||
de9b991b JR |
21 | import os |
22 | import shutil | |
23 | import git | |
24 | import subprocess | |
25 | import logging | |
cb87ff89 | 26 | import lttng_ivc.settings as Settings |
5908717c | 27 | import pprint |
d2a87b59 | 28 | import magic |
cb87ff89 JR |
29 | |
30 | from lttng_ivc.utils.utils import sha256_checksum | |
816f8cc1 | 31 | from lttng_ivc.utils.utils import find_dir, find_file |
de9b991b | 32 | |
fe7b987e | 33 | _logger = logging.getLogger('project') |
de9b991b JR |
34 | |
35 | class Project(object): | |
36 | ||
37 | def __init__(self, label, git_path, sha1, tmpdir): | |
38 | self.label = label | |
39 | self.git_path = git_path | |
40 | self.sha1 = sha1 | |
41 | ||
42 | """ Custom configure flags in the for of ['-x', 'arg']""" | |
43 | self.custom_configure_flags = [] | |
44 | ccache = shutil.which("ccache") | |
45 | if ccache is not None: | |
46 | self.custom_configure_flags.append("CC={} gcc".format(ccache)) | |
47 | self.custom_configure_flags.append("CXX={} g++".format(ccache)) | |
97d494b0 | 48 | self.custom_configure_flags.append("CFLAGS=-g -O0".format(ccache)) |
de9b991b JR |
49 | |
50 | """ A collection of Project dependencies """ | |
fe7b987e | 51 | self.dependencies = {} |
cb87ff89 | 52 | # used for project cache and pickle validation |
fe7b987e | 53 | self._immutable = False |
cb87ff89 | 54 | self._py_file_checksum = sha256_checksum(Settings.project_py_file_location) |
de9b991b JR |
55 | |
56 | # State | |
de9b991b JR |
57 | self.isBuilt = False |
58 | self.isConfigured = False | |
59 | self.isInstalled = False | |
ab63b97e | 60 | self.skip = False |
de9b991b | 61 | |
fe7b987e JR |
62 | self.basedir = tmpdir |
63 | self.log_path = os.path.join(tmpdir, "log") | |
64 | self.source_path = os.path.join(tmpdir, "source") | |
65 | self.installation_path = os.path.join(tmpdir, "install") | |
66 | ||
97d494b0 JR |
67 | if os.path.isdir(self.basedir): |
68 | # Perform cleanup since it should not happen | |
69 | shutil.rmtree(self.basedir) | |
70 | ||
fe7b987e | 71 | os.makedirs(self.log_path) |
de9b991b JR |
72 | os.makedirs(self.source_path) |
73 | os.makedirs(self.installation_path) | |
de9b991b JR |
74 | |
75 | self.special_env_variables = {} | |
76 | ||
77 | # Init the repo for work | |
78 | self.checkout() | |
79 | self.bootstrap() | |
80 | ||
18aedaf9 JR |
81 | def add_special_env_variable(self, key, value): |
82 | if key in self.special_env_variables: | |
83 | _logger.warning("{} Special var {} is already defined".format( | |
84 | self.label, key)) | |
85 | raise Exception("Multiple definition of a special environment variable") | |
86 | self.special_env_variables[key] = value | |
87 | ||
de9b991b | 88 | def get_cppflags(self): |
18aedaf9 JR |
89 | cppflags = ["-I{}/include".format(self.installation_path)] |
90 | for key, dep in self.dependencies.items(): | |
91 | cppflags.append(dep.get_cppflags()) | |
92 | ||
93 | return " ".join(cppflags) | |
de9b991b JR |
94 | |
95 | def get_ldflags(self): | |
18aedaf9 JR |
96 | ldflags = ["-L{}/lib".format(self.installation_path)] |
97 | for key, dep in self.dependencies.items(): | |
98 | ldflags.append(dep.get_ldflags()) | |
99 | return " ".join(ldflags) | |
de9b991b JR |
100 | |
101 | def get_ld_library_path(self): | |
18aedaf9 JR |
102 | library_path = ["{}/lib".format(self.installation_path)] |
103 | for key, dep in self.dependencies.items(): | |
104 | library_path.append(dep.get_ld_library_path()) | |
105 | return ":".join(library_path) | |
106 | ||
107 | def get_bin_path(self): | |
108 | bin_path = ["{}/bin".format(self.installation_path)] | |
109 | for key, dep in self.dependencies.items(): | |
110 | bin_path.append(dep.get_bin_path()) | |
111 | return ":".join(bin_path) | |
de9b991b JR |
112 | |
113 | def get_env(self): | |
114 | """Modify environment to reflect dependency""" | |
18aedaf9 JR |
115 | env_var = {"CPPFLAGS": (self.get_cppflags(), " "), |
116 | "LDFLAGS": (self.get_ldflags(), " "), | |
117 | "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"), | |
118 | } | |
de9b991b JR |
119 | |
120 | env = os.environ.copy() | |
121 | ||
122 | for var, value in self.special_env_variables.items(): | |
123 | if var in env: | |
ca2f2d59 JR |
124 | if var == "LD_LIBRARY_PATH": |
125 | env[var] = ":".join([env[var], value]) | |
0cd453ab JR |
126 | elif var == "CPPFLAGS" or var == "CFLAGS" or var == "LDFLAGS": |
127 | env[var] = " ".join([env[var], value]) | |
ca2f2d59 JR |
128 | else: |
129 | _logger.warning("{} Special var {} is already defined".format( | |
130 | self.label, var)) | |
131 | raise Exception("Multiple definition of a special environment variable") | |
de9b991b JR |
132 | else: |
133 | env[var] = value | |
134 | ||
fe7b987e | 135 | for key, dep in self.dependencies.items(): |
de9b991b | 136 | # Extra space just in case |
de9b991b JR |
137 | for var, value in dep.special_env_variables.items(): |
138 | if var in env: | |
de9b991b | 139 | # Raise for now since no special cases is known |
18aedaf9 JR |
140 | _logger.warning("{} Special var {} is already defined".format( |
141 | self.label, var)) | |
de9b991b JR |
142 | raise Exception("Multiple definition of a special environment variable") |
143 | else: | |
144 | env[var] = value | |
145 | ||
18aedaf9 JR |
146 | for var, (value, delimiter) in env_var.items(): |
147 | tmp = [value] | |
148 | if var in env: | |
149 | tmp.append(env[var]) | |
150 | env[var] = delimiter.join(tmp) | |
151 | ||
de9b991b JR |
152 | return env |
153 | ||
154 | def autobuild(self): | |
155 | """ | |
156 | Perform the bootstrap, configuration, build and install the | |
157 | project. Build dependencies if not already built | |
158 | """ | |
fe7b987e JR |
159 | if (self.isConfigured and self.isBuilt and self.isInstalled): |
160 | return | |
161 | ||
162 | if self._immutable: | |
163 | raise Exception("Object is immutable. Illegal autobuild") | |
164 | ||
165 | for key, dep in self.dependencies.items(): | |
de9b991b JR |
166 | dep.autobuild() |
167 | ||
fe7b987e | 168 | if self.isConfigured ^ self.isBuilt ^ self.isInstalled: |
de9b991b | 169 | raise Exception("Project steps where manually triggered. Can't autobuild") |
97d494b0 JR |
170 | _logger.debug("{} Autobuild configure".format(self.label)) |
171 | try: | |
172 | self.configure() | |
173 | except subprocess.CalledProcessError as e: | |
174 | _logger.error("{} Configure failed. See {} for more details.".format(self.label, self.log_path)) | |
175 | raise e | |
176 | ||
177 | _logger.debug("{} Autobuild build".format(self.label)) | |
178 | try: | |
179 | self.build() | |
180 | except subprocess.CalledProcessError as e: | |
181 | _logger.error("{} Build failed. See {} for more details.".format(self.label, self.log_path)) | |
182 | raise e | |
183 | ||
184 | _logger.debug("{} Autobuild install".format(self.label)) | |
185 | try: | |
186 | self.install() | |
187 | except subprocess.CalledProcessError as e: | |
188 | _logger.error("{} Install failed. See {} for more details.".format(self.label, self.log_path)) | |
189 | raise e | |
de9b991b | 190 | |
d2a87b59 JR |
191 | _logger.debug("{} Autobuild rpath strip".format(self.label)) |
192 | try: | |
193 | self.rpath_strip() | |
194 | except subprocess.CalledProcessError as e: | |
195 | _logger.error("{} Rpath stripping failed. See {} for more details.".format(self.label, self.log_path)) | |
196 | raise e | |
197 | ||
de9b991b | 198 | def checkout(self): |
fe7b987e JR |
199 | if self._immutable: |
200 | raise Exception("Object is immutable. Illegal checkout") | |
201 | ||
de9b991b JR |
202 | repo = git.Repo.clone_from(self.git_path, self.source_path) |
203 | commit = repo.commit(self.sha1) | |
204 | repo.head.reference = commit | |
205 | assert repo.head.is_detached | |
206 | repo.head.reset(index=True, working_tree=True) | |
207 | ||
208 | def bootstrap(self): | |
209 | """ | |
210 | Bootstap the project. Raise subprocess.CalledProcessError on | |
211 | bootstrap error. | |
212 | """ | |
fe7b987e JR |
213 | if self._immutable: |
214 | raise Exception("Object is immutable. Illegal bootstrap") | |
215 | ||
216 | out = os.path.join(self.log_path, "bootstrap.out") | |
217 | err = os.path.join(self.log_path, "bootstrap.err") | |
218 | ||
de9b991b | 219 | os.chdir(self.source_path) |
fe7b987e JR |
220 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
221 | p = subprocess.run(['./bootstrap'], stdout=stdout, stderr=stderr) | |
de9b991b JR |
222 | p.check_returncode() |
223 | return p | |
224 | ||
225 | def configure(self): | |
226 | """ | |
227 | Configure the project. | |
228 | Raises subprocess.CalledProcessError on configure error | |
229 | """ | |
fe7b987e JR |
230 | if self._immutable: |
231 | raise Exception("Object is immutable. Illegal configure") | |
232 | ||
de9b991b | 233 | # Check that all our dependencies were actually installed |
fe7b987e | 234 | for key, dep in self.dependencies.items(): |
de9b991b JR |
235 | if not dep.isInstalled: |
236 | # TODO: Custom exception here Dependency Error | |
237 | raise Exception("Dependency project flagged as not installed") | |
238 | ||
fe7b987e JR |
239 | out = os.path.join(self.log_path, "configure.out") |
240 | err = os.path.join(self.log_path, "configure.err") | |
5908717c JR |
241 | env_file = os.path.join(self.log_path, "configure.env") |
242 | ||
243 | env = self.get_env() | |
244 | ||
245 | with open(env_file, 'w') as tmp: | |
246 | pprint.pprint(env, stream=tmp) | |
fe7b987e | 247 | |
de9b991b JR |
248 | os.chdir(self.source_path) |
249 | args = ['./configure'] | |
250 | prefix = '--prefix={}'.format(self.installation_path) | |
251 | args.append(prefix) | |
252 | args.extend(self.custom_configure_flags) | |
253 | ||
254 | # TODO: log output and add INFO log point | |
fe7b987e | 255 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
5908717c | 256 | p = subprocess.run(args, env=env, stdout=stdout, |
fe7b987e | 257 | stderr=stderr) |
de9b991b JR |
258 | p.check_returncode() |
259 | self.isConfigured = True | |
260 | return p | |
261 | ||
262 | def build(self): | |
263 | """ | |
264 | Build the project. Raise subprocess.CalledProcessError on build | |
265 | error. | |
266 | """ | |
fe7b987e JR |
267 | if self._immutable: |
268 | raise Exception("Object is immutable. Illegal build") | |
269 | ||
270 | out = os.path.join(self.log_path, "build.out") | |
271 | err = os.path.join(self.log_path, "build.err") | |
272 | ||
de9b991b JR |
273 | os.chdir(self.source_path) |
274 | args = ['make'] | |
275 | env = self.get_env() | |
de9b991b JR |
276 | |
277 | # Number of usable cpu | |
278 | # https://docs.python.org/3/library/os.html#os.cpu_count | |
279 | num_cpu = str(len(os.sched_getaffinity(0))) | |
280 | args.append('-j') | |
281 | args.append(num_cpu) | |
97d494b0 | 282 | args.append('V=1') |
de9b991b JR |
283 | |
284 | # TODO: log output and add INFO log point with args | |
fe7b987e JR |
285 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
286 | p = subprocess.run(args, env=env, stdout=stdout, | |
287 | stderr=stderr) | |
de9b991b JR |
288 | p.check_returncode() |
289 | self.isBuilt = True | |
290 | return p | |
291 | ||
292 | def install(self): | |
293 | """ | |
294 | Install the project. Raise subprocess.CalledProcessError on | |
295 | bootstrap error | |
296 | """ | |
fe7b987e JR |
297 | if self._immutable: |
298 | raise Exception("Object is immutable. Illegal install") | |
299 | ||
97d494b0 JR |
300 | out = os.path.join(self.log_path, "install.out") |
301 | err = os.path.join(self.log_path, "install.err") | |
fe7b987e | 302 | |
de9b991b JR |
303 | os.chdir(self.source_path) |
304 | args = ['make', 'install'] | |
305 | ||
306 | # TODO: log output and add INFO log point | |
fe7b987e JR |
307 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
308 | p = subprocess.run(args, env=self.get_env(), stdout=stdout, | |
309 | stderr=stderr) | |
de9b991b JR |
310 | p.check_returncode() |
311 | self.isInstalled = True | |
312 | return p | |
313 | ||
d2a87b59 JR |
314 | def rpath_strip(self): |
315 | to_strip = [os.path.join(self.installation_path, "bin"), | |
316 | os.path.join(self.installation_path, "lib")] | |
317 | ||
318 | out = os.path.join(self.log_path, "rpath-strip.out") | |
319 | err = os.path.join(self.log_path, "rpath-strip.err") | |
320 | ||
321 | for path in to_strip: | |
322 | for base, dirs, files in os.walk(path): | |
323 | for tmp in files: | |
324 | abs_path = os.path.abspath(os.path.join(base, tmp)) | |
325 | magic_str = magic.from_file(abs_path) | |
326 | # Skip all non-elf file | |
327 | if "ELF" not in magic_str.split(): | |
328 | with open(err, 'a') as stderr: | |
329 | stderr.write("{} skip, is not an ELF, file type: {}\n".format(abs_path, magic_str)) | |
330 | continue | |
331 | cmd = ["chrpath", "-d", abs_path] | |
332 | with open(out, 'a') as stdout, open(err, 'a') as stderr: | |
333 | stdout.write("Running {}\n".format(cmd)) | |
334 | stderr.write("Running {}\n".format(cmd)) | |
335 | p = subprocess.run(cmd, stdout=stdout, stderr=stderr) | |
336 | p.check_returncode() | |
337 | ||
de9b991b JR |
338 | def cleanup(self): |
339 | if os.path.exists(self.source_path): | |
340 | shutil.rmtree(self.source_path) | |
341 | if os.path.exists(self.installation_path): | |
342 | shutil.rmtree(self.installation_path) | |
343 | ||
344 | ||
345 | class Lttng_modules(Project): | |
97d494b0 JR |
346 | def __init__(self, label, git_path, sha1, tmpdir): |
347 | super(Lttng_modules, self).__init__(label=label, git_path=git_path, | |
348 | sha1=sha1, tmpdir=tmpdir) | |
f383ed3e | 349 | self.add_special_env_variable("MODPROBE_OPTIONS","-v -d {}".format(self.installation_path)) |
97d494b0 | 350 | |
de9b991b JR |
351 | def bootstrap(self): |
352 | pass | |
353 | ||
354 | def configure(self): | |
355 | pass | |
356 | ||
357 | def install(self): | |
fe7b987e JR |
358 | if self._immutable: |
359 | raise Exception("Object is immutable. Illegal install") | |
de9b991b JR |
360 | os.chdir(self.source_path) |
361 | args = ['make', 'INSTALL_MOD_PATH={}'.format(self.installation_path), | |
362 | 'modules_install'] | |
363 | p = subprocess.run(args, env=self.get_env(), stdout=subprocess.PIPE, | |
364 | stderr=subprocess.PIPE) | |
365 | p.check_returncode() | |
366 | ||
367 | # Perform a local depmod | |
368 | args = ['depmod', '-b', self.installation_path] | |
369 | p = subprocess.run(args, env=self.get_env()) | |
370 | p.check_returncode() | |
371 | self.isInstalled = True | |
372 | ||
2dd375bf JR |
373 | def autobuild(self): |
374 | try: | |
375 | super(Lttng_modules, self).autobuild() | |
376 | except subprocess.CalledProcessError as e: | |
377 | self.skip = True | |
378 | ||
d2a87b59 JR |
379 | def rpath_strip(self): |
380 | pass | |
381 | ||
de9b991b JR |
382 | |
383 | class Lttng_ust(Project): | |
384 | def __init__(self, label, git_path, sha1, tmpdir): | |
385 | super(Lttng_ust, self).__init__(label=label, git_path=git_path, | |
386 | sha1=sha1, tmpdir=tmpdir) | |
387 | self.custom_configure_flags.extend(['--disable-man-pages']) | |
e5cbfcca JR |
388 | self.custom_configure_flags.extend(['--enable-python-agent']) |
389 | self.custom_configure_flags.extend(['--enable-java-agent-jul']) | |
390 | ||
74eb7096 JR |
391 | jul_path = os.path.join(self.installation_path, |
392 | "share/java/liblttng-ust-agent.jar") | |
8e2c5f79 | 393 | classpath = ":".join([jul_path, '.']) |
74eb7096 | 394 | self.add_special_env_variable("CLASSPATH", classpath) |
de9b991b | 395 | |
2094d672 JR |
396 | def install(self): |
397 | super(Lttng_ust, self).install() | |
398 | python_path = find_dir(self.installation_path, "lttngust") | |
399 | if python_path: | |
400 | # Fetch the parent of lttngust folder | |
401 | python_path = os.path.dirname(python_path) | |
402 | self.add_special_env_variable("PYTHONPATH", python_path) | |
403 | ||
404 | ||
de9b991b JR |
405 | |
406 | class Lttng_tools(Project): | |
18aedaf9 JR |
407 | def __init__(self, label, git_path, sha1, tmpdir): |
408 | super(Lttng_tools, self).__init__(label=label, git_path=git_path, | |
409 | sha1=sha1, tmpdir=tmpdir) | |
410 | self.add_special_env_variable("LTTNG_SESSION_CONFIG_XSD_PATH", | |
411 | os.path.join(self.installation_path, "share/xml/lttng/")) | |
de9b991b | 412 | |
816f8cc1 JR |
413 | # Find the mi xsd |
414 | for xsd in Settings.mi_xsd_file_name: | |
415 | mi = find_file(self.source_path, xsd) | |
416 | if mi: | |
417 | break | |
418 | if not mi: | |
419 | raise Exception("MI xsd not found") | |
420 | self.mi_xsd = mi | |
421 | ||
de9b991b JR |
422 | |
423 | class Babeltrace(Project): | |
424 | pass |