Commit | Line | Data |
---|---|---|
de9b991b JR |
1 | import os |
2 | import shutil | |
3 | import git | |
4 | import subprocess | |
5 | import logging | |
cb87ff89 JR |
6 | import lttng_ivc.settings as Settings |
7 | ||
8 | from lttng_ivc.utils.utils import sha256_checksum | |
de9b991b | 9 | |
fe7b987e | 10 | _logger = logging.getLogger('project') |
de9b991b JR |
11 | |
12 | class Project(object): | |
13 | ||
14 | def __init__(self, label, git_path, sha1, tmpdir): | |
15 | self.label = label | |
16 | self.git_path = git_path | |
17 | self.sha1 = sha1 | |
18 | ||
19 | """ Custom configure flags in the for of ['-x', 'arg']""" | |
20 | self.custom_configure_flags = [] | |
21 | ccache = shutil.which("ccache") | |
22 | if ccache is not None: | |
23 | self.custom_configure_flags.append("CC={} gcc".format(ccache)) | |
24 | self.custom_configure_flags.append("CXX={} g++".format(ccache)) | |
97d494b0 | 25 | self.custom_configure_flags.append("CFLAGS=-g -O0".format(ccache)) |
de9b991b JR |
26 | |
27 | """ A collection of Project dependencies """ | |
fe7b987e | 28 | self.dependencies = {} |
cb87ff89 | 29 | # used for project cache and pickle validation |
fe7b987e | 30 | self._immutable = False |
cb87ff89 | 31 | self._py_file_checksum = sha256_checksum(Settings.project_py_file_location) |
de9b991b JR |
32 | |
33 | # State | |
de9b991b JR |
34 | self.isBuilt = False |
35 | self.isConfigured = False | |
36 | self.isInstalled = False | |
ab63b97e | 37 | self.skip = False |
de9b991b | 38 | |
fe7b987e JR |
39 | self.basedir = tmpdir |
40 | self.log_path = os.path.join(tmpdir, "log") | |
41 | self.source_path = os.path.join(tmpdir, "source") | |
42 | self.installation_path = os.path.join(tmpdir, "install") | |
43 | ||
97d494b0 JR |
44 | if os.path.isdir(self.basedir): |
45 | # Perform cleanup since it should not happen | |
46 | shutil.rmtree(self.basedir) | |
47 | ||
fe7b987e | 48 | os.makedirs(self.log_path) |
de9b991b JR |
49 | os.makedirs(self.source_path) |
50 | os.makedirs(self.installation_path) | |
de9b991b JR |
51 | |
52 | self.special_env_variables = {} | |
53 | ||
54 | # Init the repo for work | |
55 | self.checkout() | |
56 | self.bootstrap() | |
57 | ||
18aedaf9 JR |
58 | def add_special_env_variable(self, key, value): |
59 | if key in self.special_env_variables: | |
60 | _logger.warning("{} Special var {} is already defined".format( | |
61 | self.label, key)) | |
62 | raise Exception("Multiple definition of a special environment variable") | |
63 | self.special_env_variables[key] = value | |
64 | ||
de9b991b | 65 | def get_cppflags(self): |
18aedaf9 JR |
66 | cppflags = ["-I{}/include".format(self.installation_path)] |
67 | for key, dep in self.dependencies.items(): | |
68 | cppflags.append(dep.get_cppflags()) | |
69 | ||
70 | return " ".join(cppflags) | |
de9b991b JR |
71 | |
72 | def get_ldflags(self): | |
18aedaf9 JR |
73 | ldflags = ["-L{}/lib".format(self.installation_path)] |
74 | for key, dep in self.dependencies.items(): | |
75 | ldflags.append(dep.get_ldflags()) | |
76 | return " ".join(ldflags) | |
de9b991b JR |
77 | |
78 | def get_ld_library_path(self): | |
18aedaf9 JR |
79 | library_path = ["{}/lib".format(self.installation_path)] |
80 | for key, dep in self.dependencies.items(): | |
81 | library_path.append(dep.get_ld_library_path()) | |
82 | return ":".join(library_path) | |
83 | ||
84 | def get_bin_path(self): | |
85 | bin_path = ["{}/bin".format(self.installation_path)] | |
86 | for key, dep in self.dependencies.items(): | |
87 | bin_path.append(dep.get_bin_path()) | |
88 | return ":".join(bin_path) | |
de9b991b JR |
89 | |
90 | def get_env(self): | |
91 | """Modify environment to reflect dependency""" | |
18aedaf9 JR |
92 | env_var = {"CPPFLAGS": (self.get_cppflags(), " "), |
93 | "LDFLAGS": (self.get_ldflags(), " "), | |
94 | "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"), | |
95 | } | |
de9b991b JR |
96 | |
97 | env = os.environ.copy() | |
98 | ||
99 | for var, value in self.special_env_variables.items(): | |
100 | if var in env: | |
de9b991b | 101 | # Raise for now since no special cases is known |
18aedaf9 JR |
102 | _logger.warning("{} Special var {} is already defined".format( |
103 | self.label, var)) | |
de9b991b JR |
104 | raise Exception("Multiple definition of a special environment variable") |
105 | else: | |
106 | env[var] = value | |
107 | ||
fe7b987e | 108 | for key, dep in self.dependencies.items(): |
de9b991b | 109 | # Extra space just in case |
de9b991b JR |
110 | for var, value in dep.special_env_variables.items(): |
111 | if var in env: | |
de9b991b | 112 | # Raise for now since no special cases is known |
18aedaf9 JR |
113 | _logger.warning("{} Special var {} is already defined".format( |
114 | self.label, var)) | |
de9b991b JR |
115 | raise Exception("Multiple definition of a special environment variable") |
116 | else: | |
117 | env[var] = value | |
118 | ||
18aedaf9 JR |
119 | for var, (value, delimiter) in env_var.items(): |
120 | tmp = [value] | |
121 | if var in env: | |
122 | tmp.append(env[var]) | |
123 | env[var] = delimiter.join(tmp) | |
124 | ||
de9b991b JR |
125 | return env |
126 | ||
127 | def autobuild(self): | |
128 | """ | |
129 | Perform the bootstrap, configuration, build and install the | |
130 | project. Build dependencies if not already built | |
131 | """ | |
fe7b987e JR |
132 | if (self.isConfigured and self.isBuilt and self.isInstalled): |
133 | return | |
134 | ||
135 | if self._immutable: | |
136 | raise Exception("Object is immutable. Illegal autobuild") | |
137 | ||
138 | for key, dep in self.dependencies.items(): | |
de9b991b JR |
139 | dep.autobuild() |
140 | ||
fe7b987e | 141 | if self.isConfigured ^ self.isBuilt ^ self.isInstalled: |
de9b991b JR |
142 | raise Exception("Project steps where manually triggered. Can't autobuild") |
143 | ||
97d494b0 JR |
144 | _logger.debug("{} Autobuild configure".format(self.label)) |
145 | try: | |
146 | self.configure() | |
147 | except subprocess.CalledProcessError as e: | |
148 | _logger.error("{} Configure failed. See {} for more details.".format(self.label, self.log_path)) | |
149 | raise e | |
150 | ||
151 | _logger.debug("{} Autobuild build".format(self.label)) | |
152 | try: | |
153 | self.build() | |
154 | except subprocess.CalledProcessError as e: | |
155 | _logger.error("{} Build failed. See {} for more details.".format(self.label, self.log_path)) | |
156 | raise e | |
157 | ||
158 | _logger.debug("{} Autobuild install".format(self.label)) | |
159 | try: | |
160 | self.install() | |
161 | except subprocess.CalledProcessError as e: | |
162 | _logger.error("{} Install failed. See {} for more details.".format(self.label, self.log_path)) | |
163 | raise e | |
de9b991b JR |
164 | |
165 | def checkout(self): | |
fe7b987e JR |
166 | if self._immutable: |
167 | raise Exception("Object is immutable. Illegal checkout") | |
168 | ||
de9b991b JR |
169 | repo = git.Repo.clone_from(self.git_path, self.source_path) |
170 | commit = repo.commit(self.sha1) | |
171 | repo.head.reference = commit | |
172 | assert repo.head.is_detached | |
173 | repo.head.reset(index=True, working_tree=True) | |
174 | ||
175 | def bootstrap(self): | |
176 | """ | |
177 | Bootstap the project. Raise subprocess.CalledProcessError on | |
178 | bootstrap error. | |
179 | """ | |
fe7b987e JR |
180 | if self._immutable: |
181 | raise Exception("Object is immutable. Illegal bootstrap") | |
182 | ||
183 | out = os.path.join(self.log_path, "bootstrap.out") | |
184 | err = os.path.join(self.log_path, "bootstrap.err") | |
185 | ||
de9b991b | 186 | os.chdir(self.source_path) |
fe7b987e JR |
187 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
188 | p = subprocess.run(['./bootstrap'], stdout=stdout, stderr=stderr) | |
de9b991b JR |
189 | p.check_returncode() |
190 | return p | |
191 | ||
192 | def configure(self): | |
193 | """ | |
194 | Configure the project. | |
195 | Raises subprocess.CalledProcessError on configure error | |
196 | """ | |
fe7b987e JR |
197 | if self._immutable: |
198 | raise Exception("Object is immutable. Illegal configure") | |
199 | ||
de9b991b | 200 | # Check that all our dependencies were actually installed |
fe7b987e | 201 | for key, dep in self.dependencies.items(): |
de9b991b JR |
202 | if not dep.isInstalled: |
203 | # TODO: Custom exception here Dependency Error | |
204 | raise Exception("Dependency project flagged as not installed") | |
205 | ||
fe7b987e JR |
206 | out = os.path.join(self.log_path, "configure.out") |
207 | err = os.path.join(self.log_path, "configure.err") | |
208 | ||
de9b991b JR |
209 | os.chdir(self.source_path) |
210 | args = ['./configure'] | |
211 | prefix = '--prefix={}'.format(self.installation_path) | |
212 | args.append(prefix) | |
213 | args.extend(self.custom_configure_flags) | |
214 | ||
215 | # TODO: log output and add INFO log point | |
fe7b987e JR |
216 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
217 | p = subprocess.run(args, env=self.get_env(), stdout=stdout, | |
218 | stderr=stderr) | |
de9b991b JR |
219 | p.check_returncode() |
220 | self.isConfigured = True | |
221 | return p | |
222 | ||
223 | def build(self): | |
224 | """ | |
225 | Build the project. Raise subprocess.CalledProcessError on build | |
226 | error. | |
227 | """ | |
fe7b987e JR |
228 | if self._immutable: |
229 | raise Exception("Object is immutable. Illegal build") | |
230 | ||
231 | out = os.path.join(self.log_path, "build.out") | |
232 | err = os.path.join(self.log_path, "build.err") | |
233 | ||
de9b991b JR |
234 | os.chdir(self.source_path) |
235 | args = ['make'] | |
236 | env = self.get_env() | |
de9b991b JR |
237 | |
238 | # Number of usable cpu | |
239 | # https://docs.python.org/3/library/os.html#os.cpu_count | |
240 | num_cpu = str(len(os.sched_getaffinity(0))) | |
241 | args.append('-j') | |
242 | args.append(num_cpu) | |
97d494b0 | 243 | args.append('V=1') |
de9b991b JR |
244 | |
245 | # TODO: log output and add INFO log point with args | |
fe7b987e JR |
246 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
247 | p = subprocess.run(args, env=env, stdout=stdout, | |
248 | stderr=stderr) | |
de9b991b JR |
249 | p.check_returncode() |
250 | self.isBuilt = True | |
251 | return p | |
252 | ||
253 | def install(self): | |
254 | """ | |
255 | Install the project. Raise subprocess.CalledProcessError on | |
256 | bootstrap error | |
257 | """ | |
fe7b987e JR |
258 | if self._immutable: |
259 | raise Exception("Object is immutable. Illegal install") | |
260 | ||
97d494b0 JR |
261 | out = os.path.join(self.log_path, "install.out") |
262 | err = os.path.join(self.log_path, "install.err") | |
fe7b987e | 263 | |
de9b991b JR |
264 | os.chdir(self.source_path) |
265 | args = ['make', 'install'] | |
266 | ||
267 | # TODO: log output and add INFO log point | |
fe7b987e JR |
268 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
269 | p = subprocess.run(args, env=self.get_env(), stdout=stdout, | |
270 | stderr=stderr) | |
de9b991b JR |
271 | p.check_returncode() |
272 | self.isInstalled = True | |
273 | return p | |
274 | ||
275 | def cleanup(self): | |
276 | if os.path.exists(self.source_path): | |
277 | shutil.rmtree(self.source_path) | |
278 | if os.path.exists(self.installation_path): | |
279 | shutil.rmtree(self.installation_path) | |
280 | ||
281 | ||
282 | class Lttng_modules(Project): | |
97d494b0 JR |
283 | def __init__(self, label, git_path, sha1, tmpdir): |
284 | super(Lttng_modules, self).__init__(label=label, git_path=git_path, | |
285 | sha1=sha1, tmpdir=tmpdir) | |
f0acf3f3 | 286 | self.add_special_env_variable("MODPROBE_OPTIONS","-d {}".format(self.installation_path)) |
97d494b0 | 287 | |
de9b991b JR |
288 | def bootstrap(self): |
289 | pass | |
290 | ||
291 | def configure(self): | |
292 | pass | |
293 | ||
294 | def install(self): | |
fe7b987e JR |
295 | if self._immutable: |
296 | raise Exception("Object is immutable. Illegal install") | |
de9b991b JR |
297 | os.chdir(self.source_path) |
298 | args = ['make', 'INSTALL_MOD_PATH={}'.format(self.installation_path), | |
299 | 'modules_install'] | |
300 | p = subprocess.run(args, env=self.get_env(), stdout=subprocess.PIPE, | |
301 | stderr=subprocess.PIPE) | |
302 | p.check_returncode() | |
303 | ||
304 | # Perform a local depmod | |
305 | args = ['depmod', '-b', self.installation_path] | |
306 | p = subprocess.run(args, env=self.get_env()) | |
307 | p.check_returncode() | |
308 | self.isInstalled = True | |
309 | ||
310 | ||
311 | class Lttng_ust(Project): | |
312 | def __init__(self, label, git_path, sha1, tmpdir): | |
313 | super(Lttng_ust, self).__init__(label=label, git_path=git_path, | |
314 | sha1=sha1, tmpdir=tmpdir) | |
315 | self.custom_configure_flags.extend(['--disable-man-pages']) | |
316 | ||
317 | ||
318 | class Lttng_tools(Project): | |
18aedaf9 JR |
319 | def __init__(self, label, git_path, sha1, tmpdir): |
320 | super(Lttng_tools, self).__init__(label=label, git_path=git_path, | |
321 | sha1=sha1, tmpdir=tmpdir) | |
322 | self.add_special_env_variable("LTTNG_SESSION_CONFIG_XSD_PATH", | |
323 | os.path.join(self.installation_path, "share/xml/lttng/")) | |
de9b991b JR |
324 | |
325 | ||
326 | class Babeltrace(Project): | |
327 | pass |