Commit | Line | Data |
---|---|---|
b0810081 | 1 | import sys |
de9b991b JR |
2 | import os |
3 | import yaml | |
4 | import logging | |
fe7b987e | 5 | import hashlib |
de9b991b JR |
6 | |
7 | from git import Repo | |
8 | ||
b0810081 JR |
9 | dir_path = os.path.dirname(os.path.realpath(__file__)) |
10 | sys.path.insert(0, os.path.join(dir_path, "..")) | |
11 | ||
fe7b987e | 12 | import settings as Settings |
b0810081 | 13 | import utils.ProjectFactory as ProjectFactory |
de9b991b JR |
14 | |
15 | def is_ref_branch(repo, ref): | |
16 | try: | |
17 | repo.remote().refs[ref] | |
18 | is_branch = True | |
19 | except: | |
20 | is_branch = False | |
21 | ||
22 | return is_branch | |
23 | ||
24 | ||
25 | def is_ref_tag(repo, ref): | |
26 | try: | |
27 | repo.tags[ref] | |
28 | is_tag = True | |
29 | except: | |
30 | is_tag = False | |
31 | ||
32 | return is_tag | |
33 | ||
34 | ||
35 | def is_ref_commit(repo, ref): | |
36 | try: | |
37 | Repo.rev_parse(repo, ref) | |
38 | is_commit = True | |
39 | except: | |
40 | is_commit = False | |
41 | ||
42 | return is_commit | |
43 | ||
44 | ||
45 | def logging_setup(): | |
46 | logger_format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' | |
47 | logging.basicConfig(level=logging.DEBUG, | |
48 | format=logger_format, | |
49 | datefmt='%m-%d %H:%M', | |
50 | filename='./debug.log', | |
51 | filemode='w') | |
52 | # define a Handler which writes INFO messages or higher to the sys.stderr | |
53 | console = logging.StreamHandler() | |
54 | console.setLevel(logging.DEBUG) | |
55 | # set a format which is simpler for console use | |
56 | formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') | |
57 | # tell the handler to use this format | |
58 | console.setFormatter(formatter) | |
59 | # add the handler to the root logger | |
60 | logging.getLogger('').addHandler(console) | |
61 | ||
62 | ||
63 | logging_setup() | |
64 | ||
65 | # Remote setup | |
66 | logger_git = logging.getLogger('setup.git') | |
67 | ||
68 | # Fetch local base repository | |
fe7b987e | 69 | with open(Settings.configuration_file, 'r') as stream: |
de9b991b JR |
70 | config = yaml.load(stream) |
71 | ||
fe7b987e JR |
72 | # Validate that all default dependancy are present. |
73 | # TODO: move to function | |
74 | projects_markers = set() | |
75 | for project, markers in config.items(): | |
76 | if markers is None: | |
77 | continue | |
78 | for marker in markers: | |
79 | projects_markers.add(marker['marker']) | |
80 | ||
81 | for project, markers in config.items(): | |
82 | if markers is None: | |
83 | continue | |
84 | for marker in markers: | |
85 | if 'precook_deps' in marker: | |
86 | for dep in marker['precook_deps']: | |
87 | if dep not in projects_markers: | |
88 | raise Exception("{} is not defined".format(dep)) | |
89 | ||
90 | ||
de9b991b JR |
91 | # Retrieve all possibles remotes and clean url for path |
92 | remotes = {} | |
93 | for project, markers in config.items(): | |
94 | if markers is None: | |
95 | continue | |
96 | for marker in markers: | |
97 | url = marker['url'] | |
fe7b987e JR |
98 | url2path = hashlib.sha1(url.encode('utf8')).hexdigest() |
99 | path = os.path.abspath(Settings.git_remote_folder + '/' + url2path) | |
de9b991b JR |
100 | remotes[url] = path |
101 | ||
102 | logger_git.info('Remotes to be fetched {}'.format(remotes)) | |
103 | ||
fe7b987e JR |
104 | if not os.path.isdir(Settings.git_remote_folder): |
105 | os.makedirs(Settings.git_remote_folder) | |
de9b991b JR |
106 | |
107 | # Fetch the remote | |
108 | for url, path in remotes.items(): | |
109 | if os.path.exists(path): | |
110 | if not os.path.isdir(path): | |
111 | logger_git.error('Remote path {} exists and is not a folder'.format(path)) | |
112 | exit() | |
113 | repo = Repo(path) | |
114 | else: | |
115 | repo = Repo.clone_from(url, path) | |
116 | ||
117 | # TODO: might be necessary to actually update the base branch, to validate | |
118 | repo.remote().fetch() | |
119 | ||
120 | # Create marker definition for test runners | |
121 | runnable_markers = {} | |
122 | for project, markers in config.items(): | |
123 | if markers is None: | |
124 | continue | |
125 | for marker in markers: | |
126 | name = marker['marker'] | |
127 | ref = marker['ref'] | |
128 | url = marker['url'] | |
fe7b987e JR |
129 | if 'precook_deps' in marker: |
130 | deps = marker['precook_deps'] | |
131 | else: | |
132 | deps = [] | |
133 | ||
de9b991b JR |
134 | path = remotes[url] |
135 | repo = Repo(path) | |
136 | ||
137 | git_object = None | |
138 | if is_ref_branch(repo, ref): | |
139 | git_object = Repo.rev_parse(repo, repo.remote().refs[ref].name) | |
140 | elif is_ref_tag(repo, ref): | |
141 | git_object = repo.tags[ref].commit | |
142 | elif is_ref_commit(repo, ref): | |
143 | git_object = repo.commit(ref) | |
144 | ||
145 | if git_object is None: | |
146 | logger_git.error('Invalid git reference for marker "{}"'.format(name)) | |
147 | exit(1) | |
148 | ||
149 | logger_git.info('Marker:{: <30} Sha1 {: <20}'.format(name, git_object.hexsha)) | |
150 | ||
151 | if name in runnable_markers: | |
152 | logger_git.error('Duplicate for entry for marker "{}"'.format(name)) | |
153 | exit(1) | |
154 | ||
155 | runnable_markers[name] = { | |
156 | 'project': project, | |
157 | 'sha1': git_object.hexsha, | |
158 | 'url': url, | |
fe7b987e JR |
159 | 'path': path, |
160 | 'deps': deps | |
de9b991b JR |
161 | } |
162 | ||
fe7b987e | 163 | with open(Settings.run_configuration_file, 'w') as run_configuration: |
de9b991b | 164 | yaml.dump(runnable_markers, run_configuration, default_flow_style=False) |
b0810081 JR |
165 | |
166 | # Prebuild all projects | |
167 | for key in runnable_markers: | |
168 | logger_git.info('Preparing and building {}'.format(key)) | |
169 | ProjectFactory.get_precook(key) |