lttng_ivc/bootstrap.py

   1 import sys
   2 import os
   3 import yaml
   4 import logging
   5 import hashlib
   6
   7 from git import Repo
   8
   9 dir_path = os.path.dirname(os.path.realpath(__file__))
  10 sys.path.insert(0, os.path.join(dir_path, ".."))
  11
  12 import settings as Settings
  13 import utils.ProjectFactory as ProjectFactory
  14
  15 def is_ref_branch(repo, ref):
  16     try:
  17         repo.remote().refs[ref]
  18         is_branch = True
  19     except:
  20         is_branch = False
  21
  22     return is_branch
  23
  24
  25 def is_ref_tag(repo, ref):
  26     try:
  27         repo.tags[ref]
  28         is_tag = True
  29     except:
  30         is_tag = False
  31
  32     return is_tag
  33
  34
  35 def is_ref_commit(repo, ref):
  36     try:
  37         Repo.rev_parse(repo, ref)
  38         is_commit = True
  39     except:
  40         is_commit = False
  41
  42     return is_commit
  43
  44
  45 def logging_setup():
  46     logger_format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s'
  47     logging.basicConfig(level=logging.DEBUG,
  48                         format=logger_format,
  49                         datefmt='%m-%d %H:%M',
  50                         filename='./debug.log',
  51                         filemode='w')
  52     # define a Handler which writes INFO messages or higher to the sys.stderr
  53     console = logging.StreamHandler()
  54     console.setLevel(logging.DEBUG)
  55     # set a format which is simpler for console use
  56     formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
  57     # tell the handler to use this format
  58     console.setFormatter(formatter)
  59     # add the handler to the root logger
  60     logging.getLogger('').addHandler(console)
  61
  62
  63 logging_setup()
  64
  65 # Remote setup
  66 logger_git = logging.getLogger('setup.git')
  67
  68 # Fetch local base repository
  69 with open(Settings.configuration_file, 'r') as stream:
  70     config = yaml.load(stream)
  71
  72 # Validate that all default dependancy are present.
  73 # TODO: move to function
  74 projects_markers = set()
  75 for project, markers in config.items():
  76     if markers is None:
  77         continue
  78     for marker in markers:
  79         projects_markers.add(marker['marker'])
  80
  81 for project, markers in config.items():
  82     if markers is None:
  83         continue
  84     for marker in markers:
  85         if 'precook_deps' in marker:
  86             for dep in marker['precook_deps']:
  87                 if dep not in projects_markers:
  88                     raise Exception("{} is not defined".format(dep))
  89
  90
  91 # Retrieve all possibles remotes and clean url for path
  92 remotes = {}
  93 for project, markers in config.items():
  94     if markers is None:
  95         continue
  96     for marker in markers:
  97         url = marker['url']
  98         url2path = hashlib.sha1(url.encode('utf8')).hexdigest()
  99         path = os.path.abspath(Settings.git_remote_folder + '/' + url2path)
 100         remotes[url] = path
 101
 102 logger_git.info('Remotes to be fetched {}'.format(remotes))
 103
 104 if not os.path.isdir(Settings.git_remote_folder):
 105     os.makedirs(Settings.git_remote_folder)
 106
 107 # Fetch the remote
 108 for url, path in remotes.items():
 109     if os.path.exists(path):
 110         if not os.path.isdir(path):
 111             logger_git.error('Remote path {} exists and is not a folder'.format(path))
 112             exit()
 113         repo = Repo(path)
 114     else:
 115         repo = Repo.clone_from(url, path)
 116
 117     # TODO: might be necessary to actually update the base branch, to validate
 118     repo.remote().fetch()
 119
 120 # Create marker definition for test runners
 121 runnable_markers = {}
 122 for project, markers in config.items():
 123     if markers is None:
 124         continue
 125     for marker in markers:
 126         name = marker['marker']
 127         ref = marker['ref']
 128         url = marker['url']
 129         if 'precook_deps' in marker:
 130             deps = marker['precook_deps']
 131         else:
 132             deps = []
 133
 134         path = remotes[url]
 135         repo = Repo(path)
 136
 137         git_object = None
 138         if is_ref_branch(repo, ref):
 139             git_object = Repo.rev_parse(repo, repo.remote().refs[ref].name)
 140         elif is_ref_tag(repo, ref):
 141             git_object = repo.tags[ref].commit
 142         elif is_ref_commit(repo, ref):
 143             git_object = repo.commit(ref)
 144
 145         if git_object is None:
 146             logger_git.error('Invalid git reference for marker "{}"'.format(name))
 147             exit(1)
 148
 149         logger_git.info('Marker:{: <30}  Sha1 {: <20}'.format(name, git_object.hexsha))
 150
 151         if name in runnable_markers:
 152             logger_git.error('Duplicate for entry for marker "{}"'.format(name))
 153             exit(1)
 154
 155         runnable_markers[name] = {
 156                 'project': project,
 157                 'sha1': git_object.hexsha,
 158                 'url': url,
 159                 'path': path,
 160                 'deps': deps
 161         }
 162
 163 with open(Settings.run_configuration_file, 'w') as run_configuration:
 164     yaml.dump(runnable_markers, run_configuration, default_flow_style=False)
 165
 166 # Prebuild all projects
 167 for key in runnable_markers:
 168     logger_git.info('Preparing and building {}'.format(key))
 169     ProjectFactory.get_precook(key)