[deliverable/linux.git] / scripts / checkkconfigsymbols.py

#!/usr/bin/env python

"""Find Kconfig symbols that are referenced but not defined."""

# (c) 2014-2015 Valentin Rothberg <Valentin.Rothberg@lip6.fr>
# (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de>
#
# Licensed under the terms of the GNU GPL License version 2


import os
import re
import sys
from subprocess import Popen, PIPE, STDOUT
from optparse import OptionParser


# regex expressions
OPERATORS = r"&|\(|\)|\||\!"
FEATURE = r"(?:\w*[A-Z0-9]\w*){2,}"
DEF = r"^\s*(?:menu){,1}config\s+(" + FEATURE + r")\s*"
EXPR = r"(?:" + OPERATORS + r"|\s|" + FEATURE + r")+"
STMT = r"^\s*(?:if|select|depends\s+on)\s+" + EXPR
SOURCE_FEATURE = r"(?:\W|\b)+[D]{,1}CONFIG_(" + FEATURE + r")"

# regex objects
REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$")
REGEX_FEATURE = re.compile(r"(" + FEATURE + r")")
REGEX_SOURCE_FEATURE = re.compile(SOURCE_FEATURE)
REGEX_KCONFIG_DEF = re.compile(DEF)
REGEX_KCONFIG_EXPR = re.compile(EXPR)
REGEX_KCONFIG_STMT = re.compile(STMT)
REGEX_KCONFIG_HELP = re.compile(r"^\s+(help|---help---)\s*$")
REGEX_FILTER_FEATURES = re.compile(r"[A-Za-z0-9]$")


def parse_options():
    """The user interface of this module."""
    usage = "%prog [options]\n\n"                                              \
            "Run this tool to detect Kconfig symbols that are referenced but " \
            "not defined in\nKconfig.  The output of this tool has the "       \
            "format \'Undefined symbol\\tFile list\'\n\n"                      \
            "If no option is specified, %prog will default to check your\n"    \
            "current tree.  Please note that specifying commits will "         \
            "\'git reset --hard\'\nyour current tree!  You may save "          \
            "uncommitted changes to avoid losing data."

    parser = OptionParser(usage=usage)

    parser.add_option('-c', '--commit', dest='commit', action='store',
                      default="",
                      help="Check if the specified commit (hash) introduces "
                           "undefined Kconfig symbols.")

    parser.add_option('-d', '--diff', dest='diff', action='store',
                      default="",
                      help="Diff undefined symbols between two commits.  The "
                           "input format bases on Git log's "
                           "\'commmit1..commit2\'.")

    parser.add_option('', '--force', dest='force', action='store_true',
                      default=False,
                      help="Reset current Git tree even when it's dirty.")

    (opts, _) = parser.parse_args()

    if opts.commit and opts.diff:
        sys.exit("Please specify only one option at once.")

    if opts.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", opts.diff):
        sys.exit("Please specify valid input in the following format: "
                 "\'commmit1..commit2\'")

    if opts.commit or opts.diff:
        if not opts.force and tree_is_dirty():
            sys.exit("The current Git tree is dirty (see 'git status').  "
                     "Running this script may\ndelete important data since it "
                     "calls 'git reset --hard' for some performance\nreasons. "
                     " Please run this script in a clean Git tree or pass "
                     "'--force' if you\nwant to ignore this warning and "
                     "continue.")

    return opts


def main():
    """Main function of this module."""
    opts = parse_options()

    if opts.commit or opts.diff:
        head = get_head()

        # get commit range
        commit_a = None
        commit_b = None
        if opts.commit:
            commit_a = opts.commit + "~"
            commit_b = opts.commit
        elif opts.diff:
            split = opts.diff.split("..")
            commit_a = split[0]
            commit_b = split[1]
            undefined_a = {}
            undefined_b = {}

        # get undefined items before the commit
        execute("git reset --hard %s" % commit_a)
        undefined_a = check_symbols()

        # get undefined items for the commit
        execute("git reset --hard %s" % commit_b)
        undefined_b = check_symbols()

        # report cases that are present for the commit but not before
        for feature in undefined_b:
            # feature has not been undefined before
            if not feature in undefined_a:
                files = undefined_b.get(feature)
                print "%s\t%s" % (feature, ", ".join(files))
            # check if there are new files that reference the undefined feature
            else:
                files = undefined_b.get(feature) - undefined_a.get(feature)
                if files:
                    print "%s\t%s" % (feature, ", ".join(files))

        # reset to head
        execute("git reset --hard %s" % head)

    # default to check the entire tree
    else:
        undefined = check_symbols()
        for feature in undefined:
            files = undefined.get(feature)


def execute(cmd):
    """Execute %cmd and return stdout.  Exit in case of error."""
    pop = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True)
    (stdout, _) = pop.communicate()  # wait until finished
    if pop.returncode != 0:
        sys.exit(stdout)
    return stdout


def tree_is_dirty():
    """Return true if the current working tree is dirty (i.e., if any file has
    been added, deleted, modified, renamed or copied but not committed)."""
    stdout = execute("git status --porcelain")
    for line in stdout:
        if re.findall(r"[URMADC]{1}", line[:2]):
            return True
    return False


def get_head():
    """Return commit hash of current HEAD."""
    stdout = execute("git rev-parse HEAD")
    return stdout.strip('\n')


def check_symbols():
    """Find undefined Kconfig symbols and return a dict with the symbol as key
    and a list of referencing files as value."""
    source_files = []
    kconfig_files = []
    defined_features = set()
    referenced_features = dict()  # {feature: [files]}

    # use 'git ls-files' to get the worklist
    stdout = execute("git ls-files")
    if len(stdout) > 0 and stdout[-1] == "\n":
        stdout = stdout[:-1]

    for gitfile in stdout.rsplit("\n"):
        if ".git" in gitfile or "ChangeLog" in gitfile or      \
                ".log" in gitfile or os.path.isdir(gitfile) or \
                gitfile.startswith("tools/"):
            continue
        if REGEX_FILE_KCONFIG.match(gitfile):
            kconfig_files.append(gitfile)
        else:
            # all non-Kconfig files are checked for consistency
            source_files.append(gitfile)

    for sfile in source_files:
        parse_source_file(sfile, referenced_features)

    for kfile in kconfig_files:
        parse_kconfig_file(kfile, defined_features, referenced_features)

    undefined = {}  # {feature: [files]}
    for feature in sorted(referenced_features):
        # filter some false positives
        if feature == "FOO" or feature == "BAR" or \
                feature == "FOO_BAR" or feature == "XXX":
            continue
        if feature not in defined_features:
            if feature.endswith("_MODULE"):
                # avoid false positives for kernel modules
                if feature[:-len("_MODULE")] in defined_features:
                    continue
            undefined[feature] = referenced_features.get(feature)
    return undefined


def parse_source_file(sfile, referenced_features):
    """Parse @sfile for referenced Kconfig features."""
    lines = []
    with open(sfile, "r") as stream:
        lines = stream.readlines()

    for line in lines:
        if not "CONFIG_" in line:
            continue
        features = REGEX_SOURCE_FEATURE.findall(line)
        for feature in features:
            if not REGEX_FILTER_FEATURES.search(feature):
                continue
            sfiles = referenced_features.get(feature, set())
            sfiles.add(sfile)
            referenced_features[feature] = sfiles


def get_features_in_line(line):
    """Return mentioned Kconfig features in @line."""
    return REGEX_FEATURE.findall(line)


def parse_kconfig_file(kfile, defined_features, referenced_features):
    """Parse @kfile and update feature definitions and references."""
    lines = []
    skip = False

    with open(kfile, "r") as stream:
        lines = stream.readlines()

    for i in range(len(lines)):
        line = lines[i]
        line = line.strip('\n')
        line = line.split("#")[0]  # ignore comments

        if REGEX_KCONFIG_DEF.match(line):
            feature_def = REGEX_KCONFIG_DEF.findall(line)
            defined_features.add(feature_def[0])
            skip = False
        elif REGEX_KCONFIG_HELP.match(line):
            skip = True
        elif skip:
            # ignore content of help messages
            pass
        elif REGEX_KCONFIG_STMT.match(line):
            features = get_features_in_line(line)
            # multi-line statements
            while line.endswith("\\"):
                i += 1
                line = lines[i]
                line = line.strip('\n')
                features.extend(get_features_in_line(line))
            for feature in set(features):
                paths = referenced_features.get(feature, set())
                paths.add(kfile)
                referenced_features[feature] = paths


if __name__ == "__main__":
    main()
Commit	Line	Data
24fe1f03 VR	1	#!/usr/bin/env python
24fe1f03 VR	2
b1a3f243	3	"""Find Kconfig symbols that are referenced but not defined."""
24fe1f03	4
208d5115	5	# (c) 2014-2015 Valentin Rothberg <Valentin.Rothberg@lip6.fr>
cc641d55	6	# (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de>
24fe1f03	7	#
cc641d55	8	# Licensed under the terms of the GNU GPL License version 2
24fe1f03 VR	9
	10
	11	import os
	12	import re
b1a3f243	13	import sys
24fe1f03	14	from subprocess import Popen, PIPE, STDOUT
b1a3f243	15	from optparse import OptionParser
24fe1f03	16
cc641d55 VR	17
cc641d55 VR	18	# regex expressions
24fe1f03	19	OPERATORS = r"&\|\(\|\)\|\\|\|\!"
cc641d55 VR	20	FEATURE = r"(?:\w[A-Z0-9]\w){2,}"
cc641d55 VR	21	DEF = r"^\s(?:menu){,1}config\s+(" + FEATURE + r")\s"
24fe1f03 VR	22	EXPR = r"(?:" + OPERATORS + r"\|\s\|" + FEATURE + r")+"
24fe1f03 VR	23	STMT = r"^\s*(?:if\|select\|depends\s+on)\s+" + EXPR
cc641d55	24	SOURCE_FEATURE = r"(?:\W\|\b)+[D]{,1}CONFIG_(" + FEATURE + r")"
24fe1f03	25
cc641d55	26	# regex objects
24fe1f03 VR	27	REGEX_FILE_KCONFIG = re.compile(r".Kconfig[\.\w+\-]$")
24fe1f03 VR	28	REGEX_FEATURE = re.compile(r"(" + FEATURE + r")")
cc641d55 VR	29	REGEX_SOURCE_FEATURE = re.compile(SOURCE_FEATURE)
cc641d55 VR	30	REGEX_KCONFIG_DEF = re.compile(DEF)
24fe1f03 VR	31	REGEX_KCONFIG_EXPR = re.compile(EXPR)
	32	REGEX_KCONFIG_STMT = re.compile(STMT)
	33	REGEX_KCONFIG_HELP = re.compile(r"^\s+(help\|---help---)\s*$")
	34	REGEX_FILTER_FEATURES = re.compile(r"[A-Za-z0-9]$")
	35
	36
b1a3f243 VR	37	def parse_options():
	38	"""The user interface of this module."""
	39	usage = "%prog [options]\n\n" \
	40	"Run this tool to detect Kconfig symbols that are referenced but " \
	41	"not defined in\nKconfig. The output of this tool has the " \
	42	"format \'Undefined symbol\\tFile list\'\n\n" \
	43	"If no option is specified, %prog will default to check your\n" \
	44	"current tree. Please note that specifying commits will " \
	45	"\'git reset --hard\'\nyour current tree! You may save " \
	46	"uncommitted changes to avoid losing data."
	47
	48	parser = OptionParser(usage=usage)
	49
	50	parser.add_option('-c', '--commit', dest='commit', action='store',
	51	default="",
	52	help="Check if the specified commit (hash) introduces "
	53	"undefined Kconfig symbols.")
	54
	55	parser.add_option('-d', '--diff', dest='diff', action='store',
	56	default="",
	57	help="Diff undefined symbols between two commits. The "
	58	"input format bases on Git log's "
	59	"\'commmit1..commit2\'.")
	60
	61	parser.add_option('', '--force', dest='force', action='store_true',
	62	default=False,
	63	help="Reset current Git tree even when it's dirty.")
	64
	65	(opts, _) = parser.parse_args()
	66
	67	if opts.commit and opts.diff:
	68	sys.exit("Please specify only one option at once.")
	69
	70	if opts.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", opts.diff):
	71	sys.exit("Please specify valid input in the following format: "
	72	"\'commmit1..commit2\'")
	73
	74	if opts.commit or opts.diff:
	75	if not opts.force and tree_is_dirty():
	76	sys.exit("The current Git tree is dirty (see 'git status'). "
	77	"Running this script may\ndelete important data since it "
	78	"calls 'git reset --hard' for some performance\nreasons. "
	79	" Please run this script in a clean Git tree or pass "
	80	"'--force' if you\nwant to ignore this warning and "
	81	"continue.")
	82
	83	return opts
	84
	85
24fe1f03 VR	86	def main():
24fe1f03 VR	87	"""Main function of this module."""
b1a3f243 VR	88	opts = parse_options()
	89
	90	if opts.commit or opts.diff:
	91	head = get_head()
	92
	93	# get commit range
	94	commit_a = None
	95	commit_b = None
	96	if opts.commit:
	97	commit_a = opts.commit + "~"
	98	commit_b = opts.commit
	99	elif opts.diff:
	100	split = opts.diff.split("..")
	101	commit_a = split[0]
	102	commit_b = split[1]
	103	undefined_a = {}
	104	undefined_b = {}
	105
	106	# get undefined items before the commit
	107	execute("git reset --hard %s" % commit_a)
	108	undefined_a = check_symbols()
	109
	110	# get undefined items for the commit
	111	execute("git reset --hard %s" % commit_b)
	112	undefined_b = check_symbols()
	113
	114	# report cases that are present for the commit but not before
	115	for feature in undefined_b:
	116	# feature has not been undefined before
	117	if not feature in undefined_a:
	118	files = undefined_b.get(feature)
	119	print "%s\t%s" % (feature, ", ".join(files))
	120	# check if there are new files that reference the undefined feature
	121	else:
	122	files = undefined_b.get(feature) - undefined_a.get(feature)
	123	if files:
	124	print "%s\t%s" % (feature, ", ".join(files))
	125
	126	# reset to head
	127	execute("git reset --hard %s" % head)
	128
	129	# default to check the entire tree
	130	else:
	131	undefined = check_symbols()
	132	for feature in undefined:
	133	files = undefined.get(feature)
	134
	135
	136	def execute(cmd):
	137	"""Execute %cmd and return stdout. Exit in case of error."""
	138	pop = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True)
	139	(stdout, _) = pop.communicate() # wait until finished
	140	if pop.returncode != 0:
	141	sys.exit(stdout)
	142	return stdout
	143
	144
	145	def tree_is_dirty():
	146	"""Return true if the current working tree is dirty (i.e., if any file has
	147	been added, deleted, modified, renamed or copied but not committed)."""
	148	stdout = execute("git status --porcelain")
	149	for line in stdout:
	150	if re.findall(r"[URMADC]{1}", line[:2]):
	151	return True
152	return False
153
154
155	def get_head():
156	"""Return commit hash of current HEAD."""
157	stdout = execute("git rev-parse HEAD")
158	return stdout.strip('\n')
159
160
161	def check_symbols():
162	"""Find undefined Kconfig symbols and return a dict with the symbol as key
163	and a list of referencing files as value."""
24fe1f03 VR	164	source_files = []
	165	kconfig_files = []
	166	defined_features = set()
cc641d55	167	referenced_features = dict() # {feature: [files]}
24fe1f03 VR	168
24fe1f03 VR	169	# use 'git ls-files' to get the worklist
b1a3f243	170	stdout = execute("git ls-files")
24fe1f03 VR	171	if len(stdout) > 0 and stdout[-1] == "\n":
	172	stdout = stdout[:-1]
	173
	174	for gitfile in stdout.rsplit("\n"):
208d5115 VR	175	if ".git" in gitfile or "ChangeLog" in gitfile or \
	176	".log" in gitfile or os.path.isdir(gitfile) or \
	177	gitfile.startswith("tools/"):
24fe1f03 VR	178	continue
	179	if REGEX_FILE_KCONFIG.match(gitfile):
	180	kconfig_files.append(gitfile)
	181	else:
cc641d55	182	# all non-Kconfig files are checked for consistency
24fe1f03 VR	183	source_files.append(gitfile)
	184
	185	for sfile in source_files:
	186	parse_source_file(sfile, referenced_features)
	187
	188	for kfile in kconfig_files:
	189	parse_kconfig_file(kfile, defined_features, referenced_features)
	190
b1a3f243	191	undefined = {} # {feature: [files]}
24fe1f03	192	for feature in sorted(referenced_features):
cc641d55 VR	193	# filter some false positives
	194	if feature == "FOO" or feature == "BAR" or \
	195	feature == "FOO_BAR" or feature == "XXX":
	196	continue
24fe1f03 VR	197	if feature not in defined_features:
24fe1f03 VR	198	if feature.endswith("_MODULE"):
cc641d55	199	# avoid false positives for kernel modules
24fe1f03 VR	200	if feature[:-len("_MODULE")] in defined_features:
24fe1f03 VR	201	continue
b1a3f243 VR	202	undefined[feature] = referenced_features.get(feature)
b1a3f243 VR	203	return undefined
24fe1f03 VR	204
	205
	206	def parse_source_file(sfile, referenced_features):
	207	"""Parse @sfile for referenced Kconfig features."""
	208	lines = []
	209	with open(sfile, "r") as stream:
	210	lines = stream.readlines()
	211
	212	for line in lines:
	213	if not "CONFIG_" in line:
	214	continue
	215	features = REGEX_SOURCE_FEATURE.findall(line)
	216	for feature in features:
	217	if not REGEX_FILTER_FEATURES.search(feature):
	218	continue
cc641d55 VR	219	sfiles = referenced_features.get(feature, set())
	220	sfiles.add(sfile)
	221	referenced_features[feature] = sfiles
24fe1f03 VR	222
	223
	224	def get_features_in_line(line):
	225	"""Return mentioned Kconfig features in @line."""
	226	return REGEX_FEATURE.findall(line)
	227
	228
	229	def parse_kconfig_file(kfile, defined_features, referenced_features):
	230	"""Parse @kfile and update feature definitions and references."""
	231	lines = []
	232	skip = False
	233
	234	with open(kfile, "r") as stream:
	235	lines = stream.readlines()
	236
	237	for i in range(len(lines)):
	238	line = lines[i]
	239	line = line.strip('\n')
cc641d55	240	line = line.split("#")[0] # ignore comments
24fe1f03 VR	241
	242	if REGEX_KCONFIG_DEF.match(line):
	243	feature_def = REGEX_KCONFIG_DEF.findall(line)
	244	defined_features.add(feature_def[0])
	245	skip = False
	246	elif REGEX_KCONFIG_HELP.match(line):
	247	skip = True
	248	elif skip:
cc641d55	249	# ignore content of help messages
24fe1f03 VR	250	pass
	251	elif REGEX_KCONFIG_STMT.match(line):
	252	features = get_features_in_line(line)
cc641d55	253	# multi-line statements
24fe1f03 VR	254	while line.endswith("\\"):
	255	i += 1
	256	line = lines[i]
	257	line = line.strip('\n')
	258	features.extend(get_features_in_line(line))
	259	for feature in set(features):
	260	paths = referenced_features.get(feature, set())
	261	paths.add(kfile)
	262	referenced_features[feature] = paths
	263
	264
	265	if __name__ == "__main__":
	266	main()