scripts/checkkconfigsymbols.py

   1 #!/usr/bin/env python2
   2
   3 """Find Kconfig symbols that are referenced but not defined."""
   4
   5 # (c) 2014-2015 Valentin Rothberg <valentinrothberg@gmail.com>
   6 # (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de>
   7 #
   8 # Licensed under the terms of the GNU GPL License version 2
   9
  10
  11 import os
  12 import re
  13 import sys
  14 from subprocess import Popen, PIPE, STDOUT
  15 from optparse import OptionParser
  16
  17
  18 # regex expressions
  19 OPERATORS = r"&|\(|\)|\||\!"
  20 FEATURE = r"(?:\w*[A-Z0-9]\w*){2,}"
  21 DEF = r"^\s*(?:menu){,1}config\s+(" + FEATURE + r")\s*"
  22 EXPR = r"(?:" + OPERATORS + r"|\s|" + FEATURE + r")+"
  23 DEFAULT = r"default\s+.*?(?:if\s.+){,1}"
  24 STMT = r"^\s*(?:if|select|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR
  25 SOURCE_FEATURE = r"(?:\W|\b)+[D]{,1}CONFIG_(" + FEATURE + r")"
  26
  27 # regex objects
  28 REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$")
  29 REGEX_FEATURE = re.compile(r'(?!\B"[^"]*)' + FEATURE + r'(?![^"]*"\B)')
  30 REGEX_SOURCE_FEATURE = re.compile(SOURCE_FEATURE)
  31 REGEX_KCONFIG_DEF = re.compile(DEF)
  32 REGEX_KCONFIG_EXPR = re.compile(EXPR)
  33 REGEX_KCONFIG_STMT = re.compile(STMT)
  34 REGEX_KCONFIG_HELP = re.compile(r"^\s+(help|---help---)\s*$")
  35 REGEX_FILTER_FEATURES = re.compile(r"[A-Za-z0-9]$")
  36 REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+")
  37
  38
  39 def parse_options():
  40     """The user interface of this module."""
  41     usage = "%prog [options]\n\n"                                              \
  42             "Run this tool to detect Kconfig symbols that are referenced but " \
  43             "not defined in\nKconfig.  The output of this tool has the "       \
  44             "format \'Undefined symbol\\tFile list\'\n\n"                      \
  45             "If no option is specified, %prog will default to check your\n"    \
  46             "current tree.  Please note that specifying commits will "         \
  47             "\'git reset --hard\'\nyour current tree!  You may save "          \
  48             "uncommitted changes to avoid losing data."
  49
  50     parser = OptionParser(usage=usage)
  51
  52     parser.add_option('-c', '--commit', dest='commit', action='store',
  53                       default="",
  54                       help="Check if the specified commit (hash) introduces "
  55                            "undefined Kconfig symbols.")
  56
  57     parser.add_option('-d', '--diff', dest='diff', action='store',
  58                       default="",
  59                       help="Diff undefined symbols between two commits.  The "
  60                            "input format bases on Git log's "
  61                            "\'commmit1..commit2\'.")
  62
  63     parser.add_option('-f', '--find', dest='find', action='store_true',
  64                       default=False,
  65                       help="Find and show commits that may cause symbols to be "
  66                            "missing.  Required to run with --diff.")
  67
  68     parser.add_option('-i', '--ignore', dest='ignore', action='store',
  69                       default="",
  70                       help="Ignore files matching this pattern.  Note that "
  71                            "the pattern needs to be a Python regex.  To "
  72                            "ignore defconfigs, specify -i '.*defconfig'.")
  73
  74     parser.add_option('', '--force', dest='force', action='store_true',
  75                       default=False,
  76                       help="Reset current Git tree even when it's dirty.")
  77
  78     (opts, _) = parser.parse_args()
  79
  80     if opts.commit and opts.diff:
  81         sys.exit("Please specify only one option at once.")
  82
  83     if opts.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", opts.diff):
  84         sys.exit("Please specify valid input in the following format: "
  85                  "\'commmit1..commit2\'")
  86
  87     if opts.commit or opts.diff:
  88         if not opts.force and tree_is_dirty():
  89             sys.exit("The current Git tree is dirty (see 'git status').  "
  90                      "Running this script may\ndelete important data since it "
  91                      "calls 'git reset --hard' for some performance\nreasons. "
  92                      " Please run this script in a clean Git tree or pass "
  93                      "'--force' if you\nwant to ignore this warning and "
  94                      "continue.")
  95
  96     if opts.commit:
  97         opts.find = False
  98
  99     if opts.ignore:
 100         try:
 101             re.match(opts.ignore, "this/is/just/a/test.c")
 102         except:
 103             sys.exit("Please specify a valid Python regex.")
 104
 105     return opts
 106
 107
 108 def main():
 109     """Main function of this module."""
 110     opts = parse_options()
 111
 112     if opts.commit or opts.diff:
 113         head = get_head()
 114
 115         # get commit range
 116         commit_a = None
 117         commit_b = None
 118         if opts.commit:
 119             commit_a = opts.commit + "~"
 120             commit_b = opts.commit
 121         elif opts.diff:
 122             split = opts.diff.split("..")
 123             commit_a = split[0]
 124             commit_b = split[1]
 125             undefined_a = {}
 126             undefined_b = {}
 127
 128         # get undefined items before the commit
 129         execute("git reset --hard %s" % commit_a)
 130         undefined_a = check_symbols(opts.ignore)
 131
 132         # get undefined items for the commit
 133         execute("git reset --hard %s" % commit_b)
 134         undefined_b = check_symbols(opts.ignore)
 135
 136         # report cases that are present for the commit but not before
 137         for feature in sorted(undefined_b):
 138             # feature has not been undefined before
 139             if not feature in undefined_a:
 140                 files = sorted(undefined_b.get(feature))
 141                 print "%s\t%s" % (yel(feature), ", ".join(files))
 142                 if opts.find:
 143                     commits = find_commits(feature, opts.diff)
 144                     print red(commits)
 145             # check if there are new files that reference the undefined feature
 146             else:
 147                 files = sorted(undefined_b.get(feature) -
 148                                undefined_a.get(feature))
 149                 if files:
 150                     print "%s\t%s" % (yel(feature), ", ".join(files))
 151                     if opts.find:
 152                         commits = find_commits(feature, opts.diff)
 153                         print red(commits)
 154
 155         # reset to head
 156         execute("git reset --hard %s" % head)
 157
 158     # default to check the entire tree
 159     else:
 160         undefined = check_symbols(opts.ignore)
 161         for feature in sorted(undefined):
 162             files = sorted(undefined.get(feature))
 163             print "%s\t%s" % (yel(feature), ", ".join(files))
 164
 165
 166 def yel(string):
 167     """
 168     Color %string yellow.
 169     """
 170     return "\033[33m%s\033[0m" % string
 171
 172
 173 def red(string):
 174     """
 175     Color %string red.
 176     """
 177     return "\033[31m%s\033[0m" % string
 178
 179
 180 def execute(cmd):
 181     """Execute %cmd and return stdout.  Exit in case of error."""
 182     pop = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True)
 183     (stdout, _) = pop.communicate()  # wait until finished
 184     if pop.returncode != 0:
 185         sys.exit(stdout)
 186     return stdout
 187
 188
 189 def find_commits(symbol, diff):
 190     """Find commits changing %symbol in the given range of %diff."""
 191     commits = execute("git log --pretty=oneline --abbrev-commit -G %s %s"
 192                       % (symbol, diff))
 193     return commits
 194
 195
 196 def tree_is_dirty():
 197     """Return true if the current working tree is dirty (i.e., if any file has
 198     been added, deleted, modified, renamed or copied but not committed)."""
 199     stdout = execute("git status --porcelain")
 200     for line in stdout:
 201         if re.findall(r"[URMADC]{1}", line[:2]):
 202             return True
 203     return False
 204
 205
 206 def get_head():
 207     """Return commit hash of current HEAD."""
 208     stdout = execute("git rev-parse HEAD")
 209     return stdout.strip('\n')
 210
 211
 212 def check_symbols(ignore):
 213     """Find undefined Kconfig symbols and return a dict with the symbol as key
 214     and a list of referencing files as value.  Files matching %ignore are not
 215     checked for undefined symbols."""
 216     source_files = []
 217     kconfig_files = []
 218     defined_features = set()
 219     referenced_features = dict()  # {feature: [files]}
 220
 221     # use 'git ls-files' to get the worklist
 222     stdout = execute("git ls-files")
 223     if len(stdout) > 0 and stdout[-1] == "\n":
 224         stdout = stdout[:-1]
 225
 226     for gitfile in stdout.rsplit("\n"):
 227         if ".git" in gitfile or "ChangeLog" in gitfile or      \
 228                 ".log" in gitfile or os.path.isdir(gitfile) or \
 229                 gitfile.startswith("tools/"):
 230             continue
 231         if REGEX_FILE_KCONFIG.match(gitfile):
 232             kconfig_files.append(gitfile)
 233         else:
 234             # all non-Kconfig files are checked for consistency
 235             source_files.append(gitfile)
 236
 237     for sfile in source_files:
 238         if ignore and re.match(ignore, sfile):
 239             # do not check files matching %ignore
 240             continue
 241         parse_source_file(sfile, referenced_features)
 242
 243     for kfile in kconfig_files:
 244         if ignore and re.match(ignore, kfile):
 245             # do not collect references for files matching %ignore
 246             parse_kconfig_file(kfile, defined_features, dict())
 247         else:
 248             parse_kconfig_file(kfile, defined_features, referenced_features)
 249
 250     undefined = {}  # {feature: [files]}
 251     for feature in sorted(referenced_features):
 252         # filter some false positives
 253         if feature == "FOO" or feature == "BAR" or \
 254                 feature == "FOO_BAR" or feature == "XXX":
 255             continue
 256         if feature not in defined_features:
 257             if feature.endswith("_MODULE"):
 258                 # avoid false positives for kernel modules
 259                 if feature[:-len("_MODULE")] in defined_features:
 260                     continue
 261             undefined[feature] = referenced_features.get(feature)
 262     return undefined
 263
 264
 265 def parse_source_file(sfile, referenced_features):
 266     """Parse @sfile for referenced Kconfig features."""
 267     lines = []
 268     with open(sfile, "r") as stream:
 269         lines = stream.readlines()
 270
 271     for line in lines:
 272         if not "CONFIG_" in line:
 273             continue
 274         features = REGEX_SOURCE_FEATURE.findall(line)
 275         for feature in features:
 276             if not REGEX_FILTER_FEATURES.search(feature):
 277                 continue
 278             sfiles = referenced_features.get(feature, set())
 279             sfiles.add(sfile)
 280             referenced_features[feature] = sfiles
 281
 282
 283 def get_features_in_line(line):
 284     """Return mentioned Kconfig features in @line."""
 285     return REGEX_FEATURE.findall(line)
 286
 287
 288 def parse_kconfig_file(kfile, defined_features, referenced_features):
 289     """Parse @kfile and update feature definitions and references."""
 290     lines = []
 291     skip = False
 292
 293     with open(kfile, "r") as stream:
 294         lines = stream.readlines()
 295
 296     for i in range(len(lines)):
 297         line = lines[i]
 298         line = line.strip('\n')
 299         line = line.split("#")[0]  # ignore comments
 300
 301         if REGEX_KCONFIG_DEF.match(line):
 302             feature_def = REGEX_KCONFIG_DEF.findall(line)
 303             defined_features.add(feature_def[0])
 304             skip = False
 305         elif REGEX_KCONFIG_HELP.match(line):
 306             skip = True
 307         elif skip:
 308             # ignore content of help messages
 309             pass
 310         elif REGEX_KCONFIG_STMT.match(line):
 311             features = get_features_in_line(line)
 312             # multi-line statements
 313             while line.endswith("\\"):
 314                 i += 1
 315                 line = lines[i]
 316                 line = line.strip('\n')
 317                 features.extend(get_features_in_line(line))
 318             for feature in set(features):
 319                 if REGEX_NUMERIC.match(feature):
 320                     # ignore numeric values
 321                     continue
 322                 paths = referenced_features.get(feature, set())
 323                 paths.add(kfile)
 324                 referenced_features[feature] = paths
 325
 326
 327 if __name__ == "__main__":
 328     main()