"Fossies" - the Fresh Open Source Software Archive

Member "pytorch-1.8.2/tools/clang_tidy.py" (23 Jul 2021, 10115 Bytes) of package /linux/misc/pytorch-1.8.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "clang_tidy.py" see the Fossies "Dox" file reference documentation.

    1 #!/usr/bin/env python
    2 """
    3 A driver script to run clang-tidy on changes detected via git.
    4 
    5 By default, clang-tidy runs on all files you point it at. This means that even
    6 if you changed only parts of that file, you will get warnings for the whole
    7 file. This script has the ability to ask git for the exact lines that have
    8 changed since a particular git revision, and makes clang-tidy only lint those.
    9 This makes it much less overhead to integrate in CI and much more relevant to
   10 developers. This git-enabled mode is optional, and full scans of a directory
   11 tree are also possible. In both cases, the script allows filtering files via
   12 glob or regular expressions.
   13 """
   14 
   15 
   16 
   17 import argparse
   18 import collections
   19 import fnmatch
   20 import json
   21 import os
   22 import os.path
   23 import re
   24 import shlex
   25 import subprocess
   26 import sys
   27 import tempfile
   28 
   29 try:
   30     from shlex import quote
   31 except ImportError:
   32     from pipes import quote
   33 
   34 Patterns = collections.namedtuple("Patterns", "positive, negative")
   35 
   36 
   37 # NOTE: Clang-tidy cannot lint headers directly, because headers are not
   38 # compiled -- translation units are, of which there is one per implementation
   39 # (c/cc/cpp) file.
   40 DEFAULT_FILE_PATTERN = re.compile(r".*\.c(c|pp)?")
   41 
   42 # @@ -start,count +start,count @@
   43 CHUNK_PATTERN = r"^@@\s+-\d+(?:,\d+)?\s+\+(\d+)(?:,(\d+))?\s+@@"
   44 
   45 
   46 # Set from command line arguments in main().
   47 VERBOSE = False
   48 
   49 
   50 def run_shell_command(arguments):
   51     """Executes a shell command."""
   52     if VERBOSE:
   53         print(" ".join(arguments))
   54     try:
   55         output = subprocess.check_output(arguments).decode().strip()
   56     except subprocess.CalledProcessError:
   57         _, error, _ = sys.exc_info()
   58         error_output = error.output.decode().strip()
   59         raise RuntimeError("Error executing {}: {}".format(" ".join(arguments), error_output))
   60 
   61     return output
   62 
   63 
   64 def split_negative_from_positive_patterns(patterns):
   65     """Separates negative patterns (that start with a dash) from positive patterns"""
   66     positive, negative = [], []
   67     for pattern in patterns:
   68         if pattern.startswith("-"):
   69             negative.append(pattern[1:])
   70         else:
   71             positive.append(pattern)
   72 
   73     return Patterns(positive, negative)
   74 
   75 
   76 def get_file_patterns(globs, regexes):
   77     """Returns a list of compiled regex objects from globs and regex pattern strings."""
   78     # fnmatch.translate converts a glob into a regular expression.
   79     # https://docs.python.org/2/library/fnmatch.html#fnmatch.translate
   80     glob = split_negative_from_positive_patterns(globs)
   81     regexes = split_negative_from_positive_patterns(regexes)
   82 
   83     positive_regexes = regexes.positive + [fnmatch.translate(g) for g in glob.positive]
   84     negative_regexes = regexes.negative + [fnmatch.translate(g) for g in glob.negative]
   85 
   86     positive_patterns = [re.compile(regex) for regex in positive_regexes] or [
   87         DEFAULT_FILE_PATTERN
   88     ]
   89     negative_patterns = [re.compile(regex) for regex in negative_regexes]
   90 
   91     return Patterns(positive_patterns, negative_patterns)
   92 
   93 
   94 def filter_files(files, file_patterns):
   95     """Returns all files that match any of the patterns."""
   96     if VERBOSE:
   97         print("Filtering with these file patterns: {}".format(file_patterns))
   98     for file in files:
   99         if not any(n.match(file) for n in file_patterns.negative):
  100             if any(p.match(file) for p in file_patterns.positive):
  101                 yield file
  102                 continue
  103         if VERBOSE:
  104             print("{} omitted due to file filters".format(file))
  105 
  106 
  107 def get_changed_files(revision, paths):
  108     """Runs git diff to get the paths of all changed files."""
  109     # --diff-filter AMU gets us files that are (A)dded, (M)odified or (U)nmerged (in the working copy).
  110     # --name-only makes git diff return only the file paths, without any of the source changes.
  111     command = "git diff-index --diff-filter=AMU --ignore-all-space --name-only"
  112     output = run_shell_command(shlex.split(command) + [revision] + paths)
  113     return output.split("\n")
  114 
  115 
  116 def get_all_files(paths):
  117     """Returns all files that are tracked by git in the given paths."""
  118     output = run_shell_command(["git", "ls-files"] + paths)
  119     return output.split("\n")
  120 
  121 
  122 def get_changed_lines(revision, filename):
  123     """Runs git diff to get the line ranges of all file changes."""
  124     command = shlex.split("git diff-index --unified=0") + [revision, filename]
  125     output = run_shell_command(command)
  126     changed_lines = []
  127     for chunk in re.finditer(CHUNK_PATTERN, output, re.MULTILINE):
  128         start = int(chunk.group(1))
  129         count = int(chunk.group(2) or 1)
  130         # If count == 0, a chunk was removed and can be ignored.
  131         if count == 0:
  132             continue
  133         changed_lines.append([start, start + count])
  134 
  135     return {"name": filename, "lines": changed_lines}
  136 
  137 ninja_template = """
  138 rule do_cmd
  139   command = $cmd
  140   description = Running clang-tidy
  141 
  142 {build_rules}
  143 """
  144 
  145 build_template = """
  146 build {i}: do_cmd
  147   cmd = {cmd}
  148 """
  149 
  150 
  151 def run_shell_commands_in_parallel(commands):
  152     """runs all the commands in parallel with ninja, commands is a List[List[str]]"""
  153     build_entries = [build_template.format(i=i, cmd=' '.join([quote(s) for s in command]))
  154                      for i, command in enumerate(commands)]
  155 
  156     file_contents = ninja_template.format(build_rules='\n'.join(build_entries)).encode()
  157     f = tempfile.NamedTemporaryFile(delete=False)
  158     try:
  159         f.write(file_contents)
  160         f.close()
  161         return run_shell_command(['ninja', '-f', f.name])
  162     finally:
  163         os.unlink(f.name)
  164 
  165 
  166 def run_clang_tidy(options, line_filters, files):
  167     """Executes the actual clang-tidy command in the shell."""
  168     command = [options.clang_tidy_exe, "-p", options.compile_commands_dir]
  169     if not options.config_file and os.path.exists(".clang-tidy"):
  170         options.config_file = ".clang-tidy"
  171     if options.config_file:
  172         import yaml
  173 
  174         with open(options.config_file) as config:
  175             # Here we convert the YAML config file to a JSON blob.
  176             command += ["-config", json.dumps(yaml.load(config, Loader=yaml.FullLoader))]
  177     command += options.extra_args
  178 
  179     if line_filters:
  180         command += ["-line-filter", json.dumps(line_filters)]
  181 
  182     if options.parallel:
  183         commands = [list(command) + [f] for f in files]
  184         output = run_shell_commands_in_parallel(commands)
  185     else:
  186         command += files
  187         if options.dry_run:
  188             command = [re.sub(r"^([{[].*[]}])$", r"'\1'", arg) for arg in command]
  189             return " ".join(command)
  190 
  191         output = run_shell_command(command)
  192 
  193     if not options.keep_going and "[clang-diagnostic-error]" in output:
  194         message = "Found clang-diagnostic-errors in clang-tidy output: {}"
  195         raise RuntimeError(message.format(output))
  196 
  197     return output
  198 
  199 
  200 def parse_options():
  201     """Parses the command line options."""
  202     parser = argparse.ArgumentParser(description="Run Clang-Tidy (on your Git changes)")
  203     parser.add_argument(
  204         "-e",
  205         "--clang-tidy-exe",
  206         default="clang-tidy",
  207         help="Path to clang-tidy executable",
  208     )
  209     parser.add_argument(
  210         "-g",
  211         "--glob",
  212         action="append",
  213         default=[],
  214         help="Only lint files that match these glob patterns "
  215         "(see documentation for `fnmatch` for supported syntax)."
  216         "If a pattern starts with a - the search is negated for that pattern.",
  217     )
  218     parser.add_argument(
  219         "-x",
  220         "--regex",
  221         action="append",
  222         default=[],
  223         help="Only lint files that match these regular expressions (from the start of the filename). "
  224         "If a pattern starts with a - the search is negated for that pattern.",
  225     )
  226     parser.add_argument(
  227         "-c",
  228         "--compile-commands-dir",
  229         default="build",
  230         help="Path to the folder containing compile_commands.json",
  231     )
  232     parser.add_argument(
  233         "-d", "--diff", help="Git revision to diff against to get changes"
  234     )
  235     parser.add_argument(
  236         "-p",
  237         "--paths",
  238         nargs="+",
  239         default=["."],
  240         help="Lint only the given paths (recursively)",
  241     )
  242     parser.add_argument(
  243         "-n",
  244         "--dry-run",
  245         action="store_true",
  246         help="Only show the command to be executed, without running it",
  247     )
  248     parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
  249     parser.add_argument(
  250         "--config-file",
  251         help="Path to a clang-tidy config file. Defaults to '.clang-tidy'.",
  252     )
  253     parser.add_argument(
  254         "-k",
  255         "--keep-going",
  256         action="store_true",
  257         help="Don't error on compiler errors (clang-diagnostic-error)",
  258     )
  259     parser.add_argument(
  260         "-j",
  261         "--parallel",
  262         action="store_true",
  263         help="Run clang tidy in parallel per-file (requires ninja to be installed).",
  264     )
  265     parser.add_argument(
  266         "extra_args", nargs="*", help="Extra arguments to forward to clang-tidy"
  267     )
  268     return parser.parse_args()
  269 
  270 
  271 def main():
  272     options = parse_options()
  273 
  274     # This flag is pervasive enough to set it globally. It makes the code
  275     # cleaner compared to threading it through every single function.
  276     global VERBOSE
  277     VERBOSE = options.verbose
  278 
  279     # Normalize the paths first.
  280     paths = [path.rstrip("/") for path in options.paths]
  281     if options.diff:
  282         files = get_changed_files(options.diff, paths)
  283     else:
  284         files = get_all_files(paths)
  285     file_patterns = get_file_patterns(options.glob, options.regex)
  286     files = list(filter_files(files, file_patterns))
  287 
  288     # clang-tidy error's when it does not get input files.
  289     if not files:
  290         print("No files detected.")
  291         sys.exit()
  292 
  293     line_filters = []
  294     if options.diff:
  295         line_filters = [get_changed_lines(options.diff, f) for f in files]
  296 
  297     pwd = os.getcwd() + "/"
  298     clang_tidy_output = run_clang_tidy(options, line_filters, files)
  299     formatted_output = []
  300 
  301     for line in clang_tidy_output.splitlines():
  302         if line.startswith(pwd):
  303             print(line[len(pwd):])
  304 
  305 if __name__ == "__main__":
  306     main()