PRESUBMIT.py

# Copyright 2019 The ANGLE Project Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Top-level presubmit script for code generation.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details on the presubmit API built into depot_tools.
"""

import itertools
import os
import re
import shutil
import subprocess
import sys
import tempfile
import textwrap
import pathlib

# This line is 'magic' in that git-cl looks for it to decide whether to
# use Python3 instead of Python2 when running the code in this file.
USE_PYTHON3 = True

# Fragment of a regular expression that matches C/C++ and Objective-C++ implementation files and headers.
_IMPLEMENTATION_AND_HEADER_EXTENSIONS = r'\.(c|cc|cpp|cxx|mm|h|hpp|hxx)$'

# Fragment of a regular expression that matches C++ and Objective-C++ header files.
_HEADER_EXTENSIONS = r'\.(h|hpp|hxx)$'

_PRIMARY_EXPORT_TARGETS = [
    '//:libEGL',
    '//:libGLESv1_CM',
    '//:libGLESv2',
    '//:translator',
]


def _SplitIntoMultipleCommits(description_text):
    paragraph_split_pattern = r"(?m)(^\s*$\n)"
    multiple_paragraphs = re.split(paragraph_split_pattern, description_text)
    multiple_commits = [""]
    change_id_pattern = re.compile(r"(?m)^Change-Id: [a-zA-Z0-9]*$")
    for paragraph in multiple_paragraphs:
        multiple_commits[-1] += paragraph
        if change_id_pattern.search(paragraph):
            multiple_commits.append("")
    if multiple_commits[-1] == "":
        multiple_commits.pop()
    return multiple_commits


def _CheckCommitMessageFormatting(input_api, output_api):

    def _IsLineBlank(line):
        return line.isspace() or line == ""

    def _PopBlankLines(lines, reverse=False):
        if reverse:
            while len(lines) > 0 and _IsLineBlank(lines[-1]):
                lines.pop()
        else:
            while len(lines) > 0 and _IsLineBlank(lines[0]):
                lines.pop(0)

    def _IsTagLine(line):
        return ":" in line

    def _CheckTabInCommit(lines):
        return all([line.find("\t") == -1 for line in lines])

    allowlist_strings = ['Revert', 'Roll', 'Manual roll', 'Reland', 'Re-land']
    summary_linelength_warning_lower_limit = 65
    summary_linelength_warning_upper_limit = 70
    description_linelength_limit = 72

    git_output = input_api.change.DescriptionText()

    multiple_commits = _SplitIntoMultipleCommits(git_output)
    errors = []

    for k in range(len(multiple_commits)):
        commit_msg_lines = multiple_commits[k].splitlines()
        commit_number = len(multiple_commits) - k
        commit_tag = "Commit " + str(commit_number) + ":"
        commit_msg_line_numbers = {}
        for i in range(len(commit_msg_lines)):
            commit_msg_line_numbers[commit_msg_lines[i]] = i + 1
        _PopBlankLines(commit_msg_lines, True)
        _PopBlankLines(commit_msg_lines, False)
        allowlisted = False
        if len(commit_msg_lines) > 0:
            for allowlist_string in allowlist_strings:
                if commit_msg_lines[0].startswith(allowlist_string):
                    allowlisted = True
                    break
        if allowlisted:
            continue

        if not _CheckTabInCommit(commit_msg_lines):
            errors.append(
                output_api.PresubmitError(commit_tag + "Tabs are not allowed in commit message."))

        # the tags paragraph is at the end of the message
        # the break between the tags paragraph is the first line without ":"
        # this is sufficient because if a line is blank, it will not have ":"
        last_paragraph_line_count = 0
        while len(commit_msg_lines) > 0 and _IsTagLine(commit_msg_lines[-1]):
            last_paragraph_line_count += 1
            commit_msg_lines.pop()
        if last_paragraph_line_count == 0:
            errors.append(
                output_api.PresubmitError(
                    commit_tag +
                    "Please ensure that there are tags (e.g., Bug:, Test:) in your description."))
        if len(commit_msg_lines) > 0:
            if not _IsLineBlank(commit_msg_lines[-1]):
                output_api.PresubmitError(commit_tag +
                                          "Please ensure that there exists 1 blank line " +
                                          "between tags and description body.")
            else:
                # pop the blank line between tag paragraph and description body
                commit_msg_lines.pop()
                if len(commit_msg_lines) > 0 and _IsLineBlank(commit_msg_lines[-1]):
                    errors.append(
                        output_api.PresubmitError(
                            commit_tag + 'Please ensure that there exists only 1 blank line '
                            'between tags and description body.'))
                    # pop all the remaining blank lines between tag and description body
                    _PopBlankLines(commit_msg_lines, True)
        if len(commit_msg_lines) == 0:
            errors.append(
                output_api.PresubmitError(commit_tag +
                                          'Please ensure that your description summary'
                                          ' and description body are not blank.'))
            continue

        if summary_linelength_warning_lower_limit <= len(commit_msg_lines[0]) \
        <= summary_linelength_warning_upper_limit:
            errors.append(
                output_api.PresubmitPromptWarning(
                    commit_tag + "Your description summary should be on one line of " +
                    str(summary_linelength_warning_lower_limit - 1) + " or less characters."))
        elif len(commit_msg_lines[0]) > summary_linelength_warning_upper_limit:
            errors.append(
                output_api.PresubmitError(
                    commit_tag + "Please ensure that your description summary is on one line of " +
                    str(summary_linelength_warning_lower_limit - 1) + " or less characters."))
        commit_msg_lines.pop(0)  # get rid of description summary
        if len(commit_msg_lines) == 0:
            continue
        if not _IsLineBlank(commit_msg_lines[0]):
            errors.append(
                output_api.PresubmitError(commit_tag +
                                          'Please ensure the summary is only 1 line and '
                                          'there is 1 blank line between the summary '
                                          'and description body.'))
        else:
            commit_msg_lines.pop(0)  # pop first blank line
            if len(commit_msg_lines) == 0:
                continue
            if _IsLineBlank(commit_msg_lines[0]):
                errors.append(
                    output_api.PresubmitError(commit_tag +
                                              'Please ensure that there exists only 1 blank line '
                                              'between description summary and description body.'))
                # pop all the remaining blank lines between
                # description summary and description body
                _PopBlankLines(commit_msg_lines)

        # loop through description body
        while len(commit_msg_lines) > 0:
            line = commit_msg_lines.pop(0)
            # lines starting with 4 spaces, quotes or lines without space(urls)
            # are exempt from length check
            if line.startswith("    ") or line.startswith("> ") or " " not in line:
                continue
            if len(line) > description_linelength_limit:
                errors.append(
                    output_api.PresubmitError(
                        commit_tag + 'Line ' + str(commit_msg_line_numbers[line]) +
                        ' is too long.\n' + '"' + line + '"\n' + 'Please wrap it to ' +
                        str(description_linelength_limit) + ' characters. ' +
                        "Lines without spaces or lines starting with 4 spaces are exempt."))
                break
    return errors


def _CheckChangeHasBugField(input_api, output_api):
    """Requires that the changelist have a Bug: field from a known project."""
    bugs = input_api.change.BugsFromDescription()

    # The bug must be in the form of "project:number".  None is also accepted, which is used by
    # rollers as well as in very minor changes.
    if len(bugs) == 1 and bugs[0] == 'None':
        return []

    projects = [
        'angleproject:', 'chromium:', 'dawn:', 'fuchsia:', 'skia:', 'swiftshader:', 'tint:', 'b/'
    ]
    bug_regex = re.compile(r"([a-z]+[:/])(\d+)")
    errors = []
    extra_help = False

    if not bugs:
        errors.append('Please ensure that your description contains\n'
                      'Bug: bugtag\n'
                      'directly above the Change-Id tag (no empty line in-between)')
        extra_help = True

    for bug in bugs:
        if bug == 'None':
            errors.append('Invalid bug tag "None" in presence of other bug tags.')
            continue

        match = re.match(bug_regex, bug)
        if match == None or bug != match.group(0) or match.group(1) not in projects:
            errors.append('Incorrect bug tag "' + bug + '".')
            extra_help = True

    if extra_help:
        change_ids = re.findall('^Change-Id:', input_api.change.FullDescriptionText(), re.M)
        if len(change_ids) > 1:
            errors.append('Note: multiple Change-Id tags found in description')

        errors.append('''Acceptable bugtags:
    project:bugnumber - where project is one of ({projects})
    b/bugnumber - for Buganizer/IssueTracker bugs
'''.format(projects=', '.join(p[:-1] for p in projects if p != 'b/')))

    return [output_api.PresubmitError('\n\n'.join(errors))] if errors else []


def _CheckCodeGeneration(input_api, output_api):

    class Msg(output_api.PresubmitError):
        """Specialized error message"""

        def __init__(self, message, **kwargs):
            super(output_api.PresubmitError, self).__init__(
                message,
                long_text='Please ensure your ANGLE repositiory is synced to tip-of-tree\n'
                'and all ANGLE DEPS are fully up-to-date by running gclient sync.\n'
                '\n'
                'If that fails, run scripts/run_code_generation.py to refresh generated hashes.\n'
                '\n'
                'If you are building ANGLE inside Chromium you must bootstrap ANGLE\n'
                'before gclient sync. See the DevSetup documentation for more details.\n',
                **kwargs)

    code_gen_path = input_api.os_path.join(input_api.PresubmitLocalPath(),
                                           'scripts/run_code_generation.py')
    cmd_name = 'run_code_generation'
    cmd = [input_api.python3_executable, code_gen_path, '--verify-no-dirty']
    test_cmd = input_api.Command(name=cmd_name, cmd=cmd, kwargs={}, message=Msg)
    if input_api.verbose:
        print('Running ' + cmd_name)
    return input_api.RunTests([test_cmd])


# Taken directly from Chromium's PRESUBMIT.py
def _CheckNewHeaderWithoutGnChange(input_api, output_api):
    """Checks that newly added header files have corresponding GN changes.
  Note that this is only a heuristic. To be precise, run script:
  build/check_gn_headers.py.
  """

    def headers(f):
        return input_api.FilterSourceFile(f, files_to_check=(r'.+%s' % _HEADER_EXTENSIONS,))

    new_headers = []
    for f in input_api.AffectedSourceFiles(headers):
        if f.Action() != 'A':
            continue
        new_headers.append(f.LocalPath())

    def gn_files(f):
        return input_api.FilterSourceFile(f, files_to_check=(r'.+\.gn',))

    all_gn_changed_contents = ''
    for f in input_api.AffectedSourceFiles(gn_files):
        for _, line in f.ChangedContents():
            all_gn_changed_contents += line

    problems = []
    for header in new_headers:
        basename = input_api.os_path.basename(header)
        if basename not in all_gn_changed_contents:
            problems.append(header)

    if problems:
        return [
            output_api.PresubmitPromptWarning(
                'Missing GN changes for new header files',
                items=sorted(problems),
                long_text='Please double check whether newly added header files need '
                'corresponding changes in gn or gni files.\nThis checking is only a '
                'heuristic. Run build/check_gn_headers.py to be precise.\n'
                'Read https://crbug.com/661774 for more info.')
        ]
    return []


def _CheckExportValidity(input_api, output_api):
    outdir = tempfile.mkdtemp()
    # shell=True is necessary on Windows, as otherwise subprocess fails to find
    # either 'gn' or 'vpython3' even if they are findable via PATH.
    use_shell = input_api.is_windows
    try:
        try:
            subprocess.check_output(['gn', 'gen', outdir], shell=use_shell)
        except subprocess.CalledProcessError as e:
            return [
                output_api.PresubmitError('Unable to run gn gen for export_targets.py: %s' %
                                          e.output.decode())
            ]
        export_target_script = os.path.join(input_api.PresubmitLocalPath(), 'scripts',
                                            'export_targets.py')
        try:
            subprocess.check_output(
                ['vpython3', export_target_script, outdir] + _PRIMARY_EXPORT_TARGETS,
                stderr=subprocess.STDOUT,
                shell=use_shell)
        except subprocess.CalledProcessError as e:
            if input_api.is_committing:
                return [
                    output_api.PresubmitError('export_targets.py failed: %s' % e.output.decode())
                ]
            return [
                output_api.PresubmitPromptWarning(
                    'export_targets.py failed, this may just be due to your local checkout: %s' %
                    e.output.decode())
            ]
        return []
    finally:
        shutil.rmtree(outdir)


def _CheckTabsInSourceFiles(input_api, output_api):
    """Forbids tab characters in source files due to a WebKit repo requirement."""

    def implementation_and_headers_including_third_party(f):
        # Check third_party files too, because WebKit's checks don't make exceptions.
        return input_api.FilterSourceFile(
            f,
            files_to_check=(r'.+%s' % _IMPLEMENTATION_AND_HEADER_EXTENSIONS,),
            files_to_skip=[f for f in input_api.DEFAULT_FILES_TO_SKIP if not "third_party" in f])

    files_with_tabs = []
    for f in input_api.AffectedSourceFiles(implementation_and_headers_including_third_party):
        for (num, line) in f.ChangedContents():
            if '\t' in line:
                files_with_tabs.append(f)
                break

    if files_with_tabs:
        return [
            output_api.PresubmitError(
                'Tab characters in source files.',
                items=sorted(files_with_tabs),
                long_text=
                'Tab characters are forbidden in ANGLE source files because WebKit\'s Subversion\n'
                'repository does not allow tab characters in source files.\n'
                'Please remove tab characters from these files.')
        ]
    return []


# https://stackoverflow.com/a/196392
def is_ascii(s):
    return all(ord(c) < 128 for c in s)


def _CheckNonAsciiInSourceFiles(input_api, output_api):
    """Forbids non-ascii characters in source files."""

    def implementation_and_headers(f):
        return input_api.FilterSourceFile(
            f, files_to_check=(r'.+%s' % _IMPLEMENTATION_AND_HEADER_EXTENSIONS,))

    files_with_non_ascii = []
    for f in input_api.AffectedSourceFiles(implementation_and_headers):
        for (num, line) in f.ChangedContents():
            if not is_ascii(line):
                files_with_non_ascii.append("%s: %s" % (f, line))
                break

    if files_with_non_ascii:
        return [
            output_api.PresubmitError(
                'Non-ASCII characters in source files.',
                items=sorted(files_with_non_ascii),
                long_text='Non-ASCII characters are forbidden in ANGLE source files.\n'
                'Please remove non-ASCII characters from these files.')
        ]
    return []


def _CheckCommentBeforeTestInTestFiles(input_api, output_api):
    """Require a comment before TEST_P() and other tests."""

    def test_files(f):
        return input_api.FilterSourceFile(
            f, files_to_check=(r'^src/tests/.+\.cpp$', r'^src/.+_unittest\.cpp$'))

    tests_with_no_comment = []
    for f in input_api.AffectedSourceFiles(test_files):
        diff = f.GenerateScmDiff()
        last_line_was_comment = False
        for line in diff.splitlines():
            # Skip removed lines
            if line.startswith('-'):
                continue

            new_line_is_comment = line.startswith(' //') or line.startswith('+//')
            new_line_is_test_declaration = (
                line.startswith('+TEST_P(') or line.startswith('+TEST(') or
                line.startswith('+TYPED_TEST('))

            if new_line_is_test_declaration and not last_line_was_comment:
                tests_with_no_comment.append(line[1:])

            last_line_was_comment = new_line_is_comment

    if tests_with_no_comment:
        return [
            output_api.PresubmitError(
                'Tests without comment.',
                items=sorted(tests_with_no_comment),
                long_text='ANGLE requires a comment describing what a test does.')
        ]
    return []


def _CheckWildcardInTestExpectationFiles(input_api, output_api):
    """Require wildcard as API tag (i.e. in foo.bar/*) in expectations when no additional feature is
    enabled."""

    def expectation_files(f):
        return input_api.FilterSourceFile(
            f, files_to_check=[r'^src/tests/angle_end2end_tests_expectations.txt$'])

    expectation_pattern = re.compile(r'^.*:\s*[a-zA-Z0-9._*]+\/([^ ]*)\s*=.*$')

    expectations_without_wildcard = []
    for f in input_api.AffectedSourceFiles(expectation_files):
        diff = f.GenerateScmDiff()
        for line in diff.splitlines():
            # Only look at new lines
            if not line.startswith('+'):
                continue

            match = re.match(expectation_pattern, line[1:].strip())
            if match is None:
                continue

            tag = match.group(1)

            # The tag is in the following general form:
            #
            #     FRONTENDAPI_BACKENDAPI[_FEATURE]*
            #
            # Any part of the above may be a wildcard.  Warn about usage of FRONTEND_BACKENDAPI as
            # the tag.  Instead, the backend should be specified before the : and `*` used as the
            # tag.  If any additional tags are present, it's a specific expectation that should
            # remain specific (and not wildcarded).  NoFixture is an exception as X_Y_NoFixture is
            # the generic form of the tags of tests that don't use the fixture.

            sections = [section for section in tag.split('_') if section != 'NoFixture']

            # Allow '*_...', or 'FRONTENDAPI_*_...'.
            if '*' in sections[0] or (len(sections) > 1 and '*' in sections[1]):
                continue

            # Warn if no additional tags are present
            if len(sections) == 2:
                expectations_without_wildcard.append(line[1:])

    if expectations_without_wildcard:
        return [
            output_api.PresubmitError(
                'Use wildcard in API tags (after /) in angle_end2end_tests_expectations.txt.',
                items=expectations_without_wildcard,
                long_text="""ANGLE prefers end2end expections to use the following form:

1234 MAC OPENGL : Foo.Bar/* = SKIP

instead of:

1234 MAC OPENGL : Foo.Bar/ES2_OpenGL = SKIP
1234 MAC OPENGL : Foo.Bar/ES3_OpenGL = SKIP

Expectatations that are specific (such as Foo.Bar/ES2_OpenGL_SomeFeature) are allowed.""")
        ]
    return []


def _CheckShaderVersionInShaderLangHeader(input_api, output_api):
    """Requires an update to ANGLE_SH_VERSION when ShaderLang.h or ShaderVars.h change."""

    def headers(f):
        return input_api.FilterSourceFile(
            f,
            files_to_check=(r'^include/GLSLANG/ShaderLang.h$', r'^include/GLSLANG/ShaderVars.h$'))

    headers_changed = input_api.AffectedSourceFiles(headers)
    if len(headers_changed) == 0:
        return []

    # Skip this check for reverts and rolls.  Unlike
    # _CheckCommitMessageFormatting, relands are still checked because the
    # original change might have incremented the version correctly, but the
    # rebase over a new version could accidentally remove that (because another
    # change in the meantime identically incremented it).
    git_output = input_api.change.DescriptionText()
    multiple_commits = _SplitIntoMultipleCommits(git_output)
    for commit in multiple_commits:
        if commit.startswith('Revert') or commit.startswith('Roll'):
            return []

    diffs = '\n'.join(f.GenerateScmDiff() for f in headers_changed)
    versions = dict(re.findall(r'^([-+])#define ANGLE_SH_VERSION\s+(\d+)', diffs, re.M))

    if len(versions) != 2 or int(versions['+']) <= int(versions['-']):
        return [
            output_api.PresubmitError(
                'ANGLE_SH_VERSION should be incremented when ShaderLang.h or ShaderVars.h change.',
            )
        ]
    return []


def _CheckGClientExists(input_api, output_api, search_limit=None):
    presubmit_path = pathlib.Path(input_api.PresubmitLocalPath())

    for current_path in itertools.chain([presubmit_path], presubmit_path.parents):
        gclient_path = current_path.joinpath('.gclient')
        if gclient_path.exists() and gclient_path.is_file():
            return []
        # search_limit parameter is used in unit tests to prevent searching all the way to root
        # directory for reproducibility.
        elif search_limit != None and current_path == search_limit:
            break

    return [
        output_api.PresubmitError(
            'Missing .gclient file.',
            long_text=textwrap.fill(
                width=100,
                text='The top level directory of the repository must contain a .gclient file.'
                ' You can follow the steps outlined in the link below to get set up for ANGLE'
                ' development:') +
            '\n\nhttps://chromium.googlesource.com/angle/angle/+/refs/heads/main/doc/DevSetup.md')
    ]

def CheckChangeOnUpload(input_api, output_api):
    results = []
    results.extend(input_api.canned_checks.CheckForCommitObjects(input_api, output_api))
    results.extend(_CheckTabsInSourceFiles(input_api, output_api))
    results.extend(_CheckNonAsciiInSourceFiles(input_api, output_api))
    results.extend(_CheckCommentBeforeTestInTestFiles(input_api, output_api))
    results.extend(_CheckWildcardInTestExpectationFiles(input_api, output_api))
    results.extend(_CheckShaderVersionInShaderLangHeader(input_api, output_api))
    results.extend(_CheckCodeGeneration(input_api, output_api))
    results.extend(_CheckChangeHasBugField(input_api, output_api))
    results.extend(input_api.canned_checks.CheckChangeHasDescription(input_api, output_api))
    results.extend(_CheckNewHeaderWithoutGnChange(input_api, output_api))
    results.extend(_CheckExportValidity(input_api, output_api))
    results.extend(
        input_api.canned_checks.CheckPatchFormatted(
            input_api, output_api, result_factory=output_api.PresubmitError))
    results.extend(_CheckCommitMessageFormatting(input_api, output_api))
    results.extend(_CheckGClientExists(input_api, output_api))

    return results


def CheckChangeOnCommit(input_api, output_api):
    return CheckChangeOnUpload(input_api, output_api)