Rebase of chromiumos fork

https://chromium.googlesource.com/chromiumos/third_party/shellcheck/

ShellCheck.cabal

@@ -97,6 +97,7 @@ library
  ShellCheck.Regex
  other-modules:
  Paths_ShellCheck
+ ShellCheck.PortageAutoInternalVariables
  default-language: Haskell98
 
 executable shellcheck

portage/get_vars.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2019 The ChromiumOS Authors
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google LLC nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# Binary License Terms
+
+"""Extract eclass variable names into Haskell list format."""
+from __future__ import print_function
+import datetime
+import os
+import re
+import sys
+import textwrap
+# Matches a line that declares a variable in an eclass.
+VAR_RE = re.compile(r'@(?:ECLASS-)?VARIABLE:\s*(\w+)$')
+# Matches a line that declares inheritance.
+INHERIT_RE = re.compile(r'^[^#]*\binherit((?:\s+[\w-]+)+)$')
+VAR_FILE_HEADER = """module ShellCheck.PortageAutoInternalVariables (
+ portageAutoInternalVariables
+ ) where
+-- This file contains the variables generated by
+-- portage/get_vars.py"""
+PORTAGE_AUTO_VAR_NAME = 'portageAutoInternalVariables'
+class Eclass:
+ """Container for eclass information"""
+ def __init__(self, name, eclass_vars, inheritances):
+ self.name = name
+ self.vars = eclass_vars
+ self.inheritances = inheritances
+ def calculate_eclass_vars(self, eclasses):
+ while self.inheritances:
+ name = self.inheritances.pop()
+ try:
+ sub_eclass = eclasses[name]
+ new_vars = sub_eclass.calculate_eclass_vars(eclasses).vars
+ self.vars = self.vars.union(new_vars)
+ except Exception:
+ pass
+ return self
+def print_var_list(eclass, eclass_vars):
+ var_list = ' '.join(['"%s",' % v for v in sorted(eclass_vars)])
+ print(' -- %s\n%s' %
+ (eclass,
+ textwrap.fill(
+ var_list, 80, initial_indent=' ', subsequent_indent=' ')))
+def process_file(eclass_path):
+ eclass_name = os.path.splitext(os.path.basename(eclass_path))[0]
+ with open(eclass_path, 'r') as f:
+ eclass_vars = set()
+ eclass_inheritances = set()
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ while line[-1] == '\\':
+ line = line[:-1] + next(f).strip()
+ match = VAR_RE.search(line)
+ if match:
+ var_name = match.group(1)
+ eclass_vars.add(var_name.strip())
+ else:
+ match = INHERIT_RE.search(line)
+ if match:
+ for inheritance in re.split(r'\s+', match.group(1)):
+ if inheritance.strip():
+ eclass_inheritances.add(inheritance.strip())
+ return Eclass(eclass_name, eclass_vars, eclass_inheritances)
+def format_eclasses_as_haskell_map(eclasses):
+ map_entries = []
+ join_string = '", "'
+ for value in sorted(eclasses, key=(lambda x: x.name)):
+ if value.vars:
+ var_list_string = f'"{join_string.join(sorted(list(value.vars)))}"'
+ map_entries.append(
+ textwrap.fill(
+ f'("{value.name}", [{var_list_string}])',
+ 80,
+ initial_indent=' ',
+ subsequent_indent=' '))
+ return_string = ',\n\n'.join(map_entries)
+ return_string = f""" Data.Map.fromList
+ [
+{return_string}
+ ]"""
+ return f"""{VAR_FILE_HEADER}\n\n
+-- Last Generated: {datetime.datetime.now().strftime("%x")}
+import qualified Data.Map
+{PORTAGE_AUTO_VAR_NAME} =
+{return_string}"""
+def main(argv):
+ eclasses = {}
+ for path in sorted(argv, key=os.path.basename):
+ if not path.endswith('.eclass'):
+ continue
+ new_eclass = process_file(path)
+ eclasses[new_eclass.name] = new_eclass
+ eclasses_list = [
+ value.calculate_eclass_vars(eclasses) for key, value in eclasses.items()
+ ]
+ print(format_eclasses_as_haskell_map(eclasses_list))
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))

portage/get_vars_diff.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+# Copyright 2020 The ChromiumOS Authors
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google LLC nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# Binary License Terms
+
+"""Generates diff of vars from get_vars.py and those existing in Data.hs."""
+import itertools
+from pathlib import Path
+import subprocess
+SCRIPT = Path(__file__).resolve()
+THIRD_PARTY = SCRIPT.parent.parent.parent.parent.parent
+# List of relative directories in which to find the eclasses.
+eclass_rel_dirs = (
+ THIRD_PARTY / 'chromiumos-overlay' / 'eclass',
+ THIRD_PARTY / 'portage-stable' / 'eclass',
+ THIRD_PARTY / 'eclass-overlay' / 'eclass',
+)
+# Runs get_vars.py with the eclass paths and store the output.
+cmd = [SCRIPT.with_name('get_vars.py')] + list(
+ itertools.chain(*(x.glob('*') for x in eclass_rel_dirs)))
+new_output = subprocess.check_output(cmd, encoding='utf-8').splitlines()
+new = []
+for line in new_output:
+ if '--' in line:
+ new.append(line.strip())
+ elif not line.strip():
+ continue
+ else:
+ new += (line.replace('"', '').replace('\n', '').split(','))
+# Reads the Data.hs relevant area and store the lines.
+data_hs = THIRD_PARTY / 'shellcheck' / 'src' / 'ShellCheck' / 'Data.hs'
+with data_hs.open('r', encoding='utf-8') as fp:
+ record = False
+ old = []
+ for line in fp:
+ if line.strip() == '-- autotest.eclass declared incorrectly':
+ break
+ if line.strip() == '-- generic ebuilds':
+ record = True
+ if record:
+ if '--' in line:
+ old.append(line.strip())
+ elif not line.strip():
+ continue
+ else:
+ old += line.replace('"', '').replace('\n', '').split(',')
+# Cleans up empty bits as a result of parsing difficulties.
+new = [x.strip() for x in new if x.strip()]
+old = [x.strip() for x in old if x.strip()]
+all_eclasses = set()
+old_vars = {}
+new_vars = {}
+current_eclass = ''
+for item in old:
+ if '--' in item:
+ # It's an eclass comment line.
+ current_eclass = item[3:]
+ all_eclasses.add(current_eclass)
+ continue
+ else:
+ # It's a var, so add it to the dict of the current eclass.
+ old_vars.setdefault(current_eclass, []).append(item)
+for item in new:
+ if '--' in item:
+ # It's an eclass comment line.
+ current_eclass = item[3:]
+ all_eclasses.add(current_eclass)
+ continue
+ else:
+ # It's a var, so add it to the dict of the current eclass.
+ new_vars.setdefault(current_eclass, []).append(item)
+for eclass in sorted(all_eclasses):
+ if eclass in old_vars:
+ if eclass not in new_vars:
+ # Checks if the entire eclass is removed.
+ print(f'{eclass} not present in new variables.')
+ for var in old_vars[eclass]:
+ print(f'\t-{var}')
+ print()
+ else:
+ # Eclass isn't removed, so check for added or removed vars.
+ toprint = '\n'.join(
+ [f'\t-{x}' for x in old_vars[eclass] if x not in new_vars[eclass]] +
+ [f'\t+{x}' for x in new_vars[eclass] if x not in old_vars[eclass]])
+ if toprint:
+ print(eclass)
+ print(toprint)
+ if eclass in new_vars:
+ if eclass not in old_vars:
+ # Checks if entire eclass is new.
+ print(f'{eclass} added in new variables.')
+ for var in new_vars[eclass]:
+ print(f'\t+{var}')
+ print()

shellcheck.1.md

@@ -87,8 +87,9 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts.
 
 : Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash* and *ksh*.
  The default is to deduce the shell from the file's `shell` directive,
- shebang, or `.bash/.bats/.dash/.ksh` extension, in that order. *sh* refers to
- POSIX `sh` (not the system's), and will warn of portability issues.
+ shebang, or `.bash/.bats/.dash/.ksh/.ebuild/.eclass` extension, in that
+ order. *sh* refers to POSIX `sh` (not the system's), and will warn of
+ portability issues.
 
 **-S**\ *SEVERITY*,\ **--severity=***severity*
 

src/ShellCheck/ASTLib.hs

@@ -36,8 +36,6 @@ import Numeric (showHex)
 
 import Test.QuickCheck
 
-arguments (T_SimpleCommand _ _ (cmd:args)) = args
-
 -- Is this a type of loop?
 isLoop t = case t of
  T_WhileExpression {} -> True
@@ -559,11 +557,29 @@ getCommandNameFromExpansion t =
  extract (T_Pipeline _ _ [cmd]) = getCommandName cmd
  extract _ = Nothing
 
+-- If a command substitution is a single command, get its argument Tokens.
+-- Return an empty list if there are no arguments or the token is not a command substitution.
+-- $(date +%s) = ["+%s"]
+getArgumentsFromExpansion :: Token -> [Token]
+getArgumentsFromExpansion t =
+ case t of
+ T_DollarExpansion _ [c] -> extract c
+ T_Backticked _ [c] -> extract c
+ T_DollarBraceCommandExpansion _ [c] -> extract c
+ _ -> []
+ where
+ extract (T_Pipeline _ _ [cmd]) = arguments cmd
+ extract _ = []
+
 -- Get the basename of a token representing a command
 getCommandBasename = fmap basename . getCommandName
 
 basename = reverse . takeWhile (/= '/') . reverse
 
+-- Get the arguments to a command
+arguments (T_SimpleCommand _ _ (cmd:args)) = args
+arguments t = maybe [] arguments (getCommand t)
+
 isAssignment t =
  case t of
  T_Redirecting _ _ w -> isAssignment w