resholve

#!/usr/bin/env python2
from __future__ import print_function

from _resholve import strings
from _resholve.version import __version__

DIRECTIVE_FORMAT = "3"

import sys
import os
import StringIO
import io
import copy
import re
import csv
import shlex

from threading import Timer
from subprocess import Popen, PIPE
from contextlib import contextmanager

import argparse as theirparse  # external command parsers
import configargparse as ourparse  # resholve's own parser
import logging

from collections import defaultdict, Iterable

import sedparse


logger = logging.getLogger(__name__)
logging.basicConfig(level=os.environ.get("LOGLEVEL", logging.WARNING))
# logging.basicConfig(filename='example.log', filemode='w', level=logging.DEBUG)

"""
NOTE:
All of the below are imports from/for oil. Try to keep careful notes
on what is used from each Oil module. Oil is still young.
It's easier to track upstream reorg w/ succinct notes.
"""

import oil  # oil.__path__[0] for grammar loading

from oil.asdl import pybase  # CompoundObj

from oil.core import (
    alloc,  # Arena
    error,  # Parse, _ErrorWithLocation
    main_loop,  # ParseWholeFile
    optview,  # Parse
    pyutil,  # LoadYshGrammar, _FileResourceLoader
    state,  # InitOpts
    ui,  # ErrorFormatter.PrettyPrintError
)

from oil.osh import (
    word_,  # StaticEval
    split,  # SplitContext
    word_eval,  # CompletionWordEvaluator, StringWordEvaluator
    sh_expr_eval,  # ArithEvaluator
)

from oil.frontend import (
    parse_lib,  # ParseContext
    reader,  # FileLineReader
)

from oil._devbuild.gen.id_kind_asdl import (
    Id,  # Left_Backtick, Left_DollarParen, Lit_TildeLike, VSub_At, VSub_DollarName, VSub_Number, VSub_Name, VSub_Star
)

from oil._devbuild.gen.syntax_asdl import (
    source,  # MainFile, Stdin
    command,  # ShFunction, Simple
    SimpleVarSub,
    BracedVarSub,
    CompoundWord,
    loc,
)

from oil.tools import ysh_ify  # Cursor


def jeepers_peepers(self, *args, **kwargs):
    self._old_init(*args, **kwargs)
    self.members.append(self)


def consume_members(cls):
    members = cls.members
    cls.members = list()
    yield members


PEEPS = (SimpleVarSub, BracedVarSub, command.Simple, command.ShFunction)
for peep in PEEPS:
    peep.members = list()
    peep._old_init = peep.__init__
    peep.__init__ = jeepers_peepers
    peep.consume_members = classmethod(contextmanager(consume_members))


def blink():
    """clear global state that interferes w/ multi-step parse"""
    for peep in PEEPS:
        peep.members = list()


class ResolutionDirectives(object):
    fake = fix = keep = None

    @property
    def varsubs(self):
        """
        It's a little wasteful if we end up needing these
        more than once, but at the moment we don't, and it
        takes more logic to "ingest" them into their own var
        in even a hackish way, let alone a principled/safe
        one...
        """
        return {
            k[1:]: v
            for k, v in filter(lambda x: isinstance(x, tuple) and len(x) == 2, self.fix)
            if k.startswith("$")
        }

    def __init__(self, fake=None, fix=None, keep=None):
        self.fake = fake or set()
        self.fix = fix or set()
        self.keep = keep or set()

    def __len__(self):
        return len(self.fake) + len(self.fix) + len(self.keep)

    def update(self, *peers):
        for peer in peers:
            assert isinstance(peer, ResolutionDirectives)
            self.fake.update(peer.fake)
            self.fix.update(peer.fix)
            self.keep.update(peer.keep)

    @staticmethod
    def _type_tuple(type, entry):
        if len(entry) == 1:
            return (type, entry[0])
        if len(entry) == 2:
            return (type, entry[0], entry[1])

    def difference(self, *peers):
        out = set()
        for peer in peers:
            assert isinstance(peer, ResolutionDirectives)
            for fake in self.fake.difference(peer.fake):
                out.add((self._type_tuple("fake", fake)))
            for fix in self.fix.difference(peer.fix):
                out.add((self._type_tuple("fix", fix)))
            for keep in self.keep.difference(peer.keep):
                out.add((self._type_tuple("keep", keep)))

        return out

    def relevant(self, name):
        def rel_filt(val):
            if len(val) == 1:
                return name == val[0]
            if len(val) == 2:
                return name == val[1]

        return self.__class__(
            fake=filter(rel_filt, self.fake),
            fix=filter(rel_filt, self.fix),
            keep=filter(rel_filt, self.keep),
        )


directives = ResolutionDirectives()


def parse_directive(value, scope_map=None):
    if not len(value):
        return set()
    out = set()
    try:
        """
        TODO:
        this used to use simple string split. I changed to shlex
        to work around cases where someone needs to shell-quote
        a directive to support spacing. Example case reported in
        https://github.com/abathur/resholve/issues/32

        Unfortunately, shlex.split is about an oom slower...
        very high fruit, but there's probably a better way to
        split this particular baby.
        """
        directives = shlex.split(value)
        logger.debug("directives: %r", directives)
        for directive in directives:
            rules = directive.split(":", 1)
            if len(rules) == 1:
                out.add((rules[0],))
                continue
            elif len(rules) == 2:
                scope = rules[0] if not scope_map else scope_map(rules[0])
                rules = rules[1]
                rules = rules.split(";")
                for rule in rules:
                    out.add((scope, rule))
            else:
                raise ourparse.ArgumentTypeError(
                    "Directives may not have more than one colon: %r" % directive
                )
    except ValueError as e:
        raise ourparse.ArgumentTypeError(
            "I don't recognize %r as a known directive. See 'man resholve'." % value
        )
    return out


FAKE_DIRECTIVE_MAP = {
    "a": "alias",
    "b": "builtin",
    "e": "external",
    "f": "function",
    "k": "keyword",
    "s": "source",
}
FAKE_DIRECTIVE_TYPES = set(FAKE_DIRECTIVE_MAP.keys()) | set(FAKE_DIRECTIVE_MAP.values())


def remap_fake_scope(scope):
    global FAKE_DIRECTIVE_MAP
    if scope in FAKE_DIRECTIVE_MAP:
        return FAKE_DIRECTIVE_MAP[scope]
    else:
        return scope


def directive_fake(value):
    global FAKE_DIRECTIVE_TYPES
    directives = parse_directive(value, remap_fake_scope)
    for directive in directives:
        if len(directive) == 1:
            raise ourparse.ArgumentTypeError(
                "valid fake directives must have at least two parts"
            )
        elif directive[0] not in FAKE_DIRECTIVE_TYPES:
            raise ourparse.ArgumentTypeError(
                "fake directive type (%r) is not one of %r"
                % (directive[0], FAKE_DIRECTIVE_TYPES)
            )

    return directives


"""
This is a set of commands that NixOS uses setuid wrappers for.
I don't know if these are all literally the same case, but I
know resholve can't safely use `bin/sudo` from a sudo package
in a Nix build--because we'll bake in a reference to the un-
wrapped and thus useless copy. Painting them all with the same
brush unless/until someone informs me otherwise.
"""
NAUGHTY_NIXERS = (
    "ping",  # macos: /sbin/ping, nix: iputils
    "chsh",  # macos: /usr/bin/chsh, nix: shadow
    "newgrp",  # macos: /usr/bin/newgrp, nix: shadow
    "passwd",  # macos: /usr/bin/passwd, nix: shadow
    "su",  # macos: /usr/bin/su, nix: shadow
    "sudo",  # macos: /usr/bin/sudo, nix: sudo
    "mount",  # macos: /sbin/mount, nix: util-linux
    "umount",  # macos: /sbin/umount, nix: util-linux
)


def directive_fix(value):
    directives = parse_directive(value)
    logger.debug("validating fix directives %r", directives)
    for directive in directives:
        if len(directive) == 1:
            logger.debug("%r %r", directive[0], directive[0] == "aliases")
            if directive[0] == "aliases":
                pass  # ok literal
            elif os.path.isabs(directive[0]):
                pass  # ok absolute path
            elif directive[0] in NAUGHTY_NIXERS:
                pass  # ok force resolution of NAUGHTY_NIXER
            else:
                raise ourparse.ArgumentTypeError(
                    "valid single-part fix directives: 'aliases', absolute path"
                )
        elif len(directive) == 2:
            # we can't validate $variable or outer_command before parse
            # we could pre-validate command lookup for a target, but we
            # have to wait for the lookup() object... but it isn't a
            # given that this will only be used to substitute commands.
            # So, we'll just let it fail naturally during the regular
            # parse.
            pass
    return directives


def directive_keep(value):
    directives = parse_directive(value)
    logger.debug("validating keep directives %r", directives)
    for directive in directives:
        if len(directive) == 1:
            if directive[0][0] == "$":
                pass  # ok literal
            elif directive[0][0:2] == "~/" and os.path.isabs(
                os.path.expanduser(directive[0])
            ):
                pass
            elif directive[0][0:2] == "./":
                pass
            elif os.path.isabs(directive[0]):
                pass  # ok absolute path
            else:
                raise ourparse.ArgumentTypeError(
                    "valid single-part keep directives: $variable, absolute path, ~/path, ./path"
                )
        elif len(directive) == 2:
            pass
            # can't do much to pre-validate these
    return directives


# TODO: action unused for now; delete if still here
#       in 2022
# class InvocationAction(theirparse.Action):
#     def __call__(self, parser, namespace, values, option_string=None):
#         setattr(namespace, self.dest, Invocation(words=values))


class InvocationsAction(theirparse._AppendAction):
    """
    Collect arg as a list of invocations.

    Use kwarg split=True when the subexec arg is a single
    shell word (usually quoted) that will be split on exec.
    In this case, the arg will be split with shlex.split.

    1-word behavior is more common with --options, while
    using a collection of loose shell words is more common
    at the end of an invocation--though exceptions, such as
    the +/;-terminated -exec/-execdir/-ok/-okdir flags in
    find, do exist.

    I think (but am not certain; I forgot to doc this earlier)
    that this should always be used with nargs (whether a
    specific number, or the remainder constant).

    TODO: there's a gremlin in here wrt the above; if it isn't
    done you can get obtuse failures like in #70. This likely
    needs fixing or a backstop or better error-handing--
    something to sand down that sharp corner.
    """

    split = False

    def __init__(self, *args, **kwargs):
        kwargs["default"] = list()
        self.split = kwargs.pop("split", self.split)
        return super(InvocationsAction, self).__init__(*args, **kwargs)

    def __call__(self, parser, namespace, values, option_string=None):
        if not values:
            # bail; see https://github.com/abathur/resholve/issues/34
            # for more context/motive
            return None
        if self.split:
            words = []
            for val in values:
                words.extend([Word(val.ast, x) for x in shlex.split(val)])
            values = words
        return super(InvocationsAction, self).__call__(
            parser,
            namespace,
            Invocation(words=values),
            option_string,
        )


def interpreter_to_shebang(interp):
    if interp == "none":
        return None

    if not os.path.exists(interp):
        raise ourparse.ArgumentTypeError(
            "Interpreter must exist or be the string 'none'"
        )

    if not os.path.isabs(interp):
        raise ourparse.ArgumentTypeError("Interpreter path must be absolute")

    if not os.access(interp, os.X_OK):
        raise ourparse.ArgumentTypeError("Interpreter must be executable")

    return "#!{:}\n".format(interp)


class SuperSerial(csv.Dialect):
    """
    I'm using : for field separating and \n
    for row/record separation for now.
    I've intertained using unit separator as
    the delimiter, but : is more humane (if
    it doesn't cause all kinds of trouble)
    """

    delimiter = ":"
    escapechar = str("\x1f")  # https://en.wikipedia.org/wiki/Unit_separator
    quotechar = None
    doublequote = False
    skipinitialspace = False
    lineterminator = "\n"
    quoting = csv.QUOTE_NONE


csv.register_dialect("superserial", SuperSerial)


class LoreType(object):
    """
    Type for working with a "lore" directory.
    """

    lore_kinds = None

    @staticmethod
    def reader(f, fields):
        return csv.DictReader(
            f,
            fields,
            dialect="superserial",
        )

    @staticmethod
    def execers(f):
        return LoreType.reader(
            f,
            [
                "verdict",  # can|cannot|might
                "executable",  # abspath
            ],
        )

    @staticmethod
    def wrappers(f):
        return LoreType.reader(
            f,
            [
                "wrapper",  # abspath
                "wrapped",  # abspath
            ],
        )

    def __init__(self, lore_kinds):
        """ """
        self.lore_kinds = lore_kinds

    # TODO: see if you can fold ingest_lore in for better
    #       invocation-time ourparse errors

    def __call__(self, value):
        if len(self.lore_kinds) == 1:
            return self._call_str(value)
        else:
            return self._call_dir(value)

    def _call_str(self, value):
        try:
            for kind in self.lore_kinds:
                reader = getattr(self, kind)
                return list(reader(value.split()))
        except Exception as e:
            raise ourparse.ArgumentTypeError(
                "%s must be a colon-delimited string. Got: %r"
                % (self.lore_kinds[0], value)
            )

    def _call_file(self, path):
        try:
            if os.path.exists(path) and not os.path.isdir(path):
                for kind in self.lore_kinds:
                    f = open(path)
                    reader = getattr(self, kind)
                    return reader(f)
            else:
                raise Exception("%s is not a file" % path)
        except Exception as e:
            raise ourparse.ArgumentTypeError(
                "%s\n lore must be a single file. %s" % (self.lore_kinds[0], e)
            )

    def _call_dir(self, path):
        try:
            if os.path.isdir(path):
                files = dict()

                for kind in self.lore_kinds:
                    f = open("{:}/{:}".format(path, kind))
                    reader = getattr(self, kind)
                    files[kind] = reader(f)
                return files
            else:
                raise Exception("%s is not a directory" % path)
        except Exception as e:
            raise ourparse.ArgumentTypeError(
                "%s\nLore must be a "
                "directory with a file named "
                "'execers' (and optionally one named "
                "'wrappers')." % e
            )


class Accumulate(ourparse.Action):
    def __call__(self, _parser, ns, vals, *args):
        setattr(ns, self.dest, (getattr(ns, self.dest) or []) + vals)


lookup = None


class LookSee(dict):
    """
    A dictionary for input (~PATH) lookups.

    Broad strokes:
    - Take ownership of lookup from distutils.spawn.find_executable to
      stop matching files in the current PWD (explicit > implicit) and
      avoid having to set PATH.
    - Support both directories and individual files in the ~PATH.
      - Pre-cache files and rm from search path
      - Lazily lookup and cache others
    """

    pwd = os.getcwd()
    paths = None
    __call__ = dict.__getitem__

    def __init__(self, path):
        """
        init from path-fmt string
        - comes via --path/--inputs or RESHOLVE_(PATH|INPUTS)
        - each element may be a dir or file
        """
        dict.__init__(self)
        paths = set()
        for path in set(path.split(os.pathsep)):
            if os.path.isabs(path):
                if os.path.isfile(path):
                    self[os.path.basename(path)] = path
                else:
                    paths.add(path)
            else:
                x = os.path.join(self.pwd, path)
                if os.path.isfile(x):
                    self[path] = x
                elif os.path.exists(x):
                    paths.add(x)
                else:
                    paths.add(path)
        self.paths = paths

    def __missing__(self, basename):
        """
        Return abspath or None
        """
        for path in self.paths:
            abspath = os.path.join(path, basename)
            if os.path.isfile(abspath):
                return abspath
        return None


def make_arg_parser(module):
    parser = module.ArgumentParser(
        description=strings.description,
        usage=strings.synopsis,
        auto_env_var_prefix="RESHOLVE_",
        add_env_var_help=False,
        add_help=False,
    )
    parser.register("action", "accumulate", Accumulate)

    req = parser.add_argument_group(
        title="Required",
    )
    req_path = req.add_mutually_exclusive_group(required=True)

    req.add_argument(
        "scripts",
        metavar="script",
        type=str,
        nargs="*",
        help=strings.scripts,
    )
    req_path.add_argument(
        "--path",
        metavar="str",
        dest="path",
        type=LookSee,
        help=strings.path,
    )
    req_path.add_argument(
        "--inputs",
        metavar="str",
        dest="path",
        type=LookSee,
        help=strings.inputs,
    )

    req.add_argument(
        "--interpreter",
        dest="shebang",
        type=interpreter_to_shebang,
        metavar="p",
        help=strings.interpreter,
        required=True,
    )

    res = parser.add_argument_group(
        title="Controlling resolution",
    )

    res.add_argument(
        "--fake",
        metavar="str",
        action="append",
        type=directive_fake,
        help=strings.fake,
    )
    res.add_argument(
        "--fix",
        metavar="str",
        action="append",
        type=directive_fix,
        help=strings.fix,
    )
    res.add_argument(
        "--keep",
        metavar="str",
        action="append",
        type=directive_keep,
        help=strings.keep,
    )
    """
    TODO: disabling some options that I sketched out but that
    do not square with the current implementation and may not
    be necessary. Can probably dump if still here in 2022.

    res.add_argument(
        "--oracle",
        dest="oracle",
        metavar="str",
        action="append",
        type=str,
        help="executable...",
    )
    res.add_argument(
        "--assayer",
        dest="assayer",
        metavar="str",
        action="append",
        type=str,
        help="executable...",
    )
    """
    res.add_argument(
        "--lore",
        type=LoreType(("execers", "wrappers")),
        metavar="d",
        help=strings.lore,
    )
    res.add_argument(
        "--execer",
        type=LoreType(("execers",)),
        metavar="str",
        action="accumulate",
        help=strings.execer,
    )
    res.add_argument(
        "--wrapper",
        type=LoreType(("wrappers",)),
        metavar="str",
        action="accumulate",
        help=strings.wrapper,
    )
    res.add_argument(
        "--assay",
        type=ourparse.FileType("r"),
        metavar="f",
        # see tests for usage; not "supported; unsure.
        help=ourparse.SUPPRESS,
    )

    other = parser.add_argument_group(title="Other")

    other.add_argument(
        "--prologue",
        type=ourparse.FileType("r"),
        metavar="f",
        help=strings.prologue,
    )
    other.add_argument(
        "--epilogue",
        type=ourparse.FileType("r"),
        metavar="f",
        help=strings.epilogue,
    )

    other.add_argument(
        "--overwrite",
        action="store_true",
        dest="overwrite",
        help=strings.overwrite,
    )
    # other.add_argument(
    #     "--faff", type=int, metavar="int", help="fuzz n times for missed commands",
    # )
    meta = parser.add_argument_group(title="Meta")
    meta.add_argument(
        "-h", "--help", action="help", help="show this help message and exit"
    )
    meta.add_argument(
        "--version",
        action="version",
        version="%(prog)s {:}".format(__version__),
    )

    return parser


parser = make_arg_parser(ourparse)

resolved_scripts = dict()


def resolve_script(script_path, shebang="", prologue=None, epilogue=None):
    if script_path in resolved_scripts:
        return resolved_scripts[script_path]

    resolved = resolved_scripts[script_path] = ResolvedScript(
        script_path, shebang=shebang, prologue=prologue, epilogue=epilogue
    )
    return resolved


def write_resolved_script(script_path, resolved, overwrite=False):
    if not script_path:
        resolved.write_to()
    elif overwrite:
        logger.info("attempting to overwrite script %r", script_path)
        resolved.write_to(script_path)
        sys.stderr.write("Overwrote %r\n" % (script_path))
    else:
        logger.info(
            "overwrite not set; will attempt to write to: %s%s",
            script_path,
            ".resolved",
        )
        resolved.write_to(script_path + ".resolved")
        sys.stderr.write(
            "Rewrote %r to %r\n" % (script_path, script_path + ".resolved")
        )


def lookup_source(word):
    """
    wrap lookup(word) to special-case source lookups

    lookup can't resolve something like `source submodule/helper.sh`
    unless we add subdirs to PATH, but that could let all kinds of
    stuff into the resolution scope.

    Instead, we'll try to resolve source calls from the list of
    input scripts before giving lookup a chance.

    This is a little weird, but this is a not-implemented stub for
    documentation. The business-end of this function is temporarily
    published the sourcePATH() context manager during punshow()
    """
    raise NotImplementedError("lookup_source called outside of sourcePATH context")


@contextmanager
def sourcePATH(script_map):
    global lookup_source

    def contextual_lookup_source(word):
        if word in script_map:
            logger.debug("source in script map %r -> %r", word, script_map)
            return script_map[word]
        logger.debug("source not in script map %r", word)
        return lookup(word)

    prev = lookup_source
    lookup_source = contextual_lookup_source
    yield
    lookup_source = prev


lore = dict()
decided_execers = defaultdict(dict)


def ingest_lore(lorefiles):
    global lore
    logger.debug("ingesting lore %r", lorefiles)
    logger.debug("ingesting lore:execers %r", lorefiles["execers"])
    for row in lorefiles["execers"]:
        logger.debug("ingesting lore:execers:row %r", row)
        try:
            lore[row["executable"]] = row["verdict"]
        except Exception as e:
            raise Exception(row, lore, e)

    if "wrappers" not in lorefiles:
        return

    for row in lorefiles["wrappers"]:
        if row["wrapped"] in lore:
            # overwrite the wrapper's verdict with the wrapped executable's
            lore[row["wrapper"]] = lore[row["wrapped"]]


def ingest_assay(assayfile):
    reader = csv.DictReader(
        # TODO: If the general practice sticks, I may move cmd to end because
        # the separator is now ":", and it'll be easier to adapt to colons in
        # invocations if we can eat 3 colons and call the rest the command.
        assayfile,
        [
            "executable",  # abspath
            "command",  # ~basename $@
            "verdict",  # yes|no|unknown
            # positions 0-indexed
            # let's try to work at the word level first?
            "wordnum",
        ],
        dialect="superserial",
    )

    for row in reader:
        logger.debug("assay ingest %r", row)
        if row["wordnum"] == "0":
            # TODO this needs to be fixed somehow; argparse
            # won't catch so it'll throw a stack AND exit 1
            raise ourparse.ArgumentTypeError(
                "assay wordnum should be 1+ (0 is the same as the invoking command itself)"
            )

        decided_execers[row["command"].split()[0]] = {k: v for k, v in row.items()}


def punshow():
    global lookup
    args = parser.parse_args()
    # if it makes sense at some point, vars(args) yields a dict of these
    # that can be passed as kwargs or something...
    logger.info("argparsed: %r", args)
    logger.debug("argument sources: \n%s", parser.format_values())

    # adopt the runtime dependency path for resolving external executables
    if args.path is None:
        # made this not-required in order to support both "path" and
        # "inputs" terminology (inputs is more nix-idiomatic) but
        # store in a single dest; fake our own error
        parser.error(
            "argument --path/--inputs (or RESHOLVE_PATH/RESHOLVE_INPUTS) is required"
        )

    lookup = args.path

    if args.fake:
        for group in args.fake:
            directives.fake.update(group)

    if args.fix:
        for group in args.fix:
            directives.fix.update(group)

    if args.keep:
        for group in args.keep:
            directives.keep.update(group)

    if args.lore:
        ingest_lore(args.lore)
        global lore
        logger.debug("lore: %r", lore)

    if args.execer or args.wrapper:
        ingest_lore(
            {
                "execers": args.execer if args.execer else tuple(),
                "wrappers": args.wrapper if args.wrapper else tuple(),
            }
        )

    if args.assay:
        ingest_assay(args.assay)

    if ("aliases",) not in directives.fix:
        RecordCommandlike.disable_alias_replacement()

    try:
        to_write = set()

        checked_scripts = list()
        script_map = dict()
        for in_script in args.scripts:
            abs_script = os.path.abspath(in_script)
            if os.path.exists(abs_script):
                checked_scripts.append(abs_script)
                script_map[in_script] = abs_script
            else:
                sys.stderr.write("Aborting due to missing file: %r\n" % abs_script)
                return 2

        if len(set(checked_scripts)) != len(checked_scripts):
            sys.stderr.write(
                "Aborting due to duplicate script targets.\n"
                "Verify your inputs and try again.\n"
            )
            sys.stderr.write("  Original: %r\n" % args.scripts)
            sys.stderr.write("  Distinct: %r\n" % set(checked_scripts))
            return 2

        if len(args.scripts) == 0:
            # None == <stdin>
            with sourcePATH(dict()):
                resolved_scripts[None] = ResolvedScript(
                    shebang=args.shebang,
                    prologue=args.prologue,
                    epilogue=args.epilogue,
                )
                to_write.add(None)

                resolve_cmdlikes()
        else:
            with sourcePATH(script_map):
                for script in checked_scripts:
                    resolve_script(
                        script,
                        shebang=args.shebang,
                        prologue=args.prologue,
                        epilogue=args.epilogue,
                    )
                    to_write.add(script)

                resolve_cmdlikes()

        for script_path in sorted(to_write):
            write_resolved_script(
                script_path, resolved_scripts[script_path], args.overwrite
            )

    except IOError as e:
        sys.stderr.write("Whoooo buddy: " + str(e))
        return 2
    except Forbid as e:
        e.print_if_needed()
        return e.exit_status
    except error._ErrorWithLocation as e:
        if hasattr(e, "print_if_needed"):
            e.print_if_needed()
        elif hasattr(e, "arena"):
            # TODO: w/ the 0.16 update I suspect we can stop attaching e.arena
            # _but_ it's our only signpost for printing these oil errors atm
            # so it's not something I can just yank out without refactoring.
            # deferring since it isn't essential.
            ui.ErrorFormatter().PrettyPrintError(e, getattr(e, "label", "error: "))
        else:
            print(
                "This error has location context I couldn't "
                "figure out how to print. Please report this "
                "@ https://github.com/abathur/resholve/issues",
                repr(e),
            )
        if hasattr(e, "exit_status") and e.exit_status is not None:
            return e.exit_status
        else:
            return 2
    except InFileDirectiveError as e:
        sys.stderr.write(str(e))
        return 2
    except ourparse.ArgumentTypeError as e:
        sys.stderr.write("Hold up there, cowpoke: " + str(e))
        return 2


def find_earliest_spid(node):
    """
    I'll focus on the goal rather than mislead you with assertions about
    what this does (I'm not confident I understand all of the underlying
    variation within oil on this point).

    The old assumption was that, when a node has an spids list, the first
    item in the list is the earliest spid under the umbrella. That is
    usually true, and has held water for many months, but I've found at
    least one case with command.AndOr nodes where the leftmost spid in
    this array is actually for the && or || token.

    In addition to above:
    - the update from oil 0.14 to 0.15 also requires doing something
      with the "left" attr to avoid inserting inside braced sections.
    - the update from oil 0.15 to 0.16 required doing something with
      the "keyword" attr to avoid missing on, say, a for loop.

    We really do want the earliest spid, because we want to be able to
    inject a prologue at that point. So, we'll try to descend the ~edge
    of the AST, keeping the lowest spid we find.
    """
    node_spid = sub_spid = None
    if hasattr(node, "spids") and node.spids and len(node.spids):
        node_spid = node.spids[0]
    elif hasattr(node, "span_id") and node.span_id:
        node_spid = node.span_id
    elif hasattr(node, "token") and node.token:
        node_spid = node.token.span_id
    elif hasattr(node, "keyword") and node.keyword:
        node_spid = node.keyword.span_id
    elif hasattr(node, "name_tok") and node.name_tok:
        node_spid = node.name_tok.span_id

    if hasattr(node, "left"):
        sub_spid = find_earliest_spid(node.left)
    elif hasattr(node, "children"):
        sub_spid = find_earliest_spid(node.children[0])
    elif hasattr(node, "words"):
        sub_spid = find_earliest_spid(node.words[0])
    elif hasattr(node, "parts"):
        sub_spid = find_earliest_spid(node.parts[0])

    # min() will return a None, so I'll be really explicit
    if node_spid is None and sub_spid is None:
        raise Exception(
            "No approach found an integer span id; please report this", node
        )
    elif node_spid is None:
        return sub_spid
    elif sub_spid is None:
        return node_spid
    else:
        return min(node_spid, sub_spid)


def get_original_text(arena, location, positions):
    text = list()
    for loc in range(location, location + positions):
        span = arena.GetToken(loc)
        line = span.line.content
        text.append(line[span.col : span.col + span.length])
        logger.debug(
            "identified span=%r, line=%r, text=%r",
            span,
            line,
            text,
        )
    return "".join(text)


# For optview.Parse. Oil does [False] * option_i.ARRAY_SIZE and
# [None] * option_i.ARRAY_SIZE; we'll cheat
class NoneListFake(object):
    def __getitem__(self, option):
        return None


NO_OPTIONS_OVERLAY = NoneListFake()


class ExceptionWithContext(error._ErrorWithLocation):
    pass


# label? code? issue? feedback? url?
class ResolutionException(ExceptionWithContext, error.FatalRuntime):
    """
    mostly same code for all 3 resolution exceptions
    """

    REPO = "https://github.com/abathur/resholve"
    ISSUES = "{:}/issues".format(REPO)
    SA_TEMPL = "{e.ISSUES}/{e.issue}"
    # ex: I could use your feedback on this at:\n

    template = None
    label = None
    printed = False
    arena = None
    problem = None
    ideas = None
    status = None
    issue = None

    @property
    def see_issue(self):
        if self.issue:
            return (
                # I'd kinda like to push info/feedback/workarounds to be like
                # options or dynamic or sommink?
                self.SA_TEMPL.format(e=self)
            )
        return NotImplementedError()

    @property
    def description(self):
        if self.problem:
            # "I think it'd be nice to structure these around a core description ({problem}) with some default rules for how to append codes, description links, feedback issues, etc."
            return self.template.format(e=self)
        else:
            raise NotImplementedError("Don't raise ResolutionException directly.")

    @property
    def steps(self):
        return "\n".join(self.ideas)

    def build_template(self):
        template = ""
        if self.label:
            template += "{e.label}"
        if self.problem:
            template += "{e.problem}"
        if self.ideas:
            if len(self.ideas) > 1:
                template += "\n\nNext steps:\n{e.steps}"
            else:
                template += "\n\nNext step: {e.steps}"
        if self.issue:
            template += "\n\n{e.see_issue}"

        return template

    def __init__(self, *args, **kwargs):
        self.arena = kwargs.pop("arena", None)
        self.template = self.build_template()
        if self.status:
            error.FatalRuntime.__init__(
                self, self.status, self.description % args, kwargs.get("location", None)
            )
        else:
            ExceptionWithContext.__init__(
                self, self.description % args, kwargs.get("location", None)
            )

    def print_if_needed(self):
        if not self.printed:
            ui.ErrorFormatter().PrettyPrintError(self)
            self.printed = True


class Forbid(ResolutionException):
    pass


class Detain(Forbid):
    pass


class Census(ResolutionException):
    label = "FEEDBACK WANTED: "

    def __init__(self, *args, **kwargs):
        ResolutionException.__init__(self, *args, **kwargs)
        self.print_if_needed()


class Caution(Census):
    label = "CAUTION: "


"""
TODO: I'm waffling a bit on messaging.

I think lucid error communication--what caused resholve to halt, why it
is a problem, and what to do--is important for shifting resholve into
its next phase/gear.

But, they're also the kind of thing I can fritter a lot of time/effort
away on (and they aren't too bad to iterate on). For now, braindump and
disable most 'ideas':
- I'd like to basically workshop the messages towards whether people
  understand what they mean.
- I'm not sure where to come down wrt to linking to a web page/wiki
  and explaining in the message. I've been inclined to see this as
  something I should give more context on here because resholve isn't
  quite a linter--most errors block as you hit them.
- I don't want to get sucked into working on "docs" or a "site".
- I don't want to fritter gobs of time on terse messages that aren't
  detailed enough for anyone to solve their own problems without
  reading the manpage or clicking on a URL anyways?
- I DO want to take time to get the full test stack working again now
  so that it's possible to iterate on these in isolation with a clearer
  ~signal.
"""


class UnresolvedCommand(Forbid):
    problem = "Couldn't resolve command '%s'"
    # ideas = (
    #     "- add executables to inputs/path",
    #     "- `fake` a function/alias defined elsewhere",
    # )
    status = 3


class UnresolvedSource(Forbid):
    problem = "Unable to resolve source target '%s' to a known file"
    # ideas = (
    #     "- add it to inputs/path",
    #     "- `fake` a source defined elsewhere",
    # )
    status = 4


class UnexpectedAbsolutePath(Forbid):
    problem = "Unexpected absolute %s path."
    ideas = ("keep, fix, or pre-patch/substitute it.",)
    status = 5


class DynamicArgument(Forbid):
    problem = "Can't resolve dynamic argument in '%s'"
    # ideas = (
    #     "Decide if this is ",
    #     "- `fake` a source defined elsewhere",
    # )
    status = 6


class DynamicCommand(Forbid):
    problem = "Can't resolve dynamic command"
    # ideas = "TODO: where should they go? Doc? Wiki? Issue?"
    status = 7


class MissingLore(Forbid):
    problem = "I don't have any lore for '%s'"
    # ideas = "For more on how to supply it, see: TODO"
    status = 8


class PotentialExecer(Forbid):
    problem = (
        "'%s' _might_ be able to execute its arguments, and I "
        "don't have any command-specific rules for figuring out if this "
        "specific invocation does or not. "
    )
    # ideas = "TODO: where should they go? Doc? Wiki? Issue?"
    status = 9


# TODO: if no other testable assay cases turn up that justify keeping this vague, dial it in; the lone case ATM is for an assay that uses a number that overshoots the length of the command
class InvalidAssay(Forbid):
    problem = "I have an assay matching this invocation, but: %s\n\nAssay: %r"
    # ideas = "TODO: where should they go? Doc? Wiki? Issue?"
    status = 10


class SedECommand(Detain):
    problem = "I don't know how to handle sed `e` commands yet--sorry :("
    ideas = ("- See the feedback issue for a workaround",)
    status = 11
    issue = 28


class SedECommandParseError(Caution):
    problem = "My sed parser couldn't handle this expression. "
    ideas = (
        "- Manually check it for exec (e commands) and patch any you find.",
        "- Report your case in the issue to help us understand these cases",
    )
    issue = 28


class NixSetuidWrapper(Detain):
    problem = "There is not yet a good way to resolve '%s' in Nix builds."
    ideas = (
        "- Your feedback will help clarify the best course of action.",
        "- See the workaround in the issue.",
    )
    status = 12
    issue = 29


class AwkSubCommand(Census):
    """
    Caution: note that this was formerly an error, but I'm backing it off
    to a simple warning for now since I'm having to convert to a method
    that we can't trust to accurately identify the problem cases.
    """

    problem = (
        "I don't know how to find and replace sub commands in awk scripts yet--sorry :("
    )
    ideas = ("- See the feedback issue for a workaround",)
    # status = 13
    issue = 31


class DcBangCommand(Detain):
    problem = "I don't know how to handle dc `!` commands yet--sorry :("
    ideas = ("- See the feedback issue for a workaround",)
    status = 14
    issue = 40


# preserving exception for now, but currently unused
# rm anytime after 2021
# class OshEvalProblem(Census):
#     problem = "OSH eval error while looking for sub-exec in %s '%s'."
#     issue = 30


class OverriddenEssentialBuiltin(Census):
    problem = "Essential builtin overridden by %s '%s'."
    issue = 27


class QuotedEval(Census):
    problem = "Letting quoted '%s' through for now."
    issue = 2


class NoParserSucceeded(Forbid):
    # this is somewhat related to PotentialExecer
    # see issue 82 for backstory
    problem = (
        "'%s' executes its arguments in some circumstances, and "
        "none of my command-specific rules for figuring out if this "
        "specific invocation does or not succeeded. Look above "
        "for warning messages with more context on what went wrong."
    )
    # ideas = "TODO: where should they go? Doc? Wiki? Issue?"
    status = 15


# emit census/feedback requests at warning+ verbosity
if logger.isEnabledFor(logging.WARNING):

    def observe(cls, *args, **kwargs):
        try:
            raise cls(*args, **kwargs)
        except Census:
            # these self-print and don't block
            pass

else:

    def observe(cls, *args, **kwargs):
        if isinstance(cls, Census):
            return
        raise cls(*args, **kwargs)


class InFileDirectiveError(Exception):
    directive = path = None
    template = (
        "There's a bad directive already in this file. You may need to\n"
        "re-resholve it with the current version? Here's the context:\n\n"
        "   parsing:  {script}\n"
        " directive:  '{directive}'\n"
        "     error:  {msg}\n"
    )

    def __init__(self, msg, path, directive, *args, **kwargs):
        Exception.__init__(
            self,
            self.template.format(
                msg=msg,
                script=os.path.relpath(path),
                directive=directive[0:-1],  # lop off newline
            ),
            *args,
            **kwargs
        )


class SingleKeyDefaultDict(dict):
    def __init__(self, factory):
        self.factory = factory

    def __missing__(self, key):
        return self.setdefault(key, self.factory(key))


class MultiKeyDefaultDict(dict):
    def __init__(self, factory):
        self.factory = factory

    def __missing__(self, key):
        return self.setdefault(key, self.factory(*key))


"""
I'm refactoring to use argparse instead of a bespoke parser.
This worked okay for a while, and let me cut some corners,
but looking at how to graft in support for -comboflags made
it obvious that I don't want to be maintaining this logic in
the short run. (Maybe reconsider once the set of covered
external commands feels very stable.)

Some general principles/notes:
- only set up parsers for externals we actually find
- cache them since multiple calls will be common
- compact and clarify the definitions
- don't add help by default (favor being explicit)
- minimize the number of bespoke handlers

Some patterns coalescing:
- generic handler uses the "command" dest to find subcommands
- init parser with a name and variant, i.e. "sed" and "gnu" or "bsd"
- use a "skip" target if there are options that are mutually-exclusive
  with sub-execution

Reminder: You could reap a performance benefit by dropping all of the
bool flags, but it breaks handling of combined -abcd args.

TODO:
- Nix Test suite just for exercising the syntax of these commands (
  demonstrating it doesn't break, that it can resolve what it's
  supposed to, and a place to add edge-cases that turn up.)
- I'd like to eventually end up with a ~config-file format to specify
  these, if it can be done clearly for the vast-majority of cases
- If the above works out, I'd also like more space to ~annotate/doc
  these without polluting the source or slowing the program down...
- I'd like to condense these expressions, but retain the flexibility
  of specifying whatever argparse supports (if the above fails?)
"""


class CommandParser(theirparse.ArgumentParser):
    """
    Changes:
    - don't raise errors; I don't *think* we want to halt here
      (because then we're going to have to have a full model of all in-use
      options, even across different programs...)

    TODO:
    - if this pattern sticks, AND resholve gains a test-suite focused on
      extensively picking at external command syntax, the time needed to
      run those tests may be able to guide some optimizations here. i.e.
      pull in function source and:
      - knock out features and safety checks
      - maybe just literally construct the end data structure
      - two for loops in parse_known_args could probably be done on init
    - regardless, I need to be keeping better notes on the cmds
      - version, variant, description of how they differ, platform
    """

    def __init__(self, name, variant="generic"):
        theirparse.ArgumentParser.__init__(
            self, prog="{:} ({:})".format(name, variant), add_help=False
        )
        self.register("action", "invocations", InvocationsAction)
        # self.register("action", "invocation", InvocationAction)

    def error(self, message):
        logger.warn("CommandParser %r passing instead of %r", self, message)

    def _parse_optional(self, arg_string):
        """
        Argparse can, under some circumstances, kick back plain strings
        with our Word() wrappers removed. I'm not sure this is the best
        way to go about fixing it (maybe just reimplementing the funcs
        and not doing what they do would be better?), but for now I'll
        try this more minimal intervention.
        """
        ret = super(CommandParser, self)._parse_optional(arg_string)
        if isinstance(ret, tuple):
            action, option_string, explicit_arg = ret
            if option_string and not isinstance(option_string, Word):
                for part in arg_string.ast.parts:
                    if getattr(part, "tval", None) == option_string:
                        option_string = Word(CompoundWord([part]))
            if explicit_arg and not isinstance(explicit_arg, Word):
                for i, part in enumerate(arg_string.ast.parts):
                    if getattr(part, "tval", None) == explicit_arg:
                        explicit_arg = Word(CompoundWord(arg_string.ast.parts[i:]))
            return action, option_string, explicit_arg
        return ret


class FindParser(CommandParser):
    """
    find's exec options (-exec -execdir -ok -okdir)
    are terminated in a way argparse can't really
    deal with (;|+), so we'll declare these with
    nargs=1, pre-parse to replace these args with
    a fake word, and then swap them back in after
    the parse.
    """

    def _parse_known_args(self, arg_strings, namespace):
        simplified = list()
        capturing = False
        captures = list()
        for i, arg in enumerate(arg_strings):
            if arg in ("-exec", "-execdir", "-ok", "-okdir"):
                capturing = True
                simplified.append(arg)
                simplified.append("fakecmd")
                captures.append(list())
            elif arg in (";", "+"):
                capturing = False
            elif capturing:
                captures[-1].append(arg)
                continue
            else:
                simplified.append(arg)
        ns, rest = super(FindParser, self)._parse_known_args(simplified, namespace)
        if captures and ns.commands:
            ns.commands = [Invocation(words=x) for x in captures]
        return ns, rest


class GnuSedParser(CommandParser):
    """
    gnused's -i/--in-place options can be bare, or have the
    argument directly attached (it CANNOT be a separate
    shell word).
    """

    def _parse_known_args(self, arg_strings, namespace):
        simplified = list()
        capturing = False
        captures = list()
        for i, arg in enumerate(arg_strings):
            if arg.startswith("--in-place="):
                continue
            elif arg.startswith("-i"):
                if len(arg) > 2:
                    continue
            else:
                simplified.append(arg)
        ns, rest = super(GnuSedParser, self)._parse_known_args(simplified, namespace)
        return ns, rest


"""
supporting sub-shell invocation for a wide slate of shells
that I have access to via nix/macOS for now. i.e.,
<sh> -c "subcommand".
May have to revert some of these if they shadow a
_more important_ command of the same name?
"""

INVOKABLE_SHELLS = (
    "bash",
    "csh",
    "dash",
    "dgsh",
    "elvish",
    "es",
    "fish",
    "hilbish",
    "ion",
    "jush",
    "ksh",
    "mksh",
    "mrsh",
    "nu",
    "oh",
    "oil",
    "oksh",
    "osh",
    "rc",
    "rush",
    "sh",
    "tcsh",
    "zsh",
)

"""
The next two classes collect functions for returning
argparse parsers for some builtins and externals. We
use the functions so that we don't have to pay setup
costs for all of the parsers on every run. After this,
two defaultdicts cache parsers that get initialized.

All of the commands start with an underscore just to
avoid collisions with python keywords (for example
`def exec():` will just cause an obtuse error)

TODO:
    - I'm not terribly sure when I should use nargs="?" vs nargs="*"
    - --help/--version add_help -h -v consistency?

    Caution: argparse defaults may end up being gotchas...
    - allow_abbrev is on by default
    - nargs=?/* on options will want to eat subsequent args and may
      be a smell that you should really be using more than one parser
      (i.e. one with 0, one with 1, etc.)
"""


class BuiltinCommandParsers(object):
    @staticmethod
    def _alias():
        generic = CommandParser("alias", "builtin")
        generic.add_argument("-p", action="store_true")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _builtin():
        generic = CommandParser("builtin", "builtin")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _coproc():
        generic = CommandParser("coproc", "builtin")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _command():
        generic = CommandParser("command", "builtin")
        generic.add_argument("-p", action="store_true")
        generic.add_argument("-v", action="store_true")
        generic.add_argument("-V", action="store_true")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _eval():
        generic = CommandParser("eval", "builtin")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _exec():
        """
            exec: exec [-cl] [-a name] [command [argument ...]] [redirection ...]
        Replace the shell with the given command.

        Execute COMMAND, replacing this shell with the specified program.
        ARGUMENTS become the arguments to COMMAND.  If COMMAND is not specified,
        any redirections take effect in the current shell.

        Options:
          -a name   pass NAME as the zeroth argument to COMMAND
          -c    execute COMMAND with an empty environment
          -l    place a dash in the zeroth argument to COMMAND
        """
        generic = CommandParser("exec", "builtin")
        generic.add_argument("-a")
        generic.add_argument("-c", action="store_true")
        generic.add_argument("-l", action="store_true")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _source():
        generic = CommandParser("source", "builtin")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _type():
        generic = CommandParser("type", "builtin")
        generic.add_argument("-p", action="store_true")
        generic.add_argument("-v", action="store_true")
        generic.add_argument("-V", action="store_true")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)


class ExternalCommandParsers(object):
    @staticmethod
    def _chroot():
        """
        coreutils src/chroot.c:execvp
        bsd: [-g group] [-G group,group,...] [-u user] newroot [command]
        gnu: [OPTION] NEWROOT [COMMAND [ARG]...]
             or:  chroot OPTION
             Run COMMAND with root directory set to NEWROOT.

             --groups=G_LIST        specify supplementary groups as g1,g2,..,gN
             --userspec=USER:GROUP  specify user and group (ID or name) to use
             --skip-chdir           do not change working directory to '/'
             --help     display this help and exit
             --version  output version information and exit
        """
        generic = CommandParser("chroot")
        # bsd
        generic.add_argument("-g")
        generic.add_argument("-G")
        generic.add_argument("-u")
        # gnu
        generic.add_argument("--groups")
        generic.add_argument("--userspec")
        generic.add_argument("--skip-chdir", action="store_true")
        generic.add_argument("--help")
        generic.add_argument("--version")

        # both
        generic.add_argument("newroot", nargs="?")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _env():
        """
        coreutils src/env.c:execvp
        Usage: %s [OPTION] [COMMAND [ARG]...]

        both gnu and bsd support "-" as a synonym for -i
        argparse appears to support, but IDK if it'll cause trouble

        assignment-alikes should be stripped out already
        """
        generic = CommandParser("env")
        generic.add_argument("-P")  # bsd altpath
        # gnu
        generic.add_argument(
            # -i and - are also bsd
            "-i",
            "--ignore-environment",
            "-",
            action="store_true",
        )
        generic.add_argument("-0", "--null", action="store_true")
        generic.add_argument("-u", "--unset")  # -u is bsd
        generic.add_argument("-C", "--chdir")
        generic.add_argument("-S", "--split-string")  # -S is bsd
        generic.add_argument("--block-signal")
        generic.add_argument("--default-signal")
        generic.add_argument("--ignore-signal")
        generic.add_argument("--list-signal-handling")
        generic.add_argument("-v", "--debug", action="store_true")  # -v is bsd
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _install():
        """
        coreutils src/install.c:execlp
        """
        generic = CommandParser("install")
        generic.add_argument(
            "--strip-program",
            dest="commands",
            action="invocations",
            nargs=1,
        )
        return (generic,)

    @staticmethod
    def _nice():
        """
        coreutils src/nice.c:execvp
        Usage: %s [OPTION] [COMMAND [ARG]...]
        """
        generic = CommandParser("nice")
        generic.add_argument("-n")
        generic.add_argument("--adjustment")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _nohup():
        """
        coreutils src/nohup.c:execvp
        Usage: %s [OPTION] [COMMAND [ARG]...]
        """
        generic = CommandParser("nohup")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _runcon():
        """
        coreutils src/runcon.c:execvp
        Usage: %s [OPTION] [COMMAND [ARG]...]

        two mutually-exclusive forms; if opts are present, the command
        is the first arg--if not, there's a combined context first.
        """
        opt_context = CommandParser("runcon", "opt context")
        context = opt_context.add_mutually_exclusive_group(required=True)
        context.add_argument("-c", "--compute", action="store_true")
        context.add_argument("-t", "--type")
        context.add_argument("-u", "--user")
        context.add_argument("-r", "--role")
        context.add_argument("-l", "--range")
        opt_context.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )

        # these two forms are mutua
        combined_context = CommandParser("runcon", "combined context")
        combined_context.add_argument("context")
        combined_context.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (opt_context, combined_context)

    @staticmethod
    def _sort():
        """
        coreutils src/sort.c:execlp
        """
        generic = CommandParser("sort")
        generic.add_argument(
            "--compress-program",
            dest="commands",
            action="invocations",
            split=True,
            nargs=1,
        )
        return (generic,)

    @staticmethod
    def _split():
        """
        coreutils src/split.c:execl
        """
        generic = CommandParser("split")
        generic.add_argument(
            "--filter",
            dest="commands",
            action="invocations",
            split=True,
            nargs=1,
        )
        return (generic,)

    @staticmethod
    def _stdbuf():
        """
        coreutils src/stdbuf.c:?
        Usage: %s [OPTION] [COMMAND [ARG]...]
        """
        generic = CommandParser("stdbuf")
        generic.add_argument("-i", "--input")
        generic.add_argument("-o", "--output")
        generic.add_argument("-e", "--error")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _timeout():
        """
        coreutils src/timeout.c:execvp
        Usage: %s [OPTION] [COMMAND [ARG]...]
        """
        generic = CommandParser("timeout")
        generic.add_argument("-k", "--kill-after")
        generic.add_argument("-s", "--signal")
        generic.add_argument("-v", "--verbose", action="store_true")
        generic.add_argument("--preserve-status", action="store_true")
        generic.add_argument("--foreground", action="store_true")
        generic.add_argument("duration")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _sqlite3():
        """ """
        generic = CommandParser("sqlite3")
        generic.add_argument(
            "-cmd",
            dest="commands",
            action="invocations",
            split=True,
            nargs=1,
        )
        return (generic,)

    @staticmethod
    def _script():
        linux = CommandParser("script", "linux")
        linux.add_argument("-a", "--append", action="store_true")
        linux.add_argument("-E", "--echo")
        linux.add_argument("-e", "--return", action="store_true")
        linux.add_argument("-f", "--flush", action="store_true")
        linux.add_argument("--force", action="store_true")
        linux.add_argument("-B", "--log-io")
        linux.add_argument("-I", "--log-in")
        linux.add_argument("-O", "--log-out")
        linux.add_argument("-T", "--log-timing")
        linux.add_argument("-m", "--logging-format")
        linux.add_argument("-o", "--output-limit")
        linux.add_argument("-t", "--timing")
        linux.add_argument("-q", "--quiet", action="store_true")
        linux.add_argument("-V", "--version", action="store_true")
        linux.add_argument(
            "-c",
            "--command",
            dest="commands",
            action="invocations",
            split=True,
            nargs=1,
        )
        linux.add_argument("file", nargs="?")

        bsd = CommandParser("script", "bsd")
        bsd.add_argument("-d", action="store_true")
        bsd.add_argument("-k", action="store_true")
        bsd.add_argument("-p", action="store_true")
        bsd.add_argument("-r", action="store_true")
        bsd.add_argument("-q", action="store_true")
        bsd.add_argument("-t", action="store_true")
        bsd.add_argument("-a", action="store_true")
        bsd.add_argument("-F")
        bsd.add_argument("-T")
        bsd.add_argument("file", nargs="?")
        bsd.add_argument("commands", nargs=theirparse.REMAINDER, action="invocations")

        return (linux, bsd)

    @staticmethod
    def _sed():
        """
        see also handle_external_sed()

        It might be possible to handle this with a
        single parser, but since only one of them
        requires the GnuSedParser hack-around, I'll
        split them to be a little clearer...
        """
        gnu = GnuSedParser("sed")
        # very unsure about this group part
        gscript = gnu.add_mutually_exclusive_group()
        gnu.add_argument("-a", action="store_true")  # BSD-only
        gnu.add_argument("-n", "--quiet", "--silent", action="store_true")
        gnu.add_argument("--debug", action="store_true")
        gscript.add_argument("-e", "--expression", action="append")
        gscript.add_argument("-f", "--file")
        gnu.add_argument("--follow-symlinks", action="store_true")

        gnu.add_argument("-i", "--in-place")
        # gnu always has the arg
        gnu.add_argument("-l", "--line-length")
        gnu.add_argument("--posix", action="store_true")
        gnu.add_argument("-E", "-r", "--regexp-extended", action="store_true")
        gnu.add_argument("-s", "--separate", action="store_true")
        gnu.add_argument("-u", "--unbuffered", action="store_true")
        gnu.add_argument("-z", "--null-data", action="store_true")
        gnu.add_argument("--sandbox", action="store_true")
        gnu.add_argument("--version", action="store_true")
        gnu.add_argument("input_files", nargs="*")

        bsd = CommandParser("sed")
        # very unsure about this group part
        bscript = bsd.add_mutually_exclusive_group()
        bsd.add_argument("-a", action="store_true")  # BSD-only
        bsd.add_argument("-n", action="store_true")
        bscript.add_argument("-e", "--expression", action="append")
        bscript.add_argument("-f")
        bsd.add_argument("-i")
        # BSD uses this for line-buffered
        bsd.add_argument("-l", action="store_true")
        bsd.add_argument("-E", action="store_true")

        bsd.add_argument("input_files", nargs="*")

        return (gnu, bsd)

    @staticmethod
    def _awk():
        """
        see also handle_external_awk()
            -F fs           --field-separator=fs
            -v var=val      --assign=var=val
        Short options:      GNU long options: (extensions)
            -b          --characters-as-bytes
            -c          --traditional
            -C          --copyright
            -d[file]        --dump-variables[=file]
            -E file         --exec=file
            -g          --gen-pot
            -h          --help
            -i includefile      --include=includefile
            -l library      --load=library
            -L[fatal|invalid|no-ext]    --lint[=fatal|invalid|no-ext]
            -M          --bignum
            -N          --use-lc-numeric
            -n          --non-decimal-data
            -o[file]        --pretty-print[=file]
            -O          --optimize
            -p[file]        --profile[=file]
            -P          --posix
            -r          --re-interval
            -s          --no-optimize
            -S          --sandbox
            -t          --lint-old
            -V          --version

        """
        generic = CommandParser("awk")
        assign_fallback = CommandParser("awk")

        for parser in (generic, assign_fallback):
            # very unsure about this group part
            script = parser.add_mutually_exclusive_group(required=True)
            parser.add_argument("-D", "--debug", action="store_true")
            parser.add_argument("-F", "--field-separator")
            script.add_argument("-e", "--source", action="append")
            script.add_argument("-f", "--file")

            parser.add_argument("-b", "--characters-as-bytes", action="store_true")
            parser.add_argument("-c", "--traditional", action="store_true")
            parser.add_argument("-C", "--copyright", action="store_true")
            # technically argless form of below, but see nargs note on rlwrap...
            parser.add_argument("-d", "--dump-variables")
            parser.add_argument("-E", "--exec")
            parser.add_argument("-g", "--gen-pot", action="store_true")
            parser.add_argument("-h", "--help", action="store_true")
            parser.add_argument("-i", "--include")
            parser.add_argument("-l", "--load")
            # arg and argless forms of this; see note on rlwrap; guessing argless?
            parser.add_argument("-L", "--lint", action="store_true")
            # parser.add_argument("-L", "--lint", choices=["fatal", "invalid", "no-ext"])
            parser.add_argument("-M", "--bignum", action="store_true")
            parser.add_argument("-N", "--use-lc-numeric", action="store_true")
            parser.add_argument("-n", "--non-decimal-data", action="store_true")
            parser.add_argument("-o", "--pretty-print")
            parser.add_argument("-O", "--optimize", action="store_true")
            parser.add_argument("-p", "--profile")

            parser.add_argument("-P", "--posix", action="store_true")
            parser.add_argument("-r", "--re-interval", action="store_true")
            parser.add_argument("-s", "--no-optimize", action="store_true")
            parser.add_argument("-t", "--lint-old", action="store_true")

            parser.add_argument("-S", "--sandbox", action="store_true")
            parser.add_argument("-V", "--version", action="store_true")
            script.add_argument("script", nargs="?")
            parser.add_argument("input_file", nargs="*")

        # assign should be append, but strip_varlikes will
        # pluck out the args it is looking for...
        generic.add_argument("-v", "--assign", action="count")
        # but we need a fallback for the closed -vvarname=value form
        assign_fallback.add_argument("-v", "--assign", action="append")

        return (generic, assign_fallback)

    _gawk = _awk

    @staticmethod
    def _sudo():
        """
        sudo lists several forms, but only 2 should get resolved
        NO: sudo -h | -K | -k | -V
        NO: sudo -v [-AknS] [-g group] [-h host] [-p prompt] [-u user]
        NO: sudo -e [-AknS] [-C num] [-g group] [-h host] [-p prompt] [-T timeout] [-u user] file ...
        YES: sudo -l [-AknS] [-g group] [-h host] [-p prompt] [-U user] [-u user] [command]
        YES: sudo [-AbEHknPS] [-C num] [-g group] [-h host] [-p prompt] [-T timeout] [-u user] [VAR=value] [-i|-s] [<command>]
        """
        generic = CommandParser("sudo", "generic")
        # 1. weed out sudo forms that don't need resolving
        # note: some of these long forms aren't known correct
        #       because sudo re-uses a lot of flags
        generic.add_argument(
            # "-h", # handled separately due to overlap
            "--help",
            "-K",
            "--remove-timestamp",
            # shown in both places
            # "-k",
            # "--reset-timestamp",
            "-V",
            "--version",
            "-v",
            "--validate",
            "-e",
            "--edit",
            dest="skip",
            action="store_true",
        )
        # sudo -l form
        generic.add_argument("-l", "--list", action="count")
        generic.add_argument("-A", "--askpass", action="store_true")
        generic.add_argument("-k", "--reset-timestamp", action="store_true")
        generic.add_argument("-n", "--non-interactive", action="store_true")
        generic.add_argument("-S", "--stdin", action="store_true")
        generic.add_argument("-g", "--group")
        # nargs because `-h` (help) overlaps us...
        generic.add_argument("-h", "--host", nargs="?", const=True)
        generic.add_argument("-p", "--prompt")
        generic.add_argument("-U", "--other-user")
        generic.add_argument("-u", "--user")
        # extra from sans-l for
        generic.add_argument("-b", "--background", action="store_true")
        generic.add_argument("-E", "--preserve-env", nargs="?", const=True)
        generic.add_argument("-H", "--set-home")
        generic.add_argument("-P", "--preserve-groups", action="store_true")
        generic.add_argument("-C", "--close-from")
        generic.add_argument("-T", "--command-timeout")
        generic.add_argument("-i", "--login", action="store_true")
        generic.add_argument("-s", "--shell", action="store_true")
        # not listed in the synopsis; probably valid in most
        generic.add_argument("-B", "--bell", action="store_true")
        generic.add_argument("commands")
        return (generic,)

    @staticmethod
    def _xargs():
        """
        coreutils src/xargs.c:?
        Usage: %s [OPTION] [COMMAND [ARG]...]

        There's technically an argless form of a few args here:
        -i --replace
        -e --eof
        -l --max-lines
        all marked as deprecated or discouraged for !posix.
        We can match them with something like `nargs="?", const="{}"`,
        but it will only actually work as the last argument--otherwise
        it'll eat the next positional as the arg, never actually use
        the default, and require more handling to disambiguate between
        a bare flag with a default value and a normal flag with one
        passed. I'm going to see if we can get by w/o them...

        if this doesn't work, we'll probably need N parsers...
        """
        generic = CommandParser("xargs")
        # gnu
        generic.add_argument("-0", "--null", action="store_true")
        generic.add_argument("-a", "--arg-file")
        generic.add_argument("-d", "--delimiter")
        # see head note per both of these args
        generic.add_argument("-E", "-e", "--eof")
        generic.add_argument("-I", "-i", "--replace")

        generic.add_argument("-L", "-l", "--max-lines")
        generic.add_argument("-n", "--max-args")
        generic.add_argument("-o", "--open-tty", action="store_true")
        generic.add_argument("-P", "--max-procs")
        generic.add_argument("-p", "--interactive", action="store_true")
        generic.add_argument("--process-slot-var")
        generic.add_argument("-r", "--no-run-if-empty", action="store_true")
        generic.add_argument("-s", "--max-chars")
        generic.add_argument("--show-limits", action="store_true")
        generic.add_argument("-t", "--verbose", action="store_true")
        generic.add_argument("-x", "--exit", action="store_true")
        generic.add_argument("--help", action="store_true")
        generic.add_argument("--version", action="store_true")
        # bsd seems to have at least one extra, probably equiv of -I/-i
        generic.add_argument("-J")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _rlwrap():
        """
        coreutils src/rlwrap.c:?
        Usage: %s [OPTION] [COMMAND [ARG]...]

        There's technically an argless form of a few args here:
        -i --replace
        -e --eof
        -l --max-lines
        all marked as deprecated or discouraged for !posix.
        We can match them with something like `nargs="?", const="{}"`,
        but it will only actually work as the last argument--otherwise
        it'll eat the next positional as the arg, never actually use
        the default, and require more handling to disambiguate between
        a bare flag with a default value and a normal flag with one
        passed. I'm going to see if we can get by w/o them...

        if this doesn't work, we'll probably need N parsers...
        """
        generic = CommandParser("rlwrap")
        generic.add_argument("-a", "--always-readline", nargs="?")
        generic.add_argument("-A", "--ansi-colour-aware", action="store_true")
        generic.add_argument("-b", "--break-chars")
        generic.add_argument("-c", "--complete-filenames", action="store_true")
        generic.add_argument("-C", "--command-name")
        generic.add_argument("-D", "--history-no-dupes")
        generic.add_argument("-e", "--extra-char-after-completion")
        generic.add_argument("-f", "--file")
        generic.add_argument("-g", "--forget-matching")
        generic.add_argument("-h", "--help", action="store_true")
        generic.add_argument("-H", "--history-filename")
        generic.add_argument("-i", "--case-insensitive", action="store_true")
        generic.add_argument("-I", "--pass-sigint-as-sigterm", action="store_true")
        generic.add_argument("-l", "--logfile")
        generic.add_argument("-m", "--multi-line")
        generic.add_argument("-M", "--multi-line-ext")
        generic.add_argument("-n", "--no-warnings", action="store_true")
        generic.add_argument("-N", "--no-children", action="store_true")
        generic.add_argument("-o", "--one-shot", action="store_true")
        generic.add_argument("-O", "--only-cook")
        generic.add_argument("-p", "--prompt-colour")
        generic.add_argument("-P", "--pre-given")
        generic.add_argument("-q", "--quote-characters")
        generic.add_argument("-r", "--remember", action="store_true")
        generic.add_argument("-R", "--renice", action="store_true")
        generic.add_argument("-s", "--histsize")
        generic.add_argument("-S", "--substitute-prompt")
        generic.add_argument("-t", "--set-term-name")
        generic.add_argument("-U", "--mirror-arguments", action="store_true")
        generic.add_argument("-v", "--version", action="store_true")
        generic.add_argument("-w", "--wait-before-prompt")
        generic.add_argument("-W", "--polling", action="store_true")
        # TODO: -z/--filter may need resolution? not sure
        generic.add_argument("-z", "--filter")
        generic.add_argument(
            "commands", nargs=theirparse.REMAINDER, action="invocations"
        )
        return (generic,)

    @staticmethod
    def _find():
        """
        -exec command ;
        -exec command {} +
        -execdir command ;
        -execdir command {} +
        -ok command ;
        -okdir command ;
        """
        generic = FindParser("find")
        generic.add_argument("start")
        # boolean predicates, not quite like normal flags...
        # just capturing to ~ignore
        generic.add_argument(
            "-o",
            "-or",
            "-a",
            "-and",
            dest="ignored_boolean_predicates",
            action="store_true",
        )
        """
        This one arg doesn't use invocations;
        it has handling in the parser.
        """
        generic.add_argument(
            "-exec",
            "-execdir",
            "-ok",
            "-okdir",
            dest="commands",
            nargs=1,
            action="append",
            default=list()
            # action="invocations",
        )
        return (generic,)

    @staticmethod
    def _generic_shell():
        generic = CommandParser("generic_shell")
        generic.add_argument(
            "-c",
            # rest are nonstandard; leaving breadcrumbs;
            # possible some of these will have to be broken out
            # but let's make someone prove it first
            "--commands",  # nushell
            "--command",  # fish, hilbish, oh
            "-C",  # fish
            "--init-command",  # fish
            dest="commands",
            split=True,
            action="invocations",
            nargs=1,
        )
        return (generic,)

    whoazard = locals()
    for shell in INVOKABLE_SHELLS:
        whoazard["_" + shell] = _generic_shell

    @staticmethod
    def _tar():
        """
          `--checkpoint-action=action`
              Instruct tar to execute an action upon hitting a breakpoint. Here we give only a brief outline. See section Checkpoints, for a complete description.

              The action argument can be one of the following:

              bell
              Produce an audible bell on the console.

              dot
              .
              Print a single dot on the standard listing stream.

              echo
              Display a textual message on the standard error, with the status and number of the checkpoint. This is the default.

              echo=string
              Display string on the standard error. Before output, the string is subject to meta-character expansion.

              exec=command
              Execute the given command.

              sleep=time
              Wait for time seconds.

              ttyout=string
              Output string on the current console (`/dev/tty`).

              totals
              Print statistics (see see totals).

              wait=signo
              Wait for signal signo.

              Several `--checkpoint-action` options can be specified. The supplied actions will be executed in order of their appearance in the command line.

              Using `--checkpoint-action` without `--checkpoint` assumes default checkpoint frequency of one checkpoint per 10 records.
        -F, --info-script=COMMAND, --new-volume-script=COMMAND
          Run COMMAND at the end of each tape (implies -M).  The command can include arguments...
          --to-command=COMMAND   pipe extracted files to another program
          --rmt-command=COMMAND  use given rmt COMMAND instead of rmt
        --rsh-command=COMMAND  use remote COMMAND instead of rsh
          -I, --use-compress-program=COMMAND
          Filter data through COMMAND.  It must accept the -d option, for decompression.  The argument can contain command line options.
        """
        generic = CommandParser("generic tar")
        """
        TODO: I'm technically missing one here, but it is a
        bit harder than the rest.
        "--checkpoint-action='exec=somecommand'"

        slow-rolling this because it needs a custom parser &
        I don't see any wild examples of this on GH:
        https://github.com/search?q=%22--checkpoint-action%3Dexec%22&type=Code

        CAUTION: bsdtar flags are different enough that it
        may not tolerate living on the same parser like this
        For example, -I in gnutar can lead to subexec as we
        catch here, while -I in bsdtar is an alias for -T
        aka --files-from.
        """
        generic.add_argument(
            "-g",
            "-f",
            "-L",
            "-b",
            "-H",
            "-V",
            "-C",
            "-K",
            "-N",
            "-T",
            "-X",
            dest="arg_ignored",
            nargs=1,
        )
        generic.add_argument(
            "-v",
            "-A",
            "-c",
            "-d",
            "-r",
            "-t",
            "-u",
            "-x",
            "-G",
            "-n",
            "-S",
            "-k",
            "-U",
            "-W",
            "-O",
            "-m",
            "-p",
            "-s",
            "-M",
            "-B",
            "-i",
            "-a",
            "-j",
            "-J",
            "-z",
            "-Z",
            "-h",
            "-P",
            "-l",
            "-R",
            "-v",
            "-w",
            "-o",
            dest="argless_ignored",
            action="store_true",
        )
        generic.add_argument(
            "-F",
            "--info-script",
            "--new-volume-script",
            "--to-commands",
            "--rsh-command",
            "--rmt-command",
            "-I",
            "--use-compress-program",
            dest="commands",
            split=True,
            action="invocations",
            nargs=1,
        )
        return (generic,)

    # I think it only supports --use-compress-program
    # but over-supporting should be fine.
    _bsdtar = _tar

    @staticmethod
    def _dc():
        """
        We need to sub-search expressions for ! commands
        """
        generic = CommandParser("dc")
        generic.add_argument("-e", "--expression", action="append")
        generic.add_argument("-f", "--file")
        generic.add_argument("-V", "--version", action="store_true")
        generic.add_argument("scripts", nargs="*")

        return (generic,)

    @staticmethod
    def _msmtp():
        """
        This was based on running "msmtp --help" with msmtp v1.8.24.
        """
        generic = CommandParser("msmtp")

        # General options
        generic.add_argument(
            "--version",
            "--help",
            "-P", "--pretend",
            "-v", "-d", "--debug",
            action="store_true"
        )

        # Changing the mode of operation
        generic.add_argument("--configure", nargs=1)
        generic.add_argument("-S", "--serverinfo", action="store_true")
        generic.add_argument("--rmqs", nargs=1)

        # Configuration options
        generic.add_argument("-C", "--file", nargs=1)
        # For msmtp, "-a" is nargs=1.
        # For msmtpq, "--q-mgmt -a" is nargs=0.
        generic.add_argument("-a", nargs="?")
        generic.add_argument(
            "--account",
            "--host",
            "--port",
            nargs=1
        )
        generic.add_argument(
            "--source-ip",
            "--proxy-host",
            "--proxy-port",
            "--socket",
            "--timeout",
            "--protocol",
            "--domain",
            "--auth",
            "--user",
            nargs="?"
        )
        generic.add_argument(
            "--passwordeval",
            dest="commands",
            action="invocations",
            nargs=1
        )
        generic.add_argument(
            "--tls",
            "--tls-starttls",
            "--tls-trust-file",
            "--tls-crl-file",
            "--tls-fingerprint",
            "--tls-certcheck",
            "--tls-key-file",
            "--tls-cert-file",
            "--tls-priorities",
            "--tls-host-override",
            "--tls-min-dh-prime-bits",
            nargs="?"
        )

        # Options specific to sendmail mode
        generic.add_argument(
            "--auto-from",
            "-f", "--from=address",
            "--maildomain",
            "-N", "--dsn-notify",
            "-R", "--dsn-return",
            "-X", "--logfile",
            "--logfile-time-format",
            "--syslog",
            nargs="?"
        )
        generic.add_argument(
            "-t", "--read-recipients",
            "--read-envelope-from",
            action="store_true"
        )
        generic.add_argument(
            "--aliases",
            "--set-from-header",
            "--set-date-header",
            "--set-msgid-header",
            "--remove-bcc-headers",
            "--undisclosed-recipients",
            nargs="?"
        )
        # Accepted but ignored:
        generic.add_argument("-A", action="store_true")
        generic.add_argument("-B", nargs=1)
        generic.add_argument("-bm", action="store_true")
        generic.add_argument("-F", nargs=1)
        generic.add_argument("-G", action="store_true")
        # For msmtp, "-h" is nargs=1.
        # For msmtpq "--q-mgmt -h" is nargs=0.
        generic.add_argument("-h", nargs="?")
        generic.add_argument("-i", action="store_true")
        generic.add_argument("-L", nargs=1)
        generic.add_argument(
            "-m",
            "-n",
            action="store_true"
        )
        generic.add_argument(
            "-O",
            # See <https://github.com/abathur/resholve/pull/103#issuecomment-1710979420>
            action="count"
        )
        generic.add_argument("-o", nargs=1)

        return (generic,)

    @classmethod
    def _msmtpq(cls):
        """
        This was based on running "msmtpq --q-mgmt -h" with msmtp v1.8.24.
        """
        generic = cls._msmtp()[0]

        # -R, -d, -a and -h are already handled by the msmtp parser.
        generic.add_argument(
            "--q-mgmt",
            "-r",
            "-p",
            action="store_true"
        )

        return (generic,)


def generate_builtin_command_parser(cmdname):
    cmdname = "_" + cmdname
    if hasattr(BuiltinCommandParsers, cmdname):
        return getattr(BuiltinCommandParsers, cmdname)()
    return None


def generate_external_command_parser(cmdname):
    cmdname = "_" + cmdname
    if hasattr(ExternalCommandParsers, cmdname):
        return getattr(ExternalCommandParsers, cmdname)()
    return None


builtins = SingleKeyDefaultDict(generate_builtin_command_parser)
externals = SingleKeyDefaultDict(generate_external_command_parser)

"""
Meta:
the process documented here is ~accurate, but the implementation has been
significantly refactored a _few_ times toward two goals:

1. support recursive resolution
2. replace direct interface w/ OSH AST to resholve's own word + invocation
   abstraction to better contain the ASTs complexity.

This process was a pretty good fit for the kinds of resolution resholve did
when I initially built it out, but these changes (esp. recursive resolution)
have made me a little unsure if all of its components are still necessary.
This is mostly just to caution that it's good to view the process critically.
-- Aug 26, 2021

It's probably worth discussing the big picture before the next section. It may
be a little arcane, but it's in the service of minimizing repeated work.

When resholve process a script, it runs it through Oil's OSH parser. Early
versions walked the entire AST for command-like words, but as of 0.6.0 we just
monkey-patch a few AST nodes of interest to force __init__ to accumulate
instances of them for us, and then we come back and iterate over these nodes
and queue command-like words for resolution.

We can't resolve them all at encounter time because each could be an alias/fn
defined 3 scripts down a source chain--so we queue. There's an exception: we
do try to (recursively) process builtins that might lead to sourcing (think
paranoid `builtin source blah.sh`) and such during the initial iteration to
make it less likely that we're missing relevant facts before full resolution.

On each encounter, the visitor calls a context-specific record_ method from
the RecordCommandlike mixin. Each of these methods corresponds to a known
command resolution order--a sequence of the types of command this command-like
would resolve to. (Resolution rules differ across different contexts, like
inside an alias, and following certain commands). This call:

- Maps the position of the command-like to a list of tuples pairing 2 things:
  a "Solution" instance, and the "Word" object for it.

  self.solutions[position].append((solution, Word()))

  TODO: things have shuffled enough that it's worth re-verifying this dedupe
        whenever I have more cells to brain this with.

  - a Solution instance (obtained from the CommandLike) distinct for each
    encountered resolution order. If the visitor encounters the same
    commandlike 5 separate times but with the same resolution order, the
    solutions dictionary will map *all 5* positions to the same object.

  - a Word (a string subclass abstracting the OSH AST into something that
    better maps to resholve's concerns, and gathers logic that was formerly
    strewn all over the place). Originally, this just saved the literal AST
    because it is useful for reporting (Oil APIs can pretty-print errors
    that reference a word/token/spid.) This is one thing I'm a little
    skeptical of now. I suspect the new Invocation/Word abstractions can help
    subsume this practice, but I'm brained-out after ~6 months of refactoring
    and I just want to land this release and breathe before starting any more.
    See also the explanation before the Invocation / Word classes.

- Checks defaultdict cmdlikes[Word()] for the CommandLike instance that
  corresponds to this word. This instance will accumulate information like
  which scripts it appears in, whether it's defined as a function or alias,
  and most importantly the distinct resolution orders where it was found.

Once we visit every interesting node, we call CommandLike.resolve() on all
CommandLike objects. Each CommandLike will "solve" for the distinct contexts
it was found in, and add the appropriate resolution to the corresponding
Solution object. This process records whether each Solution should be a no-op,
raise an exception, or result in a replacement. It'll also trigger an eager
recursive resolution of invocations that may have subcommands lurking in them.

Next, we render the resolved script by looping over each of these solutions.
"""


class Solution(object):
    """
    Mutable placeholder objects to pair with commandlike encounters.

    We can't solve the commandlike until we're done parsing, but we
    know enough context at encounter time to deduplicate solutions.
    Calling code will save a reference to the context-appropriate
    solution object for each encounter, and a single solution pass
    later can identify the correct replacement for each one.
    """

    exception = kind = replacement = None
    positions = 1

    def __repr__(self):
        if self.solved:
            if self.exception:
                return "Solution(error: {:})".format(self.exception)
            elif isinstance(self.replacement, Iterable):
                return "Solution({:} -> {:})".format(*self.replacement)
            else:
                return "Solution(no-op {:})".format(self.kind)
        else:
            return "Solution(unsolved {:})".format(self.kind)

    def custom(self, kind, initial, replacement, positions):
        self.replacement = (initial, replacement)
        self.kind = kind
        self.positions = positions
        return self

    @property
    def solved(self):
        return self.replacement is not None or self.exception is not None

    def no_change(self, cmdkind):
        self.replacement = False
        self.kind = cmdkind

    def replace_name_or_alias_def_with(
        self, cmdkind, initial, replacement, external=False
    ):
        """
        Special-case cmdlikes that are known to have an alias so we can use a
        regex.sub to fix them a little more precisely.

        Caution: this might end up being fragile. It will only match the
        initial term if a space follows before the next `=`.

        TODO: should probably find a less-hacky way to handle these.
        """
        self.replacement = (
            re.compile(initial + r"(?!\S*?=)"),
            replacement,
        )
        self.kind = cmdkind

    def replace_name_with(self, cmdkind, initial, replacement, external=False):
        self.replacement = (initial, replacement)
        self.kind = cmdkind

    def replace_in(self, current):
        initial, replacement = self.replacement
        # if it's a regex (a sign this identifier is an alias), sub it
        if isinstance(initial, re._pattern_type):
            return initial.sub(replacement, current)
        else:
            return current.replace(initial, replacement)

    def causes_exception(self, exception):
        self.exception = exception


class OrderedCommandResolver(object):
    KNOWN_BUILTINS = {
        ".",
        ":",
        "[",
        "alias",
        "bg",
        "bind",
        "break",
        "builtin",
        "caller",
        "cd",
        "command",
        "compgen",
        "complete",
        "compopt",
        "continue",
        "declare",
        "dirs",
        "disown",
        "echo",
        "enable",
        "eval",
        "exec",
        "exit",
        "export",
        "false",
        "fc",
        "fg",
        "getopts",
        "hash",
        "help",
        "history",
        "jobs",
        "kill",
        "let",
        "local",
        "logout",
        "mapfile",
        "popd",
        "printf",
        "pushd",
        "pwd",
        "read",
        "readarray",
        "readonly",
        "return",
        "set",
        "shift",
        "shopt",
        "source",
        "suspend",
        "test",
        "times",
        "trap",
        "true",
        "type",
        "typeset",
        "ulimit",
        "umask",
        "unalias",
        "unset",
        "wait",
    }
    KNOWN_UNHANDLED_KEYWORDS = {"coproc"}

    @property
    def name(self):
        return self.cmd.name

    @property
    def path(self):
        return self.cmd.path

    def _true(self):
        return True

    def _false(self):
        return False

    def _unexpected_absolute_path(self):
        # any others would be errors, too, but we'll just flag the first
        observe(UnexpectedAbsolutePath, "command")

    def generate_external_resolver(self, relevant):
        if isinstance(self.name, Invocation):
            raise Exception(self.name, relevant)
        if self.name.startswith("/") and os.path.isabs(self.name):
            if (self.name,) in relevant.keep:
                # user said keep this abspath
                return self._true
            elif (self.name,) in relevant.fix:
                # user said to resolve this abspath as if it were bare
                self.cmd.path = self.name
                self.cmd.name = os.path.basename(self.name)
                return self.resolve_external
            else:
                return self._unexpected_absolute_path

        return self.resolve_external

    def __init__(self, commandlike, relevant):
        self.cmd = commandlike
        # "cro" is "command resolution order"; open to renames
        # alias > keyword > special builtin > function > builtin > external
        #                  (POSIX mode only)
        # neither special builtins nor POSIX mode have special handling atm
        # just saving space for them in case...
        self.cro = (
            # TODO: I'm turning these into tuples with a cmdkind string
            # but it might make sense to turn these into constants
            (
                "alias",
                self._true
                if ("alias", self.name) in relevant.fake
                else self.resolve_alias,
            ),
            (
                "keyword",
                self._true
                if ("keyword", self.name) in relevant.fake
                else self.resolve_keyword,
            ),
            ("special_builtin", self.resolve_special_builtin),
            (
                "function",
                self._true
                if ("function", self.name) in relevant.fake
                else self.resolve_function,
            ),
            (
                "builtin",
                self._true
                if ("builtin", self.name) in relevant.fake
                else self.resolve_builtin,
            ),
            (
                "external",
                self._true
                if ("external", self.name) in relevant.fake
                else self.generate_external_resolver(relevant),
            ),
        )

    @property
    def full(self):
        # alias > keyword > special builtin > function > builtin > external
        return self.cro

    @property
    def slash(self):
        # \command skips alias resolution
        # keyword > special builtin > function > builtin > external
        return self.cro[1:]

    @property
    def alias(self):
        return self.cro[3:]  # function > builtin > external

    @property
    def builtin(self):
        return self.cro[4:5]  # builtin

    @property
    def command(self):
        return self.cro[4:]  # builtin > external

    @property
    def external(self):
        return self.cro[5:]  # external

    """
    CommandLike.resolve() will call below methods for each distinct
    CRO encountered, and stop on the first that returns truthy.

    So they have the same basic rules:
    - return True if this is a valid resolution without change
    - return False if not (or None if unimplemented/irrelevant)
    - return a string to indicate a valid resolution with replacement
    - raise an exception to indicate a problem *(see resolve_external
      for a quirk about raising exceptions here)

    TODO: in theory these can be cached, but I'm not sure it'll be
        meaningful. The expensive part, lookup(), is already cached.
        update June 2021: walk-time lookups for some builtins and
        general recursive resolution support *might* have moved the
        needle far enough to justify this change at some point. Not
        something I'll look at until the dust settles and I do a pass
        on perf.
    """

    def resolve_alias(self):
        return self.cmd.alias

    def resolve_keyword(self):
        """
        I wanted to leave this unimplemented and let oil's parser
        shield resholve from needing to worry about it

        but oil doesn't handle the coproc keyword, so that's forcing
        my hand here.
        """
        return True if self.name in self.KNOWN_UNHANDLED_KEYWORDS else False

    def resolve_special_builtin(self):
        """
        leaving special builtin unimplemented for now; it's a
        posix-mode concept I'm hoping to avoid supporting.
        only time it really matters is if a function obscures
        a "special" builtin in posix mode

        See the file 'POSIX_mode_notes'
        """
        pass

    def resolve_function(self):
        return self.cmd.function

    def resolve_builtin(self):
        """
        I'm not sure to what extent I want to support non-bash shells,
        but if there's a push to do so it may help to post this on SO
        or make little help-wanted issues.

        The ~most-right way to do this would be to get the builtins
        from the target shell (at build-time? call-time?) but I'm not
        aware of a portable way to do it.

        If one's identified, the Nix side could accept a shell (or
        shells?) argument, run that command in each, merge the lists,
        and supply them via env.

        For now, I'm just hard-coding a list of bash builtins from
        `compgen -b` run in GNU bash,
        version 5.0.9(1)-release (x86_64-apple-darwin17.7.0)

        NOTE: if you're using resholve for some other shell and have
        a builtin not in this list, you can add an exemption with
        --fake builtin:<name>

        TODO: I don't love the ad-hoc way I'm carving out externals
        that have been ~allowed via a fix directive by checking for
        self.cmd.path here
        """
        return (
            True if (self.name in self.KNOWN_BUILTINS and not self.cmd.path) else False
        )

    def resolve_external(self):
        if (
            self.name in NAUGHTY_NIXERS
            and "NIX_BUILD_TOP" in os.environ
            and (self.name,) not in directives.fix
        ):
            # --fake 'external:sudo' or w/e will let this fall back to
            # a run-time PATH lookup.
            observe(NixSetuidWrapper, self.name)

        global lore
        executable = lookup(self.name)

        if not executable:
            # Types of ExceptionWithContext need other values in order
            # to pretty-print the error location that just aren't
            # readily available at this place using the current logic.
            #
            # Instead, we just mark the status, and code higher up the
            # stack will catch this error, inject the correct arena and
            # location information for us, and then re-raise the error.
            observe(UnresolvedCommand, self.name)
        else:
            # if we know about the executable
            # - and know it cannot exec
            #   - we can move on
            # - otherwise, we need per-invocation triage (but maybe we print a warning that they can help with hinting?)
            # if we don't
            # - either require triage, or outright fail due to the lack of lore
            if executable not in lore:
                if (
                    executable.startswith(LookSee.pwd)
                    and executable.replace(LookSee.pwd + "/", "") in lore
                ):
                    lore[executable] = lore[executable.replace(LookSee.pwd + "/", "")]
                else:
                    observe(MissingLore, executable)

            return executable


"""
We have to handle a few builtins to be able to do much else:
alias
    it can obscure all of our other builtins/commands
.|source
    we need to know we have all functions/aliases defined to be confident about what's unresolved
builtin|command|coproc|eval
    we have to look into these in case they hold instances of the above

CAUTION:
We're flying about as close to the sun as we can without tiptoeing into
symbolic execution. resholve's concept of this won't be robust to a
rapidly-evolving runtime context.
"""

FIRST_PASS_ESSENTIALS = {
    "alias",
    "builtin",
    "command",
    "coproc",
    "eval",
    ".",
    "source",
}


class CommandLike(object):
    """
    Represents each distinct command-like identifer we encounter.

    TODO: explain!
    """

    name = function = alias = path = resolved = None

    def __repr__(self):
        return "CommandLike <%s>" % self.name

    def __init__(self, name):
        self.name = name
        self.known_sources = dict()
        self.known_exemptions = list()
        self.invocations = set()
        self.order_to_source = defaultdict(set)
        self.order_to_solution = defaultdict(Solution)
        self.cros = MultiKeyDefaultDict(OrderedCommandResolver)

    """
    Chomp idiom added May 2021 to address some edge cases with the
    previous process for iterating over commandlikes and invocations.

    CommandLike resolution can now discover new commands/invocations
    to resolve. My intent is to:
    - ~chomp off an entire set of records
    - create an empty set for new records
    - yield the chomped records (to the code resolving them)
    - repeat this process if the new set has records

    See also: resolve_cmdlikes, CommandLike.{request_solution,resolve}
    """
    unresolved = set()  # intentional!

    @classmethod
    def chomp_cmdlikes(cls):
        logger.debug("unresolved clen1 %r %r", len(cls.unresolved), cls.unresolved)
        while len(cls.unresolved):
            unresolved = sorted(cls.unresolved)
            cls.unresolved = set()
            for cmdlike in unresolved:
                yield cmdlike
            logger.debug("unresolved clen2 %r %r", len(cls.unresolved), cls.unresolved)

    def chomp_invocations(self):
        logger.debug(
            "unresolved ilen1 %r %r %r", self, len(self.invocations), self.invocations
        )
        while len(self.invocations):
            invocations = self.invocations
            self.invocations = set()
            for invocation in invocations:
                yield invocation
            logger.debug(
                "unresolved ilen2 %r %r %r",
                self,
                len(self.invocations),
                self.invocations,
            )

    def scaffold_new_source(self, source):
        # only exemptions that match this commandlike's name
        # tuple only really for hashability...
        relevant = source.directives.relevant(self.name)

        if relevant in self.known_exemptions:
            # new source, but no novel exemptions
            index = self.known_exemptions.index(relevant)

            # use existing content-identical set to
            # approximate a "set of sets"
            self.known_sources[source] = self.known_exemptions[index]
        else:
            # new source with novel exemptions
            self.known_sources[source] = relevant
            self.known_exemptions.append(relevant)

        return self.cros[(self, relevant)]

    def request_solution(self, source, order, invocation=None):
        self.order_to_source[order].add(source)
        logger.debug(
            "requesting the solution for %r with len=%r",
            invocation,
            len(invocation) if invocation else 0,
        )
        CommandLike.unresolved.add(self)
        if invocation and len(invocation) > 1:
            # if there's more than one word, track invocation
            # so that we can check it for sub-executions later
            self.invocations.add((source, invocation))
        return self.order_to_solution[order]

    def find_cro(self, source):
        if source not in self.known_sources:
            return self.scaffold_new_source(source)
        return self.cros[(self, self.known_sources[source])]

    def request_full_solution(self, source, invocation=None):
        return self.request_solution(source, self.find_cro(source).full, invocation)

    def request_slash_solution(self, source, invocation=None):
        return self.request_solution(source, self.find_cro(source).slash, invocation)

    def request_alias_solution(self, source):
        return self.request_solution(source, self.find_cro(source).alias)

    def request_builtin_solution(self, source, invocation=None):
        return self.request_solution(source, self.find_cro(source).builtin, invocation)

    def request_command_solution(self, source, invocation=None):
        return self.request_solution(source, self.find_cro(source).command, invocation)

    def request_external_solution(self, source, invocation=None):
        return self.request_solution(source, self.find_cro(source).external, invocation)

    def _resolve_invocations(self, solution):
        if solution.solved:
            if solution.kind == "builtin":
                """
                TODO: there's a duplicate copy of ~90% of this logic;
                should/can it be abstracted?
                """
                global builtins
                # TODO: not sure abt this for builtins...
                # tentatively try knocking out varlikes for everyone?
                # cmd_object, args, text = self._strip_varlikes(cmd_object, args)
                name = self.name

                for source, invocation in self.chomp_invocations():
                    handler = getattr(source, "handle_builtin_{:}".format(name), None)
                    if not handler:
                        continue
                        # TODO: think about this; we can't just raise because
                        # *other* builtins land here
                        raise Exception(
                            "I've made a mistake and there's no handler for this "
                            "builtin. Please report it!",
                            invocation,
                        )
                    if handler and builtins[name]:
                        for parser in builtins[name]:
                            parsed = _unparsed = None
                            parsed = parser.parse_known_args(invocation.args)
                            if not parsed:
                                logger.debug(
                                    "parse failed (parser=%r, args=%r)",
                                    parser,
                                    invocation,
                                )
                                continue
                            else:
                                parsed, _unparsed = parsed
                                # see sudo parser for example
                                if "skip" in parsed and parsed.skip:
                                    continue
                            if handler(parsed, invocation):
                                break

            elif solution.kind == "external" and isinstance(
                solution.replacement, tuple
            ):
                executable = solution.replacement[1]

                if lore[executable] != "cannot":
                    # otherwise, walk down each execution
                    # chain until we find a terminus
                    for source, invocation in self.chomp_invocations():
                        source.look_for_external_sub_execution(self.name, invocation)

    def _resolve_cmdlike_contexts(self, distinct_cro, solution):
        # run each function, continue early on exception/true/string
        for cmdkind, func in distinct_cro:
            logger.debug("resolve(%r): trying %r %r", self.name, cmdkind, func)
            try:
                # TODO: could cache these, but not sure it'll make much diff
                test = func()
            except ExceptionWithContext as e:
                solution.causes_exception(e)
                logger.debug("break because exception")
                break

            if test == True:
                solution.no_change(cmdkind)
                logger.debug("break because no change")
                break
            elif test:
                # only external can return more than just t/f
                if self.alias:
                    solution.replace_name_or_alias_def_with(
                        cmdkind, self.path or self.name, test
                    )
                else:
                    solution.replace_name_with(cmdkind, self.path or self.name, test)

                for source in self.order_to_source[distinct_cro]:
                    source.directives.keep.add((test,))

                logger.debug("break because replacement")
                break

    def resolve(self):
        logger.debug("CALLING RESOLVE ON %r", self.name)

        # find solutions for observed types
        for distinct_cro, solution in self.order_to_solution.items():
            if not solution.solved:
                self._resolve_cmdlike_contexts(distinct_cro, solution)

            self._resolve_invocations(solution)

        self.resolved = True


# TODO: explain!
cmdlikes = SingleKeyDefaultDict(CommandLike)


def resolve_cmdlikes():
    logger.info("Resolving command-likes")
    for cmdlike in CommandLike.chomp_cmdlikes():
        if cmdlike.resolved:
            logger.debug(
                "in resolve_cmdlikes, encountered previously-resolved cmdlike %r %r %r",
                cmdlike.name,
                cmdlike,
                cmdlike.invocations,
            )
        else:
            logger.debug(
                "in resolve_cmdlikes, encountered first-time cmdlike %r %r %r",
                cmdlike.name,
                cmdlike,
                cmdlike.invocations,
            )

        logger.debug("calling cmdlike.resolve(%r) from resolve_cmdlikes", cmdlike.name)
        cmdlike.resolve()


"""
Just a bit of conceptual context along the lines of what prefaces cmdlikes.

Up through resholve 0.6.rc1, we worked directly on the AST Oil's OSH parser
kicks out for us. That was expedient because we could re-use some of Oil's
utility methods (and crib from others) to work on these nodes. It also has
some downsides that eventually undermined this approach.

- The AST is complex, and complex node-logic was getting repeated or subtly
  duplicated in many sites around resholve's code.
- The Oil routines that best fit our needs for generating strings from the AST
  also process variables. These are simpler to use, but we ~want the variables
  to stay in-place for recursive parsing.

So, on the way to 0.6.rc2, I've gone back to the mat to wrap the ASTs in our
own abstractions, roughly: Invocation([Word(AST), Word(AST)...]). Invocation
is a tuple subclass, and Word is a string subclass.

The main goals here are to:
- wrangle the complexity of navigating the AST to fish detail out (ex: which
  tokens are dynamic, what span_ids to associate with a word or invocation)
- massage the information in the AST into a form that is easier to use with
  helpful python batteries like argparse and shlex.
- help keep the AST and string/array forms correlated with less effort
"""

KEEPABLE_DYNAMIC_TOKENS = {
    Id.VSub_Name: "${}",
    Id.VSub_DollarName: "${}",
    Id.Lit_TildeLike: "{}",
}
ALLOWABLE_DYNAMIC_TOKENS = {
    Id.Left_Backtick: "`command substitution`",
    Id.Left_DollarParen: "$(command substitution)",
    Id.VSub_Number: "${}",
    Id.VSub_At: "$@",
    Id.VSub_Star: "$*",
}


def flatten_node(ast, kwargs=None):
    """
    TODO: document the lower-level token/left/ast fiddling here. In
    the process you should also validate the details. This needed a
    meaningful addition to get the 0.14->0.15 update working, but it's
    harder to know that the inverse isn't true, too (i.e., is this
    now trying to match vestigial AST patterns that we'll no longer
    encounter and don't need to maintain?)
    """
    if kwargs is None:
        kwargs = dict(
            first_spid=float("inf"),
            last_spid=float("-inf"),
            allow=None,
            keep=None,
            varlike=None,
        )
    out = list()

    get_node_positions(ast, kwargs)

    if hasattr(ast, "token"):
        out.append(flatten_node(ast.token, kwargs)[0])

    """
    Caution / TODO:
    I briefly had this enabled. AFAIK they'll always just be
    quotes, so I was repeating one on each side. But, they make
    it into the firstwords we're here to resolve. I don't *think*
    we really need them, but I also haven't really pondered this.

    If a clear need for this doesn't crop up by 2022, cut comment
    # if hasattr(ast, "left"):
    #     # AFAIK this will be a quote, so we'll repeat it
    #     out.append(ast.left.tval)

    # ... (before return)

    # if hasattr(ast, "left"):
    #     # close earlier quote
    #     out.append(ast.left.tval)

    """
    # Caution: a little unsure about this exact formulation. Got tests
    # running on the 0.14->0.15 update but I didn't document concerns
    # and pressures here, so I feel like I'm just cargo-culting.
    if hasattr(ast, "left"):
        if ast.left.id in ALLOWABLE_DYNAMIC_TOKENS:
            val = ALLOWABLE_DYNAMIC_TOKENS[ast.left.id].format(ast.left.tval)
            out.append(val)
            kwargs["allow"] = val
        elif ast.left.id in KEEPABLE_DYNAMIC_TOKENS:
            val = KEEPABLE_DYNAMIC_TOKENS[ast.left.id].format(ast.left.tval.strip("$"))
            out.append(val)
            kwargs["keep"] = val

    if hasattr(ast, "left_token"):
        # I think this is just backtick and $(
        if ast.left_token.id in ALLOWABLE_DYNAMIC_TOKENS:
            val = ALLOWABLE_DYNAMIC_TOKENS[ast.left_token.id].format(
                ast.left_token.tval
            )
            out.append(val)
            kwargs["allow"] = val

    if hasattr(ast, "tokens"):
        for token in ast.tokens:
            out.append(flatten_node(token, kwargs)[0])

    if hasattr(ast, "id"):
        # should be all Token AFAIK
        if ast.id in ALLOWABLE_DYNAMIC_TOKENS:
            val = ALLOWABLE_DYNAMIC_TOKENS[ast.id].format(ast.tval.strip("$"))
            out.append(val)
            kwargs["allow"] = val
        elif ast.id in KEEPABLE_DYNAMIC_TOKENS:
            val = KEEPABLE_DYNAMIC_TOKENS[ast.id].format(ast.tval.strip("$"))
            out.append(val)
            kwargs["keep"] = val
        elif ast.id == Id.Lit_VarLike:
            kwargs["varlike"] = ast.tval.strip("=")
            out.append(ast.tval)
        else:
            out.append(ast.tval)
    elif hasattr(ast, "tval"):
        # TODO: can probably throw this out a window if it doesn't
        # squeak by 2022
        raise Exception(
            "assumption that everything with an id is a token and has a value broke in some way",
            ast,
        )

    if hasattr(ast, "parts"):
        for part in ast.parts:
            out.append(flatten_node(part, kwargs)[0])

    return "".join(out), kwargs


def get_node_positions(node, pos):
    if hasattr(node, "span_id") and node.span_id is not None:
        pos["first_spid"] = min(pos["first_spid"], node.span_id)
        pos["last_spid"] = max(pos["last_spid"], node.span_id)

    if hasattr(node, "spids") and node.spids:
        pos["first_spid"] = min(pos["first_spid"], *node.spids)
        pos["last_spid"] = max(pos["last_spid"], *node.spids)

    if hasattr(node, "token"):
        pos["first_spid"] = min(pos["first_spid"], node.token.span_id)
        pos["last_spid"] = max(pos["last_spid"], node.token.span_id)


class PositionalObjectBase(object):
    last_spid = first_spid = None

    @property
    def spid_len(self):
        return 1 + (self.last_spid - self.first_spid)

    def _describe_spans(self):
        # TODO: this is OK. I'd like to condense but waiting
        #       in case I decide to add more detail
        if (
            self.first_spid is not None
            and self.last_spid is not None
            and self.first_spid != self.last_spid
        ):
            return "spans {:} positions from spids {:}-{:}".format(
                self.spid_len, self.first_spid, self.last_spid
            )
        elif self.first_spid is not None or self.last_spid is not None:
            return "spans {:} position at spid {:}".format(
                self.spid_len, self.first_spid or self.last_spid
            )
        else:
            return "spans unknown"

    def __repr__(self):
        return "{:}({:} {:})".format(
            self.__class__.__name__,
            super(PositionalObjectBase, self).__repr__(),
            self._describe_spans(),
        )


class Word(PositionalObjectBase, str):
    ast = None

    def __new__(cls, ast, text=None):
        _text, kwargs = flatten_node(ast)
        self = str.__new__(cls, text or _text)
        # would like to pull all reliance on the AST behind stronger
        # abstraction boundaries, but it does currently leak out in a
        # small number of places. The majority are to pass location to
        # oil's exception classes
        self.ast = ast
        for k, v in kwargs.items():
            setattr(self, k, v)
        self.ok, _non_dynamic_text, self.quoted = word_.StaticEval(ast)
        return self


class Invocation(PositionalObjectBase, tuple):
    firstword = None

    def __new__(cls, ast=None, words=None):
        self = None
        if ast:
            if hasattr(ast, "words") and len(ast.words):
                self = tuple.__new__(cls, map(Word, ast.words))
            elif hasattr(ast, "ast"):
                self = tuple.__new__(cls, (Word(ast.ast),))
            elif hasattr(ast, "redirects"):
                """
                At least one idiom, <filename, can end up
                here because it's a ~command, but we don't
                want it going down the full pipeline because
                it doesn't have any command words...
                (command.Simple
                  redirects: [
                    (redir
                      op: (Token
                        id:Id.Redir_Less
                        span_id:0
                        val:'<')
                      loc: (redir_loc.Fd fd:0)
                      arg: (CompoundWord
                        parts:[(Token
                          id:Id.Lit_Chars
                          span_id:1
                          val:hehe)]
                      )
                    )
                  ]
                  do_fork: F
                )
                """
                # skip this
                return None
        elif words:
            self = tuple.__new__(cls, words)

        self.firstword = firstword = self[0]
        self.first_spid = firstword.first_spid
        self.last_spid = self[-1].last_spid
        return self

    def __hash__(self):
        """
        Factor positions into uniqueness calculations for set/dict. We can't put this up on the positional object base because
        the Word object is used as a key to the cmdlikes dict
        (where we don't want position-sensitive uniques).
        """
        return hash((self.firstword, self.first_spid, self.last_spid))

    @property
    def args(self):
        return self[1:]

    def drop(self, args):
        words = list(self)
        for arg in reversed(sorted(args)):
            del words[arg]
        return self.__class__(words=words)

    def clip(self, x, y):
        if x and y:
            return self.__class__(words=self[x:y])
        elif x:
            return self.__class__(words=self[x:])
        elif y:
            return self.__class__(words=self[:y])


# TODO: refactor down some of this duplication (here and elsewhere)
# keeping it painfully explicit for now to make it easier to
# think/search between different cases here
class RecordCommandlike(object):
    solutions = invocations = None

    def __init__(self):
        self.solutions = defaultdict(list)

    @contextmanager
    def consume_solutions(self):
        yield self.solutions
        self.solutions.clear()

    @classmethod
    def disable_alias_replacement(cls):
        cls.record_alias_declaration = cls._disabled_recorder

    def _disabled_recorder(self, *args, **kwargs):
        pass

    def handle_builtin_alias(self, parsed, invocation):
        """
        aliases seem simple at a blush, but the defs
        can also be very complex...
        alias a=b a="b" a='b' "a=b" 'a=b' a=$b a="$b" a='$b' "a=$b" 'a=$b' 'a=echo $b', etc.

        code is ~v4 of trying to handle aliases.
        - v1: try to spot alias identifiers to add to
          allowed unresolved "commands"
        - v2: try to extract alias/expansion pairs to
          - allow unresolved commands
          - make users triage externals in low-hanging-fruit
            alias definitions (resolve? ignore?)
        - v3: roughly v2 except:
          - split out into own method
        - v4: roughly v3 except on Invocation/Word API

        TODO:
        - do anything else with high-hanging fruit?
        - there may be a generalized handler for "varlikes" here, which
          oil defines as <id>=<stuff> but I'll say is more like:
          [<alias|declare|local|set>] [<<id>=<val>>...]
          (mainly valuable if I decide to consider taking one more firm
          step into dynamic-land by resolving through refs to simple vars)
        """
        logger.debug(
            "alias invocation: %r, %r", invocation, [x.ast for x in invocation]
        )

        for word in invocation.args:
            # Caution: strings w/ varsubs pass static eval here
            # may have to unwrap them first?
            logger.debug(
                "alias (ok? %r) split %r",
                word.ok,
                word.strip("\"='").split("="),
            )
            if word.ok:
                # not dynamic; more examples in tests/aliases.sh, but:
                # 'd', 'echo $SOURCE_DATE_EPOCH'
                alias, definition = word.strip("\"='").split("=")
                # TODO: maybe below deserves an explicit API
                cmdlikes[alias].alias = True
                commandlike = definition.split()[0]

                pos = word.first_spid

                self.record_alias_declaration(word, commandlike, pos)
            # else: # dynamic
            # TODO: decide if you want to emit a warning or error on dynamic aliases?
            #       letting them slide for now.

    def handle_first_pass_essentials(self, invocation):
        global FIRST_PASS_ESSENTIALS, cmdlikes
        logger.debug("handle_first_pass_essentials %r ", invocation)

        if invocation.firstword in FIRST_PASS_ESSENTIALS:
            cmdlike = cmdlikes[invocation.firstword]
            if not cmdlike.alias and not cmdlike.function:
                self.look_for_essential_builtin_sub_execution(invocation)
                return True
            else:
                raise Exception("shadowed builtin", invocation)
        else:
            return False

    def builtin_handler(builtin, to_record):
        def temp(self, parsed, invocation):
            recorder = getattr(self, "record_{:}_invocation".format(to_record))
            logger.debug(
                "handle_builtin_%s 1%r1 2%r2",
                builtin,
                parsed,
                invocation,
            )

            for subcmd in parsed.commands:
                if subcmd.firstword.ok:
                    if not self.handle_first_pass_essentials(subcmd):
                        recorder(subcmd)

        return temp

    # TODO handle_builtin_alias = builtin_handler("alias", "alias")
    handle_builtin_builtin = builtin_handler("builtin", "builtin")
    # TODO: note migrated from elsewhere:
    # we'll need a function handler for special case
    # functionality like:
    # - substituting command -v -> echo?
    # - possibly warning on and requiring an override for `-p`
    handle_builtin_command = builtin_handler("command", "command")
    handle_builtin_type = builtin_handler("type", "command")
    # TODO: no rush, but maybe we can help the weird coproc
    # situation with a handler that throws better errors
    handle_builtin_coproc = builtin_handler("coproc", "command")
    # TODO handle_builtin_eval = builtin_handler("eval", "command")
    handle_builtin_exec = builtin_handler("exec", "external")
    # TODO handle_builtin_source = builtin_handler("source", "command")

    def handle_builtin_source(self, parsed, invocation):
        """
        CAUTION:
        in a multi-module library, we should:
        - look up targets at their install path to resolve the source statement into
          the correct absolute URI.
        - post-substitute to replace the build-time-path with the install/output path
        """

        target = None

        firstword = parsed.commands[0].firstword
        logger.debug("handle_builtin_source %r %r %r", parsed, invocation, firstword)

        if not firstword.ok:
            self.allow_dynamic_executable_word(firstword, "source")
            logger.debug(
                "Not trying to resolve/parse dynamic source statement. In theory, this should only happen when the dynamic token was exempted with a directive or is in the short list of tokens (like dollar-subs) that we automatically exempt. We obviously can't resolve into the file, but we won't object to the statement. AST: %r",
                invocation,
            )
            return

        if firstword.startswith("/") and os.path.isabs(firstword):
            # Note: logic here tracks generate_external_resolver()
            if ("source", firstword) in self.directives.keep or (
                ".",
                firstword,
            ) in self.directives.keep:
                # user said keep this abspath
                self.record_external_source(firstword)
                target = False
            elif (
                "source",
                firstword,
            ) in self.directives.fix or (
                ".",
                firstword,
            ) in self.directives.fix:
                # user said to resolve this abspath as if it were bare
                relfirstword = os.path.basename(firstword)
                target = lookup_source(relfirstword)
                logger.debug(
                    "Looked up source: %r -> %r -> %r",
                    firstword,
                    relfirstword,
                    target,
                )
            else:
                observe(
                    UnexpectedAbsolutePath,
                    "source",
                    location=loc.Word(firstword.ast),
                    arena=self.arena,
                )
        else:
            target = lookup_source(firstword)
            logger.debug("Looked up source: %r -> %r", firstword, target)

        # it was already a valid absolute path
        if target:
            self.record_external_source(firstword, replace_with=target)
        # It didn't resolve, or it was an invalid absolute path
        elif target != False:
            # in theory I could pass this exception to the new
            # self.record_external_source, but this would mean we can't
            # fail as soon. Failing soon seems better, no?
            observe(
                UnresolvedSource,
                firstword,
                location=loc.Word(firstword.ast),
                arena=self.arena,
            )

    def handle_builtin_eval(self, parsed, invocation):
        logger.debug(
            "handle builtin eval %r",
            invocation,
        )
        """
        invocation.firstword == eval
        firstword == the firstword of what we'll eval
        """
        firstword = parsed.commands[0].firstword
        if not firstword.ok:
            try:
                self.allow_dynamic_executable_word(firstword, "eval")
            except Forbid as e:
                logger.debug("allowing resolution error for eval %r", e)

        if firstword.quoted:
            observe(
                QuotedEval,
                invocation.firstword,
                location=loc.Word(firstword.ast),
                arena=self.arena,
            )
            return

    def look_for_essential_builtin_sub_execution(self, invocation):
        name = invocation.firstword
        if name == ".":  # remap
            name = "source"

        global builtins
        # TODO: not sure abt this for builtins... drop if
        # it doesn't bite anyone by 2023
        # tentatively try knocking out varlikes for everyone?
        # invocation = self._strip_varlikes(invocation)

        if builtins[name]:
            for parser in builtins[name]:
                parsed = _unparsed = None
                parsed = parser.parse_known_args(invocation.args)
                if not parsed:
                    logger.debug(
                        "parse failed (parser=%r, invocation=%r)", parser, invocation
                    )
                    continue
                else:
                    parsed, _unparsed = parsed
                    # see sudo parser for example
                    if "skip" in parsed and parsed.skip:
                        continue

                handler = getattr(self, "handle_builtin_{:}".format(name), None)
                if not handler:
                    raise Exception(
                        "I've made a mistake and there's no handler for this "
                        "builtin. Please report it!",
                        invocation,
                    )
                logger.debug(
                    "name %r has subcommand handler: %r",
                    name,
                    handler,
                )
                if handler(parsed, invocation):
                    return

    def _strip_varlikes(self, invocation):
        return invocation.drop([i for i, word in enumerate(invocation) if word.varlike])

    def _find_sed_e_cmd(self, expr):
        """
        '{ s/a/b/ ; s/b/c/e ; }'
        sed: -e expression #1, char 20: e/r/w commands disabled in sandbox mode
        1
        --expression='s/a/b/' -e 's/b/c/e'
        sed: -e expression #2, char 7: e/r/w commands disabled in sandbox mode
        1

        -e 'e echo'
        'sed: -e expression #1, char 1: e/r/w commands disabled in sandbox mode\n'

        {
            "cmd": "e",
            "line": 68,
            "x": {
                "cmd_txt": {
                    "text": [
                        "d",
                        "a",
                        "t",
                        "e",
                        "\n"
                    ]
                }
            }
        },
        """

        parsed = []
        try:
            sedparse.compile_string(parsed, expr)
            # TODO: if we can figure out how (or contribute upstream to support) to print this back out losslessly, we might actually be able to use sedparse to just edit and re-print the AST. That said, when I naively try the bundled `print_program` function for 'sed -e "e echo"' I send up with something like 'e\\\necho', which probably can't just be yeeted in
            for cmd in parsed:
                if cmd.cmd == "e":
                    return True
        except sedparse.ParseError as e:
            observe(
                SedECommandParseError,
                location=expr.ast,
                arena=self.arena,
            )

        return False

    def handle_external_sed(self, parsed, invocation):
        """
        We want to check sed expressions on the CLI. They can
        either come after an -e/--expression flag, or they can
        be the first ~positional argument as long as -e or -f
        or --expression or --file weren't used.

        (I guess there's a hole here, in that someone can construct
        a heredoc or string and pass it into the script. I'm going
        to call that the arbitrary line in the sand. I'd be tickled
        pink to be able to hand off both these expressions *and* the
        any heredocs/etc to another program that is ~resholve for
        sed scripts/expressions.)

        This is also the main example of how to override a handler
        for a command that can't skate by with the generic handler.
        - return True if this is now handled (will try any additional
          parsers before failing if it isn't handled)
        """
        if parsed.expression:
            for expression in parsed.expression:
                if self._find_sed_e_cmd(expression):
                    observe(
                        SedECommand,
                        location=expression.ast,
                        arena=self.arena,
                    )
            return True
        elif not parsed.file and parsed.input_files:
            # trailing positional expression
            first = parsed.input_files[0]
            if self._find_sed_e_cmd(first):
                observe(
                    SedECommand,
                    location=first.ast,
                    arena=self.arena,
                )
            return True
        return True

    def _find_awk_sub_cmd(self, expr):
        """
        Note: the first draft of this behavior leveraged awk's --sandbox flag
        to figure out when this behavior was present. This is good, but for
        reasons I haven't yet understood the calls out were hanging during
        cross builds (instead of producing binary format errors...):

        $ awk --sandbox 'BEGIN { system("date"); close("date")}'
        awk: cmd. line:1: fatal: 'system' function not allowed in sandbox mode

        $ awk --sandbox 'BEGIN {
            cmd = "ls -lrth"
            while ( ( cmd | getline result ) > 0 ) {
                print result
            }
            close(cmd);
        }'
        awk: cmd. line:3: fatal: redirection not allowed in sandbox mode

        Since we're having cross trouble and I haven't had much luck finding
        a Python parser for the Awk language, we'll do the dumbest-plausible
        thing here but then just warn about this for now.
        """

        if "system(" in expr:
            logger.warning(
                "awk expression may contain system command (exec): '%s'", expr
            )
            return True

        if " | " in expr:
            logger.warning("awk expression may contain pipe (exec): '%s'", expr)
            return True

        if " |& " in expr:
            logger.warning("awk expression may contain pipe (exec): '%s'", expr)
            return True

        return False

    def handle_external_awk(self, parsed, invocation):
        """
        We want to check awk expressions on the CLI. They can
        either come after an -e/--source flag, or they can
        be the first ~positional argument

        TODO: validate this behavior for awk (as long as -e or -f
        or --expression or --file weren't used.)

        (I guess there's a hole here, in that someone could construct
        a heredoc or something and pass it into the script. I'm going
        to call that the arbitrary line in the sand. I'd be tickled
        pink to be able to hand off both these expressions *and* the
        any heredocs/files to another program that is ~resholve for
        awk scripts/expressions.)
        """
        if parsed.source:
            for expression in parsed.source:
                if self._find_awk_sub_cmd(expression):
                    observe(
                        AwkSubCommand,
                        location=expression.ast,
                        arena=self.arena,
                    )
            return True
        elif not parsed.file and parsed.script:
            # trailing positional expression
            if self._find_awk_sub_cmd(parsed.script):
                observe(
                    AwkSubCommand,
                    location=parsed.script.ast,
                    arena=self.arena,
                )
            return True
        return True

    # TODO: maybe a better way for the caller to find these
    handle_external_gawk = handle_external_awk

    def handle_external_dc(self, parsed, invocation):
        """
        check dc expressions for ! cmds
        """
        if parsed.expression:
            for expression in parsed.expression:
                if "!" in expression:
                    bangpos = expression.index("!")
                    if expression[bangpos + 1] not in ("<", "=", ">"):
                        observe(
                            DcBangCommand,
                            location=expression.ast,
                            arena=self.arena,
                        )
            return True
        return True

    def handle_external_generic(self, parsed, invocation):
        # TODO: shimming shells in here may be a bigger crime than
        #       duplicating the logic in two functions would be?
        shell = invocation.firstword in INVOKABLE_SHELLS
        for subinvoke in parsed.commands:
            logger.debug("subinvoke %r", subinvoke)
            cmdlike = cmdlikes[subinvoke.firstword]
            subcmd = subinvoke.firstword
            if not subcmd.ok:
                self.allow_dynamic_executable_word(subcmd, invocation.firstword)
            else:
                pos = subinvoke.first_spid

                # TODO: 3x near-identical copies of this logic
                solution = None
                solutions = self.solutions[pos]
                if pos in self.solutions:
                    for solution, word_ob in solutions:
                        if word_ob == subcmd:
                            # keep matching solution
                            break
                        else:
                            # otherwise, don't exit w/ solution set
                            solution = None
                if not solution:
                    if shell:
                        solution = cmdlike.request_full_solution(self, subinvoke)
                    else:
                        solution = cmdlike.request_external_solution(self, subinvoke)
                    solutions.append((solution, subcmd))

                    logger.debug(
                        "while handling external flag for %r, recorded solution=%r at pos=%r",
                        subinvoke,
                        solution,
                        pos,
                    )

                if not solution.solved:
                    cmdlike.resolve()

        return True  # TODO: confidence low

    def look_for_external_sub_execution(self, outer, invocation):
        global decided_execers, lore

        logger.debug(
            "look_for_external_sub_execution0: %r outer=%r",
            invocation,
            outer,
        )

        if len(invocation) >= 1:
            pos = invocation.first_spid
            subcmd = invocation.firstword
            # TODO: handle_external_generic needed this; not certain
            # this one _doesn't_, but I haven't been able to find a
            # test case that breaks on it (yet). rm in 2022.
            #     remember that you have to indent the rest under else?
            # if not subcmd.ok:
            #     self.allow_dynamic_executable_word(subcmd, invocation.firstword)
            # else:

            cmdlike = cmdlikes[subcmd]

            solution = None
            solutions = self.solutions[pos]
            if pos in self.solutions:
                for solution, word_ob in solutions:
                    if word_ob == subcmd:
                        # keep matching solution
                        break
                    else:
                        # otherwise, don't exit w/ solution set
                        solution = None
            if not solution:
                # else:
                solution = cmdlike.request_external_solution(self, invocation)
                solutions.append((solution, subcmd))

                logger.debug(
                    "while seeking external subexec for %r, recorded solution=%r at pos=%r cmdlike=%r",
                    invocation,
                    solution,
                    pos,
                    cmdlike.name,
                )

            if not solution.solved:
                cmdlike.resolve()

            if solution.exception:
                return

            replacement = solution.replacement[1] if solution.replacement else None
            if len(invocation) == 1 or (
                replacement and replacement in lore and lore[replacement] == "cannot"
            ):
                logger.debug("allowing safe-passage for executable declared 'cannot'")
                return

        logger.debug(
            "look_for_external_sub_execution1: %r outer=%r",
            invocation,
            outer,
        )

        if outer in decided_execers:
            assay = decided_execers[outer]
            wordnum = int(assay["wordnum"])
            logger.debug("outer=%r assay=%r wordnum=%r", outer, assay, wordnum)
            # TODO: don't trust the below until verified post invocation
            # refactor (aug 24 2021)
            if assay["command"] == " ".join(invocation):
                if wordnum >= len(invocation):
                    # Caution: before oil 0.14 -> 0.15, location was
                    # word=invocation[-1].ast; I'm unsure about the parts[0]
                    # tests pass, but this interface isn't documented or used
                    # extensively so it wouldn't shock me if there are edge
                    # cases. Not going to spend energy reasoning through since
                    # this is probably going to get rooted out at some point.
                    observe(
                        InvalidAssay,
                        "the wordnum index(%d) is too large to zero-index args(%d)"
                        % (wordnum, len(invocation)),
                        assay,
                        location=invocation[-1].ast.parts[0],
                        arena=self.arena,
                    )
                pos = invocation[wordnum].first_spid
                subcmd = invocation[wordnum]
                logger.debug("assay.command == text")
                # contra to the normal process--we'll register and
                # then immediately resolve this (if it isn't already)
                cmdlike = cmdlikes[subcmd]

                solution = None
                solutions = self.solutions[pos]
                if pos in self.solutions:
                    for solution, word_ob in solutions:
                        if word_ob == subcmd:
                            # keep matching solution
                            break
                        else:
                            # otherwise, don't exit w/ solution set
                            solution = None
                if not solution:
                    # else:
                    solution = cmdlike.request_external_solution(self, invocation)
                    solutions.append((solution, invocation[wordnum]))

                logger.debug(
                    "look_for_external_sub_execution2: %r %r %r %r %r",
                    cmdlike,
                    solution,
                    pos,
                    subcmd,
                    solution.solved,
                )
                if not solution.solved:
                    cmdlike.resolve()

                logger.debug(
                    "look_for_external_sub_execution3: %r %r %r %r %r",
                    cmdlike,
                    solution,
                    pos,
                    subcmd,
                    solution.solved,
                )
                if solution.solved:
                    return

        global externals
        # tentatively try knocking out varlikes for everyone?
        invocation = self._strip_varlikes(invocation)

        if externals[subcmd]:
            for parser in externals[subcmd]:
                parsed = _unparsed = None
                parsed = parser.parse_known_args(invocation.args)
                if not parsed:
                    logger.debug(
                        "parse failed (parser=%r, args=%r)", parser, invocation
                    )
                    continue
                else:
                    parsed, _unparsed = parsed
                    # see sudo parser for example
                    if "skip" in parsed and parsed.skip:
                        continue

                handler = getattr(
                    self,
                    "handle_external_{:}".format(subcmd),
                    self.handle_external_generic,
                )
                logger.debug(
                    "parsed %r args %r (unparsed: %r)", parser, parsed, _unparsed
                )
                logger.debug(
                    "calling %r(parsed=%r, ob=%r)",
                    handler,
                    parsed,
                    invocation,
                )
                if handler(parsed, invocation):
                    return

            observe(
                NoParserSucceeded,
                invocation.firstword,
                location=loc.Word(invocation.firstword.ast),
                arena=self.arena,
            )

        observe(
            PotentialExecer,
            invocation.firstword,
            location=loc.Word(invocation.firstword.ast),
            arena=self.arena,
        )

    def record_general_invocation(self, invocation):
        """Record command found in general namespace."""
        pos = invocation.firstword.first_spid
        self.solutions[pos].append(
            (
                cmdlikes[invocation.firstword].request_full_solution(self, invocation),
                invocation.firstword,
            )
        )

    def record_slash_invocation(self, invocation):
        """Record \command in (general namespace - aliases)."""
        pos = invocation.firstword.first_spid
        self.solutions[pos].append(
            (
                cmdlikes[invocation.firstword].request_slash_solution(self, invocation),
                invocation.firstword,
            )
        )

    def record_alias_declaration(self, declaration, identifier, pos):
        """Record command found in alias definition namespace."""
        self.solutions[pos].append(
            (cmdlikes[identifier].request_alias_solution(self), declaration)
        )

    def record_builtin_invocation(self, invocation):
        """Record command argument to `builtin` builtin :)"""
        pos = invocation.firstword.first_spid
        self.solutions[pos].append(
            (
                cmdlikes[invocation.firstword].request_builtin_solution(
                    self, invocation
                ),
                invocation.firstword,
            )
        )

    def record_command_invocation(self, invocation):
        """Record command argument to `command` builtin."""
        pos = invocation.firstword.first_spid
        self.solutions[pos].append(
            (
                cmdlikes[invocation.firstword].request_command_solution(
                    self, invocation
                ),
                invocation.firstword,
            )
        )

        # TODO: I disabled this, but I'm not sure if this is right or not.
        # If this is still here after 2021, it can be removed.
        # if we can *know* whether to resolve or not, we don't need
        # to return the solution (and a lot of this gets superfluous)
        # if the resolution context will *always* be an external
        # we can go ahead and resolve; if it can be a function or alias
        # we need to wait until we've encountered all of those.
        # if not solution.solved:
        #     cmdlike.resolve()
        # return solution

    def record_external_invocation(self, invocation):
        """Record command argument to commands known to run externals."""
        pos = invocation.firstword.first_spid
        self.solutions[pos].append(
            (
                cmdlikes[invocation.firstword].request_external_solution(
                    self, invocation
                ),
                invocation.firstword,
            )
        )

    def record_external_source(self, source, replace_with=None, position=None):
        """
        Record file argument to `source` builtin.

        CAUTION:
        This has the same basic effect as the others, but it uses
        slightly different logic. In short, the others are for queueing late
        resolution of each command-like. We *could* do that here--but we have
        to parse each source *before* we can late-resolve everything else
        correctly.

        So, in this one case, we'll immediately resolve and parse these.
        """
        global resolved_scripts
        pos = source.first_spid
        sol = Solution()
        self.solutions[pos].append((sol, source))
        if replace_with:
            # TODO: see note about constant cmdkind
            sol.replace_name_with("source", source, replace_with)
            source = replace_with
        else:
            # TODO: see note about constant cmdkind
            sol.no_change("source")

        logger.info("Resolving source")

        if (
            source not in resolved_scripts
            and ("source", source) not in self.directives.fake
        ):
            logger.info("Parsing sourced script %r", source)
            resolved = self.__class__(source)
            resolved_scripts[source] = resolved

        self.directives.keep.add(
            (
                "source",
                source,
            )
        )


evaluator = None


class ResolvedScript(RecordCommandlike):
    """
    Represent each individual script that we resolve.
    """

    @staticmethod
    def _make_parser(parse_ctx, script, arena):
        """
        Do the last few steps to make a file-specific parser.
        """
        return parse_ctx.MakeOshParser(reader.FileLineReader(script, arena))

    def replace_shebang(self, fileob, shebang=False):
        """
        - skip over the shebang lines, else raise
        - prepend new shebang if available
        """
        pos = fileob.tell()
        line = fileob.readline()

        while line[0:2] == "#!":
            if shebang is False:
                raise Exception(
                    "Oh gosh :( you gotta get this shebang looked at", line, pos
                )
            elif shebang or shebang is None:
                logger.debug("Skipping shebang line: %r", line)

            pos = fileob.tell()
            line = fileob.readline()

        logger.debug("First non-shebang line: %r", line)
        fileob.seek(pos)
        logger.debug("appending shebang: %r", shebang)
        return StringIO.StringIO((shebang if shebang else "") + fileob.read())

    def __init__(
        self,
        script_path=None,
        shebang="",
        prologue=None,
        epilogue=None,
    ):
        # generally, defer work until we know the script loaded
        with (
            open(script_path) if script_path else io.BytesIO(sys.stdin.read())
        ) as script:
            if shebang:
                script = self.replace_shebang(script, shebang)

            # each directive type is roughly:
            # set([ (scope, ), (scope, identifier), ... ])
            self.directives = ResolutionDirectives()
            self.parse_directives(script, script_path)

            # actually initialize
            RecordCommandlike.__init__(self)
            self.prologue = prologue
            self.epilogue = epilogue

            node = self.parse_script(script, script_path)
            updated_script = self.substitute_vars()
            if updated_script:
                # if we substitute:
                # - clear some global state
                blink()
                # - parse again
                node = self.parse_script(updated_script, script_path)
                updated_script.close()

        # "resolve"
        try:
            self.visit_functions()
            self.visit_commands()
            self.handle_potential_redefiners()
        except Forbid as e:
            e.print_if_needed()
            raise e

    def parse_script(self, script, script_path):
        global evaluator
        self.arena = arena = alloc.Arena(save_tokens=True)

        parse_ctx = parse_lib.ParseContext(
            arena=arena,
            parse_opts=optview.Parse(state.InitOpts(), NO_OPTIONS_OVERLAY),
            aliases={},  # dummy
            ysh_grammar=pyutil.LoadYshGrammar(
                pyutil._FileResourceLoader(oil.__path__[0])
            ),
            one_pass_parse=True,
        )

        if script_path:
            # TODO: is there a real difference between using mainfile and
            # sourcedfile? (this gets re-used for sourced scripts)
            arena.PushSource(source.MainFile(script_path))
        else:
            arena.PushSource(source.Stdin(""))

        mem = state.Mem("", list(), arena, list())
        parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
        mem.exec_opts = exec_opts
        """
        TODO:
        I noted the dict below is for envs like {"HOME": "/noplacelike"}
        but the underlying oil APIs have changed a little and I'm less
        sure if this is still true or not. In any case, it wasn't the
        place to deal with substituting vars.

        Eventually, investigate/doc what this is doing (maybe it only
        works for eval -> string?)
        """
        state._InitVarsFromEnv(mem, dict())

        splitter = split.SplitContext(mem)
        errfmt = ui.ErrorFormatter()
        tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
        # TODO: ask andy about these as well
        evaluator = word_eval.CompletionWordEvaluator(
            mem, exec_opts, mutable_opts, tilde_ev, splitter, errfmt
        )
        evaluator.arith_ev = sh_expr_eval.ArithEvaluator(
            mem, exec_opts, mutable_opts, parse_ctx, errfmt
        )
        evaluator.arith_ev.word_ev = evaluator

        try:
            # TODO: sort out this node/root thing
            self.root = node = main_loop.ParseWholeFile(
                self._make_parser(parse_ctx, script, arena)
            )
        except error.Parse as e:
            e.arena = arena
            raise

        return node

    def substitute_vars(self):
        """
        This ties together a few parts, so here's the big picture:
        1. monkey-patch oil's SimpleVarSub and BracedVarSub to
           get oil to compile a set of these for us so we don't have
           to walk the node tree to find them.
        2. iterate over these and create custom Solution objects for
           any vars we got a --fix $var:sub directive for.
        3. perform a distinct write cycle to flatten these subs back
           down into the script and re-parse it before doing the main
           resolution pass.

        Caution:
        - This replaces the entire varsub with what we specify, which
        overwrites every braced expansion indiscriminately. It roughly
        bets on a shaky assumption that the kinds of variables users
        need to substitute won't often be subject to expansions that
        change their case or otherwise substitute characters.
        - This has no impact on vars that are set for the benefit of
        executables that we'll invoke. We're just substituting *uses*
        because we can do that ~statelessly without needing a complex
        model for understanding what a variable is, and how it can
        change and how it can't. I'm not sure how much of a problem
        this will prove to be in practice. Fingers crossed.

        TODO: I'm not sure what all of the tendrils are, but ideally
        we make this as clear/salient for the user as possible, via
        some combination of messaging when we encounter dynamic vars
        in firstwords (i.e., when the user will discover the problem
        and when we'll first suggest the replacement approach), and
        "warnings" that we might print when we do the substitution.
        These might do things like enumerate all lines that reference
        this var
        """

        file_ob = None
        # there are probably not any already, but we'll consume them
        # just-in-case to make sure we don't
        with self.consume_solutions() as solutions:
            replace_varsubs = self.directives.varsubs
            with BracedVarSub.consume_members() as brace_subs:
                for sub in brace_subs:
                    location = sub.left.span_id
                    positions = 1 + sub.right.span_id - sub.left.span_id
                    # Caution: less confident about this one than the SimpleVarSub copies coming up
                    token = sub.token.tval

                    # if there's an explicit replacement, we'll use it
                    if token in replace_varsubs:
                        sol = Solution().custom(
                            "var",
                            get_original_text(self.arena, location, positions),
                            replace_varsubs[token],
                            positions,
                        )
                        self.solutions[location].append((sol, Word(sub)))

            with SimpleVarSub.consume_members() as simple_subs:
                # just do simple/braced separate; their structures differ
                for sub in simple_subs:
                    location = sub.left.span_id
                    positions = 1
                    token = sub.left.tval.lstrip("$")  # normalize simple w/ braced

                    # if there's an explicit replacement, we'll use it
                    if token in replace_varsubs:
                        sol = Solution().custom(
                            "var",
                            get_original_text(self.arena, location, positions),
                            replace_varsubs[token],
                            positions,
                        )
                        self.solutions[location].append((sol, Word(sub)))

            if len(self.solutions):
                file_ob = io.BytesIO()
                self.render(to=file_ob, full=False)
                file_ob.seek(0)  # be kind

        return file_ob

    def parse_directives(self, script, path):
        """
        read the tail of the file for directives;
        abort as soon as we find a nonmatching line

        # resholve: <type> scope:<identifier>
        """
        global directives
        path = path if path else "<stdin>"
        self.in_doc_directives = ResolutionDirectives()
        for line in script:
            if line.startswith("# resholve: "):
                try:
                    if line.startswith("# resholve: allow"):
                        raise InFileDirectiveError(
                            "While parsing resholve directives already in this "
                            "script, I encountered an 'allow' directive from "
                            "resholve(d)'s pre-history. The program no longer "
                            "supports this directive format--you'll have to "
                            "re-resholve this script with a modern version. ",
                            path,
                            line,
                        )
                    elif line.startswith("# resholve: fake"):
                        self.in_doc_directives.fake.update(
                            directive_fake(line[17:].strip())
                        )
                    elif line.startswith("# resholve: fix"):
                        self.in_doc_directives.fix.update(
                            directive_fix(line[16:].strip())
                        )
                    elif line.startswith("# resholve: keep"):
                        self.in_doc_directives.keep.update(
                            directive_keep(line[17:].strip())
                        )
                except ourparse.ArgumentTypeError as e:
                    raise InFileDirectiveError(e.message, path, line)
            elif line.startswith(
                "### resholve directives (auto-generated) ## format_version: "
            ):
                # some modern version
                version = line[60:].strip()
                if version != DIRECTIVE_FORMAT:
                    logger.warning(
                        "resholve version {:} with *native* directive "
                        "fmt v{:} parsing directive fmt v{:}".format(
                            __version__, DIRECTIVE_FORMAT, version
                        )
                    )
            # older directive separator
            elif line.startswith("### resholve directives (auto-generated)"):
                # some modern version
                version = "1"
                if version != DIRECTIVE_FORMAT:
                    logger.warning(
                        "resholve version {:} with *native* directive "
                        "fmt v{:} parsing directive fmt v{:}".format(
                            __version__, DIRECTIVE_FORMAT, version
                        )
                    )
        script.seek(0)
        self.directives.update(self.in_doc_directives, directives)

    def write_directives(self, cursor):
        global DIRECTIVE_FORMAT
        template = "\n### {:} ## {:}: {:}\n"
        # we only want items that weren't already in the file
        if len(self.in_doc_directives) < len(self.directives):
            if len(self.in_doc_directives) == 0:
                cursor.f.write(
                    template.format(
                        "resholve directives (auto-generated)",
                        "format_version",
                        DIRECTIVE_FORMAT,
                    )
                )
            new_directives = self.directives.difference(self.in_doc_directives)
            cursor.f.writelines(
                [
                    self.write_directive(map(self.spacequote, directive))
                    for directive in sorted(new_directives)
                ]
                + ["\n"]
            )

    @staticmethod
    def spacequote(text):
        """single-quote a string if it contains a space"""
        if " " in text:
            return repr(text)
        else:
            return text

    @staticmethod
    def write_directive(directive):
        """
        Format a single directive for the file tail.

        Assumes input directive is already spacequoted.
        """
        if len(directive) == 3:
            return "# resholve: {:} {:}:{:}\n".format(*directive)
        elif len(directive) == 2:
            return "# resholve: {:} {:}\n".format(*directive)

    def write_to(self, path=None):
        f = open(path, "w") if path else sys.stdout
        try:
            self.render(to=f)
        except Forbid as e:
            e.print_if_needed()
            raise e
        finally:
            f.close()

    # TODO: this ~full idiom smells like something that won't age well
    # if the new 2-phase write flow holds (and doesn't turn into a 3?)
    # look for an excuse to refactor render/write responsibilities.
    # Probably a 2022 thing.
    def render(self, to=sys.stdout, full=True):
        cursor = ysh_ify.Cursor(self.arena, to)

        replacements = sorted(self.solutions.keys())
        if len(replacements):
            logger.info("Making replacements")
            logger.debug("  The replacements are: %r", self.solutions)
            logger.debug("  They'll be made in this order: %r", replacements)

        # if the user has specified a prepend string, find the first
        # actual code span, so that we can inject before it.
        if full and self.prologue:
            first_span = find_earliest_spid(self.root)
            cursor.PrintUntilSpid(first_span)
            to.write("\n# begin prologue inserted by resholve\n")
            to.write(self.prologue.read())
            to.write("\n# end prologue inserted by resholve\n")
            cursor.PrintUntilSpid(first_span)
            # rewind in case we need to use again
            self.prologue.seek(0)

        last = -1

        for location in replacements:
            solutions = self.solutions[location]
            """
            Making this a little wonky to support the possibility of
            multiple replacements at the same SPID (i.e., edge-case
            things like `bash -c "ls | xargs"`). So we'll retain
            positions and current in case there are extra iterations
            and then finally write to file once we know we're done
            """
            positions = current = None
            for i, (solution, word_object) in enumerate(solutions):
                if last > location:
                    logger.warning(
                        "skipping solution %r because the cursor (%r) is already past its location (%r)",
                        solution,
                        last,
                        location,
                    )
                    continue

                logger.debug("handling replacements: %r", location)

                e = solution.exception
                if e:
                    # In some cases we need to raise an error from somewhere
                    # we lack access to location context
                    e.location = e.word if hasattr(e, "word") else word_object.ast
                    e.arena = e.arena or self.arena
                    raise e
                elif solution.replacement:
                    logger.debug("pre-replacement %r", location)
                    if not current:
                        """
                        TODO: oil e2f1598c40d1755885f1aacbe808a00bab89884d
                        refactors PrintUntil to use tokens and renamed the old
                        behavior to PrintUntilSpid. I'll use that for expedience
                        for now, but it's worth thinking through whether using
                        tokens would simplify any here.
                        """
                        cursor.PrintUntilSpid(location)
                        logger.debug("printed up to %r", location)
                        positions = max(solution.positions, word_object.spid_len)
                        current = get_original_text(self.arena, location, positions)
                    logger.debug(
                        "identified current=%r, replacement=%r",
                        current,
                        solution.replace_in(current),
                    )
                    if current[0:1] == "\\":
                        current = current.replace(current[1:], solution.replacement[1])
                        location += 1
                    else:
                        current = solution.replace_in(current)

            if current:
                to.write(current)

                last = location + positions
                logger.debug(
                    "replacement written and skipping to %r",
                    last,
                )

                # we already wrote this one; leapfrog
                cursor.SkipUntilSpid(last)

        cursor.PrintUntilSpid(self.arena.LastSpanId())

        if full:
            if self.epilogue:
                to.write("\n# begin epilogue inserted by resholve\n")
                to.write(self.epilogue.read())
                to.write("\n# end epilogue inserted by resholve\n")

                # rewind in case we need to use again
                self.epilogue.seek(0)

            self.write_directives(cursor)

        self.arena.PopSource()
        return self

    def allow_dynamic_executable_word(self, word, outer=None):
        """
        Handle a word that didn't pass StaticEval
        """
        logger.debug("allow dynamic executable word? '%s' -> %r", outer, word)
        logger.debug("dynamic word keep=%r, allow=%r", word.keep, word.allow)
        if outer == "eval":
            logger.info(
                "allowing eval with dynamic argument %r",
                word,
            )
            return True
        elif word.allow:
            logger.info(
                "allowing dynamic command (first-word variable) with positional varsub: %r",
                word.allow,
            )
            return True
        elif word.keep:
            if outer:
                # Letting ${name}-style subs through only if they're in
                # a list of allowed names. (goal: require conscious
                # exceptions, but make them easy to add)
                if (
                    outer,
                    word,
                ) in self.directives.keep:
                    logger.debug(
                        "[strict] keep dynamic argument %r:%r",
                        outer,
                        word,
                    )
                    return True
                elif (
                    outer,
                    word.keep,
                ) in self.directives.keep:
                    logger.info(
                        "[loose] keep dynamic argument %r:%r",
                        outer,
                        word,
                    )
                    return True

                observe(
                    DynamicArgument,
                    outer,
                    location=loc.Word(word.ast),
                    arena=self.arena,
                )
            else:
                if (word,) in self.directives.keep:
                    logger.debug(
                        "[strict] keep dynamic invocation %r",
                        word,
                    )
                    return True
                elif (word.keep,) in self.directives.keep:
                    logger.info(
                        "[loose] keep dynamic invocation %r",
                        word.keep,
                    )
                    return True

                observe(
                    DynamicCommand,
                    location=loc.Word(word.ast),
                    arena=self.arena,
                )

        return False

    def handle_potential_redefiners(self):
        logger.info(
            "TODO: this is where to process builtins that might mod execution context (enable, unset, unalias?)"
        )

    def visit_command_invocation(self, invocation):
        global FIRST_PASS_ESSENTIALS
        if not len(invocation):
            raise Exception("No-length invocation? Does this happen?")

        if not invocation.firstword.ok:
            self.allow_dynamic_executable_word(invocation.firstword)
        else:
            # special-case slash-commands
            if invocation.firstword.startswith("\\"):
                words = [Word(invocation.firstword.ast, text=invocation.firstword[1:])]
                for i, word in enumerate(invocation):
                    if i > 0:
                        words.append(word)
                invocation = Invocation(words=words)
                if invocation.firstword not in FIRST_PASS_ESSENTIALS:
                    self.record_slash_invocation(invocation)
            elif invocation.firstword not in FIRST_PASS_ESSENTIALS:
                self.record_general_invocation(invocation)

            # handle essential builtins immediately
            if invocation.firstword in FIRST_PASS_ESSENTIALS:
                cmdlike = cmdlikes[invocation.firstword]
                logger.debug(
                    "FIRST_PASS_ESSENTIAL (%r): %r %r",
                    invocation.firstword,
                    cmdlike,
                    invocation,
                )
                if not cmdlike.alias and not cmdlike.function:
                    return self.look_for_essential_builtin_sub_execution(invocation)

                observe(
                    OverriddenEssentialBuiltin,
                    "function" if cmdlike.function else "alias",
                    invocation.firstword,
                    location=loc.Word(invocation.firstword.ast),
                    arena=self.arena,
                )

    def visit_functions(self):
        with command.ShFunction.consume_members() as fns:
            for sub in fns:
                cmdlikes[sub.name].function = True

    def visit_commands(self):
        with command.Simple.consume_members() as cmds:
            for sub in cmds:
                inv = Invocation(sub)
                if inv:
                    self.visit_command_invocation(inv)


if __name__ == "__main__":
    # punshow()  # DEBUG: toggle comments here when you want exceptions
    sys.exit(punshow())