Support different versions per instance of a component

commodore/cluster.py

@@ -130,7 +130,6 @@ def render_target(
  inv: Inventory,
  target: str,
  components: dict[str, Component],
- # pylint: disable=unsubscriptable-object
  component: Optional[str] = None,
 ):
  if not component:
@@ -144,22 +143,29 @@ def render_target(
  "_instance": target,
  }
  if not bootstrap:
- parameters["_base_directory"] = str(components[component].target_directory)
+ parameters["_base_directory"] = str(
+ components[component].alias_directory(target)
+ )
 
  for c in components:
- if inv.defaults_file(c).is_file():
- classes.append(f"defaults.{c}")
+ defaults_file = inv.defaults_file(c)
+ if c == component and target != component:
+ # Special case alias defaults symlink
+ defaults_file = inv.defaults_file(target)
+
+ if defaults_file.is_file():
+ classes.append(f"defaults.{defaults_file.stem}")
  else:
  click.secho(f" > Default file for class {c} missing", fg="yellow")
 
  classes.append("global.commodore")
 
  if not bootstrap:
- if not inv.component_file(component).is_file():
+ if not inv.component_file(target).is_file():
  raise click.ClickException(
  f"Target rendering failed for {target}: component class is missing"
  )
- classes.append(f"components.{component}")
+ classes.append(f"components.{target}")
  parameters["kapitan"] = {
  "vars": {
  "target": target,
@@ -180,7 +186,6 @@ def render_target(
  }
 
 
-# pylint: disable=unsubscriptable-object
 def update_target(cfg: Config, target: str, component: Optional[str] = None):
  click.secho(f"Updating Kapitan target for {target}...", bold=True)
  file = cfg.inventory.target_file(target)

commodore/compile.py

@@ -194,7 +194,7 @@ def setup_compile_environment(config: Config) -> tuple[dict[str, Any], Iterable[
  config.register_component_deprecations(cluster_parameters)
  # Raise exception if component version override without URL is present in the
  # hierarchy.
- verify_version_overrides(cluster_parameters)
+ verify_version_overrides(cluster_parameters, config.get_component_aliases())
 
  for component in config.get_components().values():
  ckey = component.parameters_key

commodore/component/__init__.py

@@ -18,6 +18,8 @@ class Component:
  _version: Optional[str] = None
  _dir: P
  _sub_path: str
+ _aliases: dict[str, str]
+ _work_dir: Optional[P]
 
  # pylint: disable=too-many-arguments
  def __init__(
@@ -43,6 +45,8 @@ def __init__(
  self.version = version
  self._sub_path = sub_path
  self._repo = None
+ self._aliases = {self.name: self.version or ""}
+ self._work_dir = work_dir
 
  @property
  def name(self) -> str:
@@ -85,17 +89,36 @@ def version(self, version: str):
  def repo_directory(self) -> P:
  return self._dir
 
+ @property
+ def sub_path(self) -> str:
+ return self._sub_path
+
  @property
  def target_directory(self) -> P:
- return self._dir / self._sub_path
+ return self.alias_directory(self.name)
 
  @property
  def class_file(self) -> P:
- return self.target_directory / "class" / f"{self.name}.yml"
+ return self.alias_class_file(self.name)
 
  @property
  def defaults_file(self) -> P:
- return self.target_directory / "class" / "defaults.yml"
+ return self.alias_defaults_file(self.name)
+
+ def alias_directory(self, alias: str) -> P:
+ apath = self._dependency.get_component(alias)
+ if not apath:
+ raise ValueError(f"unknown alias {alias} for component {self.name}")
+ return apath / self._sub_path
+
+ def alias_class_file(self, alias: str) -> P:
+ return self.alias_directory(alias) / "class" / f"{self.name}.yml"
+
+ def alias_defaults_file(self, alias: str) -> P:
+ return self.alias_directory(alias) / "class" / "defaults.yml"
+
+ def has_alias(self, alias: str):
+ return alias in self._aliases
 
  @property
  def lib_files(self) -> Iterable[P]:
@@ -126,6 +149,26 @@ def parameters_key(self):
  def checkout(self):
  self._dependency.checkout_component(self.name, self.version)
 
+ def register_alias(self, alias: str, version: str):
+ if not self._work_dir:
+ raise ValueError(
+ f"Can't register alias on component {self.name} "
+ + "which isn't configured with a working directory"
+ )
+ if alias in self._aliases:
+ raise ValueError(
+ f"alias {alias} already registered on component {self.name}"
+ )
+ self._aliases[alias] = version
+ self._dependency.register_component(alias, component_dir(self._work_dir, alias))
+
+ def checkout_alias(self, alias: str):
+ if alias not in self._aliases:
+ raise ValueError(
+ f"alias {alias} is not registered on component {self.name}"
+ )
+ self._dependency.checkout_component(alias, self._aliases[alias])
+
  def render_jsonnetfile_json(self, component_params):
  """
  Render jsonnetfile.json from jsonnetfile.jsonnet

commodore/dependency_mgmt/__init__.py

@@ -37,6 +37,27 @@ def create_component_symlinks(cfg, component: Component):
  )
 
 
+def create_alias_symlinks(cfg, component: Component, alias: str):
+ if not component.has_alias(alias):
+ raise ValueError(
+ f"component {component.name} doesn't have alias {alias} registered"
+ )
+ relsymlink(
+ component.alias_class_file(alias),
+ cfg.inventory.components_dir,
+ dest_name=f"{alias}.yml",
+ )
+ inventory_default = cfg.inventory.defaults_file(alias)
+ relsymlink(
+ component.alias_defaults_file(alias),
+ inventory_default.parent,
+ dest_name=inventory_default.name,
+ )
+ # TODO: How do we handle lib files when symlinking aliases? Code in the component
+ # alias itself should be able to find the right library. We need to define what
+ # version of the library is visible for other components.
+
+
 def create_package_symlink(cfg, pname: str, package: Package):
  """
  Create package symlink in the inventory.
@@ -67,7 +88,7 @@ def fetch_components(cfg: Config):
  component_names, component_aliases = _discover_components(cfg)
  click.secho("Registering component aliases...", bold=True)
  cfg.register_component_aliases(component_aliases)
- cspecs = _read_components(cfg, component_names)
+ cspecs = _read_components(cfg, component_aliases)
  click.secho("Fetching components...", bold=True)
  for cn in component_names:
  cspec = cspecs[cn]
@@ -85,6 +106,26 @@ def fetch_components(cfg: Config):
  cfg.register_component(c)
  create_component_symlinks(cfg, c)
 
+ components = cfg.get_components()
+
+ for alias, component in component_aliases.items():
+ if alias == component:
+ # Nothing to setup for identity alias
+ continue
+
+ c = components[component]
+ aspec = cspecs[alias]
+ if aspec.url != c.repo_url or aspec.path != c._sub_path:
+ # TODO: Figure out how we'll handle URL/subpath overrides
+ raise NotImplementedError(
+ "URL/path override for component alias not supported"
+ )
+ print(alias, aspec)
+ c.register_alias(alias, aspec.version)
+ c.checkout_alias(alias)
+
+ create_alias_symlinks(cfg, c, alias)
+
 
 def register_components(cfg: Config):
  """
@@ -96,7 +137,7 @@ def register_components(cfg: Config):
  click.secho("Discovering included components...", bold=True)
  try:
  components, component_aliases = _discover_components(cfg)
- cspecs = _read_components(cfg, components)
+ cspecs = _read_components(cfg, component_aliases)
  except KeyError as e:
  raise click.ClickException(f"While discovering components: {e}")
  click.secho("Registering components and aliases...", bold=True)
@@ -122,9 +163,9 @@ def register_components(cfg: Config):
  cfg.register_component(component)
  create_component_symlinks(cfg, component)
 
- registered_components = cfg.get_components().keys()
+ registered_components = cfg.get_components()
  pruned_aliases = {
- a: c for a, c in component_aliases.items() if c in registered_components
+ a: c for a, c in component_aliases.items() if c in registered_components.keys()
  }
  pruned = sorted(set(component_aliases.keys()) - set(pruned_aliases.keys()))
  if len(pruned) > 0:
@@ -133,6 +174,21 @@ def register_components(cfg: Config):
  )
  cfg.register_component_aliases(pruned_aliases)
 
+ for alias, cn in pruned_aliases.items():
+ if alias == cn:
+ # Nothing to setup for identity alias
+ continue
+
+ c = registered_components[cn]
+ aspec = cspecs[alias]
+ if aspec.url != c.repo_url or aspec.path != c.sub_path:
+ raise NotImplementedError("Changing alias sub path / URL NYI")
+ c.register_alias(alias, aspec.version)
+ if not component_dir(cfg.work_dir, alias).is_dir():
+ raise click.ClickException(f"Missing alias checkout for '{alias} as {cn}'")
+
+ create_alias_symlinks(cfg, c, alias)
+
 
 def fetch_packages(cfg: Config):
  """
@@ -190,9 +246,13 @@ def register_packages(cfg: Config):
  create_package_symlink(cfg, p, pkg)
 
 
-def verify_version_overrides(cluster_parameters):
+def verify_version_overrides(cluster_parameters, component_aliases: dict[str, str]):
  errors = []
+ aliases = set(component_aliases.keys()) - set(component_aliases.values())
  for cname, cspec in cluster_parameters["components"].items():
+ if cname in aliases:
+ # We don't require an url in component alias version configs
+ continue
  if "url" not in cspec:
  errors.append(f"component '{cname}'")
 

commodore/dependency_mgmt/version_parsing.py

@@ -2,6 +2,7 @@
 
 from collections.abc import Iterable
 from dataclasses import dataclass
+from typing import Optional
 
 from enum import Enum
 
@@ -33,18 +34,31 @@ class DependencySpec:
  path: str
 
  @classmethod
- def parse(cls, info: dict[str, str]) -> DependencySpec:
- if "url" not in info:
+ def parse(
+ cls,
+ info: dict[str, str],
+ base_config: Optional[DependencySpec] = None,
+ ) -> DependencySpec:
+ if "url" not in info and not base_config:
  raise DependencyParseError("url")
 
- if "version" not in info:
+ if "version" not in info and not base_config:
  raise DependencyParseError("version")
 
  path = info.get("path", "")
  if path.startswith("/"):
  path = path[1:]
 
- return DependencySpec(info["url"], info["version"], path)
+ if base_config:
+ url = info.get("url", base_config.url)
+ version = info.get("version", base_config.version)
+ if path not in info:
+ path = base_config.path
+ else:
+ url = info["url"]
+ version = info["version"]
+
+ return DependencySpec(url, version, path)
 
 
 def _read_versions(
@@ -53,6 +67,8 @@ def _read_versions(
  dependency_names: Iterable[str],
  require_key: bool = True,
  ignore_class_notfound: bool = False,
+ aliases: dict[str, str] = {},
+ fallback: dict[str, DependencySpec] = {},
 ) -> dict[str, DependencySpec]:
  deps_key = dependency_type.value
  deptype_str = dependency_type.name.lower()
@@ -71,15 +87,26 @@ def _read_versions(
  # just set deps to the empty dict.
  deps = {}
 
+ if aliases:
+ all_dep_keys = set(aliases.keys())
+ else:
+ all_dep_keys = deps.keys()
  for depname in dependency_names:
- if depname not in deps:
+ if depname not in all_dep_keys:
  raise click.ClickException(
  f"Unknown {deptype_str} '{depname}'."
  + f" Please add it to 'parameters.{deps_key}'"
  )
 
  try:
- dep = DependencySpec.parse(deps[depname])
+ basename_for_dep = aliases.get(depname, depname)
+ print(depname, basename_for_dep)
+ print(deps.get(depname, {}))
+ print(fallback.get(basename_for_dep))
+ dep = DependencySpec.parse(
+ deps.get(depname, {}),
+ base_config=fallback.get(basename_for_dep),
+ )
  except DependencyParseError as e:
  raise click.ClickException(
  f"{deptype_cap} '{depname}' is missing field '{e.field}'"
@@ -96,9 +123,26 @@ def _read_versions(
 
 
 def _read_components(
- cfg: Config, component_names: Iterable[str]
+ cfg: Config, component_aliases: dict[str, str]
 ) -> dict[str, DependencySpec]:
- return _read_versions(cfg, DepType.COMPONENT, component_names)
+ component_names = set(component_aliases.values())
+ alias_names = set(component_aliases.keys()) - component_names
+
+ component_versions = _read_versions(cfg, DepType.COMPONENT, component_names)
+ alias_versions = _read_versions(
+ cfg,
+ DepType.COMPONENT,
+ alias_names,
+ aliases=component_aliases,
+ fallback=component_versions,
+ )
+
+ for alias, aspec in alias_versions.items():
+ if alias in component_versions:
+ raise ValueError("alias name already in component_versions?")
+ component_versions[alias] = aspec
+
+ return component_versions
 
 
 def _read_packages(

commodore/helpers.py

@@ -222,7 +222,6 @@ def rm_tree_contents(basedir):
  os.unlink(f)
 
 
-# pylint: disable=unsubscriptable-object
 def relsymlink(src: P, dest_dir: P, dest_name: Optional[str] = None):
  if dest_name is None:
  dest_name = src.name