Merge pull request #42 from higgsfield-ai/feat/invoker_exec-of_master…

…_host-of-no-python

Feat/invoker exec of master host of no python

higgsfield/internal/cfg.py

@@ -32,10 +32,14 @@ class AppConfig:
  user: str
  key: str
  port: int
+ master_host: Optional[str]
+ no_python: Optional[str]
+ invoker_exec: str = "invoker"
  number_of_processes_per_node: int
 
  def __init__(self, **kwargs):
  self.__dict__.update(kwargs)
+
  @classmethod
  def from_path(cls, path: Path) -> "AppConfig":
  config_path = path / "src" / "config.py"
@@ -66,6 +70,10 @@ def from_path(cls, path: Path) -> "AppConfig":
 
  key = get_key_from_path_or_key(os.getenv("SSH_KEY"))
 
+ master_host = module.__dict__.get("MASTER_HOST", None)
+ no_python = module.__dict__.get("NO_PYTHON", None)
+ invoker_exec = module.__dict__.get("INVOKER_EXEC", "invoker")
+
  return AppConfig(
  name=name,
  github_repo_url=github_repo_url,
@@ -74,6 +82,9 @@ def from_path(cls, path: Path) -> "AppConfig":
  key=key,
  port=port,
  number_of_processes_per_node=number_of_processes_per_node,
+ master_host=master_host,
+ no_python=no_python,
+ invoker_exec=invoker_exec,
  )
 
  def get_git_origin_url(self, path) -> Optional[str]:
@@ -106,7 +117,7 @@ def set_git_origin_url(self, path: Path):
  with open(config_path, "w") as f:
  # Write back the modified lines
  f.writelines(lines)
- 
+
  if lines[-1] != "\n":
  f.write("\n")
 

higgsfield/internal/experiment/builder.py

@@ -105,6 +105,18 @@ def generate(self):
 class ExperimentBuilder(ActionBuilder):
  template_name = "experiment_action.j2"
 
+ def of_master_host(self) -> str:
+ master_host = self.app_config.master_host
+ if master_host is None:
+ return ""
+ return f"--master_host {master_host}"
+
+ def of_no_python(self) -> str:
+ no_python = self.app_config.no_python
+ if no_python is None:
+ return ""
+ return f"--no_python {no_python}"
+
  def generate(self, experiment_name: str, params: List[Param]):
  (self.wf_dir / f"run_{experiment_name}.yml").write_text(
  self.template.render(
@@ -113,12 +125,14 @@ def generate(self, experiment_name: str, params: List[Param]):
  project_name=self.app_config.name,
  params=build_gh_action_inputs(params),
  rest=build_run_params(params),
+ invoker_exec=self.app_config.invoker_exec,
+ of_master_host=self.of_master_host(),
+ of_no_python=self.of_no_python(),
  env_gen=env_keys_as_action(
  self.wf_dir.parent.parent / "env", echo_indent
  ),
  )
  )
-
 
  print("Updated experiment action", experiment_name)
 
@@ -130,7 +144,13 @@ def _source_experiments(base_path: Path):
  the same dependencies (aka environment) as the docker container.
  """
  for file in base_path.glob("**/*.py"):
- module_name = os.path.basename(file).split(".py")[0].split(".py")[0].replace(" ", "_").replace("-", "_")
+ module_name = (
+ os.path.basename(file)
+ .split(".py")[0]
+ .split(".py")[0]
+ .replace(" ", "_")
+ .replace("-", "_")
+ )
  SourceFileLoader(module_name, str(file)).load_module()
 
 

higgsfield/internal/util.py

@@ -9,8 +9,8 @@
 
 
 def check_name(name: str):
- if len(name) < 1 or len(name) > 20:
- raise ValueError("Name must be between 1 and 20 characters long")
+ if len(name) < 1 or len(name) > 30:
+ raise ValueError("Name must be between 1 and 30 characters long")
 
  if not regex.match(name):
  raise ValueError("Name must match regex ^[a-zA-Z_][a-zA-Z0-9_]*$")

higgsfield/static/templates/deploy_action.j2

@@ -37,8 +37,7 @@ jobs:
  host: ${{ env.HOSTS }}
  username: ${{ env.USER }}
  key: ${{ secrets.SSH_KEY }}
- port: ${{ env.PORT }}
- sync: true {% endraw %}
+ port: ${{ env.PORT }} {% endraw %}
  script: | 
  mkdir -p ~/higgsfield/
  cd ~/higgsfield
@@ -48,5 +47,6 @@ jobs:
  git reset --hard origin/main && \
  git pull origin main) || git clone {{ keyed_repo_url }} {{ project_name }} || rm -rf ~/higgsfield/{{ project_name }} && git clone {{ keyed_repo_url }} {{ project_name }}
 
+ cd ~/higgsfield/{{ project_name }}
  echo "SSH_KEY=NOTHING" > env
 {{ env_gen }}

higgsfield/static/templates/experiment_action.j2

@@ -68,7 +68,7 @@ jobs:
  cd ~/higgsfield/{% endraw %}{{ project_name }}
  echo "SSH_KEY=NOTHING" > env
 {{ env_gen }}
- invoker experiment run --project_name {{ project_name }} --experiment_name {{ experiment_name }} {% raw %} --run_name ${{ env.RUN_NAME }} --port ${{ env.RUN_PORT }} --nproc_per_node ${{ env.NPROC_PER_NODE }} --hosts ${{ env.HOSTS }} {% endraw %} {{ rest }}
+ {{ invoker_exec }} experiment run {{ of_master_host }} {{ of_no_python }} --project_name {{ project_name }} --experiment_name {{ experiment_name }} {% raw %} --run_name ${{ env.RUN_NAME }} --port ${{ env.RUN_PORT }} --nproc_per_node ${{ env.NPROC_PER_NODE }} --hosts ${{ env.HOSTS }} {% endraw %} {{ rest }}