WIP: Test optimizations

src/app.py

@@ -1025,6 +1025,7 @@ def app_install(
  args=None,
  no_remove_on_failure=False,
  force=False,
+ sync_perm=True,
 ):
  """
  Install apps
@@ -1201,6 +1202,7 @@ def app_install(
  label=manifest["name"],
  show_tile=False,
  protected=False,
+ sync_perm=sync_perm
  )
 
  # Prepare env. var. to pass to script
@@ -1377,7 +1379,7 @@ def app_install(
 
 
 @is_unit_operation()
-def app_remove(operation_logger, app, purge=False, force_workdir=None):
+def app_remove(operation_logger, app, purge=False, force_workdir=None, sync_perm=True):
  """
  Remove app
 
@@ -1476,7 +1478,8 @@ def app_remove(operation_logger, app, purge=False, force_workdir=None):
  else:
  logger.warning(m18n.n("app_not_properly_removed", app=app))
 
- permission_sync_to_user()
+ if sync_perm:
+ permission_sync_to_user()
  _assert_system_is_sane_for_app(manifest, "post")
 
 

src/certificate.py

@@ -122,7 +122,7 @@ def certificate_install(domain_list, force=False, no_checks=False, self_signed=F
  _certificate_install_letsencrypt(domain_list, force, no_checks)
 
 
-def _certificate_install_selfsigned(domain_list, force=False):
+def _certificate_install_selfsigned(domain_list, force=False, do_regen_conf=True):
  failed_cert_install = []
  for domain in domain_list:
  operation_logger = OperationLogger(
@@ -201,7 +201,7 @@ def _certificate_install_selfsigned(domain_list, force=False):
  _set_permissions(conf_file, "root", "root", 0o600)
 
  # Actually enable the certificate we created
- _enable_certificate(domain, new_cert_folder)
+ _enable_certificate(domain, new_cert_folder, do_regen_conf=do_regen_conf)
 
  # Check new status indicate a recently created self-signed certificate
  status = _get_status(domain)
@@ -627,6 +627,9 @@ def _prepare_certificate_signing_request(domain, key_file, output_folder):
  with open(csr_file, "wb") as f:
  f.write(crypto.dump_certificate_request(crypto.FILETYPE_PEM, csr))
 
+def _cert_exists(domain) -> bool:
+ cert_file = os.path.join(CERT_FOLDER, domain, "crt.pem")
+ return os.path.isfile(cert_file)
 
 def _get_status(domain):
  cert_file = os.path.join(CERT_FOLDER, domain, "crt.pem")
@@ -723,7 +726,7 @@ def _set_permissions(path, user, group, permissions):
  chmod(path, permissions)
 
 
-def _enable_certificate(domain, new_cert_folder):
+def _enable_certificate(domain, new_cert_folder, do_regen_conf=True):
  logger.debug("Enabling the certificate for domain %s ...", domain)
 
  live_link = os.path.join(CERT_FOLDER, domain)
@@ -741,6 +744,10 @@ def _enable_certificate(domain, new_cert_folder):
 
  os.symlink(new_cert_folder, live_link)
 
+ # We are in a higher operation such as domains_add for bulk manipulation
+ # that will take care of service / hooks later
+ if not do_regen_conf: return
+
  logger.debug("Restarting services...")
 
  for service in ("dovecot", "metronome"):

src/domain.py

@@ -240,10 +240,72 @@ def _get_parent_domain_of(domain, return_self=False, topest=False):
 
  return domain if return_self else None
 
+def domains_regen(domains: List[str]):
+ for domain in domains:
+ _force_clear_hashes([f"/etc/nginx/conf.d/{domain}.conf"])
+
+ from yunohost.app import app_ssowatconf
+ from yunohost.service import _run_service_command
+ logger.debug("Restarting services...")
+ for service in ("dovecot", "metronome"):
+ # Ugly trick to not restart metronome if it's not installed or no domain configured for XMPP
+ if service == "metronome" and (
+ os.system("dpkg --list | grep -q 'ii *metronome'") != 0
+ or not glob("/etc/metronome/conf.d/*.cfg.lua")
+ ):
+ continue
+ _run_service_command("restart", service)
+
+ if os.path.isfile("/etc/yunohost/installed"):
+ # regen nginx conf to be sure it integrates OCSP Stapling
+ # (We don't do this yet if postinstall is not finished yet)
+ # We also regenconf for postfix to propagate the SNI hash map thingy
+ regen_conf(
+ names=[
+ "nginx",
+ "metronome",
+ "dnsmasq",
+ "postfix",
+ "rspamd",
+ "mdns",
+ "dovecot",
+ ]
+ )
+ app_ssowatconf()
+ _run_service_command("reload", "nginx")
+
+# Used in tests to delete many domains at once.
+# The permissions/configuration are synchronized at the end of the entire operation.
+@is_unit_operation()
+def domains_remove(operation_logger, domains: List[str]):
+ for domain in domains:
+ domain_remove(domain, do_regen_conf=False)
+
+ domains_regen(domains)
+
+ from yunohost.hook import hook_callback
+ for domain in domains:
+ hook_callback("post_domain_remove", args=[domain])
+ logger.success(m18n.n("domain_deleted"))
+
+# Used in tests to create many domains at once.
+# The permissions/configuration are synchronized at the end of the entire operation.
+@is_unit_operation()
+def domains_add(operation_logger, domains: List[str]):
+ for domain in domains:
+ domain_add(domain, do_regen_conf=False)
+
+ domains_regen(domains)
+
+ from yunohost.hook import hook_callback
+ for domain in domains:
+ hook_callback("post_cert_update", args=[domain])
+ hook_callback("post_domain_add", args=[domain])
+ logger.success(m18n.n("domain_created"))
 
 @is_unit_operation(exclude=["dyndns_recovery_password"])
 def domain_add(
- operation_logger, domain, dyndns_recovery_password=None, ignore_dyndns=False
+ operation_logger, domain, dyndns_recovery_password=None, ignore_dyndns=False, do_regen_conf=True
 ):
  """
  Create a custom domain
@@ -258,7 +320,7 @@ def domain_add(
  from yunohost.app import app_ssowatconf
  from yunohost.utils.ldap import _get_ldap_interface
  from yunohost.utils.password import assert_password_is_strong_enough
- from yunohost.certificate import _certificate_install_selfsigned
+ from yunohost.certificate import _certificate_install_selfsigned, _cert_exists
  from yunohost.utils.dns import is_yunohost_dyndns_domain
 
  if dyndns_recovery_password:
@@ -306,7 +368,8 @@ def domain_add(
  domain=domain, recovery_password=dyndns_recovery_password
  )
 
- _certificate_install_selfsigned([domain], True)
+ if not _cert_exists(domain):
+ _certificate_install_selfsigned([domain], True, do_regen_conf=False)
 
  try:
  attr_dict = {
@@ -323,7 +386,8 @@ def domain_add(
  domain_list_cache = []
 
  # Don't regen these conf if we're still in postinstall
- if os.path.exists("/etc/yunohost/installed"):
+ # or if we're in a higher operation that will take care of it, such as domains_add
+ if os.path.exists("/etc/yunohost/installed") and do_regen_conf:
  # Sometime we have weird issues with the regenconf where some files
  # appears as manually modified even though they weren't touched ...
  # There are a few ideas why this happens (like backup/restore nginx
@@ -356,9 +420,9 @@ def domain_add(
  pass
  raise e
 
- hook_callback("post_domain_add", args=[domain])
-
- logger.success(m18n.n("domain_created"))
+ if do_regen_conf:
+ hook_callback("post_domain_add", args=[domain])
+  logger.success(m18n.n("domain_created"))
 
 
 @is_unit_operation(exclude=["dyndns_recovery_password"])
@@ -369,6 +433,7 @@ def domain_remove(
  force=False,
  dyndns_recovery_password=None,
  ignore_dyndns=False,
+ do_regen_conf=True,
 ):
  """
  Delete domains
@@ -491,6 +556,10 @@ def domain_remove(
  rm(key_file, force=True)
  rm(f"{DOMAIN_SETTINGS_DIR}/{domain}.yml", force=True)
 
+ # We are in a bulk domains_remove so don't regen_conf immediately
+ if not do_regen_conf:
+ return
+
  # Sometime we have weird issues with the regenconf where some files
  # appears as manually modified even though they weren't touched ...
  # There are a few ideas why this happens (like backup/restore nginx

src/tests/test_sso_and_portalapi.py

@@ -6,8 +6,8 @@
 
 from .conftest import message, raiseYunohostError, get_test_apps_dir
 
-from yunohost.domain import _get_maindomain, domain_add, domain_remove, domain_list
-from yunohost.user import user_create, user_list, user_delete
+from yunohost.domain import _get_maindomain, domain_add, domain_remove, domain_list, domains_add, domains_remove
+from yunohost.user import user_create, user_list, user_delete, User, users_add, users_remove
 from yunohost.authenticators.ldap_ynhuser import Authenticator, SESSION_FOLDER, short_hash
 from yunohost.app import app_install, app_remove, app_setting, app_ssowatconf, app_change_url
 from yunohost.permission import user_permission_list, user_permission_update
@@ -43,31 +43,37 @@ def setup_module(module):
 
  assert os.system("systemctl is-active yunohost-portal-api >/dev/null") == 0
 
- if "alice" not in user_list()["users"]:
- user_create("alice", maindomain, dummy_password, fullname="Alice White", admin=True)
- if "bob" not in user_list()["users"]:
- user_create("bob", maindomain, dummy_password, fullname="Bob Marley")
+ domainlist = domain_list()["domains"]
+ domains = [ domain for domain in [ subdomain, secondarydomain ] if domain not in domainlist ]
+ domains_add(domains)
 
+ # Install app first, permissions will be synced after users_add
  app_install(
  os.path.join(get_test_apps_dir(), "hellopy_ynh"),
  args=f"domain={maindomain}&init_main_permission=visitors",
  force=True,
+ sync_perm=False,
  )
 
+ userlist = user_list()["users"]
+ users_to_add = [ user for user in [
+ User("alice", maindomain, dummy_password, fullname="Alice White", admin=True),
+ User("bob", maindomain, dummy_password, fullname="Bob Marley"),
+ ] if user.name not in userlist ]
+ users_add(users_to_add)
 
-def teardown_module(module):
- if "alice" in user_list()["users"]:
- user_delete("alice")
- if "bob" in user_list()["users"]:
- user_delete("bob")
 
- app_remove("hellopy")
+def teardown_module(module):
+ # Remove app first, permissions will be synced after users_remove
+ app_remove("hellopy", sync_perm=False)
 
- if subdomain in domain_list()["domains"]:
- domain_remove(subdomain)
- if secondarydomain in domain_list()["domains"]:
- domain_remove(secondarydomain)
+ userlist = user_list()["users"]
+ users = [ user for user in [ "alice", "bob" ] if user in userlist ]
+ users_remove(users)
 
+ domainlist = domain_list()["domains"]
+ domains = [ domain for domain in [ subdomain, secondarydomain ] if domain in domainlist ]
+ domains_remove(domains)
 
 def login(session, logged_as, logged_on=None):