OCPBUGS-18534: monitor test fix to wait before connecting to a non-existent dns on PowerVS and IBMCloud platforms

pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go

@@ -9,6 +9,7 @@ import (
  "net/http"
  "os"
  "strconv"
+ "strings"
  "time"
 
  "github.com/openshift/origin/pkg/monitortestframework"
@@ -203,6 +204,15 @@ func (w *availability) StartCollection(ctx context.Context, adminRESTConfig *res
  return fmt.Errorf("error creating PDB: %w", err)
  }
 
+ // On certain platforms hitting the hostname before it is ready leads to a blackhole, this code checks
+ // the host from the cluster's context
+ if infra.Spec.PlatformSpec.Type == configv1.PowerVSPlatformType || infra.Spec.PlatformSpec.Type == configv1.IBMCloudPlatformType {
+ nodeTgt := "node/" + nodeList.Items[0].ObjectMeta.Name
+ if err := checkHostnameReady(ctx, tcpService, nodeTgt); err != nil {
+ return err
+ }
+ }
+
  // Hit it once before considering ourselves ready
  fmt.Fprintf(os.Stderr, "hitting pods through the service's LoadBalancer\n")
  timeout := 10 * time.Minute
@@ -344,3 +354,29 @@ func httpGetNoConnectionPoolTimeout(url string, timeout time.Duration) (*http.Re
 
  return client.Get(url)
 }
+
+// Uses the first node in the cluster to verify the LoadBalancer host is active before returning
+func checkHostnameReady(ctx context.Context, tcpService *corev1.Service, nodeTgt string) error {
+ oc := exutil.NewCLIForMonitorTest(tcpService.GetObjectMeta().GetNamespace())
+
+ var (
+ stdOut string
+ err error
+ )
+
+ wait.PollUntilContextTimeout(ctx, 15*time.Second, 60*time.Minute, true, func(ctx context.Context) (bool, error) {
+ logrus.Debug("Checking load balancer host is active \n")
+ stdOut, _, err = oc.AsAdmin().WithoutNamespace().RunInMonitorTest("debug").Args(nodeTgt, "--", "dig", "+short", "+notcp", tcpService.Status.LoadBalancer.Ingress[0].Hostname).Outputs()
+ if err != nil {
+ return false, nil
+ }
+ output := strings.TrimSpace(stdOut)
+ if output == "" {
+ logrus.Debug("Waiting for the load balancer to become active")
+ return false, nil
+ }
+ logrus.Debug("Load balancer active")
+ return true, nil
+ })
+ return err
+}

test/extended/util/client.go

@@ -179,6 +179,32 @@ func NewCLIWithoutNamespace(project string) *CLI {
  return cli
 }
 
+// NewCLIForMonitorTest initializes the CLI and Kube framework helpers
+// without a namespace. Should be called outside of a Ginkgo .It()
+// function.
+func NewCLIForMonitorTest(project string) *CLI {
+ cli := &CLI{
+ kubeFramework: &framework.Framework{
+ SkipNamespaceCreation: true,
+ BaseName: project,
+ Options: framework.Options{
+ ClientQPS: 20,
+ ClientBurst: 50,
+ },
+ Timeouts: framework.NewTimeoutContext(),
+ },
+ username: "admin",
+ execPath: "oc",
+ adminConfigPath: KubeConfigPath(),
+ staticConfigManifestDir: StaticConfigManifestDir(),
+ withoutNamespace: true,
+ }
+
+ // Called only once (assumed the objects will never get modified)
+ cli.setupStaticConfigsFromManifests()
+ return cli
+}
+
 // NewHypershiftManagementCLI returns a CLI that interacts with the Hypershift management cluster.
 // Contrary to a normal CLI it does not perform any cleanup, and it must not be used for any mutating
 // operations. Also, contrary to a normal CLI it must be constructed inside an `It` block. This is
@@ -813,6 +839,22 @@ func (c *CLI) Run(commands ...string) *CLI {
  return nc.setOutput(c.stdout)
 }
 
+// Executes with the kubeconfig specified from the environment
+func (c *CLI) RunInMonitorTest(commands ...string) *CLI {
+ in, out, errout := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}
+ nc := &CLI{
+ execPath: c.execPath,
+ verb: commands[0],
+ kubeFramework: c.KubeFramework(),
+ adminConfigPath: c.adminConfigPath,
+ configPath: c.configPath,
+ username: c.username,
+ globalArgs: commands,
+ }
+ nc.stdin, nc.stdout, nc.stderr = in, out, errout
+ return nc.setOutput(c.stdout)
+}
+
 // InputString adds expected input to the command
 func (c *CLI) InputString(input string) *CLI {
  c.stdin.WriteString(input)