diff --git a/pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go b/pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go index b0c930a476b4..5afc81ad431e 100644 --- a/pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go +++ b/pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "strconv" + "strings" "time" "github.com/openshift/origin/pkg/monitortestframework" @@ -203,6 +204,15 @@ func (w *availability) StartCollection(ctx context.Context, adminRESTConfig *res return fmt.Errorf("error creating PDB: %w", err) } + // On certain platforms hitting the hostname before it is ready leads to a blackhole, this code checks + // the host from the cluster's context + if infra.Spec.PlatformSpec.Type == configv1.PowerVSPlatformType || infra.Spec.PlatformSpec.Type == configv1.IBMCloudPlatformType { + nodeTgt := "node/" + nodeList.Items[0].ObjectMeta.Name + if err := checkHostnameReady(ctx, tcpService, nodeTgt); err != nil { + return err + } + } + // Hit it once before considering ourselves ready fmt.Fprintf(os.Stderr, "hitting pods through the service's LoadBalancer\n") timeout := 10 * time.Minute @@ -348,3 +358,29 @@ func httpGetNoConnectionPoolTimeout(url string, timeout time.Duration) (*http.Re return client.Get(url) } + +// Uses the first node in the cluster to verify the LoadBalancer host is active before returning +func checkHostnameReady(ctx context.Context, tcpService *corev1.Service, nodeTgt string) error { + oc := exutil.NewCLIForMonitorTest(tcpService.GetObjectMeta().GetNamespace()) + + var ( + stdOut string + err error + ) + + wait.PollUntilContextTimeout(ctx, 15*time.Second, 60*time.Minute, true, func(ctx context.Context) (bool, error) { + logrus.Debug("Checking load balancer host is active \n") + stdOut, _, err = oc.AsAdmin().WithoutNamespace().RunInMonitorTest("debug").Args(nodeTgt, "--", "dig", "+short", "+notcp", tcpService.Status.LoadBalancer.Ingress[0].Hostname).Outputs() + if err != nil { + return false, nil + } + output := strings.TrimSpace(stdOut) + if output == "" { + logrus.Debug("Waiting for the load balancer to become active") + return false, nil + } + logrus.Debug("Load balancer active") + return true, nil + }) + return err +} diff --git a/test/extended/util/client.go b/test/extended/util/client.go index 7991d726e9d8..7913c6d1c0ae 100644 --- a/test/extended/util/client.go +++ b/test/extended/util/client.go @@ -180,6 +180,32 @@ func NewCLIWithoutNamespace(project string) *CLI { return cli } +// NewCLIForMonitorTest initializes the CLI and Kube framework helpers +// without a namespace. Should be called outside of a Ginkgo .It() +// function. +func NewCLIForMonitorTest(project string) *CLI { + cli := &CLI{ + kubeFramework: &framework.Framework{ + SkipNamespaceCreation: true, + BaseName: project, + Options: framework.Options{ + ClientQPS: 20, + ClientBurst: 50, + }, + Timeouts: framework.NewTimeoutContext(), + }, + username: "admin", + execPath: "oc", + adminConfigPath: KubeConfigPath(), + staticConfigManifestDir: StaticConfigManifestDir(), + withoutNamespace: true, + } + + // Called only once (assumed the objects will never get modified) + cli.setupStaticConfigsFromManifests() + return cli +} + // NewHypershiftManagementCLI returns a CLI that interacts with the Hypershift management cluster. // Contrary to a normal CLI it does not perform any cleanup, and it must not be used for any mutating // operations. Also, contrary to a normal CLI it must be constructed inside an `It` block. This is @@ -818,6 +844,22 @@ func (c *CLI) Run(commands ...string) *CLI { return nc.setOutput(c.stdout) } +// Executes with the kubeconfig specified from the environment +func (c *CLI) RunInMonitorTest(commands ...string) *CLI { + in, out, errout := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} + nc := &CLI{ + execPath: c.execPath, + verb: commands[0], + kubeFramework: c.KubeFramework(), + adminConfigPath: c.adminConfigPath, + configPath: c.configPath, + username: c.username, + globalArgs: commands, + } + nc.stdin, nc.stdout, nc.stderr = in, out, errout + return nc.setOutput(c.stdout) +} + // InputString adds expected input to the command func (c *CLI) InputString(input string) *CLI { c.stdin.WriteString(input)