Flake: TestFleetUpdates (#3797)

This test is our biggest e2e flake, so seeing if I can mitigate this
with some extra buffer in retry, and also adding an extra test to
prove the rollout is also completed as expected.

test/e2e/fleet_test.go

@@ -719,36 +719,45 @@ func TestFleetUpdates(t *testing.T) {
  t.Run(k, func(t *testing.T) {
  t.Parallel()
  client := framework.AgonesClient.AgonesV1()
+ log := e2e.TestLogger(t)
 
  flt := v()
  flt.Spec.Template.ObjectMeta.Annotations = map[string]string{key: red}
  flt, err := client.Fleets(framework.Namespace).Create(ctx, flt, metav1.CreateOptions{})
  require.NoError(t, err)
  defer client.Fleets(framework.Namespace).Delete(ctx, flt.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck
 
+ // gate that we have the keys we expect.
  err = framework.WaitForFleetGameServersCondition(flt, func(gs *agonesv1.GameServer) bool {
  return gs.ObjectMeta.Annotations[key] == red
  })
  require.NoError(t, err)
 
  // if the generation has been updated, it's time to try again.
- err = wait.PollUntilContextTimeout(context.Background(), time.Second, 10*time.Second, true, func(ctx context.Context) (bool, error) {
+ err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) {
  flt, err = framework.AgonesClient.AgonesV1().Fleets(framework.Namespace).Get(ctx, flt.ObjectMeta.Name, metav1.GetOptions{})
  if err != nil {
+ log.WithError(err).WithField("flt", flt.ObjectMeta.Name).Warn("Could not retrieve fleet, trying again")
  return false, err
  }
  fltCopy := flt.DeepCopy()
  fltCopy.Spec.Template.ObjectMeta.Annotations[key] = green
  _, err = framework.AgonesClient.AgonesV1().Fleets(framework.Namespace).Update(ctx, fltCopy, metav1.UpdateOptions{})
  if err != nil {
- logrus.WithError(err).Warn("Could not update fleet, trying again")
+ log.WithError(err).WithField("flt", flt.ObjectMeta.Name).Warn("Could not update fleet, trying again")
  return false, nil
  }
 
  return true, nil
  })
  require.NoError(t, err)
 
+ // let's make sure we're fully Ready
+ framework.AssertFleetCondition(t, flt, func(entry *logrus.Entry, fleet *agonesv1.Fleet) bool {
+ return flt.Spec.Replicas == fleet.Status.ReadyReplicas
+ })
+
+ // ...and fully rolled out
  err = framework.WaitForFleetGameServersCondition(flt, func(gs *agonesv1.GameServer) bool {
  return gs.ObjectMeta.Annotations[key] == green
  })