Support multiple port ranges (#3747)

* feat: add port range allocation
* feat: update install
* feat: update docs

build/Makefile

@@ -67,7 +67,7 @@ GS_TEST_IMAGE ?= us-docker.pkg.dev/agones-images/examples/simple-game-server:0.3
 BETA_FEATURE_GATES ?= "CountsAndLists=true&DisableResyncOnSDKServer=true"
 
 # Enable all alpha feature gates. Keep in sync with `false` (alpha) entries in pkg/util/runtime/features.go:featureDefaults
-ALPHA_FEATURE_GATES ?= "PlayerAllocationFilter=true&PlayerTracking=true&RollingUpdateFix=true&Example=true"
+ALPHA_FEATURE_GATES ?= "PlayerAllocationFilter=true&PlayerTracking=true&RollingUpdateFix=true&PortRanges=true&Example=true"
 
 # Build with Windows support
 WITH_WINDOWS=1

cloudbuild.yaml

@@ -276,7 +276,7 @@ steps:
  declare -A versionsAndRegions=( [1.27]=us-east1 [1.28]=us-west1 [1.29]=europe-west1 )
 
  # Keep in sync with (the inverse of) pkg/util/runtime/features.go:featureDefaults
- featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&CountsAndLists=false&RollingUpdateFix=true&DisableResyncOnSDKServer=false&Example=true"
+ featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&CountsAndLists=false&RollingUpdateFix=true&PortRanges=true&DisableResyncOnSDKServer=false&Example=true"
  featureWithoutGate=""
 
  # Use this if specific feature gates can only be supported on specific Kubernetes versions.

cmd/controller/main.go

@@ -17,13 +17,17 @@ package main
 
 import (
  "context"
+ "encoding/json"
+ "fmt"
  "io"
  "net/http"
  "os"
  "path/filepath"
  "strings"
  "time"
 
+ agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
+ "agones.dev/agones/pkg/portallocator"
  "github.com/google/uuid"
  "github.com/heptiolabs/healthcheck"
  "github.com/pkg/errors"
@@ -68,6 +72,7 @@ const (
  pullSidecarFlag = "always-pull-sidecar"
  minPortFlag = "min-port"
  maxPortFlag = "max-port"
+ additionalPortRangesFlag = "additional-port-ranges"
  certFileFlag = "cert-file"
  keyFileFlag = "key-file"
  numWorkersFlag = "num-workers"
@@ -207,7 +212,7 @@ func main() {
  gsCounter := gameservers.NewPerNodeCounter(kubeInformerFactory, agonesInformerFactory)
 
  gsController := gameservers.NewController(controllerHooks, health,
- ctlConf.MinPort, ctlConf.MaxPort, ctlConf.SidecarImage, ctlConf.AlwaysPullSidecar,
+ ctlConf.PortRanges, ctlConf.SidecarImage, ctlConf.AlwaysPullSidecar,
  ctlConf.SidecarCPURequest, ctlConf.SidecarCPULimit,
  ctlConf.SidecarMemoryRequest, ctlConf.SidecarMemoryLimit, ctlConf.SdkServiceAccount,
  kubeClient, kubeInformerFactory, extClient, agonesClient, agonesInformerFactory)
@@ -282,7 +287,8 @@ func parseEnvFlags() config {
  pflag.Bool(pullSidecarFlag, viper.GetBool(pullSidecarFlag), "For development purposes, set the sidecar image to have a ImagePullPolicy of Always. Can also use ALWAYS_PULL_SIDECAR env variable")
  pflag.String(sdkServerAccountFlag, viper.GetString(sdkServerAccountFlag), "Overwrite what service account default for GameServer Pods. Defaults to Can also use SDK_SERVICE_ACCOUNT")
  pflag.Int32(minPortFlag, 0, "Required. The minimum port that that a GameServer can be allocated to. Can also use MIN_PORT env variable.")
- pflag.Int32(maxPortFlag, 0, "Required. The maximum port that that a GameServer can be allocated to. Can also use MAX_PORT env variable")
+ pflag.Int32(maxPortFlag, 0, "Required. The maximum port that that a GameServer can be allocated to. Can also use MAX_PORT env variable.")
+ pflag.String(additionalPortRangesFlag, viper.GetString(additionalPortRangesFlag), `Optional. Named set of port ranges in JSON object format: '{"game": [5000, 6000]}'. Can Also use ADDITIONAL_PORT_RANGES env variable.`)
  pflag.String(keyFileFlag, viper.GetString(keyFileFlag), "Optional. Path to the key file")
  pflag.String(certFileFlag, viper.GetString(certFileFlag), "Optional. Path to the crt file")
  pflag.String(kubeconfigFlag, viper.GetString(kubeconfigFlag), "Optional. kubeconfig to run the controller out of the cluster. Only use it for debugging as webhook won't works.")
@@ -313,6 +319,7 @@ func parseEnvFlags() config {
  runtime.Must(viper.BindEnv(sdkServerAccountFlag))
  runtime.Must(viper.BindEnv(minPortFlag))
  runtime.Must(viper.BindEnv(maxPortFlag))
+ runtime.Must(viper.BindEnv(additionalPortRangesFlag))
  runtime.Must(viper.BindEnv(keyFileFlag))
  runtime.Must(viper.BindEnv(certFileFlag))
  runtime.Must(viper.BindEnv(kubeconfigFlag))
@@ -355,9 +362,17 @@ func parseEnvFlags() config {
  logger.WithError(err).Fatalf("could not parse %s", sidecarMemoryLimitFlag)
  }
 
+ portRanges, err := parsePortRanges(viper.GetString(additionalPortRangesFlag))
+ if err != nil {
+ logger.WithError(err).Fatalf("could not parse %s", additionalPortRangesFlag)
+ }
+ portRanges[agonesv1.DefaultPortRange] = portallocator.PortRange{
+ MinPort: int32(viper.GetInt64(minPortFlag)),
+ MaxPort: int32(viper.GetInt64(maxPortFlag)),
+ }
+
  return config{
- MinPort: int32(viper.GetInt64(minPortFlag)),
- MaxPort: int32(viper.GetInt64(maxPortFlag)),
+ PortRanges: portRanges,
  SidecarImage: viper.GetString(sidecarImageFlag),
  SidecarCPURequest: requestCPU,
  SidecarCPULimit: limitCPU,
@@ -384,10 +399,32 @@ func parseEnvFlags() config {
  }
 }
 
+func parsePortRanges(s string) (map[string]portallocator.PortRange, error) {
+ if s == "" || !runtime.FeatureEnabled(runtime.FeaturePortRanges) {
+ return map[string]portallocator.PortRange{}, nil
+ }
+
+ prs := map[string][]int32{}
+ if err := json.Unmarshal([]byte(s), &prs); err != nil {
+ return nil, fmt.Errorf("invlaid additional port range format: %w", err)
+ }
+
+ portRanges := map[string]portallocator.PortRange{}
+ for k, v := range prs {
+ if len(v) != 2 {
+ return nil, fmt.Errorf("invalid port range ports for %s: requires both min and max port", k)
+ }
+ portRanges[k] = portallocator.PortRange{
+ MinPort: v[0],
+ MaxPort: v[1],
+ }
+ }
+ return portRanges, nil
+}
+
 // config stores all required configuration to create a game server controller.
 type config struct {
- MinPort int32
- MaxPort int32
+ PortRanges map[string]portallocator.PortRange
  SidecarImage string
  SidecarCPURequest resource.Quantity
  SidecarCPULimit resource.Quantity
@@ -416,12 +453,8 @@ type config struct {
 // validate ensures the ctlConfig data is valid.
 func (c *config) validate() []error {
  validationErrors := make([]error, 0)
- if c.MinPort <= 0 || c.MaxPort <= 0 {
- validationErrors = append(validationErrors, errors.New("min Port and Max Port values are required"))
- }
- if c.MaxPort < c.MinPort {
- validationErrors = append(validationErrors, errors.New("max Port cannot be set less that the Min Port"))
- }
+ portErrors := validatePorts(c.PortRanges)
+ validationErrors = append(validationErrors, portErrors...)
  resourceErrors := validateResource(c.SidecarCPURequest, c.SidecarCPULimit, corev1.ResourceCPU)
  validationErrors = append(validationErrors, resourceErrors...)
  resourceErrors = validateResource(c.SidecarMemoryRequest, c.SidecarMemoryLimit, corev1.ResourceMemory)
@@ -449,6 +482,61 @@ func validateResource(request resource.Quantity, limit resource.Quantity, resour
  return validationErrors
 }
 
+func validatePorts(portRanges map[string]portallocator.PortRange) []error {
+ validationErrors := make([]error, 0)
+ for k, r := range portRanges {
+ portErrors := validatePortRange(r.MinPort, r.MaxPort, k)
+ validationErrors = append(validationErrors, portErrors...)
+
+ }
+
+ if len(validationErrors) > 0 {
+ return validationErrors
+ }
+
+ keys := make([]string, 0, len(portRanges))
+ values := make([]portallocator.PortRange, 0, len(portRanges))
+ for k, v := range portRanges {
+ keys = append(keys, k)
+ values = append(values, v)
+ }
+
+ for i, pr := range values {
+ for j := i + 1; j < len(values); j++ {
+ if overlaps(values[j].MinPort, values[j].MaxPort, pr.MinPort, pr.MaxPort) {
+ switch {
+ case keys[j] == agonesv1.DefaultPortRange:
+ validationErrors = append(validationErrors, errors.Errorf("port range %s overlaps with min/max port", keys[i]))
+ case keys[i] == agonesv1.DefaultPortRange:
+ validationErrors = append(validationErrors, errors.Errorf("port range %s overlaps with min/max port", keys[j]))
+ default:
+ validationErrors = append(validationErrors, errors.Errorf("port range %s overlaps with min/max port of range %s", keys[i], keys[j]))
+ }
+ }
+ }
+ }
+ return validationErrors
+}
+
+func validatePortRange(minPort, maxPort int32, rangeName string) []error {
+ validationErrors := make([]error, 0)
+ var rangeCtx string
+ if rangeName != agonesv1.DefaultPortRange {
+ rangeCtx = " for port range " + rangeName
+ }
+ if minPort <= 0 || maxPort <= 0 {
+ validationErrors = append(validationErrors, errors.New("min Port and Max Port values are required"+rangeCtx))
+ }
+ if maxPort < minPort {
+ validationErrors = append(validationErrors, errors.New("max Port cannot be set less that the Min Port"+rangeCtx))
+ }
+ return validationErrors
+}
+
+func overlaps(minA, maxA, minB, maxB int32) bool {
+ return max(minA, minB) < min(maxA, maxB)
+}
+
 type runner interface {
  Run(ctx context.Context, workers int) error
 }

cmd/controller/main_test.go

@@ -17,39 +17,63 @@ package main
 import (
  "testing"
 
+ agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
+ "agones.dev/agones/pkg/portallocator"
  "github.com/stretchr/testify/assert"
  "k8s.io/apimachinery/pkg/api/resource"
 )
 
 func TestControllerConfigValidation(t *testing.T) {
  t.Parallel()
 
- c := config{MinPort: 10, MaxPort: 2}
+ c := config{PortRanges: map[string]portallocator.PortRange{
+ agonesv1.DefaultPortRange: {MinPort: 10, MaxPort: 2},
+ }}
  errs := c.validate()
  assert.Len(t, errs, 1)
  errorsContainString(t, errs, "max Port cannot be set less that the Min Port")
 
+ c.PortRanges["game"] = portallocator.PortRange{MinPort: 20, MaxPort: 12}
+ errs = c.validate()
+ assert.Len(t, errs, 2)
+ errorsContainString(t, errs, "max Port cannot be set less that the Min Port for port range game")
+
  c.SidecarMemoryRequest = resource.MustParse("2Gi")
  c.SidecarMemoryLimit = resource.MustParse("1Gi")
  errs = c.validate()
- assert.Len(t, errs, 2)
+ assert.Len(t, errs, 3)
  errorsContainString(t, errs, "Request must be less than or equal to memory limit")
 
  c.SidecarMemoryLimit = resource.MustParse("2Gi")
  c.SidecarCPURequest = resource.MustParse("2m")
  c.SidecarCPULimit = resource.MustParse("1m")
  errs = c.validate()
- assert.Len(t, errs, 2)
+ assert.Len(t, errs, 3)
  errorsContainString(t, errs, "Request must be less than or equal to cpu limit")
 
  c.SidecarMemoryLimit = resource.MustParse("2Gi")
  c.SidecarCPURequest = resource.MustParse("-2m")
  c.SidecarCPULimit = resource.MustParse("2m")
  errs = c.validate()
- assert.Len(t, errs, 2)
+ assert.Len(t, errs, 3)
  errorsContainString(t, errs, "Resource cpu request value must be non negative")
 }
 
+func TestControllerConfigValidation_PortRangeOverlap(t *testing.T) {
+ t.Parallel()
+
+ c := config{
+ PortRanges: map[string]portallocator.PortRange{
+ agonesv1.DefaultPortRange: {MinPort: 10, MaxPort: 20},
+ "game": {MinPort: 15, MaxPort: 25},
+ "other": {MinPort: 21, MaxPort: 31},
+ },
+ }
+ errs := c.validate()
+ assert.Len(t, errs, 2)
+ errorsContainString(t, errs, "port range game overlaps with min/max port")
+}
+
 func errorsContainString(t *testing.T, errs []error, expected string) {
  found := false
  for _, v := range errs {

examples/gameserver.yaml

@@ -37,6 +37,11 @@ spec:
  ports:
  # name is a descriptive name for the port
  - name: default
+ # [Stage:Alpha]
+ # [FeatureFlag:PortRanges]
+ # range is the optional port range name from which to select a port when using a 'Dynamic' or 'Passthrough' port policy.
+ # Defaults to 'default'.
+ range: default
  # portPolicy has three options:
  # - "Dynamic" (default) the system allocates a free hostPort for the gameserver, for game clients to connect to
  # - "Static", user defines the hostPort that the game client will connect to. Then onus is on the user to ensure that the

go.mod

@@ -23,6 +23,7 @@ require (
  github.com/pkg/errors v0.9.1
  github.com/prometheus/client_golang v1.16.0
  github.com/sirupsen/logrus v1.9.0
+ github.com/spf13/cast v1.3.0
  github.com/spf13/pflag v1.0.5
  github.com/spf13/viper v1.7.0
  github.com/stretchr/testify v1.8.2
@@ -91,7 +92,6 @@ require (
  github.com/prometheus/procfs v0.10.1 // indirect
  github.com/russross/blackfriday/v2 v2.1.0 // indirect
  github.com/spf13/afero v1.9.2 // indirect
- github.com/spf13/cast v1.3.0 // indirect
  github.com/spf13/jwalterweatherman v1.0.0 // indirect
  github.com/subosito/gotenv v1.2.0 // indirect
  golang.org/x/crypto v0.21.0 // indirect

install/helm/agones/defaultfeaturegates.yaml

@@ -23,6 +23,7 @@ GKEAutopilotExtendedDurationPods: false
 PlayerAllocationFilter: false
 PlayerTracking: false
 RollingUpdateFix: false
+PortRanges: false
 
 # Dev features
 FeatureAutopilotPassthroughPort: true

install/helm/agones/templates/controller.yaml

@@ -110,6 +110,14 @@ spec:
  # maximum port that can be exposed to GameServer traffic
  - name: MAX_PORT
  value: {{ .Values.gameservers.maxPort | quote }}
+ {{- if .Values.gameservers.additionalPortRanges }}
+ {{- $featureGates := include "agones.featureGates" . | fromYaml }}
+ {{- if not $featureGates.PortRanges }}
+ {{- fail "gameservers.additionalPortRanges specified without feature gate PortRanges enabled!" }}
+ {{- end }}
+ - name: ADDITIONAL_PORT_RANGES
+ value: {{ .Values.gameservers.additionalPortRanges | toJson | quote }}
+ {{- end }}
  - name: SIDECAR_IMAGE # overwrite the GameServer sidecar image that is used
  value: "{{ .Values.agones.image.registry }}/{{ .Values.agones.image.sdk.name}}:{{ default .Values.agones.image.tag .Values.agones.image.sdk.tag }}"
  - name: ALWAYS_PULL_SIDECAR # set the sidecar imagePullPolicy to Always

install/helm/agones/templates/crds/_gameserverspecschema.yaml

@@ -52,6 +52,9 @@ properties:
  name:
  title: Name is the descriptive name of the port
  type: string
+ range:
+ title: the port range name from which to select a port when using a 'Dynamic' or 'Passthrough' port policy. Defaults to 'default'.
+ type: string
  portPolicy:
  title: the port policy that will be applied to the game server
  description: |

install/helm/agones/values.yaml

@@ -281,6 +281,9 @@ gameservers:
  - default
  minPort: 7000
  maxPort: 8000
+ additionalPortRanges: {}
+ # requires feature gate PortRanges to be enabled
+ # game: [9000, 10000]
  podPreserveUnknownFields: false
 
 helm:

install/yaml/install.yaml

@@ -5369,6 +5369,9 @@ spec:
  name:
  title: Name is the descriptive name of the port
  type: string
+ range:
+ title: the port range name from which to select a port when using a 'Dynamic' or 'Passthrough' port policy. Defaults to 'default'.
+ type: string
  portPolicy:
  title: the port policy that will be applied to the game server
  description: |
@@ -10791,6 +10794,9 @@ spec:
  name:
  title: Name is the descriptive name of the port
  type: string
+ range:
+ title: the port range name from which to select a port when using a 'Dynamic' or 'Passthrough' port policy. Defaults to 'default'.
+ type: string
  portPolicy:
  title: the port policy that will be applied to the game server
  description: |
@@ -16335,6 +16341,9 @@ spec:
  name:
  title: Name is the descriptive name of the port
  type: string
+ range:
+ title: the port range name from which to select a port when using a 'Dynamic' or 'Passthrough' port policy. Defaults to 'default'.
+ type: string
  portPolicy:
  title: the port policy that will be applied to the game server
  description: |