feat(gha-runner-scale-set): ability to scale up runners with scaleUpFactor option

apis/actions.github.com/v1alpha1/autoscalinglistener_types.go

@@ -49,6 +49,9 @@ type AutoscalingListenerSpec struct {
 	// +kubebuilder:validation:Minimum:=0
 	MinRunners int `json:"minRunners,omitempty"`
 
+	// +optional
+	ScaleUpFactor string `json:"scaleUpFactor,omitempty"`
+
 	// Required
 	Image string `json:"image,omitempty"`
 

apis/actions.github.com/v1alpha1/autoscalingrunnerset_types.go

@@ -84,6 +84,9 @@ type AutoscalingRunnerSetSpec struct {
 	// +optional
 	// +kubebuilder:validation:Minimum:=0
 	MinRunners *int `json:"minRunners,omitempty"`
+
+	// +optional
+	ScaleUpFactor string `json:"scaleUpFactor,omitempty"`
 }
 
 type GitHubServerTLSConfig struct {

charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalinglisteners.yaml

@@ -143,6 +143,8 @@ spec:
                 runnerScaleSetId:
                   description: Required
                   type: integer
+                scaleUpFactor:
+                  type: string
                 template:
                   description: PodTemplateSpec describes the data a pod should have when created from a template
                   properties:

charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalingrunnersets.yaml

@@ -6983,6 +6983,8 @@ spec:
                   type: string
                 runnerScaleSetName:
                   type: string
+                scaleUpFactor:
+                  type: string
                 template:
                   description: Required
                   properties:

charts/gha-runner-scale-set/templates/autoscalingrunnerset.yaml

@@ -88,6 +88,9 @@ spec:
     {{- end }}
   minRunners: {{ .Values.minRunners | int }}
   {{- end }}
+  {{- if .Values.scaleUpFactor }}
+  scaleUpFactor: {{ quote .Values.scaleUpFactor }}
+  {{- end }}
 
   {{- with .Values.listenerTemplate}}
   listenerTemplate:

charts/gha-runner-scale-set/tests/template_test.go

@@ -661,6 +661,41 @@ func TestTemplateRenderedAutoScalingRunnerSet_MinMaxRunners_FromValuesFile(t *te
 
 	assert.Equal(t, 5, *ars.Spec.MinRunners, "MinRunners should be 5")
 	assert.Equal(t, 10, *ars.Spec.MaxRunners, "MaxRunners should be 10")
+	assert.Equal(t, "1.2", ars.Spec.ScaleUpFactor)
+}
+
+func TestTemplateRenderedAutoScalingRunnerSet_WithScaleUpFactor(t *testing.T) {
+	t.Parallel()
+
+	// Path to the helm chart we will test
+	helmChartPath, err := filepath.Abs("../../gha-runner-scale-set")
+	require.NoError(t, err)
+
+	releaseName := "test-runners"
+	namespaceName := "test-" + strings.ToLower(random.UniqueId())
+
+	options := &helm.Options{
+		Logger: logger.Discard,
+		SetValues: map[string]string{
+			"githubConfigUrl":                    "https://github.com/actions",
+			"githubConfigSecret.github_token":    "gh_token12345",
+			"minRunners":                         "5",
+			"scaleUpFactor":                      "1.2",
+			"controllerServiceAccount.name":      "arc",
+			"controllerServiceAccount.namespace": "arc-system",
+		},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/autoscalingrunnerset.yaml"})
+
+	var ars v1alpha1.AutoscalingRunnerSet
+	helm.UnmarshalK8SYaml(t, output, &ars)
+
+	require.NotNil(t, ars.Spec.ScaleUpFactor)
+	assert.Equal(t, 5, *ars.Spec.MinRunners, "MinRunners should be 5")
+	assert.Nil(t, ars.Spec.MaxRunners, "MaxRunners should be nil")
+	assert.Equal(t, "1.2", ars.Spec.ScaleUpFactor)
 }
 
 func TestTemplateRenderedAutoScalingRunnerSet_ExtraVolumes(t *testing.T) {

charts/gha-runner-scale-set/tests/values.yaml

@@ -3,6 +3,7 @@ githubConfigSecret:
   github_token: test
 maxRunners: 10
 minRunners: 5
+scaleUpFactor: "1.2"
 controllerServiceAccount:
   name: "arc"
   namespace: "arc-system"

charts/gha-runner-scale-set/values.yaml

@@ -43,6 +43,9 @@ githubConfigSecret:
 ## calculated as a sum of minRunners and the number of jobs assigned to the scale set.
 # minRunners: 0
 
+## TODO:
+# scaleUpFactor: "1.2"
+
 # runnerGroup: "default"
 
 ## name of the runner scale set to create.  Defaults to the helm release name

cmd/ghalistener/app/app.go

@@ -78,6 +78,7 @@ func New(config config.Config) (*App, error) {
 			EphemeralRunnerSetName:      config.EphemeralRunnerSetName,
 			MaxRunners:                  config.MaxRunners,
 			MinRunners:                  config.MinRunners,
+			ScaleUpFactor:               config.ScaleUpFactor,
 		},
 		worker.WithLogger(app.logger.WithName("worker")),
 	)

cmd/ghalistener/config/config.go

@@ -25,6 +25,7 @@ type Config struct {
 	EphemeralRunnerSetName      string `json:"ephemeralRunnerSetName"`
 	MaxRunners                  int    `json:"maxRunners"`
 	MinRunners                  int    `json:"minRunners"`
+	ScaleUpFactor               string `json:"scaleUpFactor"`
 	RunnerScaleSetId            int    `json:"runnerScaleSetId"`
 	RunnerScaleSetName          string `json:"runnerScaleSetName"`
 	ServerRootCA                string `json:"serverRootCA"`

cmd/ghalistener/worker/worker.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"math"
+	"strconv"
 
 	"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
 	"github.com/actions/actions-runner-controller/cmd/ghalistener/listener"
@@ -33,6 +35,7 @@ type Config struct {
 	EphemeralRunnerSetName      string
 	MaxRunners                  int
 	MinRunners                  int
+	ScaleUpFactor               string
 }
 
 // The Worker's role is to process the messages it receives from the listener.
@@ -48,6 +51,9 @@ type Worker struct {
 var _ listener.Handler = (*Worker)(nil)
 
 func New(config Config, options ...Option) (*Worker, error) {
+	if config.ScaleUpFactor == "" {
+		config.ScaleUpFactor = "1"
+	}
 	w := &Worker{
 		config:    config,
 		lastPatch: -1,
@@ -225,7 +231,16 @@ func (w *Worker) HandleDesiredRunnerCount(ctx context.Context, count, jobsComple
 func (w *Worker) setDesiredWorkerState(count, jobsCompleted int) int {
 	// Max runners should always be set by the resource builder either to the configured value,
 	// or the maximum int32 (resourcebuilder.newAutoScalingListener()).
-	targetRunnerCount := min(w.config.MinRunners+count, w.config.MaxRunners)
+	if w.config.ScaleUpFactor == "" {
+		w.config.ScaleUpFactor = "1"
+	}
+	scaleUpFactor, err := strconv.ParseFloat(w.config.ScaleUpFactor, 64)
+	if err != nil {
+		w.logger.Error(err, "validating autoscaling spec.scaleUpFactor cannot be parsed into a float64")
+		return 0
+	}
+	desiredRunners := w.config.MinRunners + int(math.Ceil(float64(count)*scaleUpFactor))
+	targetRunnerCount := min(desiredRunners, w.config.MaxRunners)
 	w.patchSeq++
 	desiredPatchID := w.patchSeq
 
@@ -251,6 +266,7 @@ func (w *Worker) setDesiredWorkerState(count, jobsCompleted int) int {
 		"max", w.config.MaxRunners,
 		"currentRunnerCount", w.lastPatch,
 		"jobsCompleted", jobsCompleted,
+		"scaleUpFactor", scaleUpFactor,
 	)
 
 	return desiredPatchID

cmd/ghalistener/worker/worker_test.go

@@ -324,3 +324,84 @@ func TestSetDesiredWorkerState_MinMaxSet(t *testing.T) {
 		assert.Equal(t, 2, w.patchSeq)
 	})
 }
+
+func TestSetDesiredWorkerState_ScaleUpFactorSet(t *testing.T) {
+	logger := logr.Discard()
+	newEmptyWorker := func() *Worker {
+		return &Worker{
+			config: Config{
+				MinRunners:    1,
+				MaxRunners:    10,
+				ScaleUpFactor: "1.5",
+			},
+			lastPatch: -1,
+			patchSeq:  -1,
+			logger:    &logger,
+		}
+	}
+
+	t.Run("initial scale when acquired == 0 and completed == 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+	})
+
+	t.Run("re-use the old state on count == 0 and completed == 1", func(t *testing.T) {
+		// Scales up 1.5 times 1+ceil(2*1.5) = 4
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(2, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 4, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("scale to min when count == 0", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(2, 0)
+		assert.Equal(t, 0, patchID)
+		patchID = w.setDesiredWorkerState(0, 1)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+	})
+
+	t.Run("scale up to max when count > max", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(6, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 10, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+	})
+
+	t.Run("scale to max when count == max", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(3, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 6, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+	})
+
+	t.Run("force 0 on empty batch and last patch == min runners", func(t *testing.T) {
+		w := newEmptyWorker()
+		patchID := w.setDesiredWorkerState(3, 0)
+		assert.Equal(t, 0, patchID)
+		assert.Equal(t, 6, w.lastPatch)
+		assert.Equal(t, 0, w.patchSeq)
+
+		patchID = w.setDesiredWorkerState(0, 3)
+		assert.Equal(t, 1, patchID)
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 1, w.patchSeq)
+
+		// Empty batch on min runners
+		patchID = w.setDesiredWorkerState(0, 0)
+		assert.Equal(t, 0, patchID) // forcing the state
+		assert.Equal(t, 1, w.lastPatch)
+		assert.Equal(t, 2, w.patchSeq)
+	})
+
+}

cmd/githubrunnerscalesetlistener/config/config.go

@@ -16,6 +16,7 @@ type Config struct {
 	EphemeralRunnerSetName      string `json:"ephemeralRunnerSetName"`
 	MaxRunners                  int    `json:"maxRunners"`
 	MinRunners                  int    `json:"minRunners"`
+	ScaleUpFactor               string `json:"scaleUpFactor"`
 	RunnerScaleSetId            int    `json:"runnerScaleSetId"`
 	RunnerScaleSetName          string `json:"runnerScaleSetName"`
 	ServerRootCA                string `json:"serverRootCA"`

config/crd/bases/actions.github.com_autoscalinglisteners.yaml

@@ -117,6 +117,10 @@ spec:
                   description: Required
                   minimum: 0
                   type: integer
+                scaleUpFactor:
+                  description: "TODO:"
+                  default: "1"
+                  type: string
                 proxy:
                   properties:
                     http:

config/crd/bases/actions.github.com_autoscalingrunnersets.yaml

@@ -6956,6 +6956,10 @@ spec:
                 minRunners:
                   minimum: 0
                   type: integer
+                scaleUpFactor:
+                  description: "TODO:"
+                  default: "1"
+                  type: string
                 proxy:
                   properties:
                     http:

controllers/actions.github.com/resourcebuilder.go

@@ -81,12 +81,24 @@ func (b *ResourceBuilder) newAutoScalingListener(autoscalingRunnerSet *v1alpha1.
 
 	effectiveMinRunners := 0
 	effectiveMaxRunners := math.MaxInt32
+	// TODO:
+	scaleUpFactor := "1"
 	if autoscalingRunnerSet.Spec.MaxRunners != nil {
 		effectiveMaxRunners = *autoscalingRunnerSet.Spec.MaxRunners
 	}
 	if autoscalingRunnerSet.Spec.MinRunners != nil {
 		effectiveMinRunners = *autoscalingRunnerSet.Spec.MinRunners
 	}
+	if autoscalingRunnerSet.Spec.ScaleUpFactor != "" {
+		suf, err := strconv.ParseFloat(autoscalingRunnerSet.Spec.ScaleUpFactor, 64)
+		if err != nil {
+			return nil, fmt.Errorf("Validation autoscaling spec.scaleUpFactor cannot be parsed into a float64: %v", err)
+		}
+		// TODO: 0 or 1
+		if suf > 0 {
+			scaleUpFactor = autoscalingRunnerSet.Spec.ScaleUpFactor
+		}
+	}
 
 	labels := b.mergeLabels(autoscalingRunnerSet.Labels, map[string]string{
 		LabelKeyGitHubScaleSetNamespace: autoscalingRunnerSet.Namespace,
@@ -121,6 +133,7 @@ func (b *ResourceBuilder) newAutoScalingListener(autoscalingRunnerSet *v1alpha1.
 			EphemeralRunnerSetName:        ephemeralRunnerSet.Name,
 			MinRunners:                    effectiveMinRunners,
 			MaxRunners:                    effectiveMaxRunners,
+			ScaleUpFactor:                 scaleUpFactor,
 			Image:                         image,
 			ImagePullSecrets:              imagePullSecrets,
 			Proxy:                         autoscalingRunnerSet.Spec.Proxy,
@@ -191,6 +204,7 @@ func (b *ResourceBuilder) newScaleSetListenerConfig(autoscalingListener *v1alpha
 		EphemeralRunnerSetName:      autoscalingListener.Spec.EphemeralRunnerSetName,
 		MaxRunners:                  autoscalingListener.Spec.MaxRunners,
 		MinRunners:                  autoscalingListener.Spec.MinRunners,
+		ScaleUpFactor:               autoscalingListener.Spec.ScaleUpFactor,
 		RunnerScaleSetId:            autoscalingListener.Spec.RunnerScaleSetId,
 		RunnerScaleSetName:          autoscalingListener.Spec.AutoscalingRunnerSetName,
 		ServerRootCA:                cert,