Add option for minimal reboot period

cmd/kured/main.go

@@ -84,11 +84,12 @@ var (
  nodeID string
  concurrency int
 
- rebootDays []string
- rebootStart string
- rebootEnd string
- timezone string
- annotateNodes bool
+ rebootDays []string
+ rebootStart string
+ rebootEnd string
+ timezone string
+ minRebootPeriod time.Duration
+ annotateNodes bool
 
  // Metrics
  rebootRequiredGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
@@ -105,6 +106,8 @@ const (
  KuredRebootInProgressAnnotation string = "weave.works/kured-reboot-in-progress"
  // KuredMostRecentRebootNeededAnnotation is the canonical string value for the kured most-recent-reboot-needed annotation
  KuredMostRecentRebootNeededAnnotation string = "weave.works/kured-most-recent-reboot-needed"
+ // KuredLastSuccessfulRebootAnnotation is the canonical string value for the kured last-successful-reboot annotation
+ KuredLastSuccessfulRebootAnnotation string = "weave.works/kured-last-successful-reboot"
  // EnvPrefix The environment variable prefix of all environment variables bound to our command line flags.
  EnvPrefix = "KURED"
 
@@ -135,7 +138,8 @@ func NewRootCommand() *cobra.Command {
  Short: "Kubernetes Reboot Daemon",
  PersistentPreRunE: bindViper,
  PreRun: flagCheck,
- Run: root}
+ Run: root,
+ }
 
  rootCmd.PersistentFlags().StringVar(&nodeID, "node-id", "",
  "node name kured runs on, should be passed down from spec.nodeName via KURED_NODE_ID environment variable")
@@ -218,6 +222,8 @@ func NewRootCommand() *cobra.Command {
  "schedule reboot only before this time of day")
  rootCmd.PersistentFlags().StringVar(&timezone, "time-zone", "UTC",
  "use this timezone for schedule inputs")
+ rootCmd.PersistentFlags().DurationVar(&minRebootPeriod, "min-reboot-period", 0,
+ "the minimal duration between reboots of a node. Requires --annotate-nodes")
 
  rootCmd.PersistentFlags().BoolVar(&annotateNodes, "annotate-nodes", false,
  "if set, the annotations 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' will be given to nodes undergoing kured reboots")
@@ -265,6 +271,9 @@ func flagCheck(cmd *cobra.Command, args []string) {
  if !reflect.DeepEqual(preRebootNodeLabelKeys, postRebootNodeLabelKeys) {
  log.Warnf("pre-reboot-node-labels keys and post-reboot-node-labels keys do not match. This may result in unexpected behaviour.")
  }
+ if !annotateNodes && minRebootPeriod != 0 {
+ log.Warn("Cannot use --min-reboot-period without --annotate-nodes. This will ignore the min-reboot-period value")
+ }
 }
 
 // stripQuotes removes any literal single or double quote chars that surround a string
@@ -317,7 +326,6 @@ func flagToEnvVar(flag string) string {
 // buildHostCommand writes a new command to run in the host namespace
 // Rancher based need different pid
 func buildHostCommand(pid int, command []string) []string {
-
  // From the container, we nsenter into the proper PID to run the hostCommand.
  // For this, kured daemonset need to be configured with hostPID:true and privileged:true
  cmd := []string{"/usr/bin/nsenter", fmt.Sprintf("-m/proc/%d/ns/mnt", pid), "--"}
@@ -400,7 +408,8 @@ func (kb KubernetesBlockingChecker) isBlocked() bool {
  podList, err := kb.client.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{
  LabelSelector: labelSelector,
  FieldSelector: fieldSelector,
- Limit: 10})
+ Limit: 10,
+ })
  if err != nil {
  log.Warnf("Reboot blocked: pod query error: %v", err)
  return true
@@ -694,6 +703,11 @@ func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []str
  continue
  }
  }
+ if minRebootPeriod != 0 {
+ if err := addNodeAnnotations(client, nodeID, map[string]string{KuredLastSuccessfulRebootAnnotation: time.Now().Format(time.RFC3339)}); err != nil {
+ continue
+ }
+ }
  }
  throttle(releaseDelay)
  release(lock, concurrency > 1)
@@ -725,16 +739,26 @@ func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []str
  continue
  }
 
+ node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+ if err != nil {
+ log.Fatalf("Error retrieving node object via k8s API: %v", err)
+ }
+
+ if minRebootPeriod > 0 {
+ if t, err := nextAllowedReboot(node); err != nil {
+ log.Warnf("Failed to determine next allowed reboot time: %s", err.Error())
+ } else if diff := t.Sub(time.Now()); diff > 0 {
+ log.Infof("Reboot not allowed until %s (%s)", t.String(), diff.String())
+ continue
+ }
+ }
+
  if !rebootRequired(sentinelCommand) {
  log.Infof("Reboot not required")
  preferNoScheduleTaint.Disable()
  continue
  }
 
- node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
- if err != nil {
- log.Fatalf("Error retrieving node object via k8s API: %v", err)
- }
  nodeMeta.Unschedulable = node.Spec.Unschedulable
 
  var timeNowString string
@@ -804,6 +828,17 @@ func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []str
  }
 }
 
+func nextAllowedReboot(node *v1.Node) (time.Time, error) {
+ if v, ok := node.Annotations[KuredLastSuccessfulRebootAnnotation]; ok {
+ t, err := time.Parse(time.RFC3339, v)
+ if err != nil {
+ return time.Time{}, fmt.Errorf("failed to parse time %q in annotation %q: %w", v, KuredLastSuccessfulRebootAnnotation, err)
+ }
+ return t.Add(minRebootPeriod), nil
+ }
+ return time.Time{}, fmt.Errorf("missing annotation %s", KuredLastSuccessfulRebootAnnotation)
+}
+
 // buildSentinelCommand creates the shell command line which will need wrapping to escape
 // the container boundaries
 func buildSentinelCommand(rebootSentinelFile string, rebootSentinelCommand string) []string {

kured-ds-signal.yaml

@@ -98,3 +98,4 @@ spec:
 # - --annotate-nodes=false
 # - --lock-release-delay=30m
 # - --log-format=text
+# - --min-reboot-period=336h

kured-ds.yaml

@@ -100,3 +100,4 @@ spec:
 # - --metrics-host=""
 # - --metrics-port=8080
 # - --concurrency=1
+# - --min-reboot-period=336h