test(main): add test for k3s (labring#3875)

* test(main): add test for k3s Signed-off-by: cuisongliu <[email protected]> * test(main): add test for k3s Signed-off-by: cuisongliu <[email protected]> --------- Signed-off-by: cuisongliu <[email protected]>
sealos-ci-robot · Sep 10, 2023 · 7659046 · 7659046
1 parent 9b2eced
commit 7659046
Show file tree

Hide file tree

Showing 9 changed files with 264 additions and 87 deletions.
diff --git a/.github/workflows/e2e_test_core.yml b/.github/workflows/e2e_test_core.yml
@@ -3,7 +3,7 @@ name: E2E Sealos Core Test
 on:
  workflow_dispatch:
  push:
- branches: [ "main" ]
+ branches: [ "**" ]
  paths:
  - ".github/workflows/e2e_test_core.yml"
  - "cmd/**"

diff --git a/.github/workflows/e2e_test_core_k3s.yml b/.github/workflows/e2e_test_core_k3s.yml
@@ -0,0 +1,77 @@
+name: E2E Sealos Core Test for K3s
+
+on:
+ workflow_dispatch:
+ push:
+ branches: [ "**" ]
+ paths:
+ - ".github/workflows/e2e_test_core_k3s.yml"
+ - "cmd/**"
+ - "pkg/**"
+ - "test/**"
+ pull_request:
+ branches: [ "*" ]
+ paths:
+ - ".github/workflows/e2e_test_core_k3s.yml"
+ - "cmd/**"
+ - "pkg/**"
+ - "test/**"
+
+
+jobs:
+ call_ci_workflow:
+ uses: ./.github/workflows/import-patch-image.yml
+ with:
+ arch: amd64
+ e2e: true
+ image: false
+ e2e-core-test:
+ needs: [ call_ci_workflow ]
+ strategy:
+ fail-fast: false
+ matrix:
+ unit:
+ [
+ E2E_sealos_k3s_basic_test,
+ ]
+ runs-on: ubuntu-20.04
+ steps:
+ - name: Download image-cri-shim
+ uses: actions/download-artifact@v3
+ with:
+ name: image-cri-shim-amd64
+ path: /tmp/
+ - name: Download sealctl
+ uses: actions/download-artifact@v3
+ with:
+ name: sealctl-amd64
+ path: /tmp/
+ - name: Download sealos
+ uses: actions/download-artifact@v3
+ with:
+ name: sealos-amd64
+ path: /tmp/
+ - name: Download e2e test
+ uses: actions/download-artifact@v3
+ with:
+ name: e2e.test
+ path: /tmp/
+ - name: Verify sealos
+ run: |
+ sudo chmod a+x /tmp/{sealos,image-cri-shim,sealctl}
+ sudo mv /tmp/sealos /usr/bin/
+ sudo sealos version
+ - name: Remove containerd && docker
+ uses: labring/[email protected]
+ with:
+ type: prune
+
+ - name: Verify E2e test
+ run: |
+ sudo apt-get remove docker docker-engine docker.io containerd runc
+ sudo apt-get purge docker-ce docker-ce-cli containerd.io # docker-compose-plugin
+ sudo apt-get remove -y moby-engine moby-cli moby-buildx moby-compose
+ sudo rm -rf /var/run/docker.sock
+ sudo rm -rf /run/containerd/containerd.sock
+ sudo chmod a+x /tmp/e2e.test
+ sudo /tmp/e2e.test --ginkgo.v --ginkgo.focus="${{ matrix.unit }}"
diff --git a/cmd/sealos/cmd/cert.go b/cmd/sealos/cmd/cert.go
@@ -81,6 +81,7 @@ func newCertCmd() *cobra.Command {
  return fmt.Errorf("create runtime failed: %v", err)
  }
  if cm, ok := rt.(runtime.CertManager); ok {
+ logger.Info("using %s cert update implement", cf.GetCluster().GetDistribution())
  return cm.UpdateCertSANs(altNames)
  }
  return nil

diff --git a/pkg/runtime/kubernetes/certs.go b/pkg/runtime/kubernetes/certs.go
@@ -37,6 +37,10 @@ const (
  KubeletConf = "kubelet.conf"
 )
 
+func (k *KubeadmRuntime) Renew() error {
+ return errors.New("not implement")
+}
+
 func (k *KubeadmRuntime) UpdateCertSANs(certSans []string) error {
  // set extra cert SANs for kubeadm configmap object
  if err := k.CompleteKubeadmConfig(setCGroupDriverAndSocket, setCertificateKey); err != nil {

diff --git a/test/e2e/cert_test.go b/test/e2e/cert_test.go
@@ -20,6 +20,8 @@ import (
  "fmt"
  "time"
 
+ "github.com/labring/sealos/test/e2e/testhelper/cmd"
+
  "github.com/labring/sealos/test/e2e/testhelper/utils"
 
  "github.com/labring/sealos/test/e2e/suites/operators"
@@ -36,6 +38,7 @@ var _ = Describe("E2E_sealos_cert_test", func() {
  err error
  )
  fakeClient = operators.NewFakeClient("")
+ cmd.SetDebug()
  BeforeEach(func() {
  images := []string{"labring/kubernetes:v1.25.0", "labring/helm:v3.8.2", "labring/calico:v3.24.1"}
  err = fakeClient.Cluster.Run(images...)

diff --git a/test/e2e/k3s_basic_test.go b/test/e2e/k3s_basic_test.go
@@ -0,0 +1,122 @@
+/*
+Copyright 2023 [email protected].
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "errors"
+ "fmt"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/google/go-containerregistry/pkg/name"
+
+ v1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/util/yaml"
+
+ "github.com/labring/sealos/pkg/utils/logger"
+
+ "github.com/labring/sealos/test/e2e/suites/operators"
+ "github.com/labring/sealos/test/e2e/testhelper/config"
+ "github.com/labring/sealos/test/e2e/testhelper/utils"
+
+ . "github.com/onsi/ginkgo/v2"
+)
+
+var _ = Describe("E2E_sealos_k3s_basic_test", func() {
+ var (
+ fakeClient *operators.FakeClient
+ err error
+ )
+ fakeClient = operators.NewFakeClient("")
+
+ Context("sealos k3s suit", func() {
+ BeforeEach(func() {
+ By("build rootfs")
+ dFile := config.RootfsDockerfile{
+ BaseImage: "labring/k3s:latest",
+ Copys: []string{"sealctl opt/"},
+ }
+ tmpdir, err := dFile.Write()
+ utils.CheckErr(err, fmt.Sprintf("failed to create dockerfile: %v", err))
+
+ By("copy sealctl to rootfs")
+ err = fakeClient.CmdInterface.Copy("/tmp/sealctl", path.Join(tmpdir, "sealctl"))
+ utils.CheckErr(err, fmt.Sprintf("failed to copy sealctl to rootfs: %v", err))
+
+ By("build image")
+ err = fakeClient.Image.BuildImage("k3s:buildin", tmpdir, operators.BuildOptions{
+ MaxPullProcs: 5,
+ })
+ utils.CheckErr(err, fmt.Sprintf("failed to build image: %v", err))
+ })
+ AfterEach(func() {
+ err = fakeClient.Cluster.Reset()
+ utils.CheckErr(err, fmt.Sprintf("failed to reset cluster for single: %v", err))
+ })
+ It("sealos normal filesystem suit", func() {
+ By("run k3s for normal")
+ err = fakeClient.Cluster.Run("k3s:buildin")
+ utils.CheckErr(err)
+ fn := func() []byte {
+ data, err := fakeClient.CmdInterface.Exec("kubectl", "get", "pods", "-A", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml", "-o", "yaml")
+ utils.CheckErr(err)
+ return data
+ }
+ count := 0
+ for {
+ pods := fn()
+ podList := &v1.PodList{}
+ _ = yaml.Unmarshal(pods, podList)
+ running := 0
+ for _, pod := range podList.Items {
+ logger.Info("k3s pods is: %s: %s", pod.Name, pod.Status.Phase)
+ if pod.Status.Phase == v1.PodRunning {
+ running++
+ }
+ }
+ if running == len(podList.Items) && running != 0 {
+ break
+ }
+ time.Sleep(2 * time.Second)
+ logger.Info("k3s pods is empty,retry %d", count+1)
+ count++
+ if count == 20 {
+ utils.CheckErr(errors.New("k3s pods is empty, for timeout"))
+ }
+ }
+ err = fakeClient.CmdInterface.AsyncExec("kubectl", "get", "nodes", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml")
+ utils.CheckErr(err)
+ displayImages, err := fakeClient.CRI.ImageList()
+ utils.CheckErr(err)
+ if displayImages != nil {
+ for _, image := range displayImages.Images {
+ for _, tag := range image.RepoTags {
+ logger.Info("image tag is %s", tag)
+ ref, err := name.ParseReference(tag)
+ utils.CheckErr(err)
+ logger.Info("image registry is %s", ref.Context().RegistryStr())
+ if ref.Context().RegistryStr() != "sealos.hub:5000" {
+ utils.CheckErr(fmt.Errorf("crictl image is not sealos.hub, %+v", strings.TrimSpace(tag)))
+ }
+ }
+ }
+ }
+ })
+ })
+
+})
diff --git a/test/e2e/suites/checkers/cluster_cert_sans.go b/test/e2e/suites/checkers/cluster_cert_sans.go
@@ -19,6 +19,8 @@ package checkers
 import (
  "fmt"
 
+ "github.com/labring/sealos/pkg/utils/logger"
+
  "github.com/labring/sealos/test/e2e/testhelper/utils"
 
  "k8s.io/apimachinery/pkg/util/sets"
@@ -28,7 +30,7 @@ var _ FakeInterface = &fakeCertSansClient{}
 
 type fakeCertSansClient struct {
  *fakeClient
- data string
+ data []string
 }
 
 func (f *fakeCertSansClient) Verify() error {
@@ -45,9 +47,14 @@ func (f *fakeCertSansClient) Verify() error {
  if !sets.NewString(f.ClusterConfiguration.APIServer.CertSANs...).Has(localIP) {
  return fmt.Errorf("cert SANs not match %s", localIP)
  }
- if f.data != "" {
- if !sets.NewString(f.ClusterConfiguration.APIServer.CertSANs...).Has(f.data) {
- return fmt.Errorf("cert SANs not match %s", f.data)
+ if len(f.data) != 0 {
+ logger.Info("cert SANs %+v", f.data)
+ for _, v := range f.data {
+ if v != "" {
+ if !sets.NewString(f.ClusterConfiguration.APIServer.CertSANs...).Has(v) {
+ return fmt.Errorf("cert SANs %+v not match %s", f.ClusterConfiguration.APIServer.CertSANs, v)
+ }
+ }
  }
  }
  return nil

diff --git a/test/e2e/suites/checkers/cluster_cert_sans_update.go b/test/e2e/suites/checkers/cluster_cert_sans_update.go