From 7659046d4b8783621ea6509f2512c5fb07246619 Mon Sep 17 00:00:00 2001 From: cuisongliu Date: Sun, 10 Sep 2023 20:50:15 +0800 Subject: [PATCH] test(main): add test for k3s (#3875) * test(main): add test for k3s Signed-off-by: cuisongliu * test(main): add test for k3s Signed-off-by: cuisongliu --------- Signed-off-by: cuisongliu --- .github/workflows/e2e_test_core.yml | 2 +- .github/workflows/e2e_test_core_k3s.yml | 77 +++++++++++ cmd/sealos/cmd/cert.go | 1 + pkg/runtime/kubernetes/certs.go | 4 + test/e2e/cert_test.go | 3 + test/e2e/k3s_basic_test.go | 122 ++++++++++++++++++ test/e2e/suites/checkers/cluster_cert_sans.go | 15 ++- .../checkers/cluster_cert_sans_update.go | 42 ------ test/e2e/suites/checkers/fake.go | 85 ++++++------ 9 files changed, 264 insertions(+), 87 deletions(-) create mode 100644 .github/workflows/e2e_test_core_k3s.yml create mode 100644 test/e2e/k3s_basic_test.go delete mode 100644 test/e2e/suites/checkers/cluster_cert_sans_update.go diff --git a/.github/workflows/e2e_test_core.yml b/.github/workflows/e2e_test_core.yml index cbdcf28180f..4b28253d0ad 100644 --- a/.github/workflows/e2e_test_core.yml +++ b/.github/workflows/e2e_test_core.yml @@ -3,7 +3,7 @@ name: E2E Sealos Core Test on: workflow_dispatch: push: - branches: [ "main" ] + branches: [ "**" ] paths: - ".github/workflows/e2e_test_core.yml" - "cmd/**" diff --git a/.github/workflows/e2e_test_core_k3s.yml b/.github/workflows/e2e_test_core_k3s.yml new file mode 100644 index 00000000000..05bb5c415f4 --- /dev/null +++ b/.github/workflows/e2e_test_core_k3s.yml @@ -0,0 +1,77 @@ +name: E2E Sealos Core Test for K3s + +on: + workflow_dispatch: + push: + branches: [ "**" ] + paths: + - ".github/workflows/e2e_test_core_k3s.yml" + - "cmd/**" + - "pkg/**" + - "test/**" + pull_request: + branches: [ "*" ] + paths: + - ".github/workflows/e2e_test_core_k3s.yml" + - "cmd/**" + - "pkg/**" + - "test/**" + + +jobs: + call_ci_workflow: + uses: ./.github/workflows/import-patch-image.yml + with: + arch: amd64 + e2e: true + image: false + e2e-core-test: + needs: [ call_ci_workflow ] + strategy: + fail-fast: false + matrix: + unit: + [ + E2E_sealos_k3s_basic_test, + ] + runs-on: ubuntu-20.04 + steps: + - name: Download image-cri-shim + uses: actions/download-artifact@v3 + with: + name: image-cri-shim-amd64 + path: /tmp/ + - name: Download sealctl + uses: actions/download-artifact@v3 + with: + name: sealctl-amd64 + path: /tmp/ + - name: Download sealos + uses: actions/download-artifact@v3 + with: + name: sealos-amd64 + path: /tmp/ + - name: Download e2e test + uses: actions/download-artifact@v3 + with: + name: e2e.test + path: /tmp/ + - name: Verify sealos + run: | + sudo chmod a+x /tmp/{sealos,image-cri-shim,sealctl} + sudo mv /tmp/sealos /usr/bin/ + sudo sealos version + - name: Remove containerd && docker + uses: labring/sealos-action@v0.0.7 + with: + type: prune + + - name: Verify E2e test + run: | + sudo apt-get remove docker docker-engine docker.io containerd runc + sudo apt-get purge docker-ce docker-ce-cli containerd.io # docker-compose-plugin + sudo apt-get remove -y moby-engine moby-cli moby-buildx moby-compose + sudo rm -rf /var/run/docker.sock + sudo rm -rf /run/containerd/containerd.sock + sudo chmod a+x /tmp/e2e.test + sudo /tmp/e2e.test --ginkgo.v --ginkgo.focus="${{ matrix.unit }}" diff --git a/cmd/sealos/cmd/cert.go b/cmd/sealos/cmd/cert.go index 095b1ce0d35..241727fc167 100644 --- a/cmd/sealos/cmd/cert.go +++ b/cmd/sealos/cmd/cert.go @@ -81,6 +81,7 @@ func newCertCmd() *cobra.Command { return fmt.Errorf("create runtime failed: %v", err) } if cm, ok := rt.(runtime.CertManager); ok { + logger.Info("using %s cert update implement", cf.GetCluster().GetDistribution()) return cm.UpdateCertSANs(altNames) } return nil diff --git a/pkg/runtime/kubernetes/certs.go b/pkg/runtime/kubernetes/certs.go index 66e3dcc38c0..df0e379f6b1 100644 --- a/pkg/runtime/kubernetes/certs.go +++ b/pkg/runtime/kubernetes/certs.go @@ -37,6 +37,10 @@ const ( KubeletConf = "kubelet.conf" ) +func (k *KubeadmRuntime) Renew() error { + return errors.New("not implement") +} + func (k *KubeadmRuntime) UpdateCertSANs(certSans []string) error { // set extra cert SANs for kubeadm configmap object if err := k.CompleteKubeadmConfig(setCGroupDriverAndSocket, setCertificateKey); err != nil { diff --git a/test/e2e/cert_test.go b/test/e2e/cert_test.go index 874a61ca354..94a55b4bef6 100644 --- a/test/e2e/cert_test.go +++ b/test/e2e/cert_test.go @@ -20,6 +20,8 @@ import ( "fmt" "time" + "github.com/labring/sealos/test/e2e/testhelper/cmd" + "github.com/labring/sealos/test/e2e/testhelper/utils" "github.com/labring/sealos/test/e2e/suites/operators" @@ -36,6 +38,7 @@ var _ = Describe("E2E_sealos_cert_test", func() { err error ) fakeClient = operators.NewFakeClient("") + cmd.SetDebug() BeforeEach(func() { images := []string{"labring/kubernetes:v1.25.0", "labring/helm:v3.8.2", "labring/calico:v3.24.1"} err = fakeClient.Cluster.Run(images...) diff --git a/test/e2e/k3s_basic_test.go b/test/e2e/k3s_basic_test.go new file mode 100644 index 00000000000..9cadd2ebbad --- /dev/null +++ b/test/e2e/k3s_basic_test.go @@ -0,0 +1,122 @@ +/* +Copyright 2023 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "errors" + "fmt" + "path" + "strings" + "time" + + "github.com/google/go-containerregistry/pkg/name" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/yaml" + + "github.com/labring/sealos/pkg/utils/logger" + + "github.com/labring/sealos/test/e2e/suites/operators" + "github.com/labring/sealos/test/e2e/testhelper/config" + "github.com/labring/sealos/test/e2e/testhelper/utils" + + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("E2E_sealos_k3s_basic_test", func() { + var ( + fakeClient *operators.FakeClient + err error + ) + fakeClient = operators.NewFakeClient("") + + Context("sealos k3s suit", func() { + BeforeEach(func() { + By("build rootfs") + dFile := config.RootfsDockerfile{ + BaseImage: "labring/k3s:latest", + Copys: []string{"sealctl opt/"}, + } + tmpdir, err := dFile.Write() + utils.CheckErr(err, fmt.Sprintf("failed to create dockerfile: %v", err)) + + By("copy sealctl to rootfs") + err = fakeClient.CmdInterface.Copy("/tmp/sealctl", path.Join(tmpdir, "sealctl")) + utils.CheckErr(err, fmt.Sprintf("failed to copy sealctl to rootfs: %v", err)) + + By("build image") + err = fakeClient.Image.BuildImage("k3s:buildin", tmpdir, operators.BuildOptions{ + MaxPullProcs: 5, + }) + utils.CheckErr(err, fmt.Sprintf("failed to build image: %v", err)) + }) + AfterEach(func() { + err = fakeClient.Cluster.Reset() + utils.CheckErr(err, fmt.Sprintf("failed to reset cluster for single: %v", err)) + }) + It("sealos normal filesystem suit", func() { + By("run k3s for normal") + err = fakeClient.Cluster.Run("k3s:buildin") + utils.CheckErr(err) + fn := func() []byte { + data, err := fakeClient.CmdInterface.Exec("kubectl", "get", "pods", "-A", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml", "-o", "yaml") + utils.CheckErr(err) + return data + } + count := 0 + for { + pods := fn() + podList := &v1.PodList{} + _ = yaml.Unmarshal(pods, podList) + running := 0 + for _, pod := range podList.Items { + logger.Info("k3s pods is: %s: %s", pod.Name, pod.Status.Phase) + if pod.Status.Phase == v1.PodRunning { + running++ + } + } + if running == len(podList.Items) && running != 0 { + break + } + time.Sleep(2 * time.Second) + logger.Info("k3s pods is empty,retry %d", count+1) + count++ + if count == 20 { + utils.CheckErr(errors.New("k3s pods is empty, for timeout")) + } + } + err = fakeClient.CmdInterface.AsyncExec("kubectl", "get", "nodes", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml") + utils.CheckErr(err) + displayImages, err := fakeClient.CRI.ImageList() + utils.CheckErr(err) + if displayImages != nil { + for _, image := range displayImages.Images { + for _, tag := range image.RepoTags { + logger.Info("image tag is %s", tag) + ref, err := name.ParseReference(tag) + utils.CheckErr(err) + logger.Info("image registry is %s", ref.Context().RegistryStr()) + if ref.Context().RegistryStr() != "sealos.hub:5000" { + utils.CheckErr(fmt.Errorf("crictl image is not sealos.hub, %+v", strings.TrimSpace(tag))) + } + } + } + } + }) + }) + +}) diff --git a/test/e2e/suites/checkers/cluster_cert_sans.go b/test/e2e/suites/checkers/cluster_cert_sans.go index a3d67273a7f..4c9bfe1d8e7 100644 --- a/test/e2e/suites/checkers/cluster_cert_sans.go +++ b/test/e2e/suites/checkers/cluster_cert_sans.go @@ -19,6 +19,8 @@ package checkers import ( "fmt" + "github.com/labring/sealos/pkg/utils/logger" + "github.com/labring/sealos/test/e2e/testhelper/utils" "k8s.io/apimachinery/pkg/util/sets" @@ -28,7 +30,7 @@ var _ FakeInterface = &fakeCertSansClient{} type fakeCertSansClient struct { *fakeClient - data string + data []string } func (f *fakeCertSansClient) Verify() error { @@ -45,9 +47,14 @@ func (f *fakeCertSansClient) Verify() error { if !sets.NewString(f.ClusterConfiguration.APIServer.CertSANs...).Has(localIP) { return fmt.Errorf("cert SANs not match %s", localIP) } - if f.data != "" { - if !sets.NewString(f.ClusterConfiguration.APIServer.CertSANs...).Has(f.data) { - return fmt.Errorf("cert SANs not match %s", f.data) + if len(f.data) != 0 { + logger.Info("cert SANs %+v", f.data) + for _, v := range f.data { + if v != "" { + if !sets.NewString(f.ClusterConfiguration.APIServer.CertSANs...).Has(v) { + return fmt.Errorf("cert SANs %+v not match %s", f.ClusterConfiguration.APIServer.CertSANs, v) + } + } } } return nil diff --git a/test/e2e/suites/checkers/cluster_cert_sans_update.go b/test/e2e/suites/checkers/cluster_cert_sans_update.go deleted file mode 100644 index 22d1dad3184..00000000000 --- a/test/e2e/suites/checkers/cluster_cert_sans_update.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2023 cuisongliu@qq.com. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package checkers - -import ( - "fmt" - - "k8s.io/apimachinery/pkg/util/sets" -) - -var _ FakeInterface = &fakeCertSansUpdateClient{} - -type fakeCertSansUpdateClient struct { - *fakeClient - data string -} - -func (f *fakeCertSansUpdateClient) Verify() error { - if f.UpdateConfiguration == nil { - return nil - } - if f.data != "" { - if !sets.NewString(f.UpdateConfiguration.ClusterConfiguration.APIServer.CertSANs...).Has(f.data) { - return fmt.Errorf("cert SANs not match %s", f.data) - } - } - return nil -} diff --git a/test/e2e/suites/checkers/fake.go b/test/e2e/suites/checkers/fake.go index 8eae43ccb09..1f61533c168 100644 --- a/test/e2e/suites/checkers/fake.go +++ b/test/e2e/suites/checkers/fake.go @@ -22,6 +22,8 @@ import ( "os" "path/filepath" + v1 "k8s.io/api/core/v1" + "github.com/labring/sealos/test/e2e/testhelper/utils" "github.com/labring/sealos/pkg/types/v1beta1" @@ -54,9 +56,6 @@ type fakeClient struct { JoinConfiguration kubeadm.JoinConfiguration KubeletConfiguration kubelet.KubeletConfiguration } - UpdateConfiguration *struct { - ClusterConfiguration kubeadm.ClusterConfiguration - } clusterName string cmd cmd.Interface @@ -86,9 +85,6 @@ func NewFakeGroupClient(name string, opt *FakeOpts) (*FakeClientGroup, error) { if err := client.loadInitConfig(); err != nil { return nil, err } - if err := client.loadUpdateConfig(); err != nil { - return nil, err - } if opt == nil { opt = &FakeOpts{} } @@ -114,12 +110,11 @@ func NewFakeGroupClient(name string, opt *FakeOpts) (*FakeClientGroup, error) { &fakeCgroupClient{fakeClient: client, data: opt.Cgroup}, &fakePodCIDRClient{fakeClient: client, data: opt.PodCIDR}, &fakeServiceCIDRClient{fakeClient: client, data: opt.ServiceCIDR}, - &fakeCertSansClient{fakeClient: client, data: opt.CertSan}, + &fakeCertSansClient{fakeClient: client, data: []string{opt.CertSan, opt.CertDomain}}, &fakeSingleTaintsClient{fakeClient: client}, &fakeTaintsClient{fakeClient: client, data: opt.Taints}, &fakeImageClient{fakeClient: client, data: opt.Images}, &fakeEtcdClient{fakeClient: client, etcd: opt.Etcd}, - &fakeCertSansUpdateClient{fakeClient: client, data: opt.CertDomain}, }}, nil } @@ -156,51 +151,61 @@ func (f *fakeClient) loadInitConfig() error { return err } yamls := utils.ToYalms(string(data)) + clusterConfigFn := func() ([]byte, error) { + cfg, err := f.cmd.Exec("kubectl", "get", "cm", "-n", "kube-system", "kubeadm-config", "-o", "yaml") + if err != nil { + return nil, err + } + cm := &v1.ConfigMap{} + _ = yaml.Unmarshal(cfg, cm) + return []byte(cm.Data["ClusterConfiguration"]), nil + } + kubeproxyConfigFn := func() ([]byte, error) { + cfg, err := f.cmd.Exec("kubectl", "get", "cm", "-n", "kube-system", "kube-proxy", "-o", "yaml") + if err != nil { + return nil, err + } + cm := &v1.ConfigMap{} + _ = yaml.Unmarshal(cfg, cm) + return []byte(cm.Data["config.conf"]), nil + } + kubeletConfigFn := func() ([]byte, error) { + cfg, err := f.cmd.Exec("kubectl", "get", "cm", "-n", "kube-system", "kubelet-config", "-o", "yaml") + if err != nil { + return nil, err + } + cm := &v1.ConfigMap{} + _ = yaml.Unmarshal(cfg, cm) + return []byte(cm.Data["kubelet"]), nil + } + for _, yamlString := range yamls { obj, _ := utils.UnmarshalData([]byte(yamlString)) kind, _, _ := unstructured.NestedString(obj, "kind") switch kind { case "InitConfiguration": _ = yaml.Unmarshal([]byte(yamlString), &f.InitConfiguration) - case "ClusterConfiguration": - _ = yaml.Unmarshal([]byte(yamlString), &f.ClusterConfiguration) - case "KubeProxyConfiguration": - _ = yaml.Unmarshal([]byte(yamlString), &f.KubeProxyConfiguration) - case "KubeletConfiguration": - _ = yaml.Unmarshal([]byte(yamlString), &f.KubeletConfiguration) } } - - clusterConfig := fmt.Sprintf("/root/.sealos/%s/Clusterfile", f.clusterName) - if !utils.IsFileExist(clusterConfig) { - return fmt.Errorf("file %s not exist", clusterConfig) + kubeadmCfg, err := clusterConfigFn() + if err != nil { + return err } - return utils.UnmarshalYamlFile(clusterConfig, &f.Cluster) -} - -func (f *fakeClient) loadUpdateConfig() error { - logger.Info("verify default cluster info") - initFile, err := f.getFilePath("kubeadm-update.yaml") + _ = yaml.Unmarshal(kubeadmCfg, &f.ClusterConfiguration) + kubeProxyCfg, err := kubeproxyConfigFn() if err != nil { - logger.Error(err) - f.UpdateConfiguration = nil - return nil + return err } - f.UpdateConfiguration = &struct { - ClusterConfiguration kubeadm.ClusterConfiguration - }{} - data, err := os.ReadFile(filepath.Clean(initFile)) + _ = yaml.Unmarshal(kubeProxyCfg, &f.KubeProxyConfiguration) + kubeletCfg, err := kubeletConfigFn() if err != nil { return err } - yamls := utils.ToYalms(string(data)) - for _, yamlString := range yamls { - obj, _ := utils.UnmarshalData([]byte(yamlString)) - kind, _, _ := unstructured.NestedString(obj, "kind") - switch kind { - case "ClusterConfiguration": - _ = yaml.Unmarshal([]byte(yamlString), &f.UpdateConfiguration.ClusterConfiguration) - } + _ = yaml.Unmarshal(kubeletCfg, &f.KubeletConfiguration) + + clusterConfig := fmt.Sprintf("/root/.sealos/%s/Clusterfile", f.clusterName) + if !utils.IsFileExist(clusterConfig) { + return fmt.Errorf("file %s not exist", clusterConfig) } - return nil + return utils.UnmarshalYamlFile(clusterConfig, &f.Cluster) }