feat: resources monitor, metering controller, billing controller, bil…

…lingrecordquery, pricequery controller (labring#3008)

* change accountbalance struct
add billing controller;
add new query CRD: billingresordquery and pricequery, and query controller;
modify the storage amount ratio to 1:1000000
optimize mongo query code

* hide the old metering controller

* add query gvk to debt webhook white list;
monitor data table by day, eg: monitor_20230517;
billing collection cancels the timing table and changes it to an index type table;
optimize the billing logic and bill by the hour;
optimize the use of gonanoid to generate 12-digit bill id;
payment crd adds the PaymentMethod field, which is used to identify wechat or other billing methods in the future.

* add resources controller workflow
fix payment controller panic error
add resources controller cluster image workflow

.github/workflows/controllers.yml

@@ -88,6 +88,7 @@ jobs:
  - { name: app, path: app }
  - { name: db-bytebase, path: db/bytebase }
  - { name: db-adminer, path: db/adminer }
+ - { name: resources, path: resources }
  steps:
  - name: Checkout
  uses: actions/checkout@v3
@@ -183,6 +184,7 @@ jobs:
  - { name: app, path: app }
  - { name: db-bytebase, path: db/bytebase }
  - { name: db-adminer, path: db/adminer }
+ - { name: resources, path: resources }
  steps:
  - name: Checkout
  uses: actions/checkout@v3

controllers/account/PROJECT

@@ -42,4 +42,26 @@ resources:
  webhooks:
  conversion: true
  webhookVersion: v1
+- controller: true
+ domain: sealos.io
+ group: account
+ kind: Billing
+ version: v1
+- api:
+ crdVersion: v1
+ namespaced: true
+ controller: true
+ domain: sealos.io
+ group: account
+ kind: BillingRecordQuery
+ path: github.com/labring/sealos/controllers/account/api/v1
+ version: v1
+- api:
+ crdVersion: v1
+ namespaced: true
+ domain: sealos.io
+ group: account
+ kind: PriceQuery
+ path: github.com/labring/sealos/controllers/account/api/v1
+ version: v1
 version: "3"

controllers/account/api/v1/account_types.go

@@ -66,8 +66,9 @@ type AccountStatus struct {
  Balance int64 `json:"balance,omitempty"`
 
  //Deduction amount
- DeductionBalance int64 `json:"deductionBalance,omitempty"`
- ChargeList []Charge `json:"chargeList,omitempty"`
+ DeductionBalance int64 `json:"deductionBalance,omitempty"`
+ // delete in the future
+ ChargeList []Charge `json:"chargeList,omitempty"`
 }
 
 //+kubebuilder:object:root=true

controllers/account/api/v1/accountbalance_types.go

@@ -17,25 +17,48 @@ limitations under the License.
 package v1
 
 import (
- meteringcommonv1 "github.com/labring/sealos/controllers/common/metering/api/v1"
- meteringv1 "github.com/labring/sealos/controllers/metering/api/v1"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const AccountBalancePrefix = "accountbalance"
 
+type (
+ Status string
+ Type int
+)
+
+const (
+ // Consumption 消费
+ Consumption Type = iota
+ // Recharge 充值
+ Recharge
+)
+
+const (
+ Completed Status = "completed"
+ Create Status = "create"
+ Failed Status = "failed"
+)
+
+type Costs map[string]int64
+
 // AccountBalanceSpec defines the desired state of AccountBalance
 type AccountBalanceSpec struct {
- Owner string `json:"owner"`
- Timestamp int64 `json:"timestamp,omitempty"`
- Amount int64 `json:"amount,omitempty"`
- Details string `json:"details,omitempty"`
- ResourceInfoList meteringcommonv1.ResourceInfoList `json:"resourceInfoList,omitempty"`
+ OrderID string `json:"order_id" bson:"order_id"`
+ Owner string `json:"owner" bson:"owner"`
+ Time metav1.Time `json:"time" bson:"time"`
+ Type Type `json:"type" bson:"type"`
+ Costs Costs `json:"costs,omitempty" bson:"costs,omitempty"`
+ // TODO will delete field in future
+ //Timestamp int64 `json:"timestamp,omitempty"`
+ Amount int64 `json:"amount,omitempty" bson:"amount"`
+ Details string `json:"details,omitempty" bson:"details,omitempty"`
+ //ResourceInfoList meteringcommonv1.ResourceInfoList `json:"resourceInfoList,omitempty"`
 }
 
 // AccountBalanceStatus defines the observed state of AccountBalance
 type AccountBalanceStatus struct {
- Status meteringv1.Status `json:"status,omitempty"`
+ Status Status `json:"billingStatus,omitempty"`
 }
 
 //+kubebuilder:object:root=true

controllers/account/api/v1/billingrecordquery_types.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
+
+// BillingRecordQuerySpec defines the desired state of BillingRecordQuery
+type BillingRecordQuerySpec struct {
+ // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
+ // Important: Run "make" to regenerate code after modifying this file
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+ StartTime metav1.Time `json:"startTime"`
+ EndTime metav1.Time `json:"endTime"`
+ OrderID string `json:"orderID,omitempty"`
+ Type Type `json:"type"`
+}
+
+// BillingRecordQueryStatus defines the observed state of BillingRecordQuery
+type BillingRecordQueryStatus struct {
+ // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
+ // Important: Run "make" to regenerate code after modifying this file
+ PageLength int `json:"pageLength"`
+ RechargeAmount int64 `json:"rechargeAmount"`
+ DeductionAmount Costs `json:"deductionAmount,omitempty"`
+ Items []AccountBalanceSpec `json:"item,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+
+// BillingRecordQuery is the Schema for the billingrecordqueries API
+type BillingRecordQuery struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ObjectMeta `json:"metadata,omitempty"`
+
+ Spec BillingRecordQuerySpec `json:"spec,omitempty"`
+ Status BillingRecordQueryStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// BillingRecordQueryList contains a list of BillingRecordQuery
+type BillingRecordQueryList struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ListMeta `json:"metadata,omitempty"`
+ Items []BillingRecordQuery `json:"items"`
+}
+
+func init() {
+ SchemeBuilder.Register(&BillingRecordQuery{}, &BillingRecordQueryList{})
+}

controllers/account/api/v1/debt_webhook.go

@@ -22,6 +22,9 @@ import (
  "os"
  "strings"
 
+ admissionV1 "k8s.io/api/admission/v1"
+ "k8s.io/apimachinery/pkg/types"
+
  "github.com/go-logr/logr"
  userv1 "github.com/labring/sealos/controllers/user/api/v1"
  corev1 "k8s.io/api/core/v1"
@@ -31,10 +34,10 @@ import (
 )
 
 const (
- saPrefix = "system:serviceaccount"
- mastersGroup = "system:masters"
- kubeSystemNamespace = "kube-system"
- defaultUserNamespace = "user-system"
+ saPrefix  = "system:serviceaccount"
+ mastersGroup  = "system:masters"
+ kubeSystemNamespace  = "kube-system"
+ defaultUserSystemNamespace = "user-system"
 )
 
 var logger = logf.Log.WithName("debt-resource")
@@ -49,7 +52,7 @@ type DebtValidate struct {
 func getUserNamespace() string {
  userNamespace := os.Getenv("USER_NAMESPACE")
  if userNamespace == "" {
- return defaultUserNamespace
+ return defaultUserSystemNamespace
  }
  return userNamespace
 }
@@ -62,6 +65,10 @@ func init() {
 }
 func (d DebtValidate) Handle(ctx context.Context, req admission.Request) admission.Response {
  logger.V(1).Info("checking user", "userInfo", req.UserInfo, "req.Namespace", req.Namespace, "req.Name", req.Name, "req.gvrk", getGVRK(req))
+ // skip delete request
+ if req.Operation == admissionV1.Delete {
+ return admission.Allowed("")
+ }
  for _, g := range req.UserInfo.Groups {
  switch g {
  // if user is kubernetes-admin, pass it
@@ -95,6 +102,8 @@ func getGVRK(req admission.Request) string {
 }
 
 func isWhiteList(req admission.Request) bool {
+ // check if it is in whitelist
+ // default: "terminals.Terminal.terminal.sealos.io/v1,payments.Payment.account.sealos.io/v1,billingrecordqueries.BillingRecordQuery.account.sealos.io/v1,pricequeries.PriceQuery.account.sealos.io/v1"
  whitelists := os.Getenv("WHITELIST")
  if whitelists == "" {
  return false
@@ -112,16 +121,22 @@ func isWhiteList(req admission.Request) bool {
 }
 
 func checkOption(ctx context.Context, logger logr.Logger, c client.Client, nsName string) admission.Response {
- nsList := &corev1.NamespaceList{}
- if err := c.List(ctx, nsList, client.MatchingFields{"name": nsName}); err != nil {
- logger.Error(err, "list ns error", "naName", nsName, "nsList", nsList)
- return admission.ValidationResponse(true, nsName)
+ //nsList := &corev1.NamespaceList{}
+ //if err := c.List(ctx, nsList, client.MatchingFields{"name": nsName}); err != nil {
+ // logger.Error(err, "list ns error", "naName", nsName, "nsList", nsList)
+ // return admission.ValidationResponse(true, nsName)
+ //}
+ // skip check if nsName is empty or equal to user system namespace
+ if nsName == "" || nsName == getUserNamespace() {
+ return admission.Allowed("")
+ }
+ ns := &corev1.Namespace{}
+ if err := c.Get(ctx, types.NamespacedName{Name: nsName, Namespace: nsName}, ns); err != nil {
+ return admission.Allowed("namespace not found")
  }
-
- //ns name unique in k8s
- ns := nsList.Items[0]
  // Check if it is a user namespace
  user, ok := ns.Annotations[userv1.UserAnnotationOwnerKey]
+ logger.V(1).Info("check user namespace", "ns.name", ns.Name, "ns", ns)
  if !ok {
  return admission.ValidationResponse(true, fmt.Sprintf("this namespace is not user namespace %s,or have not create", ns.Name))
  }
@@ -133,9 +148,9 @@ func checkOption(ctx context.Context, logger logr.Logger, c client.Client, nsNam
  }
 
  for _, account := range accountList.Items {
- if account.Status.Balance-account.Status.DeductionBalance < 0 {
- return admission.ValidationResponse(false, fmt.Sprintf("account balance less than 0,now account is %.2f¥", float64(account.Status.Balance-account.Status.DeductionBalance)/10000))
+ if account.Status.Balance < account.Status.DeductionBalance {
+ return admission.ValidationResponse(false, fmt.Sprintf("account balance less than 0,now account is %.2f¥", float64(account.Status.Balance-account.Status.DeductionBalance)/1000000))
  }
  }
- return admission.ValidationResponse(true, "")
+ return admission.Allowed("pass user " + user)
 }

controllers/account/api/v1/payment_types.go

@@ -17,6 +17,8 @@ limitations under the License.
 package v1
 
 import (
+ "fmt"
+
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -45,6 +47,8 @@ type PaymentSpec struct {
  UserID string `json:"userID,omitempty"`
  // Amount is the amount of recharge
  Amount int64 `json:"amount,omitempty"`
+ // e.g. wechat, alipay, creditcard, etc.
+ PaymentMethod string `json:"paymentMethod,omitempty"`
 }
 
 // PaymentStatus defines the observed state of Payment
@@ -78,6 +82,21 @@ type PaymentList struct {
  Items []Payment `json:"items"`
 }
 
+func (p *Payment) ToJSON() string {
+ return `{
+ "spec": {
+ "userID": "` + p.Spec.UserID + `",
+ "amount": ` + fmt.Sprint(p.Spec.Amount) + `,
+ "paymentMethod": "` + p.Spec.PaymentMethod + `"
+ },
+ "status": {
+ "tradeNO": "` + p.Status.TradeNO + `",
+ "paymentURL": "` + p.Status.CodeURL + `",
+ "status": "` + p.Status.Status + `"
+ }
+}`
+}
+
 func init() {
  SchemeBuilder.Register(&Payment{}, &PaymentList{})
 }

controllers/account/api/v1/pricequery_types.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
+
+// PriceQuerySpec defines the desired state of PriceQuery
+type PriceQuerySpec struct {
+}
+
+// PriceQueryStatus defines the observed state of PriceQuery
+type PriceQueryStatus struct {
+ BillingRecords []BillingRecord `json:"billingRecords,omitempty"`
+}
+
+type BillingRecord struct {
+ ResourceType string `json:"resourceType"`
+ Price int64 `json:"price"`
+ DiscountType string `json:"discountType,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+
+// PriceQuery is the Schema for the pricequeries API
+type PriceQuery struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ObjectMeta `json:"metadata,omitempty"`
+
+ Spec PriceQuerySpec `json:"spec,omitempty"`
+ Status PriceQueryStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// PriceQueryList contains a list of PriceQuery
+type PriceQueryList struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ListMeta `json:"metadata,omitempty"`
+ Items []PriceQuery `json:"items"`
+}
+
+func init() {
+ SchemeBuilder.Register(&PriceQuery{}, &PriceQueryList{})
+}