From 3aa20ae55cb447c08070a7e495463b621cdc8f40 Mon Sep 17 00:00:00 2001 From: Congqi Xia Date: Fri, 16 Dec 2022 16:47:53 +0800 Subject: [PATCH] Support workspace for backup files Signed-off-by: Congqi Xia --- configs/config.go | 110 +++++ go.mod | 9 +- go.sum | 1 - models/workspace.go | 26 + states/{ => autocomplete}/auto_complete.go | 9 +- .../{ => autocomplete}/auto_complete_file.go | 2 +- states/{cmd.go => autocomplete/comp.go} | 2 +- states/autocomplete/input.go | 79 +++ states/backup_mock_connect.go | 236 ++++----- states/download_pk.go | 7 +- states/download_segment.go | 5 +- states/etcd/commands.go | 148 ++++++ states/etcd/common/channel.go | 19 + states/etcd/common/collection.go | 116 +++++ states/etcd/common/index.go | 31 ++ states/etcd/common/legacy.go | 106 +++++ states/etcd/common/list.go | 42 ++ states/etcd/common/pair.go | 14 + states/etcd/common/path.go | 31 ++ states/etcd/common/segment.go | 121 +++++ states/etcd/common/session.go | 43 ++ states/etcd/common/sort.go | 14 + .../repair/channel.go} | 17 +- .../repair/checkpoint.go} | 18 +- .../repair/segment.go} | 99 ++-- states/etcd/repair/segment_empty.go | 87 ++++ .../show/channel_watched.go} | 34 +- .../show/checkpoint.go} | 15 +- .../show/collection.go} | 73 +-- states/etcd/show/collection_loaded.go | 59 +++ states/etcd/show/index.go | 109 +++++ .../show/legacy_qc_channel.go} | 85 +--- .../show/legacy_qc_cluster.go} | 26 +- .../show/legacy_qc_task.go} | 31 +- .../show/legacy_qc_task_model.go} | 39 +- .../{show_replica.go => etcd/show/replica.go} | 25 +- states/etcd/show/segment.go | 181 +++++++ states/etcd/show/segment_index.go | 152 ++++++ states/etcd/show/segment_loaded.go | 51 ++ states/etcd/show/session.go | 30 ++ states/etcd_backup.go | 7 +- states/etcd_connect.go | 4 +- states/etcd_restore.go | 39 +- states/force_release.go | 17 +- states/garbage_collect.go | 5 +- states/grpc/commands.go | 1 + states/inspect_primary_key.go | 9 +- states/instance.go | 41 +- states/load_backup.go | 180 +++++++ states/metrics.go | 3 +- states/open.go | 54 +++ states/pulsarctl_connect.go | 150 ++++++ states/show.go | 59 --- states/show_collection_load.go | 88 ---- states/show_index.go | 285 ----------- states/show_segment.go | 450 ------------------ states/show_session.go | 37 -- states/start.go | 58 +++ states/states.go | 105 +--- states/update_log_level.go | 5 +- states/util.go | 2 +- states/visit.go | 3 +- utils/util.go | 15 + 63 files changed, 2413 insertions(+), 1506 deletions(-) create mode 100644 configs/config.go create mode 100644 models/workspace.go rename states/{ => autocomplete}/auto_complete.go (94%) rename states/{ => autocomplete}/auto_complete_file.go (98%) rename states/{cmd.go => autocomplete/comp.go} (96%) create mode 100644 states/autocomplete/input.go create mode 100644 states/etcd/commands.go create mode 100644 states/etcd/common/channel.go create mode 100644 states/etcd/common/collection.go create mode 100644 states/etcd/common/index.go create mode 100644 states/etcd/common/legacy.go create mode 100644 states/etcd/common/list.go create mode 100644 states/etcd/common/pair.go create mode 100644 states/etcd/common/path.go create mode 100644 states/etcd/common/segment.go create mode 100644 states/etcd/common/session.go create mode 100644 states/etcd/common/sort.go rename states/{repair_channel.go => etcd/repair/channel.go} (84%) rename states/{reset_checkpoint.go => etcd/repair/checkpoint.go} (92%) rename states/{repair_segment.go => etcd/repair/segment.go} (76%) create mode 100644 states/etcd/repair/segment_empty.go rename states/{show_channel_watch.go => etcd/show/channel_watched.go} (66%) rename states/{show_checkpoint.go => etcd/show/checkpoint.go} (84%) rename states/{show_collection.go => etcd/show/collection.go} (61%) create mode 100644 states/etcd/show/collection_loaded.go create mode 100644 states/etcd/show/index.go rename states/{querycoord_channels.go => etcd/show/legacy_qc_channel.go} (53%) rename states/{querycoord_cluster.go => etcd/show/legacy_qc_cluster.go} (53%) rename states/{show_querycoord_task.go => etcd/show/legacy_qc_task.go} (84%) rename states/{querycoord_task.go => etcd/show/legacy_qc_task_model.go} (93%) rename states/{show_replica.go => etcd/show/replica.go} (61%) create mode 100644 states/etcd/show/segment.go create mode 100644 states/etcd/show/segment_index.go create mode 100644 states/etcd/show/segment_loaded.go create mode 100644 states/etcd/show/session.go create mode 100644 states/grpc/commands.go create mode 100644 states/load_backup.go create mode 100644 states/open.go create mode 100644 states/pulsarctl_connect.go delete mode 100644 states/show.go delete mode 100644 states/show_collection_load.go delete mode 100644 states/show_index.go delete mode 100644 states/show_segment.go delete mode 100644 states/show_session.go create mode 100644 states/start.go create mode 100644 utils/util.go diff --git a/configs/config.go b/configs/config.go new file mode 100644 index 0000000..b137f74 --- /dev/null +++ b/configs/config.go @@ -0,0 +1,110 @@ +package configs + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path" + + "gopkg.in/yaml.v3" +) + +const ( + configFileName = `birdwatcher.yaml` + defaultWorkspace = `bw_workspace` +) + +var ( + errConfigPathNotExist = errors.New("config path not exist") + errConfigPathIsFile = errors.New("config path is file") +) + +// Config stores birdwatcher config items. +type Config struct { + // birdwatcher configuration folder path + // default $PWD/.bw_config + ConfigPath string `yaml:"-"` + // backup workspace path, default $PWD/bw_workspace + WorkspacePath string `yaml:"WorkspacePath"` +} + +func (c *Config) load() error { + err := c.checkConfigPath() + if err != nil { + return err + } + + f, err := os.Open(c.getConfigPath()) + if err != nil { + return err + } + defer f.Close() + bs, err := ioutil.ReadAll(f) + if err != nil { + return err + } + + return yaml.Unmarshal(bs, c) +} + +func (c *Config) getConfigPath() string { + return path.Join(c.ConfigPath, configFileName) +} + +// checkConfigPath exists and is a directory. +func (c *Config) checkConfigPath() error { + info, err := os.Stat(c.ConfigPath) + if err != nil { + // not exist, return specified type to handle + if os.IsNotExist(err) { + return errConfigPathNotExist + } + return err + } + if !info.IsDir() { + fmt.Printf("%s is not a directory\n", c.ConfigPath) + return fmt.Errorf("%w(%s)", errConfigPathIsFile, configFileName) + } + + return nil +} + +func (c *Config) createDefault() error { + err := os.MkdirAll(c.ConfigPath, os.ModePerm) + if err != nil { + return err + } + + file, err := os.Create(c.getConfigPath()) + + if err != nil { + return err + } + defer file.Close() + + // setup default value + c.WorkspacePath = defaultWorkspace + + bs, err := yaml.Marshal(c) + if err != nil { + fmt.Println("failed to marshal config", err.Error()) + return err + } + + file.Write(bs) + return nil +} + +func NewConfig(configPath string) (*Config, error) { + config := &Config{ + ConfigPath: configPath, + } + err := config.load() + // config path not exist, may first time to run + if errors.Is(err, errConfigPathNotExist) { + return config, config.createDefault() + } + + return config, err +} diff --git a/go.mod b/go.mod index fcd2c8f..c26b064 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/samber/lo v1.28.2 github.com/spf13/cobra v1.5.0 github.com/spf13/pflag v1.0.5 + github.com/streamnative/pulsarctl v0.5.0 github.com/stretchr/testify v1.8.0 go.etcd.io/etcd/api/v3 v3.5.4 go.etcd.io/etcd/client/v3 v3.5.4 @@ -45,9 +46,7 @@ require ( github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a // indirect - github.com/fatih/color v1.7.0 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect - github.com/ghodss/yaml v1.0.0 // indirect github.com/goccy/go-json v0.9.6 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -59,7 +58,6 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect - github.com/imdario/mergo v0.3.8 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -68,10 +66,7 @@ require ( github.com/klauspost/compress v1.14.2 // indirect github.com/klauspost/cpuid v1.3.1 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect - github.com/kris-nova/logger v0.0.0-20181127235838-fd0d87064b06 // indirect - github.com/kris-nova/lolgopher v0.0.0-20180921204813-313b3abb0d9b // indirect github.com/linkedin/goavro/v2 v2.11.1 // indirect - github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect @@ -84,7 +79,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/olekukonko/tablewriter v0.0.1 // indirect github.com/pierrec/lz4 v2.0.5+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/term v1.2.0-beta.2 // indirect @@ -97,7 +91,6 @@ require ( github.com/sirupsen/logrus v1.8.1 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/streamnative/pulsarctl v0.5.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/zeebo/xxh3 v1.0.1 // indirect diff --git a/go.sum b/go.sum index 1307107..3bbea03 100644 --- a/go.sum +++ b/go.sum @@ -388,7 +388,6 @@ github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= diff --git a/models/workspace.go b/models/workspace.go new file mode 100644 index 0000000..4c01744 --- /dev/null +++ b/models/workspace.go @@ -0,0 +1,26 @@ +package models + +import "github.com/golang/protobuf/proto" + +// WorkspaceMeta stores birdwatcher workspace information. +type WorkspaceMeta struct { + // Version semver for workspace meta + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + // instance name, as rootPath for key prefix + Instance string `protobuf:"bytes,2,opt,name=instance,proto3" json:"instance,omitempty"` + // MetaPath used in keys + MetaPath string `protobuf:"bytes,3,opt,name=meta_path,proto3" json:"meta_path,omitempty"` +} + +// Reset implements protoiface.MessageV1 +func (v *WorkspaceMeta) Reset() { + *v = WorkspaceMeta{} +} + +// String implements protoiface.MessageV1 +func (v *WorkspaceMeta) String() string { + return proto.CompactTextString(v) +} + +// String implements protoiface.MessageV1 +func (v *WorkspaceMeta) ProtoMessage() {} diff --git a/states/auto_complete.go b/states/autocomplete/auto_complete.go similarity index 94% rename from states/auto_complete.go rename to states/autocomplete/auto_complete.go index b363020..038faa3 100644 --- a/states/auto_complete.go +++ b/states/autocomplete/auto_complete.go @@ -1,4 +1,4 @@ -package states +package autocomplete import ( "fmt" @@ -112,6 +112,13 @@ func (c *flagCandidate) NextCandidates(current []acCandidate) []acCandidate { return current } +// SuggestInputCommands returns suggestions based on command setup. +func SuggestInputCommands(input string, commands []*cobra.Command) map[string]string { + iResult := parseInput(input) + + return findCmdSuggestions(iResult.parts, commands) +} + func findCmdSuggestions(comps []cComp, commands []*cobra.Command) map[string]string { // no suggestion if input is empty if len(comps) == 0 { diff --git a/states/auto_complete_file.go b/states/autocomplete/auto_complete_file.go similarity index 98% rename from states/auto_complete_file.go rename to states/autocomplete/auto_complete_file.go index 486f3d3..87f0ee2 100644 --- a/states/auto_complete_file.go +++ b/states/autocomplete/auto_complete_file.go @@ -1,4 +1,4 @@ -package states +package autocomplete import ( "fmt" diff --git a/states/cmd.go b/states/autocomplete/comp.go similarity index 96% rename from states/cmd.go rename to states/autocomplete/comp.go index 3f1fc5a..b95ce1f 100644 --- a/states/cmd.go +++ b/states/autocomplete/comp.go @@ -1,4 +1,4 @@ -package states +package autocomplete type cmdCompType int diff --git a/states/autocomplete/input.go b/states/autocomplete/input.go new file mode 100644 index 0000000..3c18df8 --- /dev/null +++ b/states/autocomplete/input.go @@ -0,0 +1,79 @@ +package autocomplete + +import ( + "strings" + + "github.com/samber/lo" +) + +func parseInput(input string) inputResult { + // check is end with space + isEndBlank := strings.HasSuffix(input, " ") + + parts := strings.Split(input, " ") + parts = lo.Filter(parts, func(part string, idx int) bool { + return part != "" + }) + + comps := make([]cComp, 0, len(parts)) + currentFlagValue := false + + for _, part := range parts { + // next part is flag value + // just set last comp cValue + if currentFlagValue { + comps[len(comps)-1].cValue = part + currentFlagValue = false + continue + } + // is flag + if strings.HasPrefix(part, "-") { + raw := strings.TrimLeft(part, "-") + if strings.Contains(raw, "=") { + parts := strings.Split(raw, "=") + // a=b + if len(parts) == 2 { + comps = append(comps, cComp{ + raw: part, + cTag: parts[0], + cValue: parts[1], + cType: cmdCompFlag, + }) + } + // TODO handle part len != 2 + } else { + currentFlagValue = true + comps = append(comps, cComp{ + raw: part, + cTag: raw, + cType: cmdCompFlag, + }) + } + } else { + comps = append(comps, cComp{ + raw: part, + cTag: part, + cType: cmdCompCommand, + }) + } + } + + is := inputStateCmd + if currentFlagValue { + if isEndBlank { + is = inputStateFlagValue + } else { + is = inputStateFlagTag + } + } + + // add empty comp if end with space + if isEndBlank { + comps = append(comps, cComp{cType: cmdCompCommand}) + } + + return inputResult{ + parts: comps, + state: is, + } +} diff --git a/states/backup_mock_connect.go b/states/backup_mock_connect.go index 171645b..90f7c22 100644 --- a/states/backup_mock_connect.go +++ b/states/backup_mock_connect.go @@ -2,7 +2,6 @@ package states import ( "bufio" - "compress/gzip" "encoding/binary" "fmt" "io" @@ -10,22 +9,29 @@ import ( "net/url" "os" "path" - "strings" "github.com/golang/protobuf/proto" "github.com/milvus-io/birdwatcher/models" - "github.com/mitchellh/go-homedir" + "github.com/milvus-io/birdwatcher/states/etcd" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" ) +const ( + workspaceMetaFile = `.bw_project` +) + type embedEtcdMockState struct { cmdState client *clientv3.Client server *embed.Etcd instanceName string + workDir string + + metrics map[string][]byte + defaultMetrics map[string][]byte } // Close implements State. @@ -45,35 +51,32 @@ func (s *embedEtcdMockState) Close() { func (s *embedEtcdMockState) SetupCommands() { cmd := &cobra.Command{} + rootPath := path.Join(s.instanceName, metaPath) + cmd.AddCommand( // show [subcommand] options... - getEtcdShowCmd(s.client, path.Join(s.instanceName, metaPath)), + etcd.ShowCommand(s.client, rootPath), // download-pk - getDownloadPKCmd(s.client, path.Join(s.instanceName, metaPath)), + getDownloadPKCmd(s.client, rootPath), // inspect-pk - getInspectPKCmd(s.client, path.Join(s.instanceName, metaPath)), - // clean-empty-segment - cleanEmptySegments(s.client, path.Join(s.instanceName, metaPath)), - // remove-segment-by-id - removeSegmentByID(s.client, path.Join(s.instanceName, metaPath)), + getInspectPKCmd(s.client, rootPath), + // force-release - getForceReleaseCmd(s.client, path.Join(s.instanceName, metaPath)), + getForceReleaseCmd(s.client, rootPath), // disconnect getDisconnectCmd(s), - // repair-segment - getRepairSegmentCmd(s.client, path.Join(s.instanceName, metaPath)), - // repair-channel - getRepairChannelCmd(s.client, path.Join(s.instanceName, metaPath)), + // for testing + etcd.RepairCommand(s.client, rootPath), - // raw get - getEtcdRawCmd(s.client), + getPrintMetricsCmd(s), // exit getExitCmd(s), ) cmd.AddCommand(getGlobalUtilCommands()...) + cmd.AddCommand(etcd.RawCommands(s.client)...) s.cmdState.rootCmd = cmd s.setupFn = s.SetupCommands @@ -85,15 +88,86 @@ func (s *embedEtcdMockState) SetInstance(instanceName string) { s.SetupCommands() } +func (s *embedEtcdMockState) setupWorkDir(dir string) { + s.workDir = dir + s.syncWorkspaceInfo() +} + +// syncWorkspaceInfo try to read pre-written workspace meta info. +// if not exist or version is older, write a new one. +func (s *embedEtcdMockState) syncWorkspaceInfo() { + metaFilePath := path.Join(s.workDir, workspaceMetaFile) + err := testFile(metaFilePath) + if os.IsNotExist(err) { + fmt.Printf("%s not exist, writing a new one", metaFilePath) + // meta file not exist + s.writeWorkspaceMeta(metaFilePath) + } + if err != nil { + // path is a folder + fmt.Printf("%s cannot setup as workspace meta, err: %s\n", metaFilePath, err.Error()) + return + } + + s.readWorkspaceMeta(metaFilePath) +} + +func (s *embedEtcdMockState) writeWorkspaceMeta(path string) { + file, err := os.Create(path) + if err != nil { + fmt.Println("failed to open meta file to write", err.Error()) + return + } + defer file.Close() + + meta := &models.WorkspaceMeta{ + Version: "0.0.1", + Instance: s.instanceName, + MetaPath: metaPath, + } + + bs, err := proto.Marshal(meta) + if err != nil { + fmt.Println("failed to marshal meta info", err.Error()) + return + } + r := bufio.NewWriter(file) + + writeBackupBytes(r, bs) + r.Flush() +} + +func (s *embedEtcdMockState) readWorkspaceMeta(path string) { + file, err := os.Open(path) + if err != nil { + fmt.Printf("failed to open meta file %s to read, err: %s\n", path, err.Error()) + return + } + defer file.Close() + + r := bufio.NewReader(file) + + meta := models.WorkspaceMeta{} + err = readFixLengthHeader(r, &meta) + if err != nil { + fmt.Printf("failed to read %s as meta file, %s\n", path, err.Error()) + return + } + + s.SetInstance(meta.Instance) +} + func getEmbedEtcdInstance(server *embed.Etcd, cli *clientv3.Client, instanceName string) State { state := &embedEtcdMockState{ cmdState: cmdState{ label: fmt.Sprintf("Backup(%s)", instanceName), }, - instanceName: instanceName, - server: server, - client: cli, + instanceName: instanceName, + server: server, + client: cli, + metrics: make(map[string][]byte), + defaultMetrics: make(map[string][]byte), } state.SetupCommands() @@ -105,97 +179,38 @@ func getEmbedEtcdInstanceV2(server *embed.Etcd) *embedEtcdMockState { client := v3client.New(server.Server) state := &embedEtcdMockState{ - cmdState: cmdState{}, - server: server, - client: client, + cmdState: cmdState{}, + server: server, + client: client, + metrics: make(map[string][]byte), + defaultMetrics: make(map[string][]byte), } state.SetupCommands() return state } -func getLoadBackupCmd(state State) *cobra.Command { +func getPrintMetricsCmd(state *embedEtcdMockState) *cobra.Command { cmd := &cobra.Command{ - Use: "load-backup [file]", - Short: "load etcd backup file as env", + Use: "print-metrics", + Short: "print metrics restored from backup file", Run: func(cmd *cobra.Command, args []string) { - if len(args) == 0 { - fmt.Println("No backup file provided.") - return - } - if len(args) > 1 { - fmt.Println("only one backup file is allowed") - return - } - arg := args[0] - if strings.Contains(arg, "~") { - var err error - arg, err = homedir.Expand(arg) - if err != nil { - fmt.Println("path contains tilde, but cannot find home folder", err.Error()) - return - } - } - err := testFile(arg) + node, err := cmd.Flags().GetString("node") if err != nil { fmt.Println(err.Error()) return } - - f, err := os.Open(arg) - if err != nil { - fmt.Printf("failed to open backup file %s, err: %s\n", arg, err.Error()) - return - } - r, err := gzip.NewReader(f) - if err != nil { - fmt.Println("failed to open gzip reader, err:", err.Error()) - return - } - defer r.Close() - - rd := bufio.NewReader(r) - var header models.BackupHeader - err = readFixLengthHeader(rd, &header) - if err != nil { - fmt.Println("failed to load backup header", err.Error()) + metrics, ok := state.metrics[node] + if !ok { + fmt.Printf("not metrics found for node %s\n", node) return } - - server, err := startEmbedEtcdServer() - if err != nil { - fmt.Println("failed to start embed etcd server:", err.Error()) - return - } - fmt.Println("using data dir:", server.Config().Dir) - nextState := getEmbedEtcdInstanceV2(server) - switch header.Version { - case 1: - err = restoreFromV1File(nextState.client, rd, &header) - if err != nil { - fmt.Println("failed to restore v1 backup file", err.Error()) - nextState.Close() - return - } - nextState.SetInstance(header.Instance) - case 2: - err = restoreV2File(rd, nextState) - if err != nil { - fmt.Println("failed to restore v2 backup file", err.Error()) - nextState.Close() - return - } - default: - fmt.Printf("backup version %d not supported\n", header.Version) - nextState.Close() - return - } - - state.SetNext(nextState) + fmt.Println(string(metrics)) }, } + cmd.Flags().String("node", "", "select node metrics to print") return cmd } @@ -222,25 +237,28 @@ func readFixLengthHeader[T proto.Message](rd *bufio.Reader, header T) error { return nil } -// testFile check file path exists and has access -func testFile(file string) error { - fi, err := os.Stat(file) - if err != nil { - return err +// startEmbedEtcdServer start an embed etcd server to mock with backup data +func startEmbedEtcdServer(workspaceName string, useWorkspace bool) (*embed.Etcd, error) { + var dir string + var err error + if useWorkspace { + info, err := os.Stat(workspaceName) + if err == nil { + if info.IsDir() { + dir = workspaceName + } + } else { + fmt.Println(err.Error()) + } } - // not support iterate all possible file under directory for now - if fi.IsDir() { - return fmt.Errorf("%s is a folder", file) + if dir == "" { + fmt.Println("[Start Embed Etcd]using temp dir") + dir, err = ioutil.TempDir(os.TempDir(), "birdwatcher") + if err != nil { + return nil, err + } } - return nil -} -// startEmbedEtcdServer start an embed etcd server to mock with backup data -func startEmbedEtcdServer() (*embed.Etcd, error) { - dir, err := ioutil.TempDir(os.TempDir(), "birdwatcher") - if err != nil { - return nil, err - } fmt.Println("embed etcd use dir:", dir) config := embed.NewConfig() diff --git a/states/download_pk.go b/states/download_pk.go index cf77f06..cce52c7 100644 --- a/states/download_pk.go +++ b/states/download_pk.go @@ -11,6 +11,7 @@ import ( "github.com/gosuri/uilive" "github.com/manifoldco/promptui" "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/spf13/cobra" @@ -27,7 +28,7 @@ func getDownloadPKCmd(cli *clientv3.Client, basePath string) *cobra.Command { return err } - coll, err := getCollectionByID(cli, basePath, collectionID) + coll, err := common.GetCollectionByID(cli, basePath, collectionID) if err != nil { fmt.Println("Collection not found for id", collectionID) return nil @@ -47,7 +48,7 @@ func getDownloadPKCmd(cli *clientv3.Client, basePath string) *cobra.Command { return nil } - segments, err := listSegments(cli, basePath, func(segment *datapb.SegmentInfo) bool { + segments, err := common.ListSegments(cli, basePath, func(segment *datapb.SegmentInfo) bool { return segment.CollectionID == collectionID }) @@ -80,7 +81,7 @@ func getDownloadPKCmd(cli *clientv3.Client, basePath string) *cobra.Command { } for _, segment := range segments { - fillFieldsIfV2(cli, basePath, segment) + common.FillFieldsIfV2(cli, basePath, segment) } downloadPks(minioClient, bucketName, collectionID, pkID, segments) diff --git a/states/download_segment.go b/states/download_segment.go index 142c9ee..c03e89e 100644 --- a/states/download_segment.go +++ b/states/download_segment.go @@ -14,6 +14,7 @@ import ( "github.com/manifoldco/promptui" "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" "github.com/milvus-io/birdwatcher/proto/v2.0/indexpb" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/minio/minio-go/v7" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" @@ -38,7 +39,7 @@ func getDownloadSegmentCmd(cli *clientv3.Client, basePath string) *cobra.Command } } - segments, err := listSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { + segments, err := common.ListSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { _, ok := segSet[info.ID] return ok }) @@ -55,7 +56,7 @@ func getDownloadSegmentCmd(cli *clientv3.Client, basePath string) *cobra.Command folder := fmt.Sprintf("dlsegment_%s", time.Now().Format("20060102150406")) for _, segment := range segments { - fillFieldsIfV2(cli, basePath, segment) + common.FillFieldsIfV2(cli, basePath, segment) downloadSegment(minioClient, bucketName, segment, nil, folder) } diff --git a/states/etcd/commands.go b/states/etcd/commands.go new file mode 100644 index 0000000..6826a37 --- /dev/null +++ b/states/etcd/commands.go @@ -0,0 +1,148 @@ +package etcd + +import ( + "context" + "fmt" + + "github.com/milvus-io/birdwatcher/states/etcd/repair" + "github.com/milvus-io/birdwatcher/states/etcd/show" + "github.com/spf13/cobra" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// ShowCommand returns sub command for instanceState. +// show [subCommand] [options...] +// sub command [collection|session|segment] +func ShowCommand(cli *clientv3.Client, basePath string) *cobra.Command { + showCmd := &cobra.Command{ + Use: "show", + } + + showCmd.AddCommand( + // show collection + show.CollectionCommand(cli, basePath), + // show sessions + show.SessionCommand(cli, basePath), + // show segments + show.SegmentCommand(cli, basePath), + // show segment-loaded + show.SegmentLoadedCommand(cli, basePath), + // show index + show.IndexCommand(cli, basePath), + // show segment-index + show.SegmentIndexCommand(cli, basePath), + + // show replica + show.ReplicaCommand(cli, basePath), + // show checkpoint + show.CheckpointCommand(cli, basePath), + // show channel-watched + show.ChannelWatchedCommand(cli, basePath), + + // show collection-loaded + show.CollectionLoadedCommand(cli, basePath), + + // v2.1 legacy commands + // show querycoord-tasks + show.QueryCoordTasks(cli, basePath), + // show querycoord-channels + show.QueryCoordChannelCommand(cli, basePath), + // show querycoord-cluster + show.QueryCoordClusterCommand(cli, basePath), + ) + return showCmd +} + +// RepairCommand returns etcd repair commands. +func RepairCommand(cli *clientv3.Client, basePath string) *cobra.Command { + repairCmd := &cobra.Command{ + Use: "repair", + } + + repairCmd.AddCommand( + // repair segment + repair.SegmentCommand(cli, basePath), + // repair channel + repair.ChannelCommand(cli, basePath), + // repair checkpoint + repair.CheckpointCommand(cli, basePath), + // repair empty-segment + repair.EmptySegmentCommand(cli, basePath), + ) + + return repairCmd +} + +// RawCommands provides raw "get" command to list kv in etcd +func RawCommands(cli *clientv3.Client) []*cobra.Command { + cmd := &cobra.Command{ + Use: "get", + Run: func(cmd *cobra.Command, args []string) { + for _, arg := range args { + fmt.Println("list wrth", arg) + resp, err := cli.Get(context.Background(), arg, clientv3.WithPrefix()) + if err != nil { + fmt.Println(err.Error()) + continue + } + for _, kv := range resp.Kvs { + fmt.Printf("key: %s\n", string(kv.Key)) + fmt.Printf("Value: %s\n", string(kv.Value)) + } + } + }, + } + + return []*cobra.Command{cmd} +} + +/* +func removeSegmentByID(cli *clientv3.Client, basePath string) *cobra.Command { + cmd := &cobra.Command{ + Use: "remove-segment-by-id", + Short: "Remove segment from meta with specified segment id", + RunE: func(cmd *cobra.Command, args []string) error { + targetSegmentID, err := cmd.Flags().GetInt64("segment") + if err != nil { + return err + } + run, err := cmd.Flags().GetBool("run") + if err != nil { + return err + } + segments, err := listSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { + return true + }) + if err != nil { + fmt.Println("failed to list segments", err.Error()) + return nil + } + + for _, info := range segments { + if info.GetID() == targetSegmentID { + fmt.Printf("target segment %d found:\n", info.GetID()) + printSegmentInfo(info, false) + if run { + err := removeSegment(cli, basePath, info) + if err == nil { + fmt.Printf("remove segment %d from meta succeed\n", info.GetID()) + } else { + fmt.Printf("remove segment %d failed, err: %s\n", info.GetID(), err.Error()) + } + return nil + } + if !isEmptySegment(info) { + fmt.Printf("\n[WARN] segment %d is not empty, please make sure you know what you're doing\n", targetSegmentID) + } + return nil + } + } + fmt.Printf("[WARN] cannot find segment %d\n", targetSegmentID) + return nil + }, + } + + cmd.Flags().Bool("run", false, "flags indicating whether to remove segment from meta") + cmd.Flags().Int64("segment", 0, "segment id to remove") + return cmd +}*/ diff --git a/states/etcd/common/channel.go b/states/etcd/common/channel.go new file mode 100644 index 0000000..d5dd8b0 --- /dev/null +++ b/states/etcd/common/channel.go @@ -0,0 +1,19 @@ +package common + +import ( + "context" + "path" + "time" + + "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// ListChannelWatchV1 list v2.1 channel watch info meta. +func ListChannelWatchV1(cli *clientv3.Client, basePath string, filters ...func(channel *datapb.ChannelWatchInfo) bool) ([]datapb.ChannelWatchInfo, []string, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + + prefix := path.Join(basePath, "channelwatch") + "/" + return ListProtoObjects(ctx, cli, prefix, filters...) +} diff --git a/states/etcd/common/collection.go b/states/etcd/common/collection.go new file mode 100644 index 0000000..3ba114b --- /dev/null +++ b/states/etcd/common/collection.go @@ -0,0 +1,116 @@ +package common + +import ( + "bytes" + "context" + "errors" + "fmt" + "path" + "strconv" + "time" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/birdwatcher/proto/v2.0/etcdpb" + "github.com/milvus-io/birdwatcher/proto/v2.0/querypb" + "github.com/milvus-io/birdwatcher/proto/v2.0/schemapb" + querypbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/querypb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +const ( + // CollectionLoadPrefix is prefix for querycoord collection loaded in milvus v2.1.x + CollectionLoadPrefix = "queryCoord-collectionMeta" + // CollectionLoadPrefixV2 is prefix for querycoord collection loaded in milvus v2.2.x + CollectionLoadPrefixV2 = "querycoord-collection-loadinfo" +) + +var ( + ErrCollectionDropped = errors.New("collection dropped") + // CollectionTombstone is the special mark for collection dropped. + CollectionTombstone = []byte{0xE2, 0x9B, 0xBC} +) + +// GetCollectionByID returns collection info from etcd with provided id. +func GetCollectionByID(cli *clientv3.Client, basePath string, collID int64) (*etcdpb.CollectionInfo, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + resp, err := cli.Get(ctx, path.Join(basePath, "root-coord/collection", strconv.FormatInt(collID, 10))) + + if err != nil { + return nil, err + } + + if len(resp.Kvs) != 1 { + return nil, errors.New("invalid collection id") + } + + if bytes.Equal(resp.Kvs[0].Value, CollectionTombstone) { + return nil, fmt.Errorf("%w, collection id: %d", ErrCollectionDropped, collID) + } + + coll := &etcdpb.CollectionInfo{} + + err = proto.Unmarshal(resp.Kvs[0].Value, coll) + if err != nil { + return nil, err + } + + err = FillFieldSchemaIfEmpty(cli, basePath, coll) + if err != nil { + return nil, err + } + + return coll, nil +} + +func FillFieldSchemaIfEmpty(cli *clientv3.Client, basePath string, collection *etcdpb.CollectionInfo) error { + if len(collection.GetSchema().GetFields()) == 0 { // fields separated from schema after 2.1.1 + resp, err := cli.Get(context.TODO(), path.Join(basePath, fmt.Sprintf("root-coord/fields/%d", collection.ID)), clientv3.WithPrefix()) + if err != nil { + return err + } + for _, kv := range resp.Kvs { + field := &schemapb.FieldSchema{} + err := proto.Unmarshal(kv.Value, field) + if err != nil { + fmt.Println("found error field:", string(kv.Key), err.Error()) + continue + } + collection.Schema.Fields = append(collection.Schema.Fields, field) + } + } + + return nil +} + +// ListLoadedCollectionInfoV2_1 returns collection info from querycoord milvus v2.1.x. +func ListLoadedCollectionInfoV2_1(cli *clientv3.Client, basePath string) ([]*querypb.CollectionInfo, error) { + prefix := path.Join(basePath, CollectionLoadPrefix) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + resp, err := cli.Get(ctx, prefix, clientv3.WithPrefix()) + if err != nil { + return nil, err + } + ret := make([]*querypb.CollectionInfo, 0) + for _, kv := range resp.Kvs { + collectionInfo := &querypb.CollectionInfo{} + err = proto.Unmarshal(kv.Value, collectionInfo) + if err != nil { + return nil, err + } + ret = append(ret, collectionInfo) + } + return ret, nil +} + +// ListLoadedCollectionInfoV2_1 returns collection info from querycoord milvus v2.2.x. +func ListLoadedCollectionInfoV2_2(cli *clientv3.Client, basePath string) ([]querypbv2.CollectionLoadInfo, error) { + prefix := path.Join(basePath, CollectionLoadPrefixV2) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + info, _, err := ListProtoObjects[querypbv2.CollectionLoadInfo](ctx, cli, prefix) + return info, err +} diff --git a/states/etcd/common/index.go b/states/etcd/common/index.go new file mode 100644 index 0000000..53068ec --- /dev/null +++ b/states/etcd/common/index.go @@ -0,0 +1,31 @@ +package common + +import ( + "context" + "path" + "time" + + "github.com/milvus-io/birdwatcher/proto/v2.0/etcdpb" + "github.com/milvus-io/birdwatcher/proto/v2.0/indexpb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// ListIndex list all index with all filter satified. +func ListIndex(cli *clientv3.Client, basePath string, filters ...func(index *indexpb.IndexMeta) bool) ([]indexpb.IndexMeta, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + prefix := path.Join(basePath, "indexes") + "/" + + result, _, err := ListProtoObjects(ctx, cli, prefix, filters...) + return result, err +} + +// ListSegmentIndex list segment index info. +func ListSegmentIndex(cli *clientv3.Client, basePath string, filters ...func(segIdx *etcdpb.SegmentIndexInfo) bool) ([]etcdpb.SegmentIndexInfo, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + + prefix := path.Join(basePath, "root-coord/segment-index") + "/" + result, _, err := ListProtoObjects(ctx, cli, prefix, filters...) + return result, err +} diff --git a/states/etcd/common/legacy.go b/states/etcd/common/legacy.go new file mode 100644 index 0000000..fe17122 --- /dev/null +++ b/states/etcd/common/legacy.go @@ -0,0 +1,106 @@ +package common + +import ( + "context" + "path" + "time" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + "github.com/milvus-io/birdwatcher/proto/v2.0/querypb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +const ( + QCDmChannelMetaPrefix = "queryCoord-dmChannelWatchInfo" + QCDeltaChannelMetaPrefix = "queryCoord-deltaChannel" +) + +func ListQueryCoordDMLChannelInfos(cli *clientv3.Client, basePath string) ([]*querypb.DmChannelWatchInfo, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + prefix := path.Join(basePath, QCDmChannelMetaPrefix) + + resp, err := cli.Get(ctx, prefix, clientv3.WithPrefix()) + if err != nil { + return nil, err + } + + ret := make([]*querypb.DmChannelWatchInfo, 0) + for _, kv := range resp.Kvs { + channelInfo := &querypb.DmChannelWatchInfo{} + err = proto.Unmarshal(kv.Value, channelInfo) + if err != nil { + return nil, err + } + ret = append(ret, channelInfo) + } + return ret, nil +} + +func ListQueryCoordDeltaChannelInfos(cli *clientv3.Client, basePath string) ([]*datapb.VchannelInfo, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + prefix := path.Join(basePath, QCDeltaChannelMetaPrefix) + + resp, err := cli.Get(ctx, prefix, clientv3.WithPrefix()) + if err != nil { + return nil, err + } + + ret := make([]*datapb.VchannelInfo, 0, len(resp.Kvs)) + for _, kv := range resp.Kvs { + channelInfo := &datapb.VchannelInfo{} + err = proto.Unmarshal(kv.Value, channelInfo) + if err != nil { + return nil, err + } + reviseVChannelInfo(channelInfo) + ret = append(ret, channelInfo) + } + return ret, nil +} + +// reviseVChannelInfo will revise the datapb.VchannelInfo for upgrade compatibility from 2.0.2 +func reviseVChannelInfo(vChannel *datapb.VchannelInfo) { + removeDuplicateSegmentIDFn := func(ids []int64) []int64 { + result := make([]int64, 0, len(ids)) + existDict := make(map[int64]bool) + for _, id := range ids { + if _, ok := existDict[id]; !ok { + existDict[id] = true + result = append(result, id) + } + } + return result + } + + if vChannel == nil { + return + } + // if the segment infos is not nil(generated by 2.0.2), append the corresponding IDs to segmentIDs + // and remove the segment infos, remove deplicate ids in case there are some mixed situations + if vChannel.FlushedSegments != nil && len(vChannel.FlushedSegments) > 0 { + for _, segment := range vChannel.FlushedSegments { + vChannel.FlushedSegmentIds = append(vChannel.GetFlushedSegmentIds(), segment.GetID()) + } + vChannel.FlushedSegments = []*datapb.SegmentInfo{} + } + vChannel.FlushedSegmentIds = removeDuplicateSegmentIDFn(vChannel.GetFlushedSegmentIds()) + + if vChannel.UnflushedSegments != nil && len(vChannel.UnflushedSegments) > 0 { + for _, segment := range vChannel.UnflushedSegments { + vChannel.UnflushedSegmentIds = append(vChannel.GetUnflushedSegmentIds(), segment.GetID()) + } + vChannel.UnflushedSegments = []*datapb.SegmentInfo{} + } + vChannel.UnflushedSegmentIds = removeDuplicateSegmentIDFn(vChannel.GetUnflushedSegmentIds()) + + if vChannel.DroppedSegments != nil && len(vChannel.DroppedSegments) > 0 { + for _, segment := range vChannel.DroppedSegments { + vChannel.DroppedSegmentIds = append(vChannel.GetDroppedSegmentIds(), segment.GetID()) + } + vChannel.DroppedSegments = []*datapb.SegmentInfo{} + } + vChannel.DroppedSegmentIds = removeDuplicateSegmentIDFn(vChannel.GetDroppedSegmentIds()) +} diff --git a/states/etcd/common/list.go b/states/etcd/common/list.go new file mode 100644 index 0000000..2e4df99 --- /dev/null +++ b/states/etcd/common/list.go @@ -0,0 +1,42 @@ +package common + +import ( + "context" + "fmt" + + "github.com/golang/protobuf/proto" + clientv3 "go.etcd.io/etcd/client/v3" + "google.golang.org/protobuf/runtime/protoiface" +) + +// ListProtoObjects returns proto objects with specified prefix. +func ListProtoObjects[T any, P interface { + *T + protoiface.MessageV1 +}](ctx context.Context, cli *clientv3.Client, prefix string, filters ...func(t *T) bool) ([]T, []string, error) { + resp, err := cli.Get(ctx, prefix, clientv3.WithPrefix()) + if err != nil { + return nil, nil, err + } + result := make([]T, 0, len(resp.Kvs)) + keys := make([]string, 0, len(resp.Kvs)) +LOOP: + for _, kv := range resp.Kvs { + var elem T + info := P(&elem) + err = proto.Unmarshal(kv.Value, info) + if err != nil { + fmt.Println(err.Error()) + continue + } + + for _, filter := range filters { + if !filter(&elem) { + continue LOOP + } + } + result = append(result, elem) + keys = append(keys, string(kv.Key)) + } + return result, keys, nil +} diff --git a/states/etcd/common/pair.go b/states/etcd/common/pair.go new file mode 100644 index 0000000..e22df54 --- /dev/null +++ b/states/etcd/common/pair.go @@ -0,0 +1,14 @@ +package common + +// GetKVPair iterates KV pairs to find specified key. +func GetKVPair[T interface { + GetKey() string + GetValue() string +}](pairs []T, key string) string { + for _, pair := range pairs { + if pair.GetKey() == key { + return pair.GetValue() + } + } + return "" +} diff --git a/states/etcd/common/path.go b/states/etcd/common/path.go new file mode 100644 index 0000000..d372361 --- /dev/null +++ b/states/etcd/common/path.go @@ -0,0 +1,31 @@ +package common + +import ( + "errors" + "strconv" + "strings" +) + +func PathPartInt64(p string, idx int) (int64, error) { + part, err := PathPart(p, idx) + if err != nil { + return 0, err + } + v, err := strconv.ParseInt(part, 10, 64) + if err != nil { + return 0, err + } + return v, nil +} + +func PathPart(p string, idx int) (string, error) { + parts := strings.Split(p, "/") + // -1 means last part + if idx < 0 { + idx = len(parts) + idx + } + if idx < 0 && idx >= len(parts) { + return "", errors.New("out of index") + } + return parts[idx], nil +} diff --git a/states/etcd/common/segment.go b/states/etcd/common/segment.go new file mode 100644 index 0000000..4c69ce5 --- /dev/null +++ b/states/etcd/common/segment.go @@ -0,0 +1,121 @@ +package common + +import ( + "context" + "fmt" + "path" + "sort" + "time" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + "github.com/milvus-io/birdwatcher/proto/v2.0/querypb" + datapbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/datapb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// ListSegments list segment info from etcd +func ListSegments(cli *clientv3.Client, basePath string, filter func(*datapb.SegmentInfo) bool) ([]*datapb.SegmentInfo, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + resp, err := cli.Get(ctx, path.Join(basePath, "datacoord-meta/s")+"/", clientv3.WithPrefix()) + if err != nil { + return nil, err + } + segments := make([]*datapb.SegmentInfo, 0, len(resp.Kvs)) + for _, kv := range resp.Kvs { + info := &datapb.SegmentInfo{} + err = proto.Unmarshal(kv.Value, info) + if err != nil { + continue + } + if filter == nil || filter(info) { + segments = append(segments, info) + } + } + + sort.Slice(segments, func(i, j int) bool { + return segments[i].GetID() < segments[j].GetID() + }) + return segments, nil +} + +// FillFieldsIfV2 fill binlog paths fields for v2 segment info. +func FillFieldsIfV2(cli *clientv3.Client, basePath string, segment *datapb.SegmentInfo) error { + if len(segment.Binlogs) == 0 { + prefix := path.Join(basePath, "datacoord-meta", fmt.Sprintf("binlog/%d/%d/%d", segment.CollectionID, segment.PartitionID, segment.ID)) + fields, _, err := ListProtoObjects[datapbv2.FieldBinlog](context.Background(), cli, prefix) + if err != nil { + return err + } + + segment.Binlogs = make([]*datapb.FieldBinlog, 0, len(fields)) + for _, field := range fields { + f := &datapb.FieldBinlog{ + FieldID: field.FieldID, + Binlogs: make([]*datapb.Binlog, 0, len(field.Binlogs)), + } + + for _, binlog := range field.Binlogs { + l := &datapb.Binlog{ + EntriesNum: binlog.EntriesNum, + TimestampFrom: binlog.TimestampFrom, + TimestampTo: binlog.TimestampTo, + LogPath: binlog.LogPath, + LogSize: binlog.LogSize, + } + if l.LogPath == "" { + l.LogPath = fmt.Sprintf("files/insert_log/%d/%d/%d/%d/%d", segment.CollectionID, segment.PartitionID, segment.ID, field.FieldID, binlog.LogID) + } + f.Binlogs = append(f.Binlogs, l) + } + segment.Binlogs = append(segment.Binlogs, f) + } + } + + if len(segment.Deltalogs) == 0 { + prefix := path.Join(basePath, "datacoord-meta", fmt.Sprintf("deltalog/%d/%d/%d", segment.CollectionID, segment.PartitionID, segment.ID)) + fields, _, err := ListProtoObjects[datapb.FieldBinlog](context.Background(), cli, prefix) + if err != nil { + return err + } + + segment.Deltalogs = make([]*datapb.FieldBinlog, 0, len(fields)) + for _, field := range fields { + field := field + f := proto.Clone(&field).(*datapb.FieldBinlog) + segment.Deltalogs = append(segment.Deltalogs, f) + } + } + + if len(segment.Statslogs) == 0 { + prefix := path.Join(basePath, "datacoord-meta", fmt.Sprintf("statslog/%d/%d/%d", segment.CollectionID, segment.PartitionID, segment.ID)) + fields, _, err := ListProtoObjects[datapb.FieldBinlog](context.Background(), cli, prefix) + if err != nil { + return err + } + + segment.Statslogs = make([]*datapb.FieldBinlog, 0, len(fields)) + for _, field := range fields { + field := field + f := proto.Clone(&field).(*datapb.FieldBinlog) + segment.Statslogs = append(segment.Statslogs, f) + } + } + + return nil +} + +// ListLoadedSegments list v2.1 loaded segment info. +func ListLoadedSegments(cli *clientv3.Client, basePath string, filter func(*querypb.SegmentInfo) bool) ([]querypb.SegmentInfo, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + prefix := path.Join(basePath, "queryCoord-segmentMeta") + + segments, _, err := ListProtoObjects(ctx, cli, prefix, filter) + if err != nil { + return nil, err + } + + return segments, nil +} diff --git a/states/etcd/common/session.go b/states/etcd/common/session.go new file mode 100644 index 0000000..efeec7b --- /dev/null +++ b/states/etcd/common/session.go @@ -0,0 +1,43 @@ +package common + +import ( + "context" + "encoding/json" + "path" + "time" + + "github.com/milvus-io/birdwatcher/models" + clientv3 "go.etcd.io/etcd/client/v3" +) + +const ( + sessionPrefix = `session` +) + +// ListSessions returns all session. +func ListSessions(cli *clientv3.Client, basePath string) ([]*models.Session, error) { + prefix := path.Join(basePath, sessionPrefix) + return ListSessionsByPrefix(cli, prefix) +} + +// ListSessionsByPrefix returns all session with provided prefix. +func ListSessionsByPrefix(cli *clientv3.Client, prefix string) ([]*models.Session, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + resp, err := cli.Get(ctx, prefix, clientv3.WithPrefix()) + if err != nil { + return nil, err + } + + sessions := make([]*models.Session, 0, len(resp.Kvs)) + for _, kv := range resp.Kvs { + session := &models.Session{} + err := json.Unmarshal(kv.Value, session) + if err != nil { + continue + } + + sessions = append(sessions, session) + } + return sessions, nil +} diff --git a/states/etcd/common/sort.go b/states/etcd/common/sort.go new file mode 100644 index 0000000..c7d2ca7 --- /dev/null +++ b/states/etcd/common/sort.go @@ -0,0 +1,14 @@ +package common + +import "sort" + +type WithCollectionID interface { + GetCollectionID() int64 +} + +// SortByCollection generic sort function +func SortByCollection[T WithCollectionID](items []T) { + sort.Slice(items, func(i, j int) bool { + return items[i].GetCollectionID() < items[j].GetCollectionID() + }) +} diff --git a/states/repair_channel.go b/states/etcd/repair/channel.go similarity index 84% rename from states/repair_channel.go rename to states/etcd/repair/channel.go index 76d0219..dd437f0 100644 --- a/states/repair_channel.go +++ b/states/etcd/repair/channel.go @@ -1,4 +1,4 @@ -package states +package repair import ( "context" @@ -7,15 +7,18 @@ import ( "github.com/milvus-io/birdwatcher/proto/v2.0/commonpb" "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "google.golang.org/grpc" ) -func getRepairChannelCmd(cli *clientv3.Client, basePath string) *cobra.Command { +// ChannelCommand returns repair channel command. +func ChannelCommand(cli *clientv3.Client, basePath string) *cobra.Command { cmd := &cobra.Command{ - Use: "repair-channel", - Short: "do channel watch change and try to repair", + Use: "channel", + Aliases: []string{"channels"}, + Short: "do channel watch change and try to repair", Run: func(cmd *cobra.Command, args []string) { collID, err := cmd.Flags().GetInt64("collection") if err != nil { @@ -33,7 +36,7 @@ func getRepairChannelCmd(cli *clientv3.Client, basePath string) *cobra.Command { return } - coll, err := getCollectionByID(cli, basePath, collID) + coll, err := common.GetCollectionByID(cli, basePath, collID) if err != nil { fmt.Println("collection not found") return @@ -44,7 +47,7 @@ func getRepairChannelCmd(cli *clientv3.Client, basePath string) *cobra.Command { chans[vchan] = struct{}{} } - infos, _, err := listChannelWatchV1(cli, basePath) + infos, _, err := common.ListChannelWatchV1(cli, basePath) if err != nil { fmt.Println("failed to list channel watch info", err.Error()) return @@ -77,7 +80,7 @@ func getRepairChannelCmd(cli *clientv3.Client, basePath string) *cobra.Command { } func doDatacoordWatch(cli *clientv3.Client, basePath string, collectionID int64, vchannels []string) { - sessions, err := listSessions(cli, basePath) + sessions, err := common.ListSessions(cli, basePath) if err != nil { fmt.Println("failed to list session") return diff --git a/states/reset_checkpoint.go b/states/etcd/repair/checkpoint.go similarity index 92% rename from states/reset_checkpoint.go rename to states/etcd/repair/checkpoint.go index 376268e..a166173 100644 --- a/states/reset_checkpoint.go +++ b/states/etcd/repair/checkpoint.go @@ -1,4 +1,4 @@ -package states +package repair import ( "context" @@ -15,14 +15,16 @@ import ( "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" "github.com/milvus-io/birdwatcher/proto/v2.0/etcdpb" "github.com/milvus-io/birdwatcher/proto/v2.0/internalpb" + "github.com/milvus-io/birdwatcher/states/etcd/common" + "github.com/milvus-io/birdwatcher/utils" ) // Example: // reset-checkpoint --collection 437744071571606912 --vchannel by-dev-rootcoord-dml_3_437744071571606912v1 --mq_type kafka --address localhost:9092 --set_to latest-msgid // reset-checkpoint --collection 437744071571606912 --vchannel by-dev-rootcoord-dml_3_437744071571606912v1 --mq_type pulsar --address pulsar://localhost:6650 --set_to latest-msgid -func getResetCheckpointCmd(cli *clientv3.Client, basePath string) *cobra.Command { +func CheckpointCommand(cli *clientv3.Client, basePath string) *cobra.Command { cmd := &cobra.Command{ - Use: "reset-checkpoint", + Use: "checkpoint", Short: "reset checkpoint of vchannels to latest checkpoint(or latest msgID) of physical channel", Aliases: []string{"rc"}, Run: func(cmd *cobra.Command, args []string) { @@ -44,7 +46,7 @@ func getResetCheckpointCmd(cli *clientv3.Client, basePath string) *cobra.Command return } - coll, err := getCollectionByID(cli, basePath, collID) + coll, err := common.GetCollectionByID(cli, basePath, collID) if err != nil { fmt.Println("failed to get collection", err.Error()) return @@ -94,7 +96,7 @@ func setCheckPointWithLatestMsgID(cli *clientv3.Client, basePath string, coll *e } err = saveChannelCheckpoint(cli, basePath, ch, cp) - t, _ := ParseTS(cp.GetTimestamp()) + t, _ := utils.ParseTS(cp.GetTimestamp()) if err != nil { fmt.Printf("failed to set latest msgID(ts:%v) for vchannel:%s", t, ch) return @@ -115,7 +117,7 @@ func setCheckPointWithLatestCheckPoint(cli *clientv3.Client, basePath string, co fmt.Println("list the latest checkpoint of all physical channels:") for k, v := range pChannelName2LatestCP { - t, _ := ParseTS(v.GetTimestamp()) + t, _ := utils.ParseTS(v.GetTimestamp()) fmt.Printf("pchannel: %s, the lastest checkpoint ts: %v\n", k, t) } @@ -129,7 +131,7 @@ func setCheckPointWithLatestCheckPoint(cli *clientv3.Client, basePath string, co } err := saveChannelCheckpoint(cli, basePath, ch, cp) - t, _ := ParseTS(cp.GetTimestamp()) + t, _ := utils.ParseTS(cp.GetTimestamp()) if err != nil { fmt.Printf("failed to set latest checkpoint(ts:%v) for vchannel:%s", t, ch) return @@ -153,7 +155,7 @@ func saveChannelCheckpoint(cli *clientv3.Client, basePath string, channelName st } func getLatestCheckpointFromPChannel(cli *clientv3.Client, basePath string) (map[string]*internalpb.MsgPosition, error) { - segments, err := listSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { + segments, err := common.ListSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { return true }) if err != nil { diff --git a/states/repair_segment.go b/states/etcd/repair/segment.go similarity index 76% rename from states/repair_segment.go rename to states/etcd/repair/segment.go index d277785..0b1fe6e 100644 --- a/states/repair_segment.go +++ b/states/etcd/repair/segment.go @@ -1,12 +1,9 @@ -package states +package repair import ( - "bufio" "context" "fmt" - "os" "path" - "time" "github.com/golang/protobuf/proto" "github.com/milvus-io/birdwatcher/proto/v2.0/commonpb" @@ -14,15 +11,17 @@ import ( "github.com/milvus-io/birdwatcher/proto/v2.0/etcdpb" "github.com/milvus-io/birdwatcher/proto/v2.0/indexpb" "github.com/milvus-io/birdwatcher/proto/v2.0/schemapb" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" ) -// getRepairSegmentCmd returns command for repair-segment. -func getRepairSegmentCmd(cli *clientv3.Client, basePath string) *cobra.Command { +// SegmentCommand return repair segment command. +func SegmentCommand(cli *clientv3.Client, basePath string) *cobra.Command { cmd := &cobra.Command{ - Use: "repair-segment", - Short: "do segment & index meta check and try to repair", + Use: "segment", + Aliases: []string{"segments"}, + Short: "do segment & index meta check and try to repair", Run: func(cmd *cobra.Command, args []string) { collID, err := cmd.Flags().GetInt64("collection") @@ -35,18 +34,19 @@ func getRepairSegmentCmd(cli *clientv3.Client, basePath string) *cobra.Command { fmt.Println(err.Error()) return } - skipDownload, err := cmd.Flags().GetBool("skip-download") - if err != nil { - fmt.Println(err.Error()) - return - } + /* + skipDownload, err := cmd.Flags().GetBool("skip-download") + if err != nil { + fmt.Println(err.Error()) + return + }*/ run, err := cmd.Flags().GetBool("run") if err != nil { fmt.Println(err.Error()) return } - segments, err := listSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { + segments, err := common.ListSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { return (collID == 0 || info.CollectionID == collID) && (segmentID == 0 || info.ID == segmentID) }) @@ -56,13 +56,13 @@ func getRepairSegmentCmd(cli *clientv3.Client, basePath string) *cobra.Command { } // use v1 meta for now - segmentIndexes, err := listSegmentIndex(cli, basePath) + segmentIndexes, err := common.ListSegmentIndex(cli, basePath) if err != nil { fmt.Println(err.Error()) return } - indexBuildInfo, err := listIndex(cli, basePath) + indexBuildInfo, err := common.ListIndex(cli, basePath) if err != nil { fmt.Println(err.Error()) return @@ -105,7 +105,7 @@ func getRepairSegmentCmd(cli *clientv3.Client, basePath string) *cobra.Command { coll, ok := collections[segment.CollectionID] if !ok { - coll, err = getCollectionByID(cli, basePath, segment.CollectionID) + coll, err = common.GetCollectionByID(cli, basePath, segment.CollectionID) if err != nil { fmt.Printf("failed to query collection(id=%d) info error: %s", segment.CollectionID, err.Error()) continue @@ -113,7 +113,7 @@ func getRepairSegmentCmd(cli *clientv3.Client, basePath string) *cobra.Command { collections[segment.CollectionID] = coll } - fillFieldsIfV2(cli, basePath, segment) + common.FillFieldsIfV2(cli, basePath, segment) for _, segIdx := range segIdxs { var valid bool @@ -150,51 +150,52 @@ func getRepairSegmentCmd(cli *clientv3.Client, basePath string) *cobra.Command { return } - if !skipDownload { - minioClient, bucketName, err := getMinioAccess() + /* + if !skipDownload { + minioClient, bucketName, err := getMinioAccess() - if err != nil { - fmt.Println("failed to get minio access", err.Error()) - } - - folder := fmt.Sprintf("repair_segment_%s", time.Now().Format("20060102150406")) - for segmentID, segment := range target { - fileName := fmt.Sprintf("segment_%d", segmentID) - oldSeg := targetOld[segmentID] - f, err := os.Create(path.Join(folder, fileName)) if err != nil { - fmt.Println("failed to open file", fileName, err.Error()) - return + fmt.Println("failed to get minio access", err.Error()) } - bs, err := proto.Marshal(oldSeg) - w := bufio.NewWriter(f) - w.Write(bs) - w.Flush() - f.Close() - - index, ok := targetIndex[segmentID] - if ok { - fileName = fmt.Sprintf("index_%d", segmentID) - f, err = os.Create(path.Join(folder, fileName)) + + folder := fmt.Sprintf("repair_segment_%s", time.Now().Format("20060102150406")) + for segmentID, segment := range target { + fileName := fmt.Sprintf("segment_%d", segmentID) + oldSeg := targetOld[segmentID] + f, err := os.Create(path.Join(folder, fileName)) if err != nil { fmt.Println("failed to open file", fileName, err.Error()) return } - bs, err = proto.Marshal(index) + bs, err := proto.Marshal(oldSeg) w := bufio.NewWriter(f) w.Write(bs) w.Flush() f.Close() - } + index, ok := targetIndex[segmentID] + if ok { + fileName = fmt.Sprintf("index_%d", segmentID) + f, err = os.Create(path.Join(folder, fileName)) + if err != nil { + fmt.Println("failed to open file", fileName, err.Error()) + return + } + bs, err = proto.Marshal(index) + w := bufio.NewWriter(f) + w.Write(bs) + w.Flush() + f.Close() - err = downloadSegment(minioClient, bucketName, segment, targetIndex[segmentID], folder) - if err != nil { - fmt.Println("failed to download segment", err.Error()) - return + } + + err = downloadSegment(minioClient, bucketName, segment, targetIndex[segmentID], folder) + if err != nil { + fmt.Println("failed to download segment", err.Error()) + return + } } - } - } + }*/ // row count check for segmentID, segment := range target { diff --git a/states/etcd/repair/segment_empty.go b/states/etcd/repair/segment_empty.go new file mode 100644 index 0000000..5ca18dc --- /dev/null +++ b/states/etcd/repair/segment_empty.go @@ -0,0 +1,87 @@ +package repair + +import ( + "context" + "fmt" + "path" + "time" + + "github.com/milvus-io/birdwatcher/proto/v2.0/commonpb" + "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + "github.com/milvus-io/birdwatcher/states/etcd/common" + "github.com/spf13/cobra" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// EmptySegmentCommand returns repair empty-segment command. +func EmptySegmentCommand(cli *clientv3.Client, basePath string) *cobra.Command { + cmd := &cobra.Command{ + Use: "empty-segment", + Short: "Remove empty segment from meta", + RunE: func(cmd *cobra.Command, args []string) error { + run, err := cmd.Flags().GetBool("run") + if err != nil { + return err + } + segments, err := common.ListSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { + return info.GetState() == commonpb.SegmentState_Flushed || info.GetState() == commonpb.SegmentState_Flushing || info.GetState() == commonpb.SegmentState_Sealed + }) + if err != nil { + fmt.Println("failed to list segments", err.Error()) + return nil + } + + for _, info := range segments { + common.FillFieldsIfV2(cli, basePath, info) + if isEmptySegment(info) { + fmt.Printf("suspect segment %d found:\n", info.GetID()) + fmt.Printf("SegmentID: %d State: %s, Row Count:%d\n", info.ID, info.State.String(), info.NumOfRows) + if run { + err := removeSegment(cli, basePath, info) + if err == nil { + fmt.Printf("remove segment %d from meta succeed\n", info.GetID()) + } else { + fmt.Printf("remove segment %d failed, err: %s\n", info.GetID(), err.Error()) + } + } + + } + } + + return nil + }, + } + + cmd.Flags().Bool("run", false, "flags indicating whether to remove segments from meta") + return cmd +} + +// returns whether all binlog/statslog/deltalog is empty +func isEmptySegment(info *datapb.SegmentInfo) bool { + for _, log := range info.GetBinlogs() { + if len(log.Binlogs) > 0 { + return false + } + } + for _, log := range info.GetStatslogs() { + if len(log.Binlogs) > 0 { + return false + } + } + for _, log := range info.GetDeltalogs() { + if len(log.Binlogs) > 0 { + return false + } + } + return true +} + +func removeSegment(cli *clientv3.Client, basePath string, info *datapb.SegmentInfo) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + + path := path.Join(basePath, "datacoord-meta/s", fmt.Sprintf("%d/%d/%d", info.CollectionID, info.PartitionID, info.ID)) + _, err := cli.Delete(ctx, path) + + return err +} diff --git a/states/show_channel_watch.go b/states/etcd/show/channel_watched.go similarity index 66% rename from states/show_channel_watch.go rename to states/etcd/show/channel_watched.go index 8be16a6..6829ac6 100644 --- a/states/show_channel_watch.go +++ b/states/etcd/show/channel_watched.go @@ -1,17 +1,18 @@ -package states +package show import ( - "context" "fmt" - "path" "time" "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + "github.com/milvus-io/birdwatcher/states/etcd/common" + "github.com/milvus-io/birdwatcher/utils" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" ) -func getEtcdShowChannelWatch(cli *clientv3.Client, basePath string) *cobra.Command { +// ChannelWatchedCommand return show channel-watched commands. +func ChannelWatchedCommand(cli *clientv3.Client, basePath string) *cobra.Command { cmd := &cobra.Command{ Use: "channel-watch", Short: "display channel watching info from data coord meta store", @@ -23,18 +24,16 @@ func getEtcdShowChannelWatch(cli *clientv3.Client, basePath string) *cobra.Comma return } - infos, _, err := listChannelWatchV1(cli, basePath) + infos, keys, err := common.ListChannelWatchV1(cli, basePath, func(channel *datapb.ChannelWatchInfo) bool { + return collID == 0 || channel.GetVchan().GetCollectionID() == collID + }) if err != nil { fmt.Println("failed to list channel watch info", err.Error()) return } - for _, info := range infos { - if collID > 0 && info.GetVchan().GetCollectionID() != collID { - continue - } - - printChannelWatchInfo(info) + for i, info := range infos { + printChannelWatchInfo(info, keys[i]) } fmt.Printf("--- Total Channels: %d\n", len(infos)) @@ -44,16 +43,9 @@ func getEtcdShowChannelWatch(cli *clientv3.Client, basePath string) *cobra.Comma return cmd } -func listChannelWatchV1(cli *clientv3.Client, basePath string) ([]datapb.ChannelWatchInfo, []string, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - - prefix := path.Join(basePath, "channelwatch") + "/" - return listObject[datapb.ChannelWatchInfo](ctx, cli, prefix) -} - -func printChannelWatchInfo(info datapb.ChannelWatchInfo) { +func printChannelWatchInfo(info datapb.ChannelWatchInfo, key string) { fmt.Println("=============================") + fmt.Printf("key: %s\n", key) fmt.Printf("Channel Name:%s \t WatchState: %s\n", info.GetVchan().GetChannelName(), info.GetState().String()) //t, _ := ParseTS(uint64(info.GetStartTs())) //to, _ := ParseTS(uint64(info.GetTimeoutTs())) @@ -62,7 +54,7 @@ func printChannelWatchInfo(info datapb.ChannelWatchInfo) { fmt.Printf("Channel Watch start from: %s, timeout at: %s\n", t.Format(tsPrintFormat), to.Format(tsPrintFormat)) pos := info.GetVchan().GetSeekPosition() - startTime, _ := ParseTS(pos.GetTimestamp()) + startTime, _ := utils.ParseTS(pos.GetTimestamp()) fmt.Printf("Start Position ID: %v, time: %s\n", pos.GetMsgID(), startTime.Format(tsPrintFormat)) fmt.Printf("Unflushed segments: %v\n", info.Vchan.GetUnflushedSegmentIds()) diff --git a/states/show_checkpoint.go b/states/etcd/show/checkpoint.go similarity index 84% rename from states/show_checkpoint.go rename to states/etcd/show/checkpoint.go index fb0c096..36c6e35 100644 --- a/states/show_checkpoint.go +++ b/states/etcd/show/checkpoint.go @@ -1,4 +1,4 @@ -package states +package show import ( "context" @@ -8,11 +8,14 @@ import ( "github.com/milvus-io/birdwatcher/proto/v2.0/commonpb" "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" "github.com/milvus-io/birdwatcher/proto/v2.0/internalpb" + "github.com/milvus-io/birdwatcher/states/etcd/common" + "github.com/milvus-io/birdwatcher/utils" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" ) -func getCheckpointCmd(cli *clientv3.Client, basePath string) *cobra.Command { +// CheckpointCommand returns show checkpoint command. +func CheckpointCommand(cli *clientv3.Client, basePath string) *cobra.Command { cmd := &cobra.Command{ Use: "checkpoint", Short: "list checkpoint collection vchannels", @@ -25,7 +28,7 @@ func getCheckpointCmd(cli *clientv3.Client, basePath string) *cobra.Command { return } - coll, err := getCollectionByID(cli, basePath, collID) + coll, err := common.GetCollectionByID(cli, basePath, collID) if err != nil { fmt.Println("failed to get collection", err.Error()) return @@ -44,7 +47,7 @@ func getCheckpointCmd(cli *clientv3.Client, basePath string) *cobra.Command { if cp == nil { fmt.Printf("vchannel %s position nil\n", vchannel) } else { - t, _ := ParseTS(cp.GetTimestamp()) + t, _ := utils.ParseTS(cp.GetTimestamp()) fmt.Printf("vchannel %s seek to %v", vchannel, t) if segmentID > 0 { fmt.Printf(", for segment ID:%d\n", segmentID) @@ -61,7 +64,7 @@ func getCheckpointCmd(cli *clientv3.Client, basePath string) *cobra.Command { func getChannelCheckpoint(cli *clientv3.Client, basePath string, channelName string) (*internalpb.MsgPosition, error) { prefix := path.Join(basePath, "datacoord-meta", "channel-cp", channelName) - results, _, err := listObject[internalpb.MsgPosition](context.Background(), cli, prefix) + results, _, err := common.ListProtoObjects[internalpb.MsgPosition](context.Background(), cli, prefix) if err != nil { return nil, err } @@ -74,7 +77,7 @@ func getChannelCheckpoint(cli *clientv3.Client, basePath string, channelName str } func getCheckpointFromSegments(cli *clientv3.Client, basePath string, collID int64, vchannel string) (*internalpb.MsgPosition, int64, error) { - segments, err := listSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { + segments, err := common.ListSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { return info.CollectionID == collID && info.InsertChannel == vchannel }) if err != nil { diff --git a/states/show_collection.go b/states/etcd/show/collection.go similarity index 61% rename from states/show_collection.go rename to states/etcd/show/collection.go index 42ec1e9..63d5d88 100644 --- a/states/show_collection.go +++ b/states/etcd/show/collection.go @@ -1,9 +1,8 @@ -package states +package show import ( "bytes" "context" - "errors" "fmt" "path" "sort" @@ -16,16 +15,16 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" "github.com/milvus-io/birdwatcher/proto/v2.0/etcdpb" - "github.com/milvus-io/birdwatcher/proto/v2.0/schemapb" + "github.com/milvus-io/birdwatcher/states/etcd/common" ) -// getEtcdShowCollection returns sub command for showCmd +// CollectionCommand returns sub command for showCmd. // show collection [options...] -func getEtcdShowCollection(cli *clientv3.Client, basePath string) *cobra.Command { +func CollectionCommand(cli *clientv3.Client, basePath string) *cobra.Command { cmd := &cobra.Command{ Use: "collections", Short: "list current available collection from RootCoord", - Aliases: []string{"collections"}, + Aliases: []string{"collection"}, RunE: func(cmd *cobra.Command, args []string) error { collectionID, err := cmd.Flags().GetInt64("id") if err != nil { @@ -68,46 +67,8 @@ func getEtcdShowCollection(cli *clientv3.Client, basePath string) *cobra.Command return cmd } -var ( - ErrCollectionDropped = errors.New("collection dropped") -) - -func getCollectionByID(cli *clientv3.Client, basePath string, collID int64) (*etcdpb.CollectionInfo, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - resp, err := cli.Get(ctx, path.Join(basePath, "root-coord/collection", strconv.FormatInt(collID, 10))) - - if err != nil { - return nil, err - } - - if len(resp.Kvs) != 1 { - return nil, errors.New("invalid collection id") - } - - if bytes.Equal(resp.Kvs[0].Value, CollectionTombstone) { - return nil, fmt.Errorf("%w, collection id: %d", ErrCollectionDropped, collID) - } - - coll := &etcdpb.CollectionInfo{} - - err = proto.Unmarshal(resp.Kvs[0].Value, coll) - if err != nil { - return nil, err - } - - err = fillFieldSchemaIfEmpty(cli, basePath, coll) - if err != nil { - return nil, err - } - - return coll, nil -} - -var CollectionTombstone = []byte{0xE2, 0x9B, 0xBC} - func processCollectionKV(cli *clientv3.Client, basePath string, kv *mvccpb.KeyValue, handleErr func(key string, err error)) { - if bytes.Equal(kv.Value, CollectionTombstone) { + if bytes.Equal(kv.Value, common.CollectionTombstone) { return } @@ -118,7 +79,7 @@ func processCollectionKV(cli *clientv3.Client, basePath string, kv *mvccpb.KeyVa return } - err = fillFieldSchemaIfEmpty(cli, basePath, collection) + err = common.FillFieldSchemaIfEmpty(cli, basePath, collection) if err != nil { handleErr(string(kv.Key), err) return @@ -127,26 +88,6 @@ func processCollectionKV(cli *clientv3.Client, basePath string, kv *mvccpb.KeyVa printCollection(collection) } -func fillFieldSchemaIfEmpty(cli *clientv3.Client, basePath string, collection *etcdpb.CollectionInfo) error { - if len(collection.GetSchema().GetFields()) == 0 { // fields separated from schema after 2.1.1 - resp, err := cli.Get(context.TODO(), path.Join(basePath, fmt.Sprintf("root-coord/fields/%d", collection.ID)), clientv3.WithPrefix()) - if err != nil { - return err - } - for _, kv := range resp.Kvs { - field := &schemapb.FieldSchema{} - err := proto.Unmarshal(kv.Value, field) - if err != nil { - fmt.Println("found error field:", string(kv.Key), err.Error()) - continue - } - collection.Schema.Fields = append(collection.Schema.Fields, field) - } - } - - return nil -} - func printCollection(collection *etcdpb.CollectionInfo) { fmt.Println("================================================================================") fmt.Printf("Collection ID: %d\tCollection Name: %s\n", collection.ID, collection.Schema.Name) diff --git a/states/etcd/show/collection_loaded.go b/states/etcd/show/collection_loaded.go new file mode 100644 index 0000000..faf3f30 --- /dev/null +++ b/states/etcd/show/collection_loaded.go @@ -0,0 +1,59 @@ +package show + +import ( + "fmt" + "sort" + + "github.com/milvus-io/birdwatcher/proto/v2.0/querypb" + querypbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/querypb" + "github.com/milvus-io/birdwatcher/states/etcd/common" + "github.com/spf13/cobra" + clientv3 "go.etcd.io/etcd/client/v3" +) + +const ( + ReplicaMetaPrefix = "queryCoord-ReplicaMeta" +) + +func printLoadedCollections(infos []*querypb.CollectionInfo) { + sort.Slice(infos, func(i, j int) bool { + return infos[i].GetCollectionID() < infos[j].GetCollectionID() + }) + + for _, info := range infos { + // TODO beautify output + fmt.Println(info.String()) + } +} + +func printCollectionLoadInfoV2(loadInfov2 querypbv2.CollectionLoadInfo) { + fmt.Println(loadInfov2.String()) +} + +// CollectionLoadedCommand return show collection-loaded command. +func CollectionLoadedCommand(cli *clientv3.Client, basePath string) *cobra.Command { + cmd := &cobra.Command{ + Use: "collection-loaded", + Short: "display information of loaded collection from querycoord", + Aliases: []string{"collection-load"}, + RunE: func(cmd *cobra.Command, args []string) error { + collectionLoadInfos, err := common.ListLoadedCollectionInfoV2_1(cli, basePath) + + if err != nil { + return err + } + printLoadedCollections(collectionLoadInfos) + + loadInfov2, err := common.ListLoadedCollectionInfoV2_2(cli, basePath) + if err != nil { + return err + } + for _, info := range loadInfov2 { + printCollectionLoadInfoV2(info) + } + + return nil + }, + } + return cmd +} diff --git a/states/etcd/show/index.go b/states/etcd/show/index.go new file mode 100644 index 0000000..8c6ac01 --- /dev/null +++ b/states/etcd/show/index.go @@ -0,0 +1,109 @@ +package show + +import ( + "context" + "fmt" + "path" + "time" + + "github.com/milvus-io/birdwatcher/proto/v2.0/etcdpb" + indexpbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/indexpb" + "github.com/milvus-io/birdwatcher/states/etcd/common" + "github.com/milvus-io/birdwatcher/utils" + "github.com/spf13/cobra" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// IndexCommand returns show index command. +func IndexCommand(cli *clientv3.Client, basePath string) *cobra.Command { + cmd := &cobra.Command{ + Use: "index", + Aliases: []string{"indexes"}, + Run: func(cmd *cobra.Command, args []string) { + + fmt.Println("*************2.1.x***************") + // v2.0+ + meta, err := listIndexMeta(cli, basePath) + if err != nil { + fmt.Println(err.Error()) + return + } + + for _, m := range meta { + printIndex(m) + } + + fmt.Println("*************2.2.x***************") + // v2.2+ + fieldIndexes, err := listIndexMetaV2(cli, basePath) + if err != nil { + fmt.Println(err.Error()) + return + } + + for _, index := range fieldIndexes { + printIndexV2(index) + } + }, + } + return cmd +} + +type IndexInfoV1 struct { + info etcdpb.IndexInfo + collectionID int64 +} + +func listIndexMeta(cli *clientv3.Client, basePath string) ([]IndexInfoV1, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + + prefix := path.Join(basePath, "root-coord/index") + indexes, keys, err := common.ListProtoObjects[etcdpb.IndexInfo](ctx, cli, prefix) + result := make([]IndexInfoV1, 0, len(indexes)) + for idx, info := range indexes { + collectionID, err := common.PathPartInt64(keys[idx], -2) + if err != nil { + continue + } + result = append(result, IndexInfoV1{ + info: info, + collectionID: collectionID, + }) + } + + return result, err +} + +func listIndexMetaV2(cli *clientv3.Client, basePath string) ([]indexpbv2.FieldIndex, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + indexes, _, err := common.ListProtoObjects[indexpbv2.FieldIndex](ctx, cli, path.Join(basePath, "field-index")) + return indexes, err +} + +func printIndex(index IndexInfoV1) { + fmt.Println("==================================================================") + fmt.Printf("Index ID: %d\tIndex Name: %s\tCollectionID:%d\n", index.info.GetIndexID(), index.info.GetIndexName(), index.collectionID) + indexParams := index.info.GetIndexParams() + fmt.Printf("Index Type: %s\tMetric Type: %s\n", + common.GetKVPair(indexParams, "index_type"), + common.GetKVPair(indexParams, "metric_type"), + ) + fmt.Printf("Index Params: %s\n", common.GetKVPair(index.info.GetIndexParams(), "params")) + fmt.Println("==================================================================") +} + +func printIndexV2(index indexpbv2.FieldIndex) { + fmt.Println("==================================================================") + fmt.Printf("Index ID: %d\tIndex Name: %s\tCollectionID:%d\n", index.GetIndexInfo().GetIndexID(), index.GetIndexInfo().GetIndexName(), index.GetIndexInfo().GetCollectionID()) + createTime, _ := utils.ParseTS(index.GetCreateTime()) + fmt.Printf("Create Time: %s\tDeleted: %t\n", createTime.Format(tsPrintFormat), index.GetDeleted()) + indexParams := index.GetIndexInfo().GetIndexParams() + fmt.Printf("Index Type: %s\tMetric Type: %s\n", + common.GetKVPair(indexParams, "index_type"), + common.GetKVPair(indexParams, "metric_type"), + ) + fmt.Printf("Index Params: %s\n", common.GetKVPair(index.GetIndexInfo().GetUserIndexParams(), "params")) + fmt.Println("==================================================================") +} diff --git a/states/querycoord_channels.go b/states/etcd/show/legacy_qc_channel.go similarity index 53% rename from states/querycoord_channels.go rename to states/etcd/show/legacy_qc_channel.go index 91d7247..f74b195 100644 --- a/states/querycoord_channels.go +++ b/states/etcd/show/legacy_qc_channel.go @@ -1,20 +1,4 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package states +package show import ( "context" @@ -25,6 +9,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" @@ -33,8 +18,6 @@ import ( const ( unsubscribeChannelInfoPrefix = "queryCoord-unsubscribeChannelInfo" - dmChannelMetaPrefix = "queryCoord-dmChannelWatchInfo" - deltaChannelMetaPrefix = "queryCoord-deltaChannel" ) func printNodeUnsubChannelInfos(infos []*querypb.UnsubscribeChannelInfo) { @@ -87,67 +70,23 @@ func listQueryCoordUnsubChannelInfos(cli *clientv3.Client, basePath string) ([]* } func printDMChannelWatchInfo(infos []*querypb.DmChannelWatchInfo) { - infos2 := make([]infoWithCollectionID, 0) + common.SortByCollection(infos) for _, info := range infos { - infos2 = append(infos2, info) + //TODO beautify output + fmt.Println(info.String()) } - printInfoWithCollectionID(infos2) -} - -func listQueryCoordDMLChannelInfos(cli *clientv3.Client, basePath string) ([]*querypb.DmChannelWatchInfo, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - prefix := path.Join(basePath, dmChannelMetaPrefix) - - resp, err := cli.Get(ctx, prefix, clientv3.WithPrefix()) - if err != nil { - return nil, err - } - - ret := make([]*querypb.DmChannelWatchInfo, 0) - for _, kv := range resp.Kvs { - channelInfo := &querypb.DmChannelWatchInfo{} - err = proto.Unmarshal(kv.Value, channelInfo) - if err != nil { - return nil, err - } - ret = append(ret, channelInfo) - } - return ret, nil -} - -func listQueryCoordDeltaChannelInfos(cli *clientv3.Client, basePath string) ([]*datapb.VchannelInfo, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - prefix := path.Join(basePath, deltaChannelMetaPrefix) - - resp, err := cli.Get(ctx, prefix, clientv3.WithPrefix()) - if err != nil { - return nil, err - } - - ret := make([]*datapb.VchannelInfo, 0, len(resp.Kvs)) - for _, kv := range resp.Kvs { - channelInfo := &datapb.VchannelInfo{} - err = proto.Unmarshal(kv.Value, channelInfo) - if err != nil { - return nil, err - } - reviseVChannelInfo(channelInfo) - ret = append(ret, channelInfo) - } - return ret, nil } func printDeltaChannelInfos(infos []*datapb.VchannelInfo) { - infos2 := make([]infoWithCollectionID, 0) + common.SortByCollection(infos) for _, info := range infos { - infos2 = append(infos2, info) + // TODO beautify output + fmt.Println(info.String()) } - printInfoWithCollectionID(infos2) } -func getQueryCoordChannelInfoCmd(cli *clientv3.Client, basePath string) *cobra.Command { +// QueryCoordChannelCommand returns show querycoord-channel command. +func QueryCoordChannelCommand(cli *clientv3.Client, basePath string) *cobra.Command { cmd := &cobra.Command{ Use: "querycoord-channel", Short: "display querynode information from querycoord cluster", @@ -172,14 +111,14 @@ func getQueryCoordChannelInfoCmd(cli *clientv3.Client, basePath string) *cobra.C printNodeUnsubChannelInfos(unsubInfos) } - dmWatchInfo, err := listQueryCoordDMLChannelInfos(cli, basePath) + dmWatchInfo, err := common.ListQueryCoordDMLChannelInfos(cli, basePath) if err != nil { return err } if taskType == "" || taskType == "all" || taskType == "dml" { printDMChannelWatchInfo(dmWatchInfo) } - deltaChannels, err := listQueryCoordDeltaChannelInfos(cli, basePath) + deltaChannels, err := common.ListQueryCoordDeltaChannelInfos(cli, basePath) if err != nil { return err } diff --git a/states/querycoord_cluster.go b/states/etcd/show/legacy_qc_cluster.go similarity index 53% rename from states/querycoord_cluster.go rename to states/etcd/show/legacy_qc_cluster.go index 2d9359b..a95da30 100644 --- a/states/querycoord_cluster.go +++ b/states/etcd/show/legacy_qc_cluster.go @@ -1,26 +1,11 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package states +package show import ( "fmt" "path" "github.com/milvus-io/birdwatcher/models" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" ) @@ -31,10 +16,11 @@ const ( func listQueryCoordClusterNodeInfo(cli *clientv3.Client, basePath string) ([]*models.Session, error) { prefix := path.Join(basePath, queryNodeInfoPrefix) - return listSessionsByPrefix(cli, prefix) + return common.ListSessionsByPrefix(cli, prefix) } -func getQueryCoordClusterNodeInfo(cli *clientv3.Client, basePath string) *cobra.Command { +// QueryCoordClusterCommand returns show querycoord-cluster command. +func QueryCoordClusterCommand(cli *clientv3.Client, basePath string) *cobra.Command { cmd := &cobra.Command{ Use: "querycoord-cluster", Short: "display querynode information from querycoord cluster", @@ -46,7 +32,7 @@ func getQueryCoordClusterNodeInfo(cli *clientv3.Client, basePath string) *cobra. return nil } - onlineSessons, _ := listSessions(cli, basePath) + onlineSessons, _ := common.ListSessions(cli, basePath) onlineSessionMap := make(map[UniqueID]struct{}) for _, s := range onlineSessons { onlineSessionMap[s.ServerID] = struct{}{} diff --git a/states/show_querycoord_task.go b/states/etcd/show/legacy_qc_task.go similarity index 84% rename from states/show_querycoord_task.go rename to states/etcd/show/legacy_qc_task.go index 934071b..ce2902e 100644 --- a/states/show_querycoord_task.go +++ b/states/etcd/show/legacy_qc_task.go @@ -1,20 +1,4 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package states +package show import ( "context" @@ -25,7 +9,6 @@ import ( "sort" "strconv" - //"sort" "time" "github.com/spf13/cobra" @@ -155,7 +138,9 @@ func listQueryCoordTasks(cli *clientv3.Client, basePath string, filter func(task return triggerTasks, activateTasks, nil } -func getQueryCoordTaskCmd(cli *clientv3.Client, basePath string) *cobra.Command { +// QueryCoordTasks returns show querycoord-tasks commands. +// DEPRECATED from milvus 2.2.0. +func QueryCoordTasks(cli *clientv3.Client, basePath string) *cobra.Command { cmd := &cobra.Command{ Use: "querycoord-task", Short: "display task information from querycoord", @@ -197,7 +182,13 @@ func getQueryCoordTaskCmd(cli *clientv3.Client, basePath string) *cobra.Command }) for _, tID := range tIDs { task := tasks[tID] - fmt.Printf("%s\n\n", task.String()) + fmt.Printf("%d, %s\n", task.getTaskID(), task.getType()) + + taskStr := task.String() + if len(taskStr) > 200 { + taskStr = taskStr[:200] + } + fmt.Println(taskStr) } } diff --git a/states/querycoord_task.go b/states/etcd/show/legacy_qc_task_model.go similarity index 93% rename from states/querycoord_task.go rename to states/etcd/show/legacy_qc_task_model.go index 816c0e0..31b1dc3 100644 --- a/states/querycoord_task.go +++ b/states/etcd/show/legacy_qc_task_model.go @@ -1,20 +1,4 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package states +package show import ( "errors" @@ -24,6 +8,7 @@ import ( "github.com/milvus-io/birdwatcher/proto/v2.0/commonpb" "github.com/milvus-io/birdwatcher/proto/v2.0/querypb" + "github.com/milvus-io/birdwatcher/utils" ) const ( @@ -287,7 +272,7 @@ func (lct *loadCollectionTask) String() string { baseStr := lct.baseQueryCoordTask.String() typeStr := fmt.Sprintf("type:%s", lct.msgType().String()) ts := lct.timestamp() - time, logic := ParseTS(ts) + time, logic := utils.ParseTS(ts) timeStr := fmt.Sprintf("time:%s, logicTs:%d", time, logic) ret := fmt.Sprintf("%s\t%s\t%s\ninfo:%s", baseStr, typeStr, timeStr, lct.LoadCollectionRequest.String()) return ret @@ -327,7 +312,7 @@ func (rct *releaseCollectionTask) String() string { baseStr := rct.baseQueryCoordTask.String() typeStr := fmt.Sprintf("type:%s", rct.msgType().String()) ts := rct.timestamp() - time, logic := ParseTS(ts) + time, logic := utils.ParseTS(ts) timeStr := fmt.Sprintf("time:%s, logicTs:%d", time, logic) ret := fmt.Sprintf("%s\t%s\t%s\ninfo:%s", baseStr, typeStr, timeStr, rct.ReleaseCollectionRequest.String()) return ret @@ -368,7 +353,7 @@ func (lpt *loadPartitionTask) String() string { baseStr := lpt.baseQueryCoordTask.String() typeStr := fmt.Sprintf("type:%s", lpt.msgType().String()) ts := lpt.timestamp() - time, logic := ParseTS(ts) + time, logic := utils.ParseTS(ts) timeStr := fmt.Sprintf("time:%s, logicTs:%d", time, logic) ret := fmt.Sprintf("%s\t%s\t%s\ninfo:%s", baseStr, typeStr, timeStr, lpt.LoadPartitionsRequest.String()) return ret @@ -408,7 +393,7 @@ func (rpt *releasePartitionTask) String() string { baseStr := rpt.baseQueryCoordTask.String() typeStr := fmt.Sprintf("type:%s", rpt.msgType().String()) ts := rpt.timestamp() - time, logic := ParseTS(ts) + time, logic := utils.ParseTS(ts) timeStr := fmt.Sprintf("time:%s, logicTs:%d", time, logic) ret := fmt.Sprintf("%s\t%s\t%s\ninfo:%s", baseStr, typeStr, timeStr, rpt.ReleasePartitionsRequest.String()) return ret @@ -448,7 +433,7 @@ func (lst *loadSegmentTask) String() string { baseStr := lst.baseQueryCoordTask.String() typeStr := fmt.Sprintf("type:%s", lst.msgType().String()) ts := lst.timestamp() - time, logic := ParseTS(ts) + time, logic := utils.ParseTS(ts) timeStr := fmt.Sprintf("time:%s, logicTs:%d", time, logic) ret := fmt.Sprintf("%s\t%s\t%s\ninfo:%s", baseStr, typeStr, timeStr, lst.LoadSegmentsRequest.String()) return ret @@ -487,7 +472,7 @@ func (rst *releaseSegmentTask) String() string { baseStr := rst.baseQueryCoordTask.String() typeStr := fmt.Sprintf("type:%s", rst.msgType().String()) ts := rst.timestamp() - time, logic := ParseTS(ts) + time, logic := utils.ParseTS(ts) timeStr := fmt.Sprintf("time:%s, logicTs:%d", time, logic) ret := fmt.Sprintf("%s\t%s\t%s\ninfo:%s", baseStr, typeStr, timeStr, rst.ReleaseSegmentsRequest.String()) return ret @@ -527,7 +512,7 @@ func (wdt *watchDmChannelTask) String() string { baseStr := wdt.baseQueryCoordTask.String() typeStr := fmt.Sprintf("type:%s", wdt.msgType().String()) ts := wdt.timestamp() - time, logic := ParseTS(ts) + time, logic := utils.ParseTS(ts) timeStr := fmt.Sprintf("time:%s, logicTs:%d", time, logic) ret := fmt.Sprintf("%s\t%s\t%s\ninfo:%s", baseStr, typeStr, timeStr, wdt.WatchDmChannelsRequest.String()) return ret @@ -568,7 +553,7 @@ func (ht *handoffTask) String() string { baseStr := ht.baseQueryCoordTask.String() typeStr := fmt.Sprintf("type:%s", ht.msgType().String()) ts := ht.timestamp() - time, logic := ParseTS(ts) + time, logic := utils.ParseTS(ts) timeStr := fmt.Sprintf("time:%s, logicTs:%d", time, logic) ret := fmt.Sprintf("%s\t%s\t%s\ninfo:%s", baseStr, typeStr, timeStr, ht.HandoffSegmentsRequest.String()) return ret @@ -607,9 +592,9 @@ func (lbt *loadBalanceTask) String() string { baseStr := lbt.baseQueryCoordTask.String() typeStr := fmt.Sprintf("type:%s", lbt.msgType().String()) ts := lbt.timestamp() - time, logic := ParseTS(ts) + time, logic := utils.ParseTS(ts) timeStr := fmt.Sprintf("time:%s, logicTs:%d", time, logic) - ret := fmt.Sprintf("%s\t%s\t%s\ninfo:%s", baseStr, typeStr, timeStr, lbt.LoadBalanceRequest.String()) + ret := fmt.Sprintf("%s\t%s\t%s", baseStr, typeStr, timeStr) //fmt.Sprintf("%s\t%s\t%s\ninfo:%s", baseStr, typeStr, timeStr, lbt.LoadBalanceRequest.String()) return ret } diff --git a/states/show_replica.go b/states/etcd/show/replica.go similarity index 61% rename from states/show_replica.go rename to states/etcd/show/replica.go index 4888751..fa69df1 100644 --- a/states/show_replica.go +++ b/states/etcd/show/replica.go @@ -1,4 +1,4 @@ -package states +package show import ( "context" @@ -6,14 +6,14 @@ import ( "path" "time" - "github.com/golang/protobuf/proto" "github.com/milvus-io/birdwatcher/proto/v2.0/milvuspb" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" ) -// getEtcdShowReplica returns command for show querycoord replicas -func getEtcdShowReplica(cli *clientv3.Client, basePath string) *cobra.Command { +// ReplicaCommand returns command for show querycoord replicas. +func ReplicaCommand(cli *clientv3.Client, basePath string) *cobra.Command { cmd := &cobra.Command{ Use: "replica", Short: "list current replica information from QueryCoord", @@ -35,28 +35,21 @@ func getEtcdShowReplica(cli *clientv3.Client, basePath string) *cobra.Command { return cmd } -func listReplicas(cli *clientv3.Client, basePath string) ([]*milvuspb.ReplicaInfo, error) { +func listReplicas(cli *clientv3.Client, basePath string) ([]milvuspb.ReplicaInfo, error) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) defer cancel() - resp, err := cli.Get(ctx, path.Join(basePath, "queryCoord-ReplicaMeta"), clientv3.WithPrefix()) + prefix := path.Join(basePath, "queryCoord-ReplicaMeta") + + replicas, _, err := common.ListProtoObjects[milvuspb.ReplicaInfo](ctx, cli, prefix) if err != nil { return nil, err } - replicas := make([]*milvuspb.ReplicaInfo, 0, len(resp.Kvs)) - for _, kv := range resp.Kvs { - replica := &milvuspb.ReplicaInfo{} - if err != proto.Unmarshal(kv.Value, replica) { - continue - } - replicas = append(replicas, replica) - } - return replicas, nil } -func printReplica(replica *milvuspb.ReplicaInfo) { +func printReplica(replica milvuspb.ReplicaInfo) { fmt.Println("================================================================================") fmt.Printf("ReplicaID: %d CollectionID: %d\n", replica.ReplicaID, replica.CollectionID) for _, shardReplica := range replica.ShardReplicas { diff --git a/states/etcd/show/segment.go b/states/etcd/show/segment.go new file mode 100644 index 0000000..e929d0c --- /dev/null +++ b/states/etcd/show/segment.go @@ -0,0 +1,181 @@ +package show + +import ( + "fmt" + "sort" + + "github.com/milvus-io/birdwatcher/proto/v2.0/commonpb" + "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + "github.com/milvus-io/birdwatcher/states/etcd/common" + "github.com/milvus-io/birdwatcher/utils" + + "github.com/spf13/cobra" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// SegmentCommand returns show segments command. +func SegmentCommand(cli *clientv3.Client, basePath string) *cobra.Command { + cmd := &cobra.Command{ + Use: "segment", + Short: "display segment information from data coord meta store", + Aliases: []string{"segments"}, + RunE: func(cmd *cobra.Command, args []string) error { + + collID, err := cmd.Flags().GetInt64("collection") + if err != nil { + return err + } + segmentID, err := cmd.Flags().GetInt64("segment") + if err != nil { + return err + } + format, err := cmd.Flags().GetString("format") + if err != nil { + return err + } + detail, err := cmd.Flags().GetBool("detail") + if err != nil { + return err + } + + segments, err := common.ListSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { + return (collID == 0 || info.CollectionID == collID) && + (segmentID == 0 || info.ID == segmentID) + }) + if err != nil { + fmt.Println("failed to list segments", err.Error()) + return nil + } + + totalRC := int64(0) + healthy := 0 + var statslogSize int64 + var growing, sealed, flushed int + for _, info := range segments { + + if info.State != commonpb.SegmentState_Dropped { + totalRC += info.NumOfRows + healthy++ + } + switch info.State { + case commonpb.SegmentState_Growing: + growing++ + case commonpb.SegmentState_Sealed: + sealed++ + case commonpb.SegmentState_Flushing, commonpb.SegmentState_Flushed: + flushed++ + } + + switch format { + case "table": + common.FillFieldsIfV2(cli, basePath, info) + printSegmentInfo(info, detail) + case "line": + fmt.Printf("SegmentID: %d State: %s, Row Count:%d\n", info.ID, info.State.String(), info.NumOfRows) + case "statistics": + if info.GetState() != commonpb.SegmentState_Dropped { + common.FillFieldsIfV2(cli, basePath, info) + for _, statslog := range info.GetStatslogs() { + for _, binlog := range statslog.GetBinlogs() { + statslogSize += binlog.LogSize + } + } + } + + } + + } + if format == "statistics" { + fmt.Printf("--- Total statslog size: %d\n", statslogSize) + } + + fmt.Printf("--- Growing: %d, Sealed: %d, Flushed: %d\n", growing, sealed, flushed) + fmt.Printf("--- Total Segments: %d, row count: %d\n", healthy, totalRC) + return nil + }, + } + cmd.Flags().Int64("collection", 0, "collection id to filter with") + cmd.Flags().String("format", "line", "segment display format") + cmd.Flags().Bool("detail", false, "flags indicating whether pring detail binlog info") + cmd.Flags().Int64("segment", 0, "segment id to filter with") + return cmd +} + +const ( + tsPrintFormat = "2006-01-02 15:04:05.999 -0700" +) + +func printSegmentInfo(info *datapb.SegmentInfo, detailBinlog bool) { + fmt.Println("================================================================================") + fmt.Printf("Segment ID: %d\n", info.ID) + fmt.Printf("Segment State:%v\n", info.State) + fmt.Printf("Collection ID: %d\t\tPartitionID: %d\n", info.CollectionID, info.PartitionID) + fmt.Printf("Insert Channel:%s\n", info.InsertChannel) + fmt.Printf("Num of Rows: %d\t\tMax Row Num: %d\n", info.NumOfRows, info.MaxRowNum) + lastExpireTime, _ := utils.ParseTS(info.LastExpireTime) + fmt.Printf("Last Expire Time: %s\n", lastExpireTime.Format(tsPrintFormat)) + fmt.Printf("Compact from %v \n", info.CompactionFrom) + if info.StartPosition != nil { + startTime, _ := utils.ParseTS(info.GetStartPosition().GetTimestamp()) + fmt.Printf("Start Position ID: %v, time: %s\n", info.StartPosition.MsgID, startTime.Format(tsPrintFormat)) + } else { + fmt.Println("Start Position: nil") + } + if info.DmlPosition != nil { + dmlTime, _ := utils.ParseTS(info.DmlPosition.Timestamp) + fmt.Printf("Dml Position ID: %v, time: %s, channel name: %s\n", info.DmlPosition.MsgID, dmlTime.Format(tsPrintFormat), info.GetDmlPosition().GetChannelName()) + } else { + fmt.Println("Dml Position: nil") + } + fmt.Printf("Binlog Nums %d\tStatsLog Nums: %d\tDeltaLog Nums:%d\n", + countBinlogNum(info.Binlogs), countBinlogNum(info.Statslogs), countBinlogNum(info.Deltalogs)) + + if detailBinlog { + var binlogSize int64 + fmt.Println("**************************************") + fmt.Println("Binlogs:") + sort.Slice(info.Binlogs, func(i, j int) bool { + return info.Binlogs[i].FieldID < info.Binlogs[j].FieldID + }) + for _, log := range info.Binlogs { + fmt.Printf("Field %d:\n", log.FieldID) + for _, binlog := range log.Binlogs { + fmt.Printf("Path: %s\n", binlog.LogPath) + tf, _ := utils.ParseTS(binlog.TimestampFrom) + tt, _ := utils.ParseTS(binlog.TimestampTo) + fmt.Printf("Log Size: %d \t Entry Num: %d\t TimeRange:%s-%s\n", + binlog.LogSize, binlog.EntriesNum, + tf.Format(tsPrintFormat), tt.Format(tsPrintFormat)) + binlogSize += binlog.LogSize + } + } + + fmt.Println("**************************************") + fmt.Println("Statslogs:") + sort.Slice(info.Statslogs, func(i, j int) bool { + return info.Statslogs[i].FieldID < info.Statslogs[j].FieldID + }) + for _, log := range info.Statslogs { + fmt.Printf("Field %d: %v\n", log.FieldID, log.Binlogs) + } + + fmt.Println("**************************************") + fmt.Println("Delta Logs:") + for _, log := range info.Deltalogs { + for _, l := range log.Binlogs { + fmt.Printf("Entries: %d From: %v - To: %v\n", l.EntriesNum, l.TimestampFrom, l.TimestampTo) + fmt.Printf("Path: %v\n", l.LogPath) + } + } + } + + fmt.Println("================================================================================") +} + +func countBinlogNum(fbl []*datapb.FieldBinlog) int { + result := 0 + for _, f := range fbl { + result += len(f.Binlogs) + } + return result +} diff --git a/states/etcd/show/segment_index.go b/states/etcd/show/segment_index.go new file mode 100644 index 0000000..5f5998a --- /dev/null +++ b/states/etcd/show/segment_index.go @@ -0,0 +1,152 @@ +package show + +import ( + "context" + "fmt" + "path" + "time" + + "github.com/milvus-io/birdwatcher/proto/v2.0/commonpb" + "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + "github.com/milvus-io/birdwatcher/proto/v2.0/etcdpb" + "github.com/milvus-io/birdwatcher/proto/v2.0/indexpb" + indexpbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/indexpb" + "github.com/milvus-io/birdwatcher/states/etcd/common" + "github.com/spf13/cobra" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// SegmentIndexCommand returns show segment-index command. +func SegmentIndexCommand(cli *clientv3.Client, basePath string) *cobra.Command { + cmd := &cobra.Command{ + Use: "segment-index", + Aliases: []string{"segments-index", "segment-indexes", "segments-indexes"}, + Short: "display segment index information", + Run: func(cmd *cobra.Command, args []string) { + collID, err := cmd.Flags().GetInt64("collection") + if err != nil { + fmt.Println(err.Error()) + return + } + segmentID, err := cmd.Flags().GetInt64("segment") + if err != nil { + fmt.Println(err.Error()) + return + } + + segments, err := common.ListSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { + return (collID == 0 || info.CollectionID == collID) && + (segmentID == 0 || info.ID == segmentID) + }) + if err != nil { + fmt.Println(err.Error()) + return + } + + segmentIndexes, err := common.ListSegmentIndex(cli, basePath) + if err != nil { + fmt.Println(err.Error()) + return + } + segmentIndexesV2, err := listSegmentIndexV2(cli, basePath) + if err != nil { + fmt.Println(err.Error()) + return + } + + indexBuildInfo, err := common.ListIndex(cli, basePath) + if err != nil { + fmt.Println(err.Error()) + return + } + + indexes, _, err := common.ListProtoObjects[indexpbv2.FieldIndex](context.Background(), cli, path.Join(basePath, "field-index")) + if err != nil { + fmt.Println(err.Error()) + return + } + idIdx := make(map[int64]indexpbv2.FieldIndex) + for _, idx := range indexes { + idIdx[idx.IndexInfo.IndexID] = idx + } + + seg2Idx := make(map[int64][]etcdpb.SegmentIndexInfo) + seg2Idxv2 := make(map[int64][]indexpbv2.SegmentIndex) + for _, segIdx := range segmentIndexes { + idxs, ok := seg2Idx[segIdx.SegmentID] + if !ok { + idxs = []etcdpb.SegmentIndexInfo{} + } + + idxs = append(idxs, segIdx) + + seg2Idx[segIdx.GetSegmentID()] = idxs + } + for _, segIdx := range segmentIndexesV2 { + idxs, ok := seg2Idxv2[segIdx.SegmentID] + if !ok { + idxs = []indexpbv2.SegmentIndex{} + } + + idxs = append(idxs, segIdx) + + seg2Idxv2[segIdx.GetSegmentID()] = idxs + } + + buildID2Info := make(map[int64]indexpb.IndexMeta) + for _, info := range indexBuildInfo { + buildID2Info[info.IndexBuildID] = info + } + + for _, segment := range segments { + if segment.State != commonpb.SegmentState_Flushed { + continue + } + fmt.Printf("SegmentID: %d\t State: %s", segment.GetID(), segment.GetState().String()) + segIdxs, ok := seg2Idx[segment.GetID()] + if !ok { + // try v2 index information + segIdxv2, ok := seg2Idxv2[segment.GetID()] + if !ok { + fmt.Println("\tno segment index info") + continue + } + for _, segIdx := range segIdxv2 { + fmt.Printf("\n\tIndexV2 build ID: %d, states %s", segIdx.GetBuildID(), segIdx.GetState().String()) + idx, ok := idIdx[segIdx.GetIndexID()] + if ok { + fmt.Printf("\t Index Type:%v on Field ID: %d", common.GetKVPair(idx.GetIndexInfo().GetIndexParams(), "index_type"), idx.GetIndexInfo().GetFieldID()) + } + } + fmt.Println() + continue + } + + for _, segIdx := range segIdxs { + info, ok := buildID2Info[segIdx.BuildID] + if !ok { + fmt.Printf("\tno build info found for id: %d\n", segIdx.BuildID) + fmt.Println(segIdx.String()) + } + fmt.Printf("\n\tIndex build ID: %d, state: %s", info.IndexBuildID, info.State.String()) + fmt.Printf("\t Index Type:%v on Field ID: %d", common.GetKVPair(info.GetReq().GetIndexParams(), "index_type"), segIdx.GetFieldID()) + } + fmt.Println() + } + + }, + } + + cmd.Flags().Int64("collection", 0, "collection id to filter with") + cmd.Flags().Int64("segment", 0, "segment id to filter with") + return cmd +} + +func listSegmentIndexV2(cli *clientv3.Client, basePath string) ([]indexpbv2.SegmentIndex, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + + prefix := path.Join(basePath, "segment-index") + "/" + result, _, err := common.ListProtoObjects[indexpbv2.SegmentIndex](ctx, cli, prefix) + return result, err +} diff --git a/states/etcd/show/segment_loaded.go b/states/etcd/show/segment_loaded.go new file mode 100644 index 0000000..c1e17f9 --- /dev/null +++ b/states/etcd/show/segment_loaded.go @@ -0,0 +1,51 @@ +package show + +import ( + "fmt" + + "github.com/milvus-io/birdwatcher/proto/v2.0/querypb" + "github.com/milvus-io/birdwatcher/states/etcd/common" + "github.com/spf13/cobra" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// SegmentLoadedCommand returns show segment-loaded command. +func SegmentLoadedCommand(cli *clientv3.Client, basePath string) *cobra.Command { + cmd := &cobra.Command{ + Use: "segment-loaded", + Short: "display segment information from querycoord", + Aliases: []string{"segments-loaded"}, + RunE: func(cmd *cobra.Command, args []string) error { + + collID, err := cmd.Flags().GetInt64("collection") + if err != nil { + return err + } + segmentID, err := cmd.Flags().GetInt64("segment") + if err != nil { + return err + } + + segments, err := common.ListLoadedSegments(cli, basePath, func(info *querypb.SegmentInfo) bool { + return (collID == 0 || info.CollectionID == collID) && + (segmentID == 0 || info.SegmentID == segmentID) + }) + if err != nil { + fmt.Println("failed to list segments", err.Error()) + return nil + } + + for _, info := range segments { + fmt.Printf("Segment ID: %d LegacyNodeID: %d NodeIds: %v,DmlChannel: %s\n", info.SegmentID, info.NodeID, info.NodeIds, info.DmChannel) + fmt.Printf("%#v\n", info.GetIndexInfos()) + } + + return nil + }, + } + cmd.Flags().Int64("collection", 0, "collection id to filter with") + cmd.Flags().String("format", "line", "segment display format") + cmd.Flags().Bool("detail", false, "flags indicating whether pring detail binlog info") + cmd.Flags().Int64("segment", 0, "segment id to filter with") + return cmd +} diff --git a/states/etcd/show/session.go b/states/etcd/show/session.go new file mode 100644 index 0000000..19f691f --- /dev/null +++ b/states/etcd/show/session.go @@ -0,0 +1,30 @@ +package show + +import ( + "fmt" + + "github.com/milvus-io/birdwatcher/states/etcd/common" + "github.com/spf13/cobra" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// SessionCommand returns show session command. +// usage: show session +func SessionCommand(cli *clientv3.Client, basePath string) *cobra.Command { + cmd := &cobra.Command{ + Use: "session", + Short: "list online milvus components", + Aliases: []string{"sessions"}, + RunE: func(cmd *cobra.Command, args []string) error { + sessions, err := common.ListSessions(cli, basePath) + if err != nil { + return err + } + for _, session := range sessions { + fmt.Println(session.String()) + } + return nil + }, + } + return cmd +} diff --git a/states/etcd_backup.go b/states/etcd_backup.go index 22b18b0..d3ad014 100644 --- a/states/etcd_backup.go +++ b/states/etcd_backup.go @@ -27,6 +27,7 @@ import ( internalpbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/internalpb" querypbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/querypb" rootcoordpbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/rootcoordpb" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "google.golang.org/grpc" @@ -239,7 +240,7 @@ func backupEtcdV2(cli *clientv3.Client, base, prefix string, w *bufio.Writer, ig } func backupMetrics(cli *clientv3.Client, basePath string, w *bufio.Writer) error { - sessions, err := listSessions(cli, basePath) + sessions, err := common.ListSessions(cli, basePath) if err != nil { return err } @@ -280,7 +281,7 @@ func backupMetrics(cli *clientv3.Client, basePath string, w *bufio.Writer) error } func backupAppMetrics(cli *clientv3.Client, basePath string, w *bufio.Writer) error { - sessions, err := listSessions(cli, basePath) + sessions, err := common.ListSessions(cli, basePath) if err != nil { return err } @@ -350,7 +351,7 @@ func backupAppMetrics(cli *clientv3.Client, basePath string, w *bufio.Writer) er } func backupConfiguration(cli *clientv3.Client, basePath string, w *bufio.Writer) error { - sessions, err := listSessions(cli, basePath) + sessions, err := common.ListSessions(cli, basePath) if err != nil { return err } diff --git a/states/etcd_connect.go b/states/etcd_connect.go index e304d2b..d804505 100644 --- a/states/etcd_connect.go +++ b/states/etcd_connect.go @@ -46,7 +46,7 @@ func getConnectCommand(state State) *cobra.Command { etcdCli, err := clientv3.New(clientv3.Config{ Endpoints: []string{etcdAddr}, - DialTimeout: time.Second * 2, + DialTimeout: time.Second * 10, // disable grpc logging Logger: zap.NewNop(), @@ -56,7 +56,7 @@ func getConnectCommand(state State) *cobra.Command { } // ping etcd - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() err = pingEtcd(ctx, etcdCli) if err != nil { diff --git a/states/etcd_restore.go b/states/etcd_restore.go index dc0ee4e..ae7f6df 100644 --- a/states/etcd_restore.go +++ b/states/etcd_restore.go @@ -103,7 +103,10 @@ func restoreV2File(rd *bufio.Reader, state *embedEtcdMockState) error { } state.SetInstance(instance) case int32(models.MetricsBackup): - testRestoreMetrics(rd, ph) + restoreMetrics(rd, ph, func(session *models.Session, metrics, defaultMetrics []byte) { + state.metrics[fmt.Sprintf("%s-%d", session.ServerName, session.ServerID)] = metrics + state.defaultMetrics[fmt.Sprintf("%s-%d", session.ServerName, session.ServerID)] = defaultMetrics + }) case int32(models.Configurations): testRestoreConfigurations(rd, ph) case int32(models.AppMetrics): @@ -193,6 +196,40 @@ func restoreEtcdFromBackV2(cli *clientv3.Client, rd io.Reader, ph models.PartHea } +func restoreMetrics(rd io.Reader, ph models.PartHeader, handler func(session *models.Session, metrics, defaultMetrics []byte)) error { + for { + bs, nb, err := readBackupBytes(rd) + if err != nil { + if err == io.EOF { + return nil + } + return err + } + // stopper + if nb == 0 { + return nil + } + + // session + session := &models.Session{} + err = json.Unmarshal(bs, session) + if err != nil { + return err + } + + mbs, _, err := readBackupBytes(rd) + if err != nil { + return err + } + + dmbs, _, err := readBackupBytes(rd) + if err != nil { + return err + } + handler(session, mbs, dmbs) + } +} + func testRestoreMetrics(rd io.Reader, ph models.PartHeader) error { for { bs, nb, err := readBackupBytes(rd) diff --git a/states/force_release.go b/states/force_release.go index ecac844..d829476 100644 --- a/states/force_release.go +++ b/states/force_release.go @@ -7,6 +7,7 @@ import ( "time" "github.com/milvus-io/birdwatcher/proto/v2.0/querypb" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" ) @@ -49,7 +50,7 @@ func getReleaseDroppedCollectionCmd(cli *clientv3.Client, basePath string) *cobr Use: "release-dropped-collection", Short: "Clean loaded collections meta if it's dropped from QueryCoord", Run: func(cmd *cobra.Command, args []string) { - collectionLoadInfos, err := getLoadedCollectionInfo(cli, basePath) + collectionLoadInfos, err := common.ListLoadedCollectionInfoV2_1(cli, basePath) if err != nil { fmt.Println("failed to list loaded collections", err.Error()) return @@ -57,7 +58,7 @@ func getReleaseDroppedCollectionCmd(cli *clientv3.Client, basePath string) *cobr var missing []int64 for _, info := range collectionLoadInfos { - _, err := getCollectionByID(cli, basePath, info.CollectionID) + _, err := common.GetCollectionByID(cli, basePath, info.CollectionID) if err != nil { missing = append(missing, info.CollectionID) } @@ -86,7 +87,7 @@ func getReleaseDroppedCollectionCmd(cli *clientv3.Client, basePath string) *cobr } func releaseQueryCoordLoadMeta(cli *clientv3.Client, basePath string, collectionID int64) error { - p := path.Join(basePath, collectionMetaPrefix, fmt.Sprintf("%d", collectionID)) + p := path.Join(basePath, common.CollectionLoadPrefix, fmt.Sprintf("%d", collectionID)) ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() _, err := cli.Delete(ctx, p) @@ -95,7 +96,7 @@ func releaseQueryCoordLoadMeta(cli *clientv3.Client, basePath string, collection return err } - segments, err := listLoadedSegments(cli, basePath, func(info *querypb.SegmentInfo) bool { + segments, err := common.ListLoadedSegments(cli, basePath, func(info *querypb.SegmentInfo) bool { return info.CollectionID == collectionID }) @@ -108,7 +109,7 @@ func releaseQueryCoordLoadMeta(cli *clientv3.Client, basePath string, collection cli.Delete(ctx, p) } - dmChannels, err := listQueryCoordDMLChannelInfos(cli, basePath) + dmChannels, err := common.ListQueryCoordDMLChannelInfos(cli, basePath) if err != nil { return err } @@ -117,16 +118,16 @@ func releaseQueryCoordLoadMeta(cli *clientv3.Client, basePath string, collection if dmChannel.CollectionID != collectionID { continue } - p := path.Join(basePath, dmChannelMetaPrefix, fmt.Sprintf("%d/%s", dmChannel.CollectionID, dmChannel.DmChannel)) + p := path.Join(basePath, common.QCDmChannelMetaPrefix, fmt.Sprintf("%d/%s", dmChannel.CollectionID, dmChannel.DmChannel)) cli.Delete(ctx, p) } - deltaChannels, err := listQueryCoordDeltaChannelInfos(cli, basePath) + deltaChannels, err := common.ListQueryCoordDeltaChannelInfos(cli, basePath) for _, deltaChannel := range deltaChannels { if deltaChannel.CollectionID != collectionID { continue } - p := path.Join(basePath, deltaChannelMetaPrefix, fmt.Sprintf("%d/%s", deltaChannel.CollectionID, deltaChannel.ChannelName)) + p := path.Join(basePath, common.QCDeltaChannelMetaPrefix, fmt.Sprintf("%d/%s", deltaChannel.CollectionID, deltaChannel.ChannelName)) cli.Delete(ctx, p) } diff --git a/states/garbage_collect.go b/states/garbage_collect.go index e00c9eb..f1204f0 100644 --- a/states/garbage_collect.go +++ b/states/garbage_collect.go @@ -13,6 +13,7 @@ import ( "github.com/manifoldco/promptui" "github.com/milvus-io/birdwatcher/proto/v2.0/commonpb" "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/minio/minio-go/v7" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" @@ -65,7 +66,7 @@ const ( func garbageCollect(cli *clientv3.Client, basePath string, minioClient *minio.Client, minioRootPath string, bucketName string) { - segments, err := listSegments(cli, basePath, func(*datapb.SegmentInfo) bool { return true }) + segments, err := common.ListSegments(cli, basePath, func(*datapb.SegmentInfo) bool { return true }) if err != nil { fmt.Println("failed to list segments:", err.Error()) return @@ -77,7 +78,7 @@ func garbageCollect(cli *clientv3.Client, basePath string, minioClient *minio.Cl statslog := make(map[string]struct{}) for _, segment := range segments { - fillFieldsIfV2(cli, basePath, segment) + common.FillFieldsIfV2(cli, basePath, segment) idSegments[segment.ID] = segment if !isSegmentHealthy(segment) { diff --git a/states/grpc/commands.go b/states/grpc/commands.go new file mode 100644 index 0000000..21e034e --- /dev/null +++ b/states/grpc/commands.go @@ -0,0 +1 @@ +package grpc diff --git a/states/inspect_primary_key.go b/states/inspect_primary_key.go index a53a7ec..2c9bc37 100644 --- a/states/inspect_primary_key.go +++ b/states/inspect_primary_key.go @@ -8,6 +8,7 @@ import ( "github.com/milvus-io/birdwatcher/proto/v2.0/commonpb" "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/milvus-io/birdwatcher/storage" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" @@ -22,7 +23,7 @@ func getInspectPKCmd(cli *clientv3.Client, basePath string) *cobra.Command { var segments []*datapb.SegmentInfo var err error if len(args) == 0 { - segments, err = listSegments(cli, basePath, nil) + segments, err = common.ListSegments(cli, basePath, nil) } else { var segmentID int64 segmentID, err = strconv.ParseInt(args[0], 10, 64) @@ -31,7 +32,7 @@ func getInspectPKCmd(cli *clientv3.Client, basePath string) *cobra.Command { cmd.Help() return } - segments, err = listSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { return info.ID == segmentID }) + segments, err = common.ListSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { return info.ID == segmentID }) } if err != nil { fmt.Println("failed to parse argument:", err.Error()) @@ -49,7 +50,7 @@ func getInspectPKCmd(cli *clientv3.Client, basePath string) *cobra.Command { } pkID, has := cachedCollection[segment.CollectionID] if !has { - coll, err := getCollectionByID(cli, basePath, segment.CollectionID) + coll, err := common.GetCollectionByID(cli, basePath, segment.CollectionID) if err != nil { fmt.Println("Collection not found for id", segment.CollectionID) return @@ -68,7 +69,7 @@ func getInspectPKCmd(cli *clientv3.Client, basePath string) *cobra.Command { } } - fillFieldsIfV2(cli, basePath, segment) + common.FillFieldsIfV2(cli, basePath, segment) for _, fieldBinlog := range segment.Binlogs { if fieldBinlog.FieldID != pkID { continue diff --git a/states/instance.go b/states/instance.go index a6a5206..d676b9a 100644 --- a/states/instance.go +++ b/states/instance.go @@ -4,6 +4,7 @@ import ( "fmt" "path" + "github.com/milvus-io/birdwatcher/states/etcd" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" ) @@ -25,41 +26,39 @@ func (s *instanceState) SetupCommands() { cli := s.client instanceName := s.instanceName + basePath := path.Join(instanceName, metaPath) + cmd.AddCommand( // download-segment - getDownloadSegmentCmd(cli, path.Join(instanceName, metaPath)), + getDownloadSegmentCmd(cli, basePath), // show [subcommand] options... - getEtcdShowCmd(cli, path.Join(instanceName, metaPath)), + etcd.ShowCommand(cli, basePath), + // repair [subcommand] options... + etcd.RepairCommand(cli, basePath), // backup [component] - getBackupEtcdCmd(cli, path.Join(instanceName, metaPath)), + getBackupEtcdCmd(cli, basePath), // kill --component [component] --id [id] - getEtcdKillCmd(cli, path.Join(instanceName, metaPath)), + getEtcdKillCmd(cli, basePath), // force-release - getForceReleaseCmd(cli, path.Join(instanceName, metaPath)), + getForceReleaseCmd(cli, basePath), // download-pk - getDownloadPKCmd(cli, path.Join(instanceName, metaPath)), + getDownloadPKCmd(cli, basePath), // visit [component] [id] - getVisitCmd(s, cli, path.Join(instanceName, metaPath)), + getVisitCmd(s, cli, basePath), // show-log-level - getShowLogLevelCmd(cli, path.Join(instanceName, metaPath)), + getShowLogLevelCmd(cli, basePath), // update-log-level log_level_name component serverId - getUpdateLogLevelCmd(cli, path.Join(instanceName, metaPath)), - // clean-empty-segment - cleanEmptySegments(cli, path.Join(instanceName, metaPath)), + getUpdateLogLevelCmd(cli, basePath), + // remove-segment-by-id - removeSegmentByID(cli, path.Join(instanceName, metaPath)), + //removeSegmentByID(cli, basePath), // garbage-collect - getGarbageCollectCmd(cli, path.Join(instanceName, metaPath)), + getGarbageCollectCmd(cli, basePath), // release-dropped-collection - getReleaseDroppedCollectionCmd(cli, path.Join(instanceName, metaPath)), - // repair-segment - getRepairSegmentCmd(cli, path.Join(instanceName, metaPath)), - // repair-channel - getRepairChannelCmd(cli, path.Join(instanceName, metaPath)), - // reset-checkpoint - getResetCheckpointCmd(cli, path.Join(instanceName, metaPath)), + getReleaseDroppedCollectionCmd(cli, basePath), + // fetch-metrics - getFetchMetricsCmd(cli, path.Join(instanceName, metaPath)), + getFetchMetricsCmd(cli, basePath), // dry-mode getDryModeCmd(cli, s, s.etcdState), // disconnect diff --git a/states/load_backup.go b/states/load_backup.go new file mode 100644 index 0000000..d761dfe --- /dev/null +++ b/states/load_backup.go @@ -0,0 +1,180 @@ +package states + +import ( + "bufio" + "compress/gzip" + "fmt" + "os" + "path" + "strings" + + "github.com/milvus-io/birdwatcher/configs" + "github.com/milvus-io/birdwatcher/models" + "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" +) + +func getLoadBackupCmd(state State, config *configs.Config) *cobra.Command { + cmd := &cobra.Command{ + Use: "load-backup [file]", + Short: "load etcd backup file as env", + Run: func(cmd *cobra.Command, args []string) { + useWorkspace, err := cmd.Flags().GetBool("use-workspace") + if err != nil { + fmt.Println(err.Error()) + return + } + workspaceName, err := cmd.Flags().GetString("workspace-name") + if err != nil { + fmt.Println(err.Error()) + return + } + if len(args) == 0 { + fmt.Println("No backup file provided.") + return + } + if len(args) > 1 { + fmt.Println("only one backup file is allowed") + return + } + + arg := args[0] + f, err := openBackupFile(arg) + if err != nil { + return + } + defer f.Close() + + r, err := gzip.NewReader(f) + if err != nil { + fmt.Println("failed to open gzip reader, err:", err.Error()) + return + } + defer r.Close() + + rd := bufio.NewReader(r) + var header models.BackupHeader + err = readFixLengthHeader(rd, &header) + if err != nil { + fmt.Println("failed to load backup header", err.Error()) + return + } + + if useWorkspace { + if workspaceName == "" { + fileName := path.Base(arg) + workspaceName = fileName + } + workspaceName = createWorkspaceFolder(config, workspaceName) + } + + server, err := startEmbedEtcdServer(workspaceName, useWorkspace) + if err != nil { + fmt.Println("failed to start embed etcd server:", err.Error()) + return + } + fmt.Println("using data dir:", server.Config().Dir) + // TODO + nextState := getEmbedEtcdInstanceV2(server) + switch header.Version { + case 1: + fmt.Printf("Found backup version: %d, instance name :%s\n", header.Version, header.Instance) + err = restoreFromV1File(nextState.client, rd, &header) + if err != nil { + fmt.Println("failed to restore v1 backup file", err.Error()) + nextState.Close() + return + } + nextState.SetInstance(header.Instance) + case 2: + err = restoreV2File(rd, nextState) + if err != nil { + fmt.Println("failed to restore v2 backup file", err.Error()) + nextState.Close() + return + } + default: + fmt.Printf("backup version %d not supported\n", header.Version) + nextState.Close() + return + } + nextState.setupWorkDir(server.Config().Dir) + + state.SetNext(nextState) + }, + } + + cmd.Flags().Bool("use-workspace", false, "load backup into workspace") + cmd.Flags().String("workspace-name", "", "workspace name if used") + return cmd +} + +func openBackupFile(arg string) (*os.File, error) { + if strings.Contains(arg, "~") { + var err error + arg, err = homedir.Expand(arg) + if err != nil { + fmt.Println("path contains tilde, but cannot find home folder", err.Error()) + return nil, err + } + } + err := testFile(arg) + if err != nil { + fmt.Println("backup file not valid:", err.Error()) + return nil, err + } + + f, err := os.Open(arg) + if err != nil { + fmt.Printf("failed to open backup file %s, err: %s\n", arg, err.Error()) + return nil, err + } + return f, nil +} + +func createWorkspaceFolder(config *configs.Config, workspaceName string) string { + _, err := checkIsDirOrCreate(config.WorkspacePath) + if err != nil { + return "" + } + + workPath := path.Join(config.WorkspacePath, workspaceName) + preExist, err := checkIsDirOrCreate(workPath) + if err != nil { + return "" + } + if preExist { + fmt.Printf("%s already exists!\n", workPath) + return "" + } + return workPath +} + +func checkIsDirOrCreate(path string) (bool, error) { + info, err := os.Stat(path) + if os.IsNotExist(err) { + err = os.Mkdir(path, os.ModePerm) + if err != nil { + fmt.Println("failed to create workspace folder", err.Error()) + return false, err + } + return false, nil + } + if !info.IsDir() { + return true, fmt.Errorf("%s is a folder", path) + } + return true, nil +} + +// testFile check file path exists and has access +func testFile(file string) error { + fi, err := os.Stat(file) + if err != nil { + return err + } + // not support iterate all possible file under directory for now + if fi.IsDir() { + return fmt.Errorf("%s is a folder", file) + } + return nil +} diff --git a/states/metrics.go b/states/metrics.go index d35da0c..41923c5 100644 --- a/states/metrics.go +++ b/states/metrics.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/milvus-io/birdwatcher/models" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" ) @@ -16,7 +17,7 @@ func getFetchMetricsCmd(cli *clientv3.Client, basePath string) *cobra.Command { Use: "fetch-metrics", Short: "fetch metrics from milvus instances", Run: func(cmd *cobra.Command, args []string) { - sessions, err := listSessions(cli, basePath) + sessions, err := common.ListSessions(cli, basePath) if err != nil { fmt.Println("failed to list session", err.Error()) return diff --git a/states/open.go b/states/open.go new file mode 100644 index 0000000..f4a7ed9 --- /dev/null +++ b/states/open.go @@ -0,0 +1,54 @@ +package states + +import ( + "fmt" + "os" + "path" + + "github.com/milvus-io/birdwatcher/configs" + "github.com/spf13/cobra" +) + +// getOpenWorkspaceCmd returns open-workspace command. +func getOpenWorkspaceCmd(state State, config *configs.Config) *cobra.Command { + cmd := &cobra.Command{ + //TODO add workspace auto complete + Use: "open-workspace [workspace name]", + Short: "Open workspace", + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + fmt.Println("No backup file provided.") + return + } + if len(args) > 1 { + fmt.Println("only one backup file is allowed") + return + } + + workspaceName := args[0] + workPath := path.Join(config.WorkspacePath, workspaceName) + info, err := os.Stat(workPath) + if os.IsNotExist(err) { + fmt.Printf("workspace %s not exist\n", workspaceName) + return + } + if !info.IsDir() { + fmt.Printf("workspace %s is not a directory\n", workspaceName) + return + } + + server, err := startEmbedEtcdServer(workPath, true) + if err != nil { + fmt.Printf("failed to start embed etcd server in workspace %s, err: %s\n", workspaceName, err.Error()) + return + } + + nextState := getEmbedEtcdInstanceV2(server) + nextState.setupWorkDir(workPath) + + state.SetNext(nextState) + }, + } + + return cmd +} diff --git a/states/pulsarctl_connect.go b/states/pulsarctl_connect.go new file mode 100644 index 0000000..bcf20a9 --- /dev/null +++ b/states/pulsarctl_connect.go @@ -0,0 +1,150 @@ +package states + +import ( + "fmt" + + "github.com/spf13/cobra" + pulsarctl "github.com/streamnative/pulsarctl/pkg/pulsar" + "github.com/streamnative/pulsarctl/pkg/pulsar/common" + "github.com/streamnative/pulsarctl/pkg/pulsar/utils" +) + +func getPulsarctlCmd(state State) *cobra.Command { + cmd := &cobra.Command{ + Use: "pulsarctl", + Short: "connect to pulsar admin with pulsarctl", + Run: func(cmd *cobra.Command, args []string) { + address, err := cmd.Flags().GetString("addr") + if err != nil { + fmt.Println(err.Error()) + } + authPlugin, err := cmd.Flags().GetString("authPlugin") + if err != nil { + fmt.Println(err.Error()) + } + authParams, err := cmd.Flags().GetString("authParams") + if err != nil { + fmt.Println(err.Error()) + } + + config := common.Config{ + WebServiceURL: address, + AuthPlugin: authPlugin, + AuthParams: authParams, + PulsarAPIVersion: common.V2, + } + admin, err := pulsarctl.New(&config) + if err != nil { + fmt.Println("failed to build pulsar admin client, error:", err.Error()) + } + + adminState := getPulsarAdminState(admin, address) + state.SetNext(adminState) + }, + } + + cmd.Flags().String("addr", "http://localhost:18080", "pulsar admin address") + cmd.Flags().String("authPlugin", "", "pulsar admin auth plugin") + cmd.Flags().String("authParams", "", "pulsar admin auth parameters") + + return cmd +} + +func getPulsarAdminState(admin pulsarctl.Client, addr string) State { + state := &pulsarAdminState{ + cmdState: cmdState{ + label: fmt.Sprintf("PulsarAdmin(%s)", addr), + }, + admin: admin, + addr: addr, + } + state.SetupCommands() + return state +} + +type pulsarAdminState struct { + cmdState + admin pulsarctl.Client + addr string + + topics []string +} + +// SetupCommands setups the command. +// also called after each command run to reset flag values. +func (s *pulsarAdminState) SetupCommands() { + cmd := &cobra.Command{} + + cmd.AddCommand( + + getListTopicCmd(s.admin), + + getListSubscriptionCmd(s.admin), + + getExitCmd(s), + ) + + s.cmdState.rootCmd = cmd + s.setupFn = s.SetupCommands +} + +func getListTopicCmd(admin pulsarctl.Client) *cobra.Command { + cmd := &cobra.Command{ + Use: "list-topic", + Short: "list topics in instance", + Run: func(cmd *cobra.Command, args []string) { + nsn, err := utils.GetNameSpaceName("public", "default") + if err != nil { + fmt.Println(err.Error()) + return + } + np, p, err := admin.Topics().List(*nsn) + if err != nil { + fmt.Println("failed to list topics", err.Error()) + return + } + fmt.Println("Non-persist topics:") + for _, topic := range np { + fmt.Println(topic) + } + fmt.Println("Persist topics:") + for _, topic := range p { + fmt.Println(topic) + } + }, + } + + return cmd +} + +func getListSubscriptionCmd(admin pulsarctl.Client) *cobra.Command { + cmd := &cobra.Command{ + Use: "list-subscription", + Short: "list topic subscriptions", + Run: func(cmd *cobra.Command, args []string) { + topic, err := cmd.Flags().GetString("topic") + if err != nil { + fmt.Println(err.Error()) + return + } + + topicName, err := utils.GetTopicName(topic) + if err != nil { + fmt.Println("failed to parse topic name", err.Error()) + return + } + + subscriptions, err := admin.Subscriptions().List(*topicName) + if err != nil { + fmt.Println("failed to list subscriptions", err.Error()) + return + } + for _, subName := range subscriptions { + fmt.Println(subName) + } + }, + } + + cmd.Flags().String("topic", "", "target topic to list") + return cmd +} diff --git a/states/show.go b/states/show.go deleted file mode 100644 index 5ab4ffa..0000000 --- a/states/show.go +++ /dev/null @@ -1,59 +0,0 @@ -package states - -import ( - "context" - "fmt" - - "github.com/spf13/cobra" - clientv3 "go.etcd.io/etcd/client/v3" -) - -// getEtcdShowCmd returns sub command for instanceState. -// show [subCommand] [options...] -// sub command [collection|session|segment] -func getEtcdShowCmd(cli *clientv3.Client, basePath string) *cobra.Command { - showCmd := &cobra.Command{ - Use: "show", - } - - showCmd.AddCommand( - getEtcdShowCollection(cli, basePath), - getEtcdShowSession(cli, basePath), - getEtcdShowSegments(cli, basePath), - getLoadedSegmentsCmd(cli, basePath), - getEtcdShowIndex(cli, basePath), - getEtcdShowReplica(cli, basePath), - getCheckpointCmd(cli, basePath), - getQueryCoordTaskCmd(cli, basePath), - getQueryCoordClusterNodeInfo(cli, basePath), - - getEtcdShowSegmentIndexCmd(cli, basePath), - getQueryCoordChannelInfoCmd(cli, basePath), - getShowLoadedCollectionCmd(cli, basePath), - getEtcdShowChannelWatch(cli, basePath), - ) - return showCmd -} - -// getEtcdRawCmd provides raw "get" command to list kv in etcd -func getEtcdRawCmd(cli *clientv3.Client) *cobra.Command { - cmd := &cobra.Command{ - Use: "get", - Run: func(cmd *cobra.Command, args []string) { - for _, arg := range args { - fmt.Println("list with", arg) - resp, err := cli.Get(context.Background(), arg, clientv3.WithPrefix()) - if err != nil { - fmt.Println(err.Error()) - continue - } - for _, kv := range resp.Kvs { - fmt.Printf("key: %s\n", string(kv.Key)) - fmt.Printf("Value: %s\n", string(kv.Value)) - } - } - }, - } - - return cmd -} diff --git a/states/show_collection_load.go b/states/show_collection_load.go deleted file mode 100644 index 1284cea..0000000 --- a/states/show_collection_load.go +++ /dev/null @@ -1,88 +0,0 @@ -package states - -import ( - "context" - "fmt" - "path" - "time" - - "github.com/golang/protobuf/proto" - "github.com/milvus-io/birdwatcher/proto/v2.0/querypb" - querypbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/querypb" - "github.com/spf13/cobra" - clientv3 "go.etcd.io/etcd/client/v3" -) - -const ( - collectionMetaPrefix = "queryCoord-collectionMeta" - ReplicaMetaPrefix = "queryCoord-ReplicaMeta" - collectionLoadPrefix = "querycoord-collection-loadinfo" -) - -func printLoadedCollections(infos []*querypb.CollectionInfo) { - infos2 := make([]infoWithCollectionID, 0) - for _, info := range infos { - infos2 = append(infos2, info) - } - printInfoWithCollectionID(infos2) -} - -func getLoadedCollectionInfo(cli *clientv3.Client, basePath string) ([]*querypb.CollectionInfo, error) { - prefix := path.Join(basePath, collectionMetaPrefix) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - resp, err := cli.Get(ctx, prefix, clientv3.WithPrefix()) - if err != nil { - return nil, err - } - ret := make([]*querypb.CollectionInfo, 0) - for _, kv := range resp.Kvs { - collectionInfo := &querypb.CollectionInfo{} - err = proto.Unmarshal(kv.Value, collectionInfo) - if err != nil { - return nil, err - } - ret = append(ret, collectionInfo) - } - return ret, nil -} - -func getLoadedCollectionInfoV2(cli *clientv3.Client, basePath string) ([]querypbv2.CollectionLoadInfo, error) { - prefix := path.Join(basePath, collectionLoadPrefix) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - info, _, err := listObject[querypbv2.CollectionLoadInfo](ctx, cli, prefix) - return info, err -} - -func printCollectionLoadInfoV2(loadInfov2 querypbv2.CollectionLoadInfo) { - fmt.Println(loadInfov2.String()) -} - -func getShowLoadedCollectionCmd(cli *clientv3.Client, basePath string) *cobra.Command { - cmd := &cobra.Command{ - Use: "collection-loaded", - Short: "display information of loaded collection from querycoord", - Aliases: []string{"collection-load"}, - RunE: func(cmd *cobra.Command, args []string) error { - collectionLoadInfos, err := getLoadedCollectionInfo(cli, basePath) - if err != nil { - return err - } - printLoadedCollections(collectionLoadInfos) - - loadInfov2, err := getLoadedCollectionInfoV2(cli, basePath) - if err != nil { - return err - } - for _, info := range loadInfov2 { - printCollectionLoadInfoV2(info) - } - - return nil - }, - } - return cmd -} diff --git a/states/show_index.go b/states/show_index.go deleted file mode 100644 index a4263f2..0000000 --- a/states/show_index.go +++ /dev/null @@ -1,285 +0,0 @@ -package states - -import ( - "context" - "fmt" - "path" - "time" - - "github.com/golang/protobuf/proto" - "github.com/milvus-io/birdwatcher/proto/v2.0/commonpb" - "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" - "github.com/milvus-io/birdwatcher/proto/v2.0/etcdpb" - "github.com/milvus-io/birdwatcher/proto/v2.0/indexpb" - indexpbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/indexpb" - "github.com/spf13/cobra" - clientv3 "go.etcd.io/etcd/client/v3" - "google.golang.org/protobuf/runtime/protoiface" -) - -func listObject[T any, P interface { - *T - protoiface.MessageV1 -}](ctx context.Context, cli *clientv3.Client, prefix string) ([]T, []string, error) { - resp, err := cli.Get(ctx, prefix, clientv3.WithPrefix()) - if err != nil { - return nil, nil, err - } - result := make([]T, 0, len(resp.Kvs)) - keys := make([]string, 0, len(resp.Kvs)) - for _, kv := range resp.Kvs { - var elem T - info := P(&elem) - err = proto.Unmarshal(kv.Value, info) - if err != nil { - fmt.Println(err.Error()) - continue - } - result = append(result, elem) - keys = append(keys, string(kv.Key)) - } - return result, keys, nil -} - -func listSegmentIndex(cli *clientv3.Client, basePath string) ([]etcdpb.SegmentIndexInfo, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - - prefix := path.Join(basePath, "root-coord/segment-index") + "/" - result, _, err := listObject[etcdpb.SegmentIndexInfo](ctx, cli, prefix) - return result, err -} - -func listSegmentIndexV2(cli *clientv3.Client, basePath string) ([]indexpbv2.SegmentIndex, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - - prefix := path.Join(basePath, "segment-index") + "/" - result, _, err := listObject[indexpbv2.SegmentIndex](ctx, cli, prefix) - return result, err -} - -func listIndex(cli *clientv3.Client, basePath string) ([]indexpb.IndexMeta, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - prefix := path.Join(basePath, "indexes") + "/" - - result, _, err := listObject[indexpb.IndexMeta](ctx, cli, prefix) - return result, err -} - -type IndexInfoV1 struct { - info etcdpb.IndexInfo - collectionID int64 -} - -func listIndexMeta(cli *clientv3.Client, basePath string) ([]IndexInfoV1, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - - prefix := path.Join(basePath, "root-coord/index") - indexes, keys, err := listObject[etcdpb.IndexInfo](ctx, cli, prefix) - result := make([]IndexInfoV1, 0, len(indexes)) - for idx, info := range indexes { - collectionID, err := pathPartInt64(keys[idx], -2) - if err != nil { - continue - } - result = append(result, IndexInfoV1{ - info: info, - collectionID: collectionID, - }) - } - - return result, err -} - -func listIndexMetaV2(cli *clientv3.Client, basePath string) ([]indexpbv2.FieldIndex, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - indexes, _, err := listObject[indexpbv2.FieldIndex](ctx, cli, path.Join(basePath, "field-index")) - return indexes, err -} - -func getEtcdShowIndex(cli *clientv3.Client, basePath string) *cobra.Command { - cmd := &cobra.Command{ - Use: "index", - Aliases: []string{"indexes"}, - Run: func(cmd *cobra.Command, args []string) { - - // v2.0+ - meta, err := listIndexMeta(cli, basePath) - if err != nil { - fmt.Println(err.Error()) - return - } - - for _, m := range meta { - printIndex(m) - } - - // v2.2+ - fieldIndexes, err := listIndexMetaV2(cli, basePath) - if err != nil { - fmt.Println(err.Error()) - return - } - - for _, index := range fieldIndexes { - printIndexV2(index) - } - }, - } - return cmd -} - -func printIndex(index IndexInfoV1) { - fmt.Println("==================================================================") - fmt.Printf("Index ID: %d\tIndex Name: %s\tCollectionID:%d\n", index.info.GetIndexID(), index.info.GetIndexName(), index.collectionID) - indexParams := index.info.GetIndexParams() - fmt.Printf("Index Type: %s\tMetric Type: %s\n", - getKVPair(indexParams, "index_type"), - getKVPair(indexParams, "metric_type"), - ) - fmt.Printf("Index Params: %s\n", getKVPair(index.info.GetIndexParams(), "params")) - fmt.Println("==================================================================") -} - -func printIndexV2(index indexpbv2.FieldIndex) { - fmt.Println("==================================================================") - fmt.Printf("Index ID: %d\tIndex Name: %s\tCollectionID:%d\n", index.GetIndexInfo().GetIndexID(), index.GetIndexInfo().GetIndexName(), index.GetIndexInfo().GetCollectionID()) - createTime, _ := ParseTS(index.GetCreateTime()) - fmt.Printf("Create Time: %s\tDeleted: %t\n", createTime.Format(tsPrintFormat), index.GetDeleted()) - indexParams := index.GetIndexInfo().GetIndexParams() - fmt.Printf("Index Type: %s\tMetric Type: %s\n", - getKVPair(indexParams, "index_type"), - getKVPair(indexParams, "metric_type"), - ) - fmt.Printf("Index Params: %s\n", getKVPair(index.GetIndexInfo().GetUserIndexParams(), "params")) - fmt.Println("==================================================================") -} - -func getEtcdShowSegmentIndexCmd(cli *clientv3.Client, basePath string) *cobra.Command { - cmd := &cobra.Command{ - Use: "segment-index", - Aliases: []string{"segments-index", "segment-indexes", "segments-indexes"}, - Short: "display segment index information", - Run: func(cmd *cobra.Command, args []string) { - collID, err := cmd.Flags().GetInt64("collection") - if err != nil { - fmt.Println(err.Error()) - return - } - segmentID, err := cmd.Flags().GetInt64("segment") - if err != nil { - fmt.Println(err.Error()) - return - } - - segments, err := listSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { - return (collID == 0 || info.CollectionID == collID) && - (segmentID == 0 || info.ID == segmentID) - }) - if err != nil { - fmt.Println(err.Error()) - return - } - - segmentIndexes, err := listSegmentIndex(cli, basePath) - if err != nil { - fmt.Println(err.Error()) - return - } - segmentIndexesV2, err := listSegmentIndexV2(cli, basePath) - if err != nil { - fmt.Println(err.Error()) - return - } - - indexBuildInfo, err := listIndex(cli, basePath) - if err != nil { - fmt.Println(err.Error()) - return - } - - indexes, _, err := listObject[indexpbv2.FieldIndex](context.Background(), cli, path.Join(basePath, "field-index")) - if err != nil { - fmt.Println(err.Error()) - return - } - idIdx := make(map[int64]indexpbv2.FieldIndex) - for _, idx := range indexes { - idIdx[idx.IndexInfo.IndexID] = idx - } - - seg2Idx := make(map[int64][]etcdpb.SegmentIndexInfo) - seg2Idxv2 := make(map[int64][]indexpbv2.SegmentIndex) - for _, segIdx := range segmentIndexes { - idxs, ok := seg2Idx[segIdx.SegmentID] - if !ok { - idxs = []etcdpb.SegmentIndexInfo{} - } - - idxs = append(idxs, segIdx) - - seg2Idx[segIdx.GetSegmentID()] = idxs - } - for _, segIdx := range segmentIndexesV2 { - idxs, ok := seg2Idxv2[segIdx.SegmentID] - if !ok { - idxs = []indexpbv2.SegmentIndex{} - } - - idxs = append(idxs, segIdx) - - seg2Idxv2[segIdx.GetSegmentID()] = idxs - } - - buildID2Info := make(map[int64]indexpb.IndexMeta) - for _, info := range indexBuildInfo { - buildID2Info[info.IndexBuildID] = info - } - - for _, segment := range segments { - if segment.State != commonpb.SegmentState_Flushed { - continue - } - fmt.Printf("SegmentID: %d\t State: %s", segment.GetID(), segment.GetState().String()) - segIdxs, ok := seg2Idx[segment.GetID()] - if !ok { - // try v2 index information - segIdxv2, ok := seg2Idxv2[segment.GetID()] - if !ok { - fmt.Println("\tno segment index info") - continue - } - for _, segIdx := range segIdxv2 { - fmt.Printf("\n\tIndexV2 build ID: %d, states %s", segIdx.GetBuildID(), segIdx.GetState().String()) - idx, ok := idIdx[segIdx.GetIndexID()] - if ok { - fmt.Printf("\t Index Type:%v on Field ID: %d", getKVPair(idx.GetIndexInfo().GetIndexParams(), "index_type"), idx.GetIndexInfo().GetFieldID()) - } - } - fmt.Println() - continue - } - - for _, segIdx := range segIdxs { - info, ok := buildID2Info[segIdx.BuildID] - if !ok { - fmt.Printf("\tno build info found for id: %d\n", segIdx.BuildID) - fmt.Println(segIdx.String()) - } - fmt.Printf("\n\tIndex build ID: %d, state: %s", info.IndexBuildID, info.State.String()) - fmt.Printf("\t Index Type:%v on Field ID: %d", getKVPair(info.GetReq().GetIndexParams(), "index_type"), segIdx.GetFieldID()) - } - fmt.Println() - } - - }, - } - - cmd.Flags().Int64("collection", 0, "collection id to filter with") - cmd.Flags().Int64("segment", 0, "segment id to filter with") - return cmd -} diff --git a/states/show_segment.go b/states/show_segment.go deleted file mode 100644 index 44ac25e..0000000 --- a/states/show_segment.go +++ /dev/null @@ -1,450 +0,0 @@ -package states - -import ( - "context" - "fmt" - "path" - "sort" - "time" - - "github.com/golang/protobuf/proto" - "github.com/milvus-io/birdwatcher/proto/v2.0/commonpb" - "github.com/milvus-io/birdwatcher/proto/v2.0/datapb" - "github.com/milvus-io/birdwatcher/proto/v2.0/querypb" - datapbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/datapb" - "github.com/milvus-io/birdwatcher/storage" - "github.com/spf13/cobra" - clientv3 "go.etcd.io/etcd/client/v3" -) - -func getEtcdShowSegments(cli *clientv3.Client, basePath string) *cobra.Command { - cmd := &cobra.Command{ - Use: "segment", - Short: "display segment information from data coord meta store", - Aliases: []string{"segments"}, - RunE: func(cmd *cobra.Command, args []string) error { - - collID, err := cmd.Flags().GetInt64("collection") - if err != nil { - return err - } - segmentID, err := cmd.Flags().GetInt64("segment") - if err != nil { - return err - } - format, err := cmd.Flags().GetString("format") - if err != nil { - return err - } - detail, err := cmd.Flags().GetBool("detail") - if err != nil { - return err - } - - segments, err := listSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { - return (collID == 0 || info.CollectionID == collID) && - (segmentID == 0 || info.ID == segmentID) - }) - if err != nil { - fmt.Println("failed to list segments", err.Error()) - return nil - } - - totalRC := int64(0) - healthy := 0 - var growing, sealed, flushed int - for _, info := range segments { - - if info.State != commonpb.SegmentState_Dropped { - totalRC += info.NumOfRows - healthy++ - } - switch info.State { - case commonpb.SegmentState_Growing: - growing++ - case commonpb.SegmentState_Sealed: - sealed++ - case commonpb.SegmentState_Flushing, commonpb.SegmentState_Flushed: - flushed++ - } - - switch format { - case "table": - fillFieldsIfV2(cli, basePath, info) - printSegmentInfo(info, detail) - case "line": - fmt.Printf("SegmentID: %d State: %s, Row Count:%d\n", info.ID, info.State.String(), info.NumOfRows) - } - - } - - fmt.Printf("--- Growing: %d, Sealed: %d, Flushed: %d\n", growing, sealed, flushed) - fmt.Printf("--- Total Segments: %d, row count: %d\n", healthy, totalRC) - return nil - }, - } - cmd.Flags().Int64("collection", 0, "collection id to filter with") - cmd.Flags().String("format", "line", "segment display format") - cmd.Flags().Bool("detail", false, "flags indicating whether pring detail binlog info") - cmd.Flags().Int64("segment", 0, "segment id to filter with") - return cmd -} - -func getLoadedSegmentsCmd(cli *clientv3.Client, basePath string) *cobra.Command { - cmd := &cobra.Command{ - Use: "segment-loaded", - Short: "display segment information from querycoord", - Aliases: []string{"segments-loaded"}, - RunE: func(cmd *cobra.Command, args []string) error { - - collID, err := cmd.Flags().GetInt64("collection") - if err != nil { - return err - } - segmentID, err := cmd.Flags().GetInt64("segment") - if err != nil { - return err - } - - segments, err := listLoadedSegments(cli, basePath, func(info *querypb.SegmentInfo) bool { - return (collID == 0 || info.CollectionID == collID) && - (segmentID == 0 || info.SegmentID == segmentID) - }) - if err != nil { - fmt.Println("failed to list segments", err.Error()) - return nil - } - - for _, info := range segments { - fmt.Printf("Segment ID: %d LegacyNodeID: %d NodeIds: %v,DmlChannel: %s\n", info.SegmentID, info.NodeID, info.NodeIds, info.DmChannel) - fmt.Printf("%#v\n", info.GetIndexInfos()) - } - - return nil - }, - } - cmd.Flags().Int64("collection", 0, "collection id to filter with") - cmd.Flags().String("format", "line", "segment display format") - cmd.Flags().Bool("detail", false, "flags indicating whether pring detail binlog info") - cmd.Flags().Int64("segment", 0, "segment id to filter with") - return cmd -} - -func cleanEmptySegments(cli *clientv3.Client, basePath string) *cobra.Command { - cmd := &cobra.Command{ - Use: "clean-empty-segment", - Short: "Remove empty segment from meta", - RunE: func(cmd *cobra.Command, args []string) error { - run, err := cmd.Flags().GetBool("run") - if err != nil { - return err - } - segments, err := listSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { - return info.GetState() == commonpb.SegmentState_Flushed || info.GetState() == commonpb.SegmentState_Flushing || info.GetState() == commonpb.SegmentState_Sealed - }) - if err != nil { - fmt.Println("failed to list segments", err.Error()) - return nil - } - - for _, info := range segments { - fillFieldsIfV2(cli, basePath, info) - if isEmptySegment(info) { - fmt.Printf("suspect segment %d found:\n", info.GetID()) - printSegmentInfo(info, false) - if run { - err := removeSegment(cli, basePath, info) - if err == nil { - fmt.Printf("remove segment %d from meta succeed\n", info.GetID()) - } else { - fmt.Printf("remove segment %d failed, err: %s\n", info.GetID(), err.Error()) - } - } - - } - } - - return nil - }, - } - - cmd.Flags().Bool("run", false, "flags indicating whether to remove segments from meta") - return cmd -} - -func removeSegmentByID(cli *clientv3.Client, basePath string) *cobra.Command { - cmd := &cobra.Command{ - Use: "remove-segment-by-id", - Short: "Remove segment from meta with specified segment id", - RunE: func(cmd *cobra.Command, args []string) error { - targetSegmentID, err := cmd.Flags().GetInt64("segment") - if err != nil { - return err - } - run, err := cmd.Flags().GetBool("run") - if err != nil { - return err - } - segments, err := listSegments(cli, basePath, func(info *datapb.SegmentInfo) bool { - return true - }) - if err != nil { - fmt.Println("failed to list segments", err.Error()) - return nil - } - - for _, info := range segments { - if info.GetID() == targetSegmentID { - fmt.Printf("target segment %d found:\n", info.GetID()) - printSegmentInfo(info, false) - if run { - err := removeSegment(cli, basePath, info) - if err == nil { - fmt.Printf("remove segment %d from meta succeed\n", info.GetID()) - } else { - fmt.Printf("remove segment %d failed, err: %s\n", info.GetID(), err.Error()) - } - return nil - } - if !isEmptySegment(info) { - fmt.Printf("\n[WARN] segment %d is not empty, please make sure you know what you're doing\n", targetSegmentID) - } - return nil - } - } - fmt.Printf("[WARN] cannot find segment %d\n", targetSegmentID) - return nil - }, - } - - cmd.Flags().Bool("run", false, "flags indicating whether to remove segment from meta") - cmd.Flags().Int64("segment", 0, "segment id to remove") - return cmd -} - -// returns whether all binlog/statslog/deltalog is empty -func isEmptySegment(info *datapb.SegmentInfo) bool { - for _, log := range info.GetBinlogs() { - if len(log.Binlogs) > 0 { - return false - } - } - for _, log := range info.GetStatslogs() { - if len(log.Binlogs) > 0 { - return false - } - } - for _, log := range info.GetDeltalogs() { - if len(log.Binlogs) > 0 { - return false - } - } - return true -} - -func listSegments(cli *clientv3.Client, basePath string, filter func(*datapb.SegmentInfo) bool) ([]*datapb.SegmentInfo, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - resp, err := cli.Get(ctx, path.Join(basePath, "datacoord-meta/s")+"/", clientv3.WithPrefix()) - if err != nil { - return nil, err - } - segments := make([]*datapb.SegmentInfo, 0, len(resp.Kvs)) - for _, kv := range resp.Kvs { - info := &datapb.SegmentInfo{} - err = proto.Unmarshal(kv.Value, info) - if err != nil { - continue - } - if filter == nil || filter(info) { - segments = append(segments, info) - } - } - - sort.Slice(segments, func(i, j int) bool { - return segments[i].GetID() < segments[j].GetID() - }) - return segments, nil -} - -func removeSegment(cli *clientv3.Client, basePath string, info *datapb.SegmentInfo) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - - path := path.Join(basePath, "datacoord-meta/s", fmt.Sprintf("%d/%d/%d", info.CollectionID, info.PartitionID, info.ID)) - _, err := cli.Delete(ctx, path) - - return err -} - -func listLoadedSegments(cli *clientv3.Client, basePath string, filter func(*querypb.SegmentInfo) bool) ([]*querypb.SegmentInfo, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - resp, err := cli.Get(ctx, path.Join(basePath, "queryCoord-segmentMeta"), clientv3.WithPrefix()) - if err != nil { - return nil, err - } - segments := make([]*querypb.SegmentInfo, 0, len(resp.Kvs)) - for _, kv := range resp.Kvs { - info := &querypb.SegmentInfo{} - err = proto.Unmarshal(kv.Value, info) - if err != nil { - continue - } - if filter == nil || filter(info) { - segments = append(segments, info) - } - - } - return segments, nil - -} - -func fillFieldsIfV2(cli *clientv3.Client, basePath string, segment *datapb.SegmentInfo) error { - if len(segment.Binlogs) == 0 { - prefix := path.Join(basePath, "datacoord-meta", fmt.Sprintf("binlog/%d/%d/%d", segment.CollectionID, segment.PartitionID, segment.ID)) - fields, _, err := listObject[datapbv2.FieldBinlog](context.Background(), cli, prefix) - if err != nil { - return err - } - - segment.Binlogs = make([]*datapb.FieldBinlog, 0, len(fields)) - for _, field := range fields { - f := &datapb.FieldBinlog{ - FieldID: field.FieldID, - Binlogs: make([]*datapb.Binlog, 0, len(field.Binlogs)), - } - - for _, binlog := range field.Binlogs { - l := &datapb.Binlog{ - EntriesNum: binlog.EntriesNum, - TimestampFrom: binlog.TimestampFrom, - TimestampTo: binlog.TimestampTo, - LogPath: binlog.LogPath, - LogSize: binlog.LogSize, - } - if l.LogPath == "" { - l.LogPath = fmt.Sprintf("files/insert_log/%d/%d/%d/%d/%d", segment.CollectionID, segment.PartitionID, segment.ID, field.FieldID, binlog.LogID) - } - f.Binlogs = append(f.Binlogs, l) - } - segment.Binlogs = append(segment.Binlogs, f) - } - } - - if len(segment.Deltalogs) == 0 { - prefix := path.Join(basePath, "datacoord-meta", fmt.Sprintf("deltalog/%d/%d/%d", segment.CollectionID, segment.PartitionID, segment.ID)) - fields, _, err := listObject[datapb.FieldBinlog](context.Background(), cli, prefix) - if err != nil { - return err - } - - segment.Deltalogs = make([]*datapb.FieldBinlog, 0, len(fields)) - for _, field := range fields { - field := field - f := proto.Clone(&field).(*datapb.FieldBinlog) - segment.Deltalogs = append(segment.Deltalogs, f) - } - } - - if len(segment.Statslogs) == 0 { - prefix := path.Join(basePath, "datacoord-meta", fmt.Sprintf("statslog/%d/%d/%d", segment.CollectionID, segment.PartitionID, segment.ID)) - fields, _, err := listObject[datapb.FieldBinlog](context.Background(), cli, prefix) - if err != nil { - return err - } - - segment.Statslogs = make([]*datapb.FieldBinlog, 0, len(fields)) - for _, field := range fields { - field := field - f := proto.Clone(&field).(*datapb.FieldBinlog) - segment.Statslogs = append(segment.Statslogs, f) - } - } - - return nil -} - -const ( - tsPrintFormat = "2006-01-02 15:04:05.999 -0700" -) - -func printSegmentInfo(info *datapb.SegmentInfo, detailBinlog bool) { - fmt.Println("================================================================================") - fmt.Printf("Segment ID: %d\n", info.ID) - fmt.Printf("Segment State:%v\n", info.State) - fmt.Printf("Collection ID: %d\t\tPartitionID: %d\n", info.CollectionID, info.PartitionID) - fmt.Printf("Insert Channel:%s\n", info.InsertChannel) - fmt.Printf("Num of Rows: %d\t\tMax Row Num: %d\n", info.NumOfRows, info.MaxRowNum) - lastExpireTime, _ := ParseTS(info.LastExpireTime) - fmt.Printf("Last Expire Time: %s\n", lastExpireTime.Format(tsPrintFormat)) - fmt.Printf("Compact from %v \n", info.CompactionFrom) - if info.StartPosition != nil { - startTime, _ := ParseTS(info.GetStartPosition().GetTimestamp()) - fmt.Printf("Start Position ID: %v, time: %s\n", info.StartPosition.MsgID, startTime.Format(tsPrintFormat)) - } else { - fmt.Println("Start Position: nil") - } - if info.DmlPosition != nil { - dmlTime, _ := ParseTS(info.DmlPosition.Timestamp) - fmt.Printf("Dml Position ID: %v, time: %s, channel name: %s\n", info.DmlPosition.MsgID, dmlTime.Format(tsPrintFormat), info.GetDmlPosition().GetChannelName()) - } else { - fmt.Println("Dml Position: nil") - } - fmt.Printf("Binlog Nums %d\tStatsLog Nums: %d\tDeltaLog Nums:%d\n", - countBinlogNum(info.Binlogs), countBinlogNum(info.Statslogs), countBinlogNum(info.Deltalogs)) - - if detailBinlog { - var binlogSize int64 - fmt.Println("**************************************") - fmt.Println("Binlogs:") - sort.Slice(info.Binlogs, func(i, j int) bool { - return info.Binlogs[i].FieldID < info.Binlogs[j].FieldID - }) - for _, log := range info.Binlogs { - fmt.Printf("Field %d:\n", log.FieldID) - for _, binlog := range log.Binlogs { - fmt.Printf("Path: %s\n", binlog.LogPath) - tf, _ := ParseTS(binlog.TimestampFrom) - tt, _ := ParseTS(binlog.TimestampTo) - fmt.Printf("Log Size: %d \t Entry Num: %d\t TimeRange:%s-%s\n", - binlog.LogSize, binlog.EntriesNum, - tf.Format(tsPrintFormat), tt.Format(tsPrintFormat)) - binlogSize += binlog.LogSize - } - } - - fmt.Println("**************************************") - fmt.Println("Statslogs:") - sort.Slice(info.Statslogs, func(i, j int) bool { - return info.Statslogs[i].FieldID < info.Statslogs[j].FieldID - }) - for _, log := range info.Statslogs { - fmt.Printf("Field %d: %v\n", log.FieldID, log.Binlogs) - } - - fmt.Println("**************************************") - fmt.Println("Delta Logs:") - for _, log := range info.Deltalogs { - for _, l := range log.Binlogs { - fmt.Printf("Entries: %d From: %v - To: %v\n", l.EntriesNum, l.TimestampFrom, l.TimestampTo) - fmt.Printf("Path: %v\n", l.LogPath) - } - } - } - - fmt.Println("================================================================================") -} - -func countBinlogNum(fbl []*datapb.FieldBinlog) int { - result := 0 - for _, f := range fbl { - result += len(f.Binlogs) - } - return result -} - -func analysisBinlog() { - r := &storage.ParquetPayloadReader{} - fmt.Println(r) -} diff --git a/states/show_session.go b/states/show_session.go deleted file mode 100644 index be50715..0000000 --- a/states/show_session.go +++ /dev/null @@ -1,37 +0,0 @@ -package states - -import ( - "fmt" - "path" - - "github.com/milvus-io/birdwatcher/models" - "github.com/spf13/cobra" - clientv3 "go.etcd.io/etcd/client/v3" -) - -// getEtcdShowSession returns show session command. -// usage: show session -func getEtcdShowSession(cli *clientv3.Client, basePath string) *cobra.Command { - cmd := &cobra.Command{ - Use: "session", - Short: "list online milvus components", - Aliases: []string{"sessions"}, - RunE: func(cmd *cobra.Command, args []string) error { - sessions, err := listSessions(cli, basePath) - if err != nil { - return err - } - for _, session := range sessions { - fmt.Println(session.String()) - } - return nil - }, - } - return cmd -} - -// listSessions returns all session -func listSessions(cli *clientv3.Client, basePath string) ([]*models.Session, error) { - prefix := path.Join(basePath, "session") - return listSessionsByPrefix(cli, prefix) -} diff --git a/states/start.go b/states/start.go new file mode 100644 index 0000000..81d599b --- /dev/null +++ b/states/start.go @@ -0,0 +1,58 @@ +package states + +import ( + "fmt" + + "github.com/milvus-io/birdwatcher/configs" + "github.com/spf13/cobra" +) + +// Start returns the first state - offline. +func Start() State { + root := &cobra.Command{ + Use: "", + Short: "", + } + + state := &cmdState{ + label: "Offline", + rootCmd: root, + } + app := &ApplicationState{ + State: state, + } + + var err error + app.config, err = configs.NewConfig(".bw_config") + + if err != nil { + // run by default, just printing warning. + fmt.Println("[WARN] load config file failed", err.Error()) + } + + root.AddCommand( + // connect + getConnectCommand(state), + // load-backup + getLoadBackupCmd(state, app.config), + // open-workspace + getOpenWorkspaceCmd(state, app.config), + // pulsarctl + getPulsarctlCmd(state), + // exit + getExitCmd(state)) + + root.AddCommand(getGlobalUtilCommands()...) + + return app +} + +// ApplicationState application background state. +// used for state switch/merging. +type ApplicationState struct { + // current state + State + + // config stores configuration items + config *configs.Config +} diff --git a/states/states.go b/states/states.go index 5110efd..7652c51 100644 --- a/states/states.go +++ b/states/states.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "github.com/samber/lo" + "github.com/milvus-io/birdwatcher/states/autocomplete" "github.com/spf13/cobra" ) @@ -41,9 +41,7 @@ func (s *cmdState) Label() string { } func (s *cmdState) Suggestions(input string) map[string]string { - iResult := s.parseInput(input) - - return findCmdSuggestions(iResult.parts, s.rootCmd.Commands()) + return autocomplete.SuggestInputCommands(input, s.rootCmd.Commands()) } func printCommands(cmds []*cobra.Command) { @@ -53,78 +51,6 @@ func printCommands(cmds []*cobra.Command) { fmt.Println() } -func (s *cmdState) parseInput(input string) inputResult { - // check is end with space - isEndBlank := strings.HasSuffix(input, " ") - - parts := strings.Split(input, " ") - parts = lo.Filter(parts, func(part string, idx int) bool { - return part != "" - }) - - comps := make([]cComp, 0, len(parts)) - currentFlagValue := false - - for _, part := range parts { - // next part is flag value - // just set last comp cValue - if currentFlagValue { - comps[len(comps)-1].cValue = part - currentFlagValue = false - continue - } - // is flag - if strings.HasPrefix(part, "-") { - raw := strings.TrimLeft(part, "-") - if strings.Contains(raw, "=") { - parts := strings.Split(raw, "=") - // a=b - if len(parts) == 2 { - comps = append(comps, cComp{ - raw: part, - cTag: parts[0], - cValue: parts[1], - cType: cmdCompFlag, - }) - } - // TODO handle part len != 2 - } else { - currentFlagValue = true - comps = append(comps, cComp{ - raw: part, - cTag: raw, - cType: cmdCompFlag, - }) - } - } else { - comps = append(comps, cComp{ - raw: part, - cTag: part, - cType: cmdCompCommand, - }) - } - } - - is := inputStateCmd - if currentFlagValue { - if isEndBlank { - is = inputStateFlagValue - } else { - is = inputStateFlagTag - } - } - - // add empty comp if end with space - if isEndBlank { - comps = append(comps, cComp{cType: cmdCompCommand}) - } - - return inputResult{ - parts: comps, - state: is, - } -} - // Process is the main entry for processing command. func (s *cmdState) Process(cmd string) (State, error) { args := strings.Split(cmd, " ") @@ -160,30 +86,5 @@ func (s *cmdState) SetNext(state State) { // Close empty method to implement State. func (s *cmdState) Close() {} -// Check +// Check state is ending state. func (s *cmdState) IsEnding() bool { return false } - -// Start returns the first state - offline. -func Start() State { - root := &cobra.Command{ - Use: "", - Short: "", - } - - state := &cmdState{ - label: "Offline", - rootCmd: root, - } - - root.AddCommand( - // connect - getConnectCommand(state), - // load-backup - getLoadBackupCmd(state), - // exit - getExitCmd(state)) - - root.AddCommand(getGlobalUtilCommands()...) - - return state -} diff --git a/states/update_log_level.go b/states/update_log_level.go index 9070772..7099a58 100644 --- a/states/update_log_level.go +++ b/states/update_log_level.go @@ -13,6 +13,7 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" "github.com/milvus-io/birdwatcher/models" + "github.com/milvus-io/birdwatcher/states/etcd/common" ) // TODO read port from config @@ -23,7 +24,7 @@ func getShowLogLevelCmd(cli *clientv3.Client, basePath string) *cobra.Command { Use: "show-log-level", Short: "show log level of milvus roles", RunE: func(cmd *cobra.Command, args []string) error { - sessions, err := listSessions(cli, basePath) + sessions, err := common.ListSessions(cli, basePath) if err != nil { return err } @@ -66,7 +67,7 @@ func getUpdateLogLevelCmd(cli *clientv3.Client, basePath string) *cobra.Command Use: "update-log-level log_level [component] [serverId]", Short: "update log level of milvus role ", RunE: func(cmd *cobra.Command, args []string) error { - sessions, err := listSessions(cli, basePath) + sessions, err := common.ListSessions(cli, basePath) if err != nil { return err } diff --git a/states/util.go b/states/util.go index 030cb41..c98d4eb 100644 --- a/states/util.go +++ b/states/util.go @@ -143,7 +143,7 @@ func reviseVChannelInfo(vChannel *datapb.VchannelInfo) { } type infoWithCollectionID interface { - GetCollectionID() UniqueID + GetCollectionID() int64 String() string } diff --git a/states/visit.go b/states/visit.go index 620205a..355526e 100644 --- a/states/visit.go +++ b/states/visit.go @@ -16,6 +16,7 @@ import ( "github.com/milvus-io/birdwatcher/proto/v2.0/rootcoordpb" commonpbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/commonpb" internalpbv2 "github.com/milvus-io/birdwatcher/proto/v2.2/internalpb" + "github.com/milvus-io/birdwatcher/states/etcd/common" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "google.golang.org/grpc" @@ -74,7 +75,7 @@ func setNextState(sessionType string, conn *grpc.ClientConn, statePtr *State, se } func getSessionConnect(cli *clientv3.Client, basePath string, id int64, sessionType string) (session *models.Session, conn *grpc.ClientConn, err error) { - sessions, err := listSessions(cli, basePath) + sessions, err := common.ListSessions(cli, basePath) if err != nil { fmt.Println("failed to list session, err:", err.Error()) return nil, nil, err diff --git a/utils/util.go b/utils/util.go new file mode 100644 index 0000000..53c78f2 --- /dev/null +++ b/utils/util.go @@ -0,0 +1,15 @@ +package utils + +import "time" + +const ( + logicalBits = 18 + logicalBitsMask = (1 << logicalBits) - 1 +) + +func ParseTS(ts uint64) (time.Time, uint64) { + logical := ts & logicalBitsMask + physical := ts >> logicalBits + physicalTime := time.Unix(int64(physical/1000), int64(physical)%1000*time.Millisecond.Nanoseconds()) + return physicalTime, logical +}