diff --git a/format/timecraft.go b/format/timecraft.go index 06b9d322..483ce7ee 100644 --- a/format/timecraft.go +++ b/format/timecraft.go @@ -27,13 +27,20 @@ func SHA256(b []byte) Hash { } } -func ParseHash(s string) (h Hash, err error) { - var ok bool - h.Algorithm, h.Digest, ok = strings.Cut(s, ":") +func ParseHash(s string) Hash { + algorithm, digest, ok := strings.Cut(s, ":") if !ok { - err = fmt.Errorf("malformed hash: %s", s) + algorithm, digest = "sha256", s } - return h, err + return Hash{Algorithm: algorithm, Digest: digest} +} + +func (h Hash) Short() string { + s := h.Digest + if len(s) > 12 { + s = s[:12] + } + return s } func (h Hash) String() string { @@ -123,6 +130,7 @@ func (m *Module) UnmarshalResource(b []byte) error { } type Runtime struct { + Runtime string `json:"runtime" yaml:"runtime"` Version string `json:"version" yaml:"version"` } diff --git a/internal/cmd/describe.go b/internal/cmd/describe.go new file mode 100644 index 00000000..c00981f8 --- /dev/null +++ b/internal/cmd/describe.go @@ -0,0 +1,421 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/google/uuid" + "github.com/stealthrocket/timecraft/format" + "github.com/stealthrocket/timecraft/internal/print/human" + "github.com/stealthrocket/timecraft/internal/print/jsonprint" + "github.com/stealthrocket/timecraft/internal/print/textprint" + "github.com/stealthrocket/timecraft/internal/print/yamlprint" + "github.com/stealthrocket/timecraft/internal/stream" + "github.com/stealthrocket/timecraft/internal/timemachine" + "golang.org/x/exp/slices" +) + +const describeUsage = ` +Usage: timecraft describe [options] + +Options: + -h, --help Show this usage information + -o, --ouptut format Output format, one of: text, json, yaml + -r, --registry path Path to the timecraft registry (default to ~/.timecraft) +` + +func describe(ctx context.Context, args []string) error { + var ( + output = outputFormat("text") + registryPath = human.Path("~/.timecraft") + ) + + flagSet := newFlagSet("timecraft describe", describeUsage) + customVar(flagSet, &output, "o", "output") + customVar(flagSet, ®istryPath, "r", "registry") + parseFlags(flagSet, args) + + args = flagSet.Args() + if len(args) == 0 { + return errors.New(`expected one resource id as argument`) + } + resourceTypeLookup := args[0] + resourceIDs := []string{} + args = args[1:] + + for len(args) > 0 { + parseFlags(flagSet, args) + args = flagSet.Args() + + i := slices.IndexFunc(args, func(s string) bool { + return strings.HasPrefix(s, "-") + }) + if i < 0 { + i = len(args) + } + resourceIDs = append(resourceIDs, args[:i]...) + args = args[i:] + } + + resource, ok := findResource(resourceTypeLookup, resources[:]) + if !ok { + matchingResources := findMatchingResources(resourceTypeLookup, resources[:]) + if len(matchingResources) == 0 { + return fmt.Errorf(`no resources matching '%s'`+useGet(), resourceTypeLookup) + } + return fmt.Errorf(`no resources matching '%s' + +Did you mean?%s`, resourceTypeLookup, joinResourceTypes(matchingResources, "\n ")) + } + + registry, err := openRegistry(registryPath) + if err != nil { + return err + } + + if len(resourceIDs) == 0 { + return fmt.Errorf(`no resources were specified, use 'timecraft describe %s '`, resource.typ) + } + + var lookup func(context.Context, *timemachine.Registry, string) (any, error) + var writer stream.WriteCloser[any] + switch output { + case "json": + lookup = resource.lookup + writer = jsonprint.NewWriter[any](os.Stdout) + case "yaml": + lookup = resource.lookup + writer = yamlprint.NewWriter[any](os.Stdout) + default: + lookup = resource.describe + writer = textprint.NewWriter[any](os.Stdout) + } + defer writer.Close() + + readers := make([]stream.Reader[any], len(resourceIDs)) + for i, resource := range resourceIDs { + readers[i] = &describeResourceReader{ + context: ctx, + registry: registry, + resource: resource, + lookup: lookup, + } + } + + _, err = stream.Copy[any](writer, stream.MultiReader[any](readers...)) + return err +} + +type describeResourceReader struct { + context context.Context + registry *timemachine.Registry + resource string + lookup func(context.Context, *timemachine.Registry, string) (any, error) +} + +func (r *describeResourceReader) Read(values []any) (int, error) { + if r.registry == nil { + return 0, io.EOF + } + if len(values) == 0 { + return 0, nil + } + defer func() { r.registry = nil }() + v, err := r.lookup(r.context, r.registry, r.resource) + if err != nil { + return 0, err + } + values[0] = v + return 1, io.EOF +} + +func describeConfig(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { + d, err := reg.LookupDescriptor(ctx, format.ParseHash(id)) + if err != nil { + return nil, err + } + c, err := reg.LookupConfig(ctx, d.Digest) + if err != nil { + return nil, err + } + var runtime string + var version string + r, err := reg.LookupRuntime(ctx, c.Runtime.Digest) + if err != nil { + runtime = "(unknown)" + version = "(unknown)" + } else { + runtime = r.Runtime + version = r.Version + } + desc := &configDescriptor{ + id: d.Digest.Short(), + runtime: runtimeDescriptor{ + runtime: runtime, + version: version, + }, + modules: make([]moduleDescriptor, len(c.Modules)), + args: c.Args, + env: c.Env, + } + for i, module := range c.Modules { + desc.modules[i] = moduleDescriptor{ + id: module.Digest.Short(), + name: moduleName(module), + size: human.Bytes(module.Size), + } + } + return desc, nil +} + +func describeModule(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { + d, err := reg.LookupDescriptor(ctx, format.ParseHash(id)) + if err != nil { + return nil, err + } + m, err := reg.LookupModule(ctx, d.Digest) + if err != nil { + return nil, err + } + desc := &moduleDescriptor{ + id: d.Digest.Short(), + name: moduleName(d), + size: human.Bytes(len(m.Code)), + } + return desc, nil +} + +func describeProcess(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { + processID, _, p, err := lookupProcessByLogID(ctx, reg, id) + if err != nil { + return nil, err + } + c, err := reg.LookupConfig(ctx, p.Config.Digest) + if err != nil { + return nil, err + } + + var runtime string + var version string + r, err := reg.LookupRuntime(ctx, c.Runtime.Digest) + if err != nil { + runtime = "(unknown)" + version = "(unknown)" + } else { + runtime = r.Runtime + version = r.Version + } + + desc := &processDescriptor{ + id: processID, + startTime: human.Time(p.StartTime.In(time.Local)), + runtime: runtimeDescriptor{ + runtime: runtime, + version: version, + }, + modules: make([]moduleDescriptor, len(c.Modules)), + args: c.Args, + env: c.Env, + } + + for i, module := range c.Modules { + desc.modules[i] = moduleDescriptor{ + id: module.Digest.Short(), + name: moduleName(module), + size: human.Bytes(module.Size), + } + } + + segments := reg.ListLogSegments(ctx, processID) + defer segments.Close() + + i := stream.Iter[timemachine.LogSegment](segments) + for i.Next() { + v := i.Value() + desc.log = append(desc.log, logSegment{ + number: v.Number, + size: human.Bytes(v.Size), + createdAt: human.Time(v.CreatedAt.In(time.Local)), + }) + } + if err := i.Err(); err != nil { + return nil, err + } + + return desc, nil +} + +func describeRuntime(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { + d, err := reg.LookupDescriptor(ctx, format.ParseHash(id)) + if err != nil { + return nil, err + } + r, err := reg.LookupRuntime(ctx, d.Digest) + if err != nil { + return nil, err + } + desc := &runtimeDescriptor{ + id: d.Digest.Short(), + runtime: r.Runtime, + version: r.Version, + } + return desc, nil +} + +func lookupConfig(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { + return lookup(ctx, reg, id, (*timemachine.Registry).LookupConfig) +} + +func lookupModule(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { + return lookup(ctx, reg, id, (*timemachine.Registry).LookupModule) +} + +func lookupProcess(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { + _, desc, proc, err := lookupProcessByLogID(ctx, reg, id) + if err != nil { + return nil, err + } + return descriptorAndData(desc, proc), nil +} + +func lookupRuntime(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { + return lookup(ctx, reg, id, (*timemachine.Registry).LookupRuntime) +} + +func lookup[T any](ctx context.Context, reg *timemachine.Registry, id string, lookup func(*timemachine.Registry, context.Context, format.Hash) (T, error)) (any, error) { + desc, err := reg.LookupDescriptor(ctx, format.ParseHash(id)) + if err != nil { + return nil, err + } + data, err := lookup(reg, ctx, desc.Digest) + if err != nil { + return nil, err + } + return descriptorAndData(desc, data), nil +} + +func descriptorAndData(desc *format.Descriptor, data any) any { + return &struct { + Desc *format.Descriptor `json:"descriptor" yaml:"descriptor"` + Data any `json:"data" yaml:"data"` + }{ + Desc: desc, + Data: data, + } +} + +func lookupProcessByLogID(ctx context.Context, reg *timemachine.Registry, id string) (format.UUID, *format.Descriptor, *format.Process, error) { + processID, err := uuid.Parse(id) + if err != nil { + return processID, nil, nil, errors.New(`malformed process id (not a UUID)`) + } + manifest, err := reg.LookupLogManifest(ctx, processID) + if err != nil { + return processID, nil, nil, err + } + process, err := reg.LookupProcess(ctx, manifest.Process.Digest) + if err != nil { + return processID, nil, nil, err + } + return processID, manifest.Process, process, nil +} + +type configDescriptor struct { + id string + runtime runtimeDescriptor + modules []moduleDescriptor + args []string + env []string +} + +func (desc *configDescriptor) Format(w fmt.State, _ rune) { + fmt.Fprintf(w, "ID: %s\n", desc.id) + fmt.Fprintf(w, "Runtime: %s (%s)\n", desc.runtime.runtime, desc.runtime.version) + fmt.Fprintf(w, "Modules:\n") + for _, module := range desc.modules { + fmt.Fprintf(w, " %s: %s (%v)\n", module.id, module.name, module.size) + } + fmt.Fprintf(w, "Args:\n") + for _, arg := range desc.args { + fmt.Fprintf(w, " %s\n", arg) + } + fmt.Fprintf(w, "Env:\n") + for _, env := range desc.env { + fmt.Fprintf(w, " %s\n", env) + } +} + +type moduleDescriptor struct { + id string + name string + size human.Bytes +} + +func (desc *moduleDescriptor) Format(w fmt.State, _ rune) { + fmt.Fprintf(w, "ID: %s\n", desc.id) + fmt.Fprintf(w, "Name: %s\n", desc.name) + fmt.Fprintf(w, "Size: %v\n", desc.size) +} + +type processDescriptor struct { + id format.UUID + startTime human.Time + runtime runtimeDescriptor + modules []moduleDescriptor + args []string + env []string + log []logSegment +} + +type logSegment struct { + number int + size human.Bytes + createdAt human.Time +} + +func (desc *processDescriptor) Format(w fmt.State, _ rune) { + fmt.Fprintf(w, "ID: %s\n", desc.id) + fmt.Fprintf(w, "Start Time: %s, %s\n", desc.startTime, time.Time(desc.startTime).Format(time.RFC1123)) + fmt.Fprintf(w, "Runtime: %s (%s)\n", desc.runtime.runtime, desc.runtime.version) + fmt.Fprintf(w, "Modules:\n") + for _, module := range desc.modules { + fmt.Fprintf(w, " %s: %s (%v)\n", module.id, module.name, module.size) + } + fmt.Fprintf(w, "Args:\n") + for _, arg := range desc.args { + fmt.Fprintf(w, " %s\n", arg) + } + fmt.Fprintf(w, "Env:\n") + for _, env := range desc.env { + fmt.Fprintf(w, " %s\n", env) + } + fmt.Fprintf(w, "Log:\n") + for _, log := range desc.log { + fmt.Fprintf(w, " segment %d: %s, created %s (%s)\n", log.number, log.size, log.createdAt, time.Time(log.createdAt).Format(time.RFC1123)) + } +} + +type runtimeDescriptor struct { + id string + runtime string + version string +} + +func (desc *runtimeDescriptor) Format(w fmt.State, _ rune) { + fmt.Fprintf(w, "ID: %s\n", desc.id) + fmt.Fprintf(w, "Runtime: %s\n", desc.runtime) + fmt.Fprintf(w, "Version: %s\n", desc.version) +} + +func moduleName(module *format.Descriptor) string { + name := module.Annotations["timecraft.module.name"] + if name == "" { + name = "(none)" + } + return name +} diff --git a/internal/cmd/get.go b/internal/cmd/get.go index 034dcb3a..12aa34eb 100644 --- a/internal/cmd/get.go +++ b/internal/cmd/get.go @@ -19,7 +19,7 @@ import ( ) const getUsage = ` -Usage: timecraft get [options] +Usage: timecraft get [options] The get sub-command gives access to the state of the time machine registry. The command must be followed by the name of resources to display, which must @@ -50,36 +50,46 @@ Options: ` type resource struct { - name string - alt []string - typ format.MediaType - get func(context.Context, io.Writer, *timemachine.Registry) stream.WriteCloser[*format.Descriptor] + typ string + alt []string + mediaType format.MediaType + get func(context.Context, io.Writer, *timemachine.Registry) stream.WriteCloser[*format.Descriptor] + describe func(context.Context, *timemachine.Registry, string) (any, error) + lookup func(context.Context, *timemachine.Registry, string) (any, error) } var resources = [...]resource{ { - name: "config", - alt: []string{"conf", "configs"}, - typ: format.TypeTimecraftConfig, - get: getConfigs, + typ: "config", + alt: []string{"conf", "configs"}, + mediaType: format.TypeTimecraftConfig, + get: getConfigs, + describe: describeConfig, + lookup: lookupConfig, }, { - name: "module", - alt: []string{"mo", "mod", "mods", "modules"}, - typ: format.TypeTimecraftModule, - get: getModules, + typ: "module", + alt: []string{"mo", "mod", "mods", "modules"}, + mediaType: format.TypeTimecraftModule, + get: getModules, + describe: describeModule, + lookup: lookupModule, }, { - name: "process", - alt: []string{"ps", "procs", "processes"}, - typ: format.TypeTimecraftProcess, - get: getProcesses, + typ: "process", + alt: []string{"ps", "proc", "procs", "processes"}, + mediaType: format.TypeTimecraftProcess, + get: getProcesses, + describe: describeProcess, + lookup: lookupProcess, }, { - name: "runtime", - alt: []string{"rt", "runtimes"}, - typ: format.TypeTimecraftRuntime, - get: getRuntimes, + typ: "runtime", + alt: []string{"rt", "runtimes"}, + mediaType: format.TypeTimecraftRuntime, + get: getRuntimes, + describe: describeRuntime, + lookup: lookupRuntime, }, } @@ -97,20 +107,20 @@ func get(ctx context.Context, args []string) error { args = flagSet.Args() if len(args) == 0 { - return errors.New(`expected exactly one resource name as argument` + useGet()) + return errors.New(`expected exactly one resource type as argument` + useGet()) } - resourceNamePrefix := args[0] + resourceTypeLookup := args[0] parseFlags(flagSet, args[1:]) - resource, ok := findResource(resourceNamePrefix, resources[:]) + resource, ok := findResource(resourceTypeLookup, resources[:]) if !ok { - matchingResources := findMatchingResources(resourceNamePrefix, resources[:]) + matchingResources := findMatchingResources(resourceTypeLookup, resources[:]) if len(matchingResources) == 0 { - return fmt.Errorf(`no resources matching '%s'`+useGet(), resourceNamePrefix) + return fmt.Errorf(`no resources matching '%s'`+useGet(), resourceTypeLookup) } return fmt.Errorf(`no resources matching '%s' -Did you mean?%s`, resourceNamePrefix, joinResourceNames(matchingResources, "\n ")) +Did you mean?%s`, resourceTypeLookup, joinResourceTypes(matchingResources, "\n ")) } registry, err := openRegistry(registryPath) @@ -118,7 +128,7 @@ Did you mean?%s`, resourceNamePrefix, joinResourceNames(matchingResources, "\n return err } - reader := registry.ListResources(ctx, resource.typ, timeRange) + reader := registry.ListResources(ctx, resource.mediaType, timeRange) defer reader.Close() var writer stream.WriteCloser[*format.Descriptor] @@ -153,8 +163,8 @@ func getConfigs(ctx context.Context, w io.Writer, r *timemachine.Registry) strea return config{}, err } return config{ - ID: desc.Digest.Digest[:12], - Runtime: r.Version, + ID: desc.Digest.Short(), + Runtime: r.Runtime + " (" + r.Version + ")", Modules: len(c.Modules), Size: human.Bytes(desc.Size), }, nil @@ -173,7 +183,7 @@ func getModules(ctx context.Context, w io.Writer, r *timemachine.Registry) strea name = "(none)" } return module{ - ID: desc.Digest.Digest[:12], + ID: desc.Digest.Short(), Name: name, Size: human.Bytes(desc.Size), }, nil @@ -199,23 +209,19 @@ func getProcesses(ctx context.Context, w io.Writer, r *timemachine.Registry) str func getRuntimes(ctx context.Context, w io.Writer, r *timemachine.Registry) stream.WriteCloser[*format.Descriptor] { type runtime struct { - ID string `text:"RUNTIME ID"` - Version string `text:"VERSION"` - CreatedAt human.Time `text:"CREATED"` + ID string `text:"RUNTIME ID"` + Runtime string `text:"RUNTIME NAME"` + Version string `text:"VERSION"` } return newDescTableWriter(w, func(desc *format.Descriptor) (runtime, error) { r, err := r.LookupRuntime(ctx, desc.Digest) if err != nil { return runtime{}, err } - t, err := human.ParseTime(desc.Annotations["timecraft.object.created-at"]) - if err != nil { - return runtime{}, err - } return runtime{ - ID: desc.Digest.Digest[:12], - Version: r.Version, - CreatedAt: t, + ID: desc.Digest.Short(), + Runtime: r.Runtime, + Version: r.Version, }, nil }) } @@ -226,13 +232,13 @@ func newDescTableWriter[T any](w io.Writer, conv func(*format.Descriptor) (T, er return stream.NewWriteCloser(cw, tw) } -func findResource(name string, options []resource) (resource, bool) { +func findResource(typ string, options []resource) (resource, bool) { for _, option := range options { - if option.name == name { + if option.typ == typ { return option, true } for _, alt := range option.alt { - if alt == name { + if alt == typ { return option, true } } @@ -240,9 +246,9 @@ func findResource(name string, options []resource) (resource, bool) { return resource{}, false } -func findMatchingResources(name string, options []resource) (matches []resource) { +func findMatchingResources(typ string, options []resource) (matches []resource) { for _, option := range options { - if prefixLength(option.name, name) > 1 || prefixLength(name, option.name) > 1 { + if prefixLength(option.typ, typ) > 1 || prefixLength(typ, option.typ) > 1 { matches = append(matches, option) } } @@ -257,11 +263,11 @@ func prefixLength(base, prefix string) int { return n } -func joinResourceNames(resources []resource, prefix string) string { +func joinResourceTypes(resources []resource, prefix string) string { s := new(strings.Builder) for _, r := range resources { s.WriteString(prefix) - s.WriteString(r.name) + s.WriteString(r.typ) } return s.String() } @@ -269,10 +275,10 @@ func joinResourceNames(resources []resource, prefix string) string { func useGet() string { s := new(strings.Builder) s.WriteString("\n\n") - s.WriteString(`Use 'timecraft ' where the supported resource names are:`) + s.WriteString(`Use 'timecraft get ' where the supported resource types are:`) for _, r := range resources { s.WriteString("\n ") - s.WriteString(r.name) + s.WriteString(r.typ) } return s.String() } diff --git a/internal/cmd/help.go b/internal/cmd/help.go index 758a9de3..7a0d8418 100644 --- a/internal/cmd/help.go +++ b/internal/cmd/help.go @@ -9,18 +9,19 @@ const helpUsage = ` Usage: timecraft [options] Registry Commands: - get Display resources from the time machine registry + describe Show detailed information about specific resources + get Display resources from the time machine registry Runtime Commands: - run Run a WebAssembly module, and optionally trace execution - replay Replay a recorded trace of execution + run Run a WebAssembly module, and optionally trace execution + replay Replay a recorded trace of execution Debugging Commands: - profile Generate performance profile from execution records + profile Generate performance profile from execution records Other Commands: - help Show usage information about timecraft commands - version Show the timecraft version information + help Show usage information about timecraft commands + version Show the timecraft version information For a description of each command, run 'timecraft help '.` @@ -36,6 +37,8 @@ func help(ctx context.Context, args []string) error { } switch cmd { + case "describe": + msg = describeUsage case "get": msg = getUsage case "help", "": diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 8367f67b..bcba8826 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -76,6 +76,8 @@ func Root(ctx context.Context, args ...string) int { var err error cmd, args := args[0], args[1:] switch cmd { + case "describe": + err = describe(ctx, args) case "get": err = get(ctx, args) case "help": diff --git a/internal/cmd/run.go b/internal/cmd/run.go index cc5836dc..0cf1750f 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -5,7 +5,9 @@ import ( "errors" "fmt" "os" + "os/signal" "path/filepath" + "syscall" "time" "github.com/google/uuid" @@ -122,7 +124,7 @@ func run(ctx context.Context, args []string) error { } processID := uuid.New() - startTime := time.Now() + startTime := time.Now().UTC() module, err := registry.CreateModule(ctx, &format.Module{ Code: wasmCode, @@ -135,6 +137,7 @@ func run(ctx context.Context, args []string) error { } runtime, err := registry.CreateRuntime(ctx, &format.Runtime{ + Runtime: "timecraft", Version: currentVersion(), }) if err != nil { @@ -188,6 +191,9 @@ func run(ctx context.Context, args []string) error { fmt.Println("timecraft run:", processID) } + ctx, cancel := signal.NotifyContext(ctx, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) + defer cancel() + var system wasi.System ctx, system, err = builder.Instantiate(ctx, runtime) if err != nil { diff --git a/internal/object/store.go b/internal/object/store.go index 26862204..d26503b5 100644 --- a/internal/object/store.go +++ b/internal/object/store.go @@ -195,12 +195,10 @@ func (store dirStore) CreateObject(ctx context.Context, name string, data io.Rea if err != nil { return err } - dirPath, fileName := filepath.Split(filePath) if strings.HasPrefix(fileName, ".") { return fmt.Errorf("object names cannot start with a dot: %s", name) } - if err := os.MkdirAll(dirPath, 0777); err != nil { return err } @@ -244,14 +242,12 @@ func (store dirStore) CreateObject(ctx context.Context, name string, data io.Rea } } }() - if _, err := io.Copy(objectFile, data); err != nil { return err } if err := os.Rename(tmpPath, filePath); err != nil { return err } - success = true return nil } @@ -261,31 +257,27 @@ func (store dirStore) ReadObject(ctx context.Context, name string) (io.ReadClose if err != nil { return nil, err } - file, err := os.Open(path) - if err != nil { - return nil, err - } - return file, nil + return os.Open(path) } -func (store dirStore) StatObject(ctx context.Context, name string) (Info, error) { +func (store dirStore) StatObject(ctx context.Context, name string) (info Info, err error) { path, err := store.joinPath(name) if err != nil { - return Info{}, err + return info, err } stat, err := os.Lstat(path) if err != nil { - return Info{}, err + return info, err } if !stat.Mode().IsRegular() { - return Info{}, ErrNotExist + return info, ErrNotExist } dir, base := filepath.Split(path) tags, err := readTags(filepath.Join(dir, ".tags", base)) if err != nil { - return Info{}, err + return info, err } - info := Info{ + info = Info{ Name: name, Size: stat.Size(), CreatedAt: stat.ModTime(), @@ -295,10 +287,13 @@ func (store dirStore) StatObject(ctx context.Context, name string) (Info, error) } func (store dirStore) ListObjects(ctx context.Context, prefix string, filters ...Filter) stream.ReadCloser[Info] { - if prefix != "." && !strings.HasSuffix(prefix, "/") { - prefix += "/" + var dirPath, namePrefix string + if prefix == "." || strings.HasSuffix(prefix, "/") { + dirPath = prefix + } else { + dirPath, namePrefix = path.Split(prefix) } - path, err := store.joinPath(prefix) + path, err := store.joinPath(dirPath) if err != nil { return &errorInfoReader{err: err} } @@ -312,8 +307,8 @@ func (store dirStore) ListObjects(ctx context.Context, prefix string, filters .. } return &dirReader{ dir: dir, - path: path, - prefix: prefix, + path: dirPath, + name: namePrefix, filters: slices.Clone(filters), } } @@ -323,12 +318,19 @@ func (store dirStore) DeleteObject(ctx context.Context, name string) error { if err != nil { return err } + if err := remove(path); err != nil { + return err + } + dir, file := filepath.Split(path) + return remove(filepath.Join(dir, ".tags", file)) +} + +func remove(path string) error { if err := os.Remove(path); err != nil { if !errors.Is(err, fs.ErrNotExist) { return err } } - // TODO: cleanup empty parent directories return nil } @@ -366,7 +368,7 @@ func readTags(path string) ([]Tag, error) { type dirReader struct { dir *os.File path string - prefix string + name string filters []Filter } @@ -384,25 +386,36 @@ func (r *dirReader) Read(items []Info) (n int, err error) { if strings.HasPrefix(name, ".") { continue } + if !strings.HasPrefix(name, r.name) { + continue + } info, err := dirent.Info() if err != nil { return n, err } - tagsPath := filepath.Join(r.path, ".tags", name) - tags, err := readTags(tagsPath) - if err != nil { - return n, err + var tags []Tag + if !info.IsDir() { + tagsPath := filepath.Join(r.dir.Name(), ".tags", name) + tags, err = readTags(tagsPath) + if err != nil { + return n, err + } } items[n] = Info{ - Name: path.Join(r.prefix, name), + Name: path.Join(r.path, name), Size: info.Size(), - CreatedAt: info.ModTime(), + CreatedAt: info.ModTime().UTC(), Tags: tags, } + if info.IsDir() { + items[n].Name += "/" + items[n].Size = 0 + } + if query.MatchAll(&items[n], r.filters...) { n++ } diff --git a/internal/object/store_test.go b/internal/object/store_test.go index 97da178f..177d3a01 100644 --- a/internal/object/store_test.go +++ b/internal/object/store_test.go @@ -74,6 +74,11 @@ func testObjectStore(t *testing.T, newStore func(*testing.T) (object.Store, func scenario: "tagged objects are filtered when listing", function: testObjectStoreListTaggedObjects, }, + + { + scenario: "listing matches objects by key prefixes", + function: testObjectStoreListByPrefix, + }, } for _, test := range tests { @@ -248,8 +253,69 @@ func testObjectStoreListTaggedObjects(t *testing.T, ctx context.Context, store o []object.Info{object3}) } +func testObjectStoreListByPrefix(t *testing.T, ctx context.Context, store object.Store) { + assert.OK(t, store.CreateObject(ctx, "test-1", strings.NewReader(""))) + assert.OK(t, store.CreateObject(ctx, "test-2", strings.NewReader("A"))) + assert.OK(t, store.CreateObject(ctx, "test-3", strings.NewReader("BC"))) + assert.OK(t, store.CreateObject(ctx, "sub/key-1.0", strings.NewReader("hello"))) + assert.OK(t, store.CreateObject(ctx, "sub/key-1.1", strings.NewReader("world"))) + assert.OK(t, store.CreateObject(ctx, "sub/key-2.0", strings.NewReader("!"))) + + object0 := object.Info{Name: "sub/", Size: 0} + object1 := object.Info{Name: "test-1", Size: 0} + object2 := object.Info{Name: "test-2", Size: 1} + object3 := object.Info{Name: "test-3", Size: 2} + object4 := object.Info{Name: "sub/key-1.0", Size: 5} + object5 := object.Info{Name: "sub/key-1.1", Size: 5} + object6 := object.Info{Name: "sub/key-2.0", Size: 1} + + assert.DeepEqual(t, + listObjects(t, ctx, store, "whatever"), + []object.Info{}) + + assert.DeepEqual(t, + listObjects(t, ctx, store, "."), + []object.Info{object0, object1, object2, object3}) + + assert.DeepEqual(t, + listObjects(t, ctx, store, "test"), + []object.Info{object1, object2, object3}) + + assert.DeepEqual(t, + listObjects(t, ctx, store, "test-"), + []object.Info{object1, object2, object3}) + + assert.DeepEqual(t, + listObjects(t, ctx, store, "test-1"), + []object.Info{object1}) + + assert.DeepEqual(t, + listObjects(t, ctx, store, "sub"), + []object.Info{object0}) + + assert.DeepEqual(t, + listObjects(t, ctx, store, "sub/"), + []object.Info{object4, object5, object6}) + + assert.DeepEqual(t, + listObjects(t, ctx, store, "sub/key-"), + []object.Info{object4, object5, object6}) + + assert.DeepEqual(t, + listObjects(t, ctx, store, "sub/key-1"), + []object.Info{object4, object5}) + + assert.DeepEqual(t, + listObjects(t, ctx, store, "sub/key-1.0"), + []object.Info{object4}) + + assert.DeepEqual(t, + listObjects(t, ctx, store, "sub/key-2"), + []object.Info{object6}) +} + func listObjects(t *testing.T, ctx context.Context, store object.Store, prefix string, filters ...object.Filter) []object.Info { - objects := readValues(t, store.ListObjects(ctx, ".", filters...)) + objects := readValues(t, store.ListObjects(ctx, prefix, filters...)) clearCreatedAt(objects) sortObjectInfo(objects) return objects diff --git a/internal/print/textprint/writer.go b/internal/print/textprint/writer.go new file mode 100644 index 00000000..a562e364 --- /dev/null +++ b/internal/print/textprint/writer.go @@ -0,0 +1,40 @@ +package textprint + +import ( + "bufio" + "fmt" + "io" + + "github.com/stealthrocket/timecraft/internal/stream" +) + +const ( + separator = "--------------------------------------------------------------------------------\n" +) + +func NewWriter[T any](w io.Writer) stream.WriteCloser[T] { + return &writer[T]{output: bufio.NewWriter(w)} +} + +type writer[T any] struct { + output *bufio.Writer + count int +} + +func (w *writer[T]) Write(values []T) (int, error) { + for n, v := range values { + if w.count++; w.count > 1 { + if _, err := io.WriteString(w.output, separator); err != nil { + return n, err + } + } + if _, err := fmt.Fprint(w.output, v); err != nil { + return n, err + } + } + return len(values), nil +} + +func (w *writer[T]) Close() error { + return w.output.Flush() +} diff --git a/internal/stream/multi.go b/internal/stream/multi.go new file mode 100644 index 00000000..15a6253e --- /dev/null +++ b/internal/stream/multi.go @@ -0,0 +1,36 @@ +package stream + +import ( + "io" + + "golang.org/x/exp/slices" +) + +func MultiReader[T any](readers ...Reader[T]) Reader[T] { + return &multiReader[T]{readers: slices.Clone(readers)} +} + +type multiReader[T any] struct { + readers []Reader[T] +} + +func (m *multiReader[T]) Read(values []T) (n int, err error) { + for len(m.readers) > 0 && len(values) > 0 { + rn, err := m.readers[0].Read(values) + values = values[rn:] + n += rn + if err != nil { + if err != io.EOF { + return n, err + } + m.readers = m.readers[1:] + } + if rn == 0 { + return n, io.ErrNoProgress + } + } + if len(m.readers) == 0 { + return n, io.EOF + } + return n, nil +} diff --git a/internal/timemachine/registry.go b/internal/timemachine/registry.go index f99b001a..69983845 100644 --- a/internal/timemachine/registry.go +++ b/internal/timemachine/registry.go @@ -8,6 +8,8 @@ import ( "errors" "fmt" "io" + "io/fs" + "os" "path" "strconv" "time" @@ -97,6 +99,10 @@ func (reg *Registry) LookupProcess(ctx context.Context, hash format.Hash) (*form return process, reg.lookupObject(ctx, hash, process) } +func (reg *Registry) LookupDescriptor(ctx context.Context, hash format.Hash) (*format.Descriptor, error) { + return reg.lookupDescriptor(ctx, hash) +} + func (reg *Registry) ListModules(ctx context.Context, timeRange TimeRange, tags ...object.Tag) stream.ReadCloser[*format.Descriptor] { return reg.listObjects(ctx, format.TypeTimecraftModule, timeRange, tags) } @@ -125,8 +131,8 @@ func errorLookupObject(hash format.Hash, value format.Resource, err error) error return fmt.Errorf("lookup object: %s: %s: %w", hash, value.ContentType(), err) } -func errorListObjects(mediaType format.MediaType, err error) error { - return fmt.Errorf("list objects: %s: %w", mediaType, err) +func errorLookupDescriptor(hash format.Hash, err error) error { + return fmt.Errorf("lookup descriptor: %s: %w", hash, err) } func appendTagFilters(filters []object.Filter, tags []object.Tag) []object.Filter { @@ -177,16 +183,10 @@ func (reg *Registry) createObject(ctx context.Context, value format.ResourceMars annotations := make(map[string]string, 1+len(extraTags)+len(reg.CreateTags)) assignTags(annotations, reg.CreateTags) assignTags(annotations, extraTags) - assignTags(annotations, []object.Tag{ - { - Name: "timecraft.object.media-type", - Value: mediaType.String(), - }, - { - Name: "timecraft.object.created-at", - Value: time.Now().UTC().Format(time.RFC3339), - }, - }) + assignTags(annotations, []object.Tag{{ + Name: "timecraft.object.media-type", + Value: mediaType.String(), + }}) tags := makeTags(annotations) hash := sha256Hash(b, tags) @@ -213,12 +213,29 @@ func (reg *Registry) createObject(ctx context.Context, value format.ResourceMars } func (reg *Registry) lookupObject(ctx context.Context, hash format.Hash, value format.ResourceUnmarshaler) error { + if hash.Algorithm != "sha256" || len(hash.Digest) != 64 { + return errorLookupObject(hash, value, object.ErrNotExist) + } + r, err := reg.Store.ReadObject(ctx, reg.objectKey(hash)) if err != nil { return errorLookupObject(hash, value, err) } defer r.Close() - b, err := io.ReadAll(r) + + var b []byte + switch f := r.(type) { + case *os.File: + var s fs.FileInfo + s, err = f.Stat() + if err != nil { + return errorLookupObject(hash, value, err) + } + b = make([]byte, s.Size()) + _, err = io.ReadFull(f, b) + default: + b, err = io.ReadAll(r) + } if err != nil { return errorLookupObject(hash, value, err) } @@ -228,6 +245,54 @@ func (reg *Registry) lookupObject(ctx context.Context, hash format.Hash, value f return nil } +func (reg *Registry) lookupDescriptor(ctx context.Context, hash format.Hash) (*format.Descriptor, error) { + if hash.Algorithm != "sha256" { + return nil, errorLookupDescriptor(hash, object.ErrNotExist) + } + if len(hash.Digest) > 64 { + return nil, errorLookupDescriptor(hash, object.ErrNotExist) + } + if len(hash.Digest) < 64 { + key := reg.objectKey(hash) + + r := reg.Store.ListObjects(ctx, key) + defer r.Close() + + i := stream.Iter[object.Info](r) + n := 0 + for i.Next() { + if n++; n > 1 { + return nil, errorLookupDescriptor(hash, errors.New("too many objects match the key prefix")) + } + key = i.Value().Name + } + if err := i.Err(); err != nil { + return nil, err + } + hash = format.ParseHash(path.Base(key)) + } + info, err := reg.Store.StatObject(ctx, reg.objectKey(hash)) + if err != nil { + return nil, errorLookupDescriptor(hash, err) + } + return newDescriptor(info), nil +} + +func newDescriptor(info object.Info) *format.Descriptor { + annotations := make(map[string]string, len(info.Tags)) + assignTags(annotations, info.Tags) + + mediaType := annotations["timecraft.object.media-type"] + delete(annotations, "timecraft.object.media-type") + + return &format.Descriptor{ + MediaType: format.MediaType(mediaType), + Digest: format.ParseHash(path.Base(info.Name)), + Size: info.Size, + Annotations: annotations, + } +} + func (reg *Registry) listObjects(ctx context.Context, mediaType format.MediaType, timeRange TimeRange, matchTags []object.Tag) stream.ReadCloser[*format.Descriptor] { if !timeRange.Start.IsZero() { timeRange.Start = timeRange.Start.Add(-1) @@ -243,19 +308,7 @@ func (reg *Registry) listObjects(ctx context.Context, mediaType format.MediaType reader := reg.Store.ListObjects(ctx, "obj/", filters...) return convert(reader, func(info object.Info) (*format.Descriptor, error) { - hash, err := format.ParseHash(path.Base(info.Name)) - if err != nil { - return nil, errorListObjects(mediaType, err) - } - desc := &format.Descriptor{ - MediaType: mediaType, - Digest: hash, - Size: info.Size, - Annotations: make(map[string]string, len(info.Tags)), - } - assignTags(desc.Annotations, info.Tags) - delete(desc.Annotations, "timecraft.object.media-type") - return desc, nil + return newDescriptor(info), nil }) } @@ -297,8 +350,8 @@ func (w *logSegmentWriter) Close() error { return err } -func (reg *Registry) ListLogSegments(ctx context.Context, processID format.UUID) stream.Reader[LogSegment] { - reader := reg.Store.ListObjects(ctx, "log/"+processID.String()+"/data") +func (reg *Registry) ListLogSegments(ctx context.Context, processID format.UUID) stream.ReadCloser[LogSegment] { + reader := reg.Store.ListObjects(ctx, "log/"+processID.String()+"/data/") return convert(reader, func(info object.Info) (LogSegment, error) { number := path.Base(info.Name) n, err := strconv.ParseInt(number, 16, 32) diff --git a/main.go b/main.go index c29d12c2..66720f1a 100644 --- a/main.go +++ b/main.go @@ -3,13 +3,10 @@ package main import ( "context" "os" - "os/signal" - "syscall" "github.com/stealthrocket/timecraft/internal/cmd" ) func main() { - ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) - os.Exit(cmd.Root(ctx, os.Args[1:]...)) + os.Exit(cmd.Root(context.Background(), os.Args[1:]...)) }