diff --git a/config.go b/config.go index f534b99d..8041eeae 100644 --- a/config.go +++ b/config.go @@ -14,6 +14,7 @@ import ( "github.com/stealthrocket/timecraft/internal/object" "github.com/stealthrocket/timecraft/internal/print/human" "github.com/stealthrocket/timecraft/internal/timemachine" + "github.com/tetratelabs/wazero" "gopkg.in/yaml.v3" ) @@ -139,15 +140,77 @@ func config(ctx context.Context, args []string) error { } } +type nullable[T any] struct { + value T + exist bool +} + +// Note: commented to satisfy the linter, uncomment if we need it +// +// func null[T any]() nullable[T] { +// return nullable[T]{exist: false} +// } + +func value[T any](v T) nullable[T] { + return nullable[T]{value: v, exist: true} +} + +func (v *nullable[T]) Value() (T, bool) { + return v.value, v.exist +} + +func (v *nullable[T]) MarshalJSON() ([]byte, error) { + if !v.exist { + return []byte("null"), nil + } + return json.Marshal(v.value) +} + +func (v *nullable[T]) MarshalYAML() (any, error) { + if !v.exist { + return nil, nil + } + return v.value, nil +} + +func (v *nullable[T]) UnmarshalJSON(b []byte) error { + if string(b) == "null" { + v.exist = false + return nil + } else if err := json.Unmarshal(b, &v.value); err != nil { + v.exist = false + return err + } else { + v.exist = true + return nil + } +} + +func (v *nullable[T]) UnmarshalYAML(node *yaml.Node) error { + if node.Value == "" || node.Value == "~" || node.Value == "null" { + v.exist = false + return nil + } else if err := node.Decode(&v.value); err != nil { + v.exist = false + return err + } else { + v.exist = true + return nil + } +} + type configuration struct { Registry struct { - Location string `json:"location"` + Location nullable[human.Path] `json:"location"` } `json:"registry"` + Cache struct { + Location nullable[human.Path] `json:"location"` + } `json:"cache"` } func defaultConfig() *configuration { c := new(configuration) - c.Registry.Location = "~/.timecraft" + c.Registry.Location = value[human.Path]("~/.timecraft/registry") return c } @@ -187,13 +250,56 @@ func readConfig(r io.Reader) (*configuration, error) { return c, nil } -func (c *configuration) createRegistry() (*timemachine.Registry, error) { - p, err := human.Path(c.Registry.Location).Resolve() - if err != nil { - return nil, err +func (c *configuration) newRuntime(ctx context.Context) wazero.Runtime { + config := wazero.NewRuntimeConfig() + + var cache wazero.CompilationCache + if cachePath, ok := c.Cache.Location.Value(); ok { + // The cache is an optimization, so if we encounter errors we notify the + // user but still go ahead with the runtime instantiation. + path, err := cachePath.Resolve() + if err != nil { + fmt.Fprintf(os.Stderr, "ERR: resolving timecraft cache location: %s\n", err) + } else { + cache, err = createCacheDirectory(path) + if err != nil { + fmt.Fprintf(os.Stderr, "ERR: creating timecraft cache directory: %s\n", err) + } else { + config = config.WithCompilationCache(cache) + } + } } - if err := os.Mkdir(filepath.Dir(p), 0777); err != nil { - if !errors.Is(err, fs.ErrExist) { + + runtime := wazero.NewRuntimeWithConfig(ctx, config) + if cache != nil { + runtime = &runtimeWithCompilationCache{ + Runtime: runtime, + cache: cache, + } + } + return runtime +} + +type runtimeWithCompilationCache struct { + wazero.Runtime + cache wazero.CompilationCache +} + +func (r *runtimeWithCompilationCache) Close(ctx context.Context) error { + if r.cache != nil { + defer r.cache.Close(ctx) + } + return r.Runtime.Close(ctx) +} + +func (c *configuration) createRegistry() (*timemachine.Registry, error) { + location, ok := c.Registry.Location.Value() + if ok { + path, err := location.Resolve() + if err != nil { + return nil, err + } + if err := createDirectory(path); err != nil { return nil, err } } @@ -201,13 +307,18 @@ func (c *configuration) createRegistry() (*timemachine.Registry, error) { } func (c *configuration) openRegistry() (*timemachine.Registry, error) { - p, err := human.Path(c.Registry.Location).Resolve() - if err != nil { - return nil, err - } - store, err := object.DirStore(p) - if err != nil { - return nil, err + store := object.EmptyStore() + location, ok := c.Registry.Location.Value() + if ok { + path, err := location.Resolve() + if err != nil { + return nil, err + } + dir, err := object.DirStore(path) + if err != nil { + return nil, err + } + store = dir } registry := &timemachine.Registry{ Store: store, @@ -225,3 +336,19 @@ func createTempFile(path string, r io.Reader) (string, error) { _, err = io.Copy(w, r) return w.Name(), err } + +func createDirectory(path string) error { + if err := os.MkdirAll(path, 0777); err != nil { + if !errors.Is(err, fs.ErrExist) { + return err + } + } + return nil +} + +func createCacheDirectory(path string) (wazero.CompilationCache, error) { + if err := createDirectory(path); err != nil { + return nil, err + } + return wazero.NewCompilationCacheWithDir(path) +} diff --git a/describe.go b/describe.go index ea90b9c6..499e853b 100644 --- a/describe.go +++ b/describe.go @@ -21,7 +21,6 @@ import ( "github.com/stealthrocket/timecraft/internal/print/yamlprint" "github.com/stealthrocket/timecraft/internal/stream" "github.com/stealthrocket/timecraft/internal/timemachine" - "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/api" "golang.org/x/exp/slices" ) @@ -82,7 +81,7 @@ func describe(ctx context.Context, args []string) error { return err } - var lookup func(context.Context, *timemachine.Registry, string) (any, error) + var lookup func(context.Context, *timemachine.Registry, string, *configuration) (any, error) var writer stream.WriteCloser[any] switch output { case "json": @@ -98,43 +97,23 @@ func describe(ctx context.Context, args []string) error { defer writer.Close() readers := make([]stream.Reader[any], len(resourceIDs)) - for i, resource := range resourceIDs { - readers[i] = &describeResourceReader{ - context: ctx, - registry: registry, - resource: resource, - lookup: lookup, - } + for i := range resourceIDs { + resource := resourceIDs[i] + readers[i] = stream.ReaderFunc(func(values []any) (int, error) { + v, err := lookup(ctx, registry, resource, config) + if err != nil { + return 0, err + } + values[0] = v + return 1, io.EOF + }) } _, err = stream.Copy[any](writer, stream.MultiReader[any](readers...)) return err } -type describeResourceReader struct { - context context.Context - registry *timemachine.Registry - resource string - lookup func(context.Context, *timemachine.Registry, string) (any, error) -} - -func (r *describeResourceReader) Read(values []any) (int, error) { - if r.registry == nil { - return 0, io.EOF - } - if len(values) == 0 { - return 0, nil - } - defer func() { r.registry = nil }() - v, err := r.lookup(r.context, r.registry, r.resource) - if err != nil { - return 0, err - } - values[0] = v - return 1, io.EOF -} - -func describeConfig(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { +func describeConfig(ctx context.Context, reg *timemachine.Registry, id string, config *configuration) (any, error) { d, err := reg.LookupDescriptor(ctx, format.ParseHash(id)) if err != nil { return nil, err @@ -173,7 +152,7 @@ func describeConfig(ctx context.Context, reg *timemachine.Registry, id string) ( return desc, nil } -func describeModule(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { +func describeModule(ctx context.Context, reg *timemachine.Registry, id string, config *configuration) (any, error) { d, err := reg.LookupDescriptor(ctx, format.ParseHash(id)) if err != nil { return nil, err @@ -183,7 +162,7 @@ func describeModule(ctx context.Context, reg *timemachine.Registry, id string) ( return nil, err } - runtime := wazero.NewRuntime(ctx) + runtime := config.newRuntime(ctx) defer runtime.Close(ctx) compiledModule, err := runtime.CompileModule(ctx, m.Code) @@ -237,7 +216,7 @@ func describeModule(ctx context.Context, reg *timemachine.Registry, id string) ( return desc, nil } -func describeProcess(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { +func describeProcess(ctx context.Context, reg *timemachine.Registry, id string, config *configuration) (any, error) { processID, _, p, err := lookupProcessByLogID(ctx, reg, id) if err != nil { return nil, err @@ -297,7 +276,7 @@ func describeProcess(ctx context.Context, reg *timemachine.Registry, id string) return desc, nil } -func describeProfile(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { +func describeProfile(ctx context.Context, reg *timemachine.Registry, id string, config *configuration) (any, error) { d, err := reg.LookupDescriptor(ctx, format.ParseHash(id)) if err != nil { return nil, err @@ -315,7 +294,7 @@ func describeProfile(ctx context.Context, reg *timemachine.Registry, id string) return desc, nil } -func describeRuntime(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { +func describeRuntime(ctx context.Context, reg *timemachine.Registry, id string, config *configuration) (any, error) { d, err := reg.LookupDescriptor(ctx, format.ParseHash(id)) if err != nil { return nil, err @@ -332,15 +311,15 @@ func describeRuntime(ctx context.Context, reg *timemachine.Registry, id string) return desc, nil } -func lookupConfig(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { +func lookupConfig(ctx context.Context, reg *timemachine.Registry, id string, config *configuration) (any, error) { return lookup(ctx, reg, id, (*timemachine.Registry).LookupConfig) } -func lookupModule(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { +func lookupModule(ctx context.Context, reg *timemachine.Registry, id string, config *configuration) (any, error) { return lookup(ctx, reg, id, (*timemachine.Registry).LookupModule) } -func lookupProcess(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { +func lookupProcess(ctx context.Context, reg *timemachine.Registry, id string, config *configuration) (any, error) { _, desc, proc, err := lookupProcessByLogID(ctx, reg, id) if err != nil { return nil, err @@ -348,11 +327,11 @@ func lookupProcess(ctx context.Context, reg *timemachine.Registry, id string) (a return descriptorAndData(desc, proc), nil } -func lookupProfile(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { +func lookupProfile(ctx context.Context, reg *timemachine.Registry, id string, config *configuration) (any, error) { return lookup(ctx, reg, id, (*timemachine.Registry).LookupProfile) } -func lookupRuntime(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { +func lookupRuntime(ctx context.Context, reg *timemachine.Registry, id string, config *configuration) (any, error) { return lookup(ctx, reg, id, (*timemachine.Registry).LookupRuntime) } @@ -709,7 +688,7 @@ func (desc *logDescriptor) Format(w fmt.State, _ rune) { _, _ = table.Write(desc.Segments) } -func describeLog(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { +func describeLog(ctx context.Context, reg *timemachine.Registry, id string, config *configuration) (any, error) { logSegmentNumber := -1 logID, logNumber, ok := strings.Cut(id, "/") if ok { diff --git a/get.go b/get.go index 2c7d25b7..d1aa560b 100644 --- a/get.go +++ b/get.go @@ -55,8 +55,8 @@ type resource struct { alt []string mediaType format.MediaType get func(context.Context, io.Writer, *timemachine.Registry, bool) stream.WriteCloser[*format.Descriptor] - describe func(context.Context, *timemachine.Registry, string) (any, error) - lookup func(context.Context, *timemachine.Registry, string) (any, error) + describe func(context.Context, *timemachine.Registry, string, *configuration) (any, error) + lookup func(context.Context, *timemachine.Registry, string, *configuration) (any, error) } var resources = [...]resource{ diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 23d591af..2a768223 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -80,6 +80,16 @@ func ReadAll[T any](r Reader[T]) ([]T, error) { } } +func ReaderFunc[T any](f func([]T) (int, error)) Reader[T] { + return readerFunc[T](f) +} + +type readerFunc[T any] func([]T) (int, error) + +func (f readerFunc[T]) Read(values []T) (int, error) { + return f(values) +} + // Writer is an interface implemented by types that write a stream of values of // type T. type Writer[T any] interface { diff --git a/main_test.go b/main_test.go index 7b8a9b55..6a8d270d 100644 --- a/main_test.go +++ b/main_test.go @@ -1,11 +1,12 @@ package main_test import ( + "bytes" "context" "os" "os/exec" "path/filepath" - "strings" + "sync" "testing" "golang.org/x/exp/maps" @@ -14,6 +15,7 @@ import ( ) func TestTimecraft(t *testing.T) { + t.Setenv("TIMECRAFT_TEST_CACHE", t.TempDir()) t.Run("export", export.run) t.Run("get", get.run) t.Run("help", help.run) @@ -24,12 +26,17 @@ func TestTimecraft(t *testing.T) { type configuration struct { Registry registry `yaml:"registry"` + Cache cache `yaml:"cache"` } type registry struct { Location string `yaml:"location"` } +type cache struct { + Location string `yaml:"location"` +} + type tests map[string]func(*testing.T) func (suite tests) run(t *testing.T) { @@ -39,18 +46,19 @@ func (suite tests) run(t *testing.T) { for _, name := range names { test := suite[name] t.Run(name, func(t *testing.T) { - tmp := t.TempDir() - b, err := yaml.Marshal(configuration{ Registry: registry{ - Location: tmp, + Location: t.TempDir(), + }, + Cache: cache{ + Location: os.Getenv("TIMECRAFT_TEST_CACHE"), }, }) if err != nil { t.Fatal("marshaling timecraft configuration:", err) } - configPath := filepath.Join(tmp, "config.yaml") + configPath := filepath.Join(t.TempDir(), "config.yaml") if err := os.WriteFile(configPath, b, 0666); err != nil { t.Fatal("writing timecraft configuration:", err) } @@ -71,8 +79,10 @@ func timecraft(t *testing.T, args ...string) (stdout, stderr string, err error) defer cancel() } - outbuf := new(strings.Builder) - errbuf := new(strings.Builder) + outbuf := acquireBuffer() + errbuf := acquireBuffer() + defer releaseBuffer(outbuf) + defer releaseBuffer(errbuf) cmd := exec.CommandContext(ctx, "./timecraft", args...) cmd.Stdout = outbuf @@ -81,3 +91,19 @@ func timecraft(t *testing.T, args ...string) (stdout, stderr string, err error) err = cmd.Run() return outbuf.String(), errbuf.String(), err } + +var buffers sync.Pool + +func acquireBuffer() *bytes.Buffer { + b, _ := buffers.Get().(*bytes.Buffer) + if b == nil { + b = new(bytes.Buffer) + } else { + b.Reset() + } + return b +} + +func releaseBuffer(b *bytes.Buffer) { + buffers.Put(b) +} diff --git a/profile.go b/profile.go index 91f4d66f..9e04883e 100644 --- a/profile.go +++ b/profile.go @@ -20,7 +20,6 @@ import ( "github.com/stealthrocket/wasi-go/imports/wasi_snapshot_preview1" "github.com/stealthrocket/wazergo" "github.com/stealthrocket/wzprof" - "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/experimental" "golang.org/x/exp/maps" "golang.org/x/exp/slices" @@ -152,7 +151,7 @@ func profile(ctx context.Context, args []string) error { ), ) - runtime := wazero.NewRuntime(ctx) + runtime := config.newRuntime(ctx) defer runtime.Close(ctx) compiledModule, err := runtime.CompileModule(ctx, module.Code) diff --git a/replay.go b/replay.go index 5b2f9b6c..ad196846 100644 --- a/replay.go +++ b/replay.go @@ -13,7 +13,6 @@ import ( "github.com/stealthrocket/timecraft/internal/timemachine/wasicall" "github.com/stealthrocket/wasi-go/imports/wasi_snapshot_preview1" "github.com/stealthrocket/wazergo" - "github.com/tetratelabs/wazero" ) const replayUsage = ` @@ -77,7 +76,7 @@ func replay(ctx context.Context, args []string) error { logReader := timemachine.NewLogReader(logSegment, manifest.StartTime) defer logReader.Close() - runtime := wazero.NewRuntime(ctx) + runtime := config.newRuntime(ctx) defer runtime.Close(ctx) compiledModule, err := runtime.CompileModule(ctx, module.Code) diff --git a/root.go b/root.go index 7201293d..c13eb2fd 100644 --- a/root.go +++ b/root.go @@ -22,8 +22,11 @@ import ( "fmt" _ "net/http/pprof" "os" + "runtime" + "runtime/pprof" "strings" + "github.com/stealthrocket/timecraft/internal/print/human" "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) @@ -47,7 +50,17 @@ For a list of commands available, run 'timecraft help'.` // root is the timecraft entrypoint. func root(ctx context.Context, args ...string) int { + var ( + // Secret options, we don't document them since they are only used for + // development. Since they are not part of the public interface we may + // remove or change the syntax at any time. + cpuProfile human.Path + memProfile human.Path + ) + flagSet := newFlagSet("timecraft", helpUsage) + customVar(flagSet, &cpuProfile, "cpuprofile") + customVar(flagSet, &memProfile, "memprofile") _ = flagSet.Parse(args) if args = flagSet.Args(); len(args) == 0 { @@ -55,6 +68,31 @@ func root(ctx context.Context, args ...string) int { return 0 } + if cpuProfile != "" { + path, _ := cpuProfile.Resolve() + f, err := os.Create(path) + if err != nil { + fmt.Fprintf(os.Stderr, "WARN: could not create CPU profile: %s\n", err) + } else { + defer f.Close() + _ = pprof.StartCPUProfile(f) + defer pprof.StopCPUProfile() + } + } + + if memProfile != "" { + path, _ := memProfile.Resolve() + defer func() { + f, err := os.Create(path) + if err != nil { + fmt.Fprintf(os.Stderr, "WARN: could not create memory profile: %s\n", err) + } + defer f.Close() + runtime.GC() + _ = pprof.WriteHeapProfile(f) + }() + } + var err error cmd, args := args[0], args[1:] switch cmd { diff --git a/run.go b/run.go index 4b308f4d..688bfb76 100644 --- a/run.go +++ b/run.go @@ -84,7 +84,7 @@ func run(ctx context.Context, args []string) error { return fmt.Errorf("could not read wasm file '%s': %w", wasmPath, err) } - runtime := wazero.NewRuntime(ctx) + runtime := config.newRuntime(ctx) defer runtime.Close(ctx) wasmModule, err := runtime.CompileModule(ctx, wasmCode)