diff --git a/format/logsegment/logsegment.fbs b/format/logsegment/logsegment.fbs index c624ec71..43116785 100644 --- a/format/logsegment/logsegment.fbs +++ b/format/logsegment/logsegment.fbs @@ -42,32 +42,6 @@ table Record { function_call:[ubyte]; } -// Details about a function call. -table FunctionCall { - // State of the WebAssembly stack before and after the function was called. - // The first {param_count} values are the input parameters, and the remaining - // values are the return values. - stack:[ulong]; - // Captured sections of the WebAssembly module's linear memory, stored - // contiguously and indexed by {memory_access}. - memory:[ubyte]; - // Ordered collection of memory reads and writes made by the function. - memory_access:[MemoryAccess]; -} - -// MemoryAccess represents the capture of a section of memory. -struct MemoryAccess { - // Byte offset in the WebAssembly module's linear memory where the memory - // access starts. - offset:uint; - // Byte offset into {FunctionCall.memory}. The length of the captured memory - // can be derived by comparing the {index_offset} with that of the next - // {FunctionCall.memory_access}. The length of the final memory region can - // be derived by comparing the {index_offset} with the length of - // {FunctionCall.memory}. - index_offset:uint; -} - root_type RecordBatch; file_identifier "TL.0"; diff --git a/format/logsegment/logsegment_generated.go b/format/logsegment/logsegment_generated.go index 19a09a8d..c59a3416 100644 --- a/format/logsegment/logsegment_generated.go +++ b/format/logsegment/logsegment_generated.go @@ -265,168 +265,3 @@ func RecordStartFunctionCallVector(builder *flatbuffers.Builder, numElems int) f func RecordEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - -type FunctionCall struct { - _tab flatbuffers.Table -} - -func GetRootAsFunctionCall(buf []byte, offset flatbuffers.UOffsetT) *FunctionCall { - n := flatbuffers.GetUOffsetT(buf[offset:]) - x := &FunctionCall{} - x.Init(buf, n+offset) - return x -} - -func GetSizePrefixedRootAsFunctionCall(buf []byte, offset flatbuffers.UOffsetT) *FunctionCall { - n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:]) - x := &FunctionCall{} - x.Init(buf, n+offset+flatbuffers.SizeUint32) - return x -} - -func (rcv *FunctionCall) Init(buf []byte, i flatbuffers.UOffsetT) { - rcv._tab.Bytes = buf - rcv._tab.Pos = i -} - -func (rcv *FunctionCall) Table() flatbuffers.Table { - return rcv._tab -} - -func (rcv *FunctionCall) Stack(j int) uint64 { - o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) - if o != 0 { - a := rcv._tab.Vector(o) - return rcv._tab.GetUint64(a + flatbuffers.UOffsetT(j*8)) - } - return 0 -} - -func (rcv *FunctionCall) StackLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) - if o != 0 { - return rcv._tab.VectorLen(o) - } - return 0 -} - -func (rcv *FunctionCall) MutateStack(j int, n uint64) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) - if o != 0 { - a := rcv._tab.Vector(o) - return rcv._tab.MutateUint64(a+flatbuffers.UOffsetT(j*8), n) - } - return false -} - -func (rcv *FunctionCall) Memory(j int) byte { - o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) - if o != 0 { - a := rcv._tab.Vector(o) - return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j*1)) - } - return 0 -} - -func (rcv *FunctionCall) MemoryLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) - if o != 0 { - return rcv._tab.VectorLen(o) - } - return 0 -} - -func (rcv *FunctionCall) MemoryBytes() []byte { - o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) - if o != 0 { - return rcv._tab.ByteVector(o + rcv._tab.Pos) - } - return nil -} - -func (rcv *FunctionCall) MutateMemory(j int, n byte) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) - if o != 0 { - a := rcv._tab.Vector(o) - return rcv._tab.MutateByte(a+flatbuffers.UOffsetT(j*1), n) - } - return false -} - -func (rcv *FunctionCall) MemoryAccess(obj *MemoryAccess, j int) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) - if o != 0 { - x := rcv._tab.Vector(o) - x += flatbuffers.UOffsetT(j) * 8 - obj.Init(rcv._tab.Bytes, x) - return true - } - return false -} - -func (rcv *FunctionCall) MemoryAccessLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) - if o != 0 { - return rcv._tab.VectorLen(o) - } - return 0 -} - -func FunctionCallStart(builder *flatbuffers.Builder) { - builder.StartObject(3) -} -func FunctionCallAddStack(builder *flatbuffers.Builder, stack flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(stack), 0) -} -func FunctionCallStartStackVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { - return builder.StartVector(8, numElems, 8) -} -func FunctionCallAddMemory(builder *flatbuffers.Builder, memory flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(memory), 0) -} -func FunctionCallStartMemoryVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { - return builder.StartVector(1, numElems, 1) -} -func FunctionCallAddMemoryAccess(builder *flatbuffers.Builder, memoryAccess flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(memoryAccess), 0) -} -func FunctionCallStartMemoryAccessVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { - return builder.StartVector(8, numElems, 4) -} -func FunctionCallEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { - return builder.EndObject() -} - -type MemoryAccess struct { - _tab flatbuffers.Struct -} - -func (rcv *MemoryAccess) Init(buf []byte, i flatbuffers.UOffsetT) { - rcv._tab.Bytes = buf - rcv._tab.Pos = i -} - -func (rcv *MemoryAccess) Table() flatbuffers.Table { - return rcv._tab.Table -} - -func (rcv *MemoryAccess) Offset() uint32 { - return rcv._tab.GetUint32(rcv._tab.Pos + flatbuffers.UOffsetT(0)) -} -func (rcv *MemoryAccess) MutateOffset(n uint32) bool { - return rcv._tab.MutateUint32(rcv._tab.Pos+flatbuffers.UOffsetT(0), n) -} - -func (rcv *MemoryAccess) IndexOffset() uint32 { - return rcv._tab.GetUint32(rcv._tab.Pos + flatbuffers.UOffsetT(4)) -} -func (rcv *MemoryAccess) MutateIndexOffset(n uint32) bool { - return rcv._tab.MutateUint32(rcv._tab.Pos+flatbuffers.UOffsetT(4), n) -} - -func CreateMemoryAccess(builder *flatbuffers.Builder, offset uint32, indexOffset uint32) flatbuffers.UOffsetT { - builder.Prep(4, 8) - builder.PrependUint32(indexOffset) - builder.PrependUint32(offset) - return builder.Offset() -} diff --git a/format/timecraft.go b/format/timecraft.go index 09c19fc4..3bd638ba 100644 --- a/format/timecraft.go +++ b/format/timecraft.go @@ -72,6 +72,7 @@ const ( TypeTimecraftRuntime MediaType = "application/vnd.timecraft.runtime.v1+json" TypeTimecraftConfig MediaType = "application/vnd.timecraft.config.v1+json" TypeTimecraftProcess MediaType = "application/vnd.timecraft.process.v1+json" + TypeTimecraftProfile MediaType = "application/vnd.timecraft.profile.v1+pprof" TypeTimecraftManifest MediaType = "application/vnd.timecraft.manifest.v1+json" TypeTimecraftModule MediaType = "application/vnd.timecraft.module.v1+wasm" ) diff --git a/internal/cmd/describe.go b/internal/cmd/describe.go index 06e50e4e..d2a13fc7 100644 --- a/internal/cmd/describe.go +++ b/internal/cmd/describe.go @@ -8,8 +8,10 @@ import ( "os" "strconv" "strings" + "text/tabwriter" "time" + pprof "github.com/google/pprof/profile" "github.com/google/uuid" "github.com/stealthrocket/timecraft/format" "github.com/stealthrocket/timecraft/internal/print/human" @@ -56,39 +58,20 @@ func describe(ctx context.Context, args []string) error { flagSet := newFlagSet("timecraft describe", describeUsage) customVar(flagSet, &output, "o", "output") customVar(flagSet, ®istryPath, "r", "registry") - parseFlags(flagSet, args) + args = parseFlags(flagSet, args) - args = flagSet.Args() if len(args) == 0 { - return errors.New(`expected one resource id as argument`) + return errors.New(`expected a resource type as argument`) } - resourceTypeLookup := args[0] - resourceIDs := []string{} - args = args[1:] - for len(args) > 0 { - parseFlags(flagSet, args) - args = flagSet.Args() - - i := slices.IndexFunc(args, func(s string) bool { - return strings.HasPrefix(s, "-") - }) - if i < 0 { - i = len(args) - } - resourceIDs = append(resourceIDs, args[:i]...) - args = args[i:] + resource, err := findResource("describe", args[0]) + if err != nil { + return err } - resource, ok := findResource(resourceTypeLookup, resources[:]) - if !ok { - matchingResources := findMatchingResources(resourceTypeLookup, resources[:]) - if len(matchingResources) == 0 { - return fmt.Errorf(`no resources matching '%s'`+useGet(), resourceTypeLookup) - } - return fmt.Errorf(`no resources matching '%s' - -Did you mean?%s`, resourceTypeLookup, joinResourceTypes(matchingResources, "\n ")) + resourceIDs := args[1:] + if len(resourceIDs) == 0 { + return fmt.Errorf(`no resources were specified, use 'timecraft describe %s '`, resource.typ) } registry, err := openRegistry(registryPath) @@ -96,10 +79,6 @@ Did you mean?%s`, resourceTypeLookup, joinResourceTypes(matchingResources, "\n return err } - if len(resourceIDs) == 0 { - return fmt.Errorf(`no resources were specified, use 'timecraft describe %s '`, resource.typ) - } - var lookup func(context.Context, *timemachine.Registry, string) (any, error) var writer stream.WriteCloser[any] switch output { @@ -172,7 +151,7 @@ func describeConfig(ctx context.Context, reg *timemachine.Registry, id string) ( version = r.Version } desc := &configDescriptor{ - id: d.Digest.Short(), + id: d.Digest.String(), runtime: runtimeDescriptor{ runtime: runtime, version: version, @@ -183,7 +162,7 @@ func describeConfig(ctx context.Context, reg *timemachine.Registry, id string) ( } for i, module := range c.Modules { desc.modules[i] = moduleDescriptor{ - id: module.Digest.Short(), + id: module.Digest.String(), name: moduleName(module), size: human.Bytes(module.Size), } @@ -201,7 +180,7 @@ func describeModule(ctx context.Context, reg *timemachine.Registry, id string) ( return nil, err } desc := &moduleDescriptor{ - id: d.Digest.Short(), + id: d.Digest.String(), name: moduleName(d), size: human.Bytes(len(m.Code)), } @@ -243,7 +222,7 @@ func describeProcess(ctx context.Context, reg *timemachine.Registry, id string) for i, module := range c.Modules { desc.modules[i] = moduleDescriptor{ - id: module.Digest.Short(), + id: module.Digest.String(), name: moduleName(module), size: human.Bytes(module.Size), } @@ -268,6 +247,24 @@ func describeProcess(ctx context.Context, reg *timemachine.Registry, id string) return desc, nil } +func describeProfile(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { + d, err := reg.LookupDescriptor(ctx, format.ParseHash(id)) + if err != nil { + return nil, err + } + p, err := reg.LookupProfile(ctx, d.Digest) + if err != nil { + return nil, err + } + desc := &profileDescriptor{ + id: d.Digest.String(), + processID: d.Annotations["timecraft.process.id"], + profileType: d.Annotations["timecraft.profile.type"], + profile: p, + } + return desc, nil +} + func describeRuntime(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { d, err := reg.LookupDescriptor(ctx, format.ParseHash(id)) if err != nil { @@ -278,7 +275,7 @@ func describeRuntime(ctx context.Context, reg *timemachine.Registry, id string) return nil, err } desc := &runtimeDescriptor{ - id: d.Digest.Short(), + id: d.Digest.String(), runtime: r.Runtime, version: r.Version, } @@ -301,6 +298,10 @@ func lookupProcess(ctx context.Context, reg *timemachine.Registry, id string) (a return descriptorAndData(desc, proc), nil } +func lookupProfile(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { + return lookup(ctx, reg, id, (*timemachine.Registry).LookupProfile) +} + func lookupRuntime(ctx context.Context, reg *timemachine.Registry, id string) (any, error) { return lookup(ctx, reg, id, (*timemachine.Registry).LookupRuntime) } @@ -412,6 +413,59 @@ func (desc *processDescriptor) Format(w fmt.State, _ rune) { } } +type profileDescriptor struct { + id string + processID string + profileType string + profile *pprof.Profile +} + +func (desc *profileDescriptor) Format(w fmt.State, _ rune) { + startTime := human.Time(time.Unix(0, desc.profile.TimeNanos)) + duration := human.Duration(desc.profile.DurationNanos) + + fmt.Fprintf(w, "ID: %s\n", desc.id) + fmt.Fprintf(w, "Type: %s\n", desc.profileType) + fmt.Fprintf(w, "Process: %s\n", desc.processID) + fmt.Fprintf(w, "Start: %s, %s\n", startTime, time.Time(startTime).Format(time.RFC1123)) + fmt.Fprintf(w, "Duration: %s\n", duration) + + if period := desc.profile.Period; period != 0 { + fmt.Fprintf(w, "Period: %d %s/%s\n", period, desc.profile.PeriodType.Type, desc.profile.PeriodType.Unit) + } else { + fmt.Fprintf(w, "Period: (none)\n") + } + + fmt.Fprintf(w, "Samples: %d\n", len(desc.profile.Sample)) + hasDefault := false + tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + for i, sampleType := range desc.profile.SampleType { + if i != 0 { + fmt.Fprintf(tw, "\n") + } + fmt.Fprintf(w, "- %s\t%s", sampleType.Type, sampleType.Unit) + if sampleType.Type == desc.profile.DefaultSampleType { + hasDefault = true + fmt.Fprintf(tw, " (default)") + } + } + if !hasDefault { + fmt.Fprintf(tw, " (default)\n") + } else { + fmt.Fprintf(tw, "\n") + } + _ = tw.Flush() + + if comments := desc.profile.Comments; len(comments) == 0 { + fmt.Fprintf(w, "Comments: (none)\n") + } else { + fmt.Fprintf(w, "Comments:\n") + for _, comment := range comments { + fmt.Fprintf(w, "%s\n", comment) + } + } +} + type runtimeDescriptor struct { id string runtime string diff --git a/internal/cmd/export.go b/internal/cmd/export.go new file mode 100644 index 00000000..620fea6e --- /dev/null +++ b/internal/cmd/export.go @@ -0,0 +1,98 @@ +package cmd + +import ( + "context" + "errors" + "io" + "os" + + "github.com/google/uuid" + "github.com/stealthrocket/timecraft/format" + "github.com/stealthrocket/timecraft/internal/print/human" +) + +const exportUsage = ` +Usage: timecraft export + + The export command reads resources from the time machine registry and writes + them to local files. This command is useful to extract data generated by + timecraft and visualize it with other tools. + + The last argument is the location where the resource is exported, typically + a path on the file system. The special value "-" may be set to write the + resource to stdout. + +Options: + -h, --help Show this usage information + -r, --registry path Path to the timecraft registry (default to ~/.timecraft) +` + +func export(ctx context.Context, args []string) error { + var ( + registryPath = human.Path("~/.timecraft") + ) + + flagSet := newFlagSet("timecraft export", exportUsage) + customVar(flagSet, ®istryPath, "r", "registry") + args = parseFlags(flagSet, args) + + if len(args) != 3 { + return errors.New(`expected resource type, id, and output file as argument` + useCmd("export")) + } + + resource, err := findResource("describe", args[0]) + if err != nil { + return err + } + + registry, err := openRegistry(registryPath) + if err != nil { + return err + } + + if resource.typ == "log" { + // How should we handle logs? + // - write the manifest.json + segments to a tar archive? + // - combines the segments into a single log file? + // - ??? + return errors.New(`TODO`) + } + + var hash format.Hash + if resource.typ == "process" { + processID, err := uuid.Parse(args[1]) + if err != nil { + return errors.New(`malformed process id (not a UUID)`) + } + manifest, err := registry.LookupLogManifest(ctx, processID) + if err != nil { + return err + } + hash = manifest.Process.Digest + } else { + desc, err := registry.LookupDescriptor(ctx, format.ParseHash(args[1])) + if err != nil { + return err + } + hash = desc.Digest + } + + r, err := registry.LookupResource(ctx, hash) + if err != nil { + return err + } + defer r.Close() + + w := io.Writer(os.Stdout) + if outputFile := args[2]; outputFile != "-" { + f, err := os.Create(outputFile) + if err != nil { + return err + } + defer f.Close() + w = f + } + + _, err = io.Copy(w, r) + return err +} diff --git a/internal/cmd/get.go b/internal/cmd/get.go index 67a0f6ea..412e9884 100644 --- a/internal/cmd/get.go +++ b/internal/cmd/get.go @@ -1,6 +1,7 @@ package cmd import ( + "bytes" "context" "errors" "fmt" @@ -9,6 +10,7 @@ import ( "strings" "time" + "github.com/google/uuid" "github.com/stealthrocket/timecraft/format" "github.com/stealthrocket/timecraft/internal/print/human" "github.com/stealthrocket/timecraft/internal/print/jsonprint" @@ -23,7 +25,7 @@ Usage: timecraft get [options] The get sub-command gives access to the state of the time machine registry. The command must be followed by the name of resources to display, which must - be one of config, log, module, process, or runtime. + be one of config, log, module, process, profile, or runtime. (the command also accepts plurals and abbreviations of the resource names) Examples: @@ -66,6 +68,7 @@ var resources = [...]resource{ describe: describeConfig, lookup: lookupConfig, }, + { typ: "log", alt: []string{"logs"}, @@ -73,6 +76,7 @@ var resources = [...]resource{ describe: describeLog, lookup: describeLog, }, + { typ: "module", alt: []string{"mo", "mod", "mods", "modules"}, @@ -81,6 +85,7 @@ var resources = [...]resource{ describe: describeModule, lookup: lookupModule, }, + { typ: "process", alt: []string{"ps", "proc", "procs", "processes"}, @@ -89,6 +94,16 @@ var resources = [...]resource{ describe: describeProcess, lookup: lookupProcess, }, + + { + typ: "profile", + alt: []string{"prof", "profs", "profiles"}, + mediaType: format.TypeTimecraftProfile, + get: getProfiles, + describe: describeProfile, + lookup: lookupProfile, + }, + { typ: "runtime", alt: []string{"rt", "runtimes"}, @@ -109,24 +124,14 @@ func get(ctx context.Context, args []string) error { flagSet := newFlagSet("timecraft get", getUsage) customVar(flagSet, &output, "o", "output") customVar(flagSet, ®istryPath, "r", "registry") - parseFlags(flagSet, args) + args = parseFlags(flagSet, args) - args = flagSet.Args() - if len(args) == 0 { - return errors.New(`expected exactly one resource type as argument` + useGet()) + if len(args) != 1 { + return errors.New(`expected exactly one resource type as argument` + useCmd("get")) } - resourceTypeLookup := args[0] - parseFlags(flagSet, args[1:]) - - resource, ok := findResource(resourceTypeLookup, resources[:]) - if !ok { - matchingResources := findMatchingResources(resourceTypeLookup, resources[:]) - if len(matchingResources) == 0 { - return fmt.Errorf(`no resources matching '%s'`+useGet(), resourceTypeLookup) - } - return fmt.Errorf(`no resources matching '%s' - -Did you mean?%s`, resourceTypeLookup, joinResourceTypes(matchingResources, "\n ")) + resource, err := findResource("get", args[0]) + if err != nil { + return err } registry, err := openRegistry(registryPath) @@ -228,7 +233,7 @@ func getModules(ctx context.Context, w io.Writer, reg *timemachine.Registry) str func getProcesses(ctx context.Context, w io.Writer, reg *timemachine.Registry) stream.WriteCloser[*format.Descriptor] { type process struct { ID format.UUID `text:"PROCESS ID"` - StartTime human.Time `text:"STARTED"` + StartTime human.Time `text:"START"` } return newTableWriter(w, func(p1, p2 process) bool { @@ -246,6 +251,43 @@ func getProcesses(ctx context.Context, w io.Writer, reg *timemachine.Registry) s }) } +func getProfiles(ctx context.Context, w io.Writer, reg *timemachine.Registry) stream.WriteCloser[*format.Descriptor] { + type profile struct { + ID string `text:"PROFILE ID"` + ProcessID format.UUID `text:"PROCESS ID"` + Type string `text:"TYPE"` + StartTime human.Time `text:"START"` + Duration human.Duration `text:"DURATION"` + Size human.Bytes `text:"SIZE"` + } + return newTableWriter(w, + func(p1, p2 profile) bool { + if p1.ProcessID != p2.ProcessID { + return bytes.Compare(p1.ProcessID[:], p2.ProcessID[:]) < 0 + } + if p1.Type != p2.Type { + return p1.Type < p2.Type + } + if !time.Time(p1.StartTime).Equal(time.Time(p2.StartTime)) { + return time.Time(p1.StartTime).Before(time.Time(p2.StartTime)) + } + return p1.Duration < p2.Duration + }, + func(desc *format.Descriptor) (profile, error) { + processID, _ := uuid.Parse(desc.Annotations["timecraft.process.id"]) + startTime, _ := time.Parse(time.RFC3339Nano, desc.Annotations["timecraft.profile.start"]) + endTime, _ := time.Parse(time.RFC3339Nano, desc.Annotations["timecraft.profile.end"]) + return profile{ + ID: desc.Digest.Short(), + ProcessID: processID, + Type: desc.Annotations["timecraft.profile.type"], + StartTime: human.Time(startTime), + Duration: human.Duration(endTime.Sub(startTime)), + Size: human.Bytes(desc.Size), + }, nil + }) +} + func getRuntimes(ctx context.Context, w io.Writer, reg *timemachine.Registry) stream.WriteCloser[*format.Descriptor] { type runtime struct { ID string `text:"RUNTIME ID"` @@ -273,8 +315,8 @@ func getLogs(ctx context.Context, w io.Writer, reg *timemachine.Registry) stream type manifest struct { ProcessID format.UUID `text:"PROCESS ID"` Segments human.Count `text:"SEGMENTS"` + StartTime human.Time `text:"START"` Size human.Bytes `text:"SIZE"` - StartTime human.Time `text:"STARTED"` } return newTableWriter(w, func(m1, m2 manifest) bool { @@ -299,27 +341,35 @@ func newTableWriter[T1, T2 any](w io.Writer, orderBy func(T1, T1) bool, conv fun return stream.NewWriteCloser(cw, tw) } -func findResource(typ string, options []resource) (resource, bool) { - for _, option := range options { - if option.typ == typ { - return option, true +func findResource(cmd, typ string) (*resource, error) { + for i, resource := range resources { + if resource.typ == typ { + return &resources[i], nil } - for _, alt := range option.alt { + for _, alt := range resource.alt { if alt == typ { - return option, true + return &resources[i], nil } } } - return resource{}, false -} -func findMatchingResources(typ string, options []resource) (matches []resource) { - for _, option := range options { - if prefixLength(option.typ, typ) > 1 || prefixLength(typ, option.typ) > 1 { - matches = append(matches, option) + var matchingResources []*resource + for i, resource := range resources { + if prefixLength(resource.typ, typ) > 1 || prefixLength(typ, resource.typ) > 1 { + matchingResources = append(matchingResources, &resources[i]) } } - return matches + if len(matchingResources) == 0 { + return nil, fmt.Errorf(`no resources matching '%s'%s`, typ, useCmd(cmd)) + } + + var resourceTypes strings.Builder + for _, r := range matchingResources { + resourceTypes.WriteString("\n ") + resourceTypes.WriteString(r.typ) + } + + return nil, fmt.Errorf("no resources matching '%s'\n\nDid you mean?%s", typ, &resourceTypes) } func prefixLength(base, prefix string) int { @@ -330,19 +380,10 @@ func prefixLength(base, prefix string) int { return n } -func joinResourceTypes(resources []resource, prefix string) string { - s := new(strings.Builder) - for _, r := range resources { - s.WriteString(prefix) - s.WriteString(r.typ) - } - return s.String() -} - -func useGet() string { +func useCmd(cmd string) string { s := new(strings.Builder) s.WriteString("\n\n") - s.WriteString(`Use 'timecraft get ' where the supported resource types are:`) + s.WriteString(`Use 'timecraft ` + cmd + ` ' where the supported resource types are:`) for _, r := range resources { s.WriteString("\n ") s.WriteString(r.typ) diff --git a/internal/cmd/help.go b/internal/cmd/help.go index 7a0d8418..e84e5102 100644 --- a/internal/cmd/help.go +++ b/internal/cmd/help.go @@ -10,6 +10,7 @@ Usage: timecraft [options] Registry Commands: describe Show detailed information about specific resources + export Export resources to local files get Display resources from the time machine registry Runtime Commands: @@ -27,35 +28,41 @@ For a description of each command, run 'timecraft help '.` func help(ctx context.Context, args []string) error { flagSet := newFlagSet("timecraft help", helpUsage) - parseFlags(flagSet, args) + args = parseFlags(flagSet, args) + if len(args) == 0 { + args = []string{"help"} + } - var cmd string - var msg string + for i, cmd := range args { + var msg string - if args = flagSet.Args(); len(args) > 0 { - cmd = args[0] - } + if i != 0 { + fmt.Println("---") + } - switch cmd { - case "describe": - msg = describeUsage - case "get": - msg = getUsage - case "help", "": - msg = helpUsage - case "profile": - msg = profileUsage - case "run": - msg = runUsage - case "replay": - msg = replayUsage - case "version": - msg = versionUsage - default: - fmt.Printf("timecraft help %s: unknown command\n", cmd) - return ExitCode(1) - } + switch cmd { + case "describe": + msg = describeUsage + case "export": + msg = exportUsage + case "get": + msg = getUsage + case "help": + msg = helpUsage + case "profile": + msg = profileUsage + case "run": + msg = runUsage + case "replay": + msg = replayUsage + case "version": + msg = versionUsage + default: + fmt.Printf("timecraft help %s: unknown command\n", cmd) + return ExitCode(1) + } - fmt.Println(msg) + fmt.Println(msg) + } return nil } diff --git a/internal/cmd/profile.go b/internal/cmd/profile.go index 5c913c86..4d78d7da 100644 --- a/internal/cmd/profile.go +++ b/internal/cmd/profile.go @@ -5,22 +5,25 @@ import ( "errors" "fmt" "io" - "math" + "os" "time" pprof "github.com/google/pprof/profile" "github.com/google/uuid" - + "github.com/stealthrocket/timecraft/format" "github.com/stealthrocket/timecraft/internal/print/human" + "github.com/stealthrocket/timecraft/internal/print/jsonprint" + "github.com/stealthrocket/timecraft/internal/print/yamlprint" "github.com/stealthrocket/timecraft/internal/stream" "github.com/stealthrocket/timecraft/internal/timemachine" "github.com/stealthrocket/timecraft/internal/timemachine/wasicall" "github.com/stealthrocket/wasi-go/imports/wasi_snapshot_preview1" "github.com/stealthrocket/wazergo" "github.com/stealthrocket/wzprof" - "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/experimental" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" ) const profileUsage = ` @@ -36,54 +39,54 @@ Usage: timecraft profile [options] Example: - $ timecraft profile f6e9acbc-0543-47df-9413-b99f569cfa3b - writing cpu profile: cpu.out - writing memory profile: mem.out + $ timecraft profile --export memory:mem.out f6e9acbc-0543-47df-9413-b99f569cfa3b + ==> writing memory profile to mem.out + ... - $ go tool pprof -http :4040 cpu.out - (web page opens in browser) + $ go tool pprof -http :4040 cpu.out + (web page opens in browser) Options: - --cpuprofile path Path where the CPU profile will be written (default to cpu.out) - --duration duration Amount of time that the profiler will be running for (default to the process up time) + -d, --duration duration Amount of time that the profiler will be running for (default to the process up time) + --export type:path Exports the generated profiles, type is one of cpu or memory (may be repeated) -h, --help Show this usage information - --memprofile path Path where the memory profile will be written (default to mem.out) - --sample-rate ratio Ratio of function calls recorded by the profiler, expressed as a decimal number between 0 and 1 (default to 1) - --start-time time Time at which the profiler gets started (default to the process start time) + -o, --ouptut format Output format, one of: text, json, yaml + -t, --start-time time Time at which the profiler gets started (default to 1 minute) -r, --registry path Path to the timecraft registry (default to ~/.timecraft) ` func profile(ctx context.Context, args []string) error { var ( + exports = stringMap{} + output = outputFormat("text") startTime = human.Time{} - duration = human.Duration(0) - sampleRate = human.Rate(1.0) - cpuProfile = human.Path("cpu.out") - memProfile = human.Path("mem.out") + duration = human.Duration(1 * time.Minute) registryPath = human.Path("~/.timecraft") ) flagSet := newFlagSet("timecraft profile", profileUsage) - customVar(flagSet, &startTime, "start-time") - customVar(flagSet, &duration, "duration") - customVar(flagSet, &sampleRate, "sample-rate") - customVar(flagSet, &cpuProfile, "cpuprofile") - customVar(flagSet, &memProfile, "memprofile") + customVar(flagSet, &exports, "export") + customVar(flagSet, &output, "o", "output") + customVar(flagSet, &duration, "d", "duration") + customVar(flagSet, &startTime, "t", "start-time") customVar(flagSet, ®istryPath, "r", "registry") - parseFlags(flagSet, args) - - if time.Time(startTime).IsZero() { - startTime = human.Time(time.Unix(0, 0)) - } - if duration == 0 { - duration = human.Duration(math.MaxInt64) - } + args = parseFlags(flagSet, args) - args = flagSet.Args() if len(args) != 1 { return errors.New(`expected exactly one process id as argument`) } + exportedProfileTypes := maps.Keys(exports) + slices.Sort(exportedProfileTypes) + + for _, typ := range exportedProfileTypes { + switch typ { + case "cpu", "memory": + default: + return fmt.Errorf(`unsupported profile type: %s`, typ) + } + } + processID, err := uuid.Parse(args[0]) if err != nil { return errors.New(`malformed process id passed as argument (not a UUID)`) @@ -98,6 +101,14 @@ func profile(ctx context.Context, args []string) error { if err != nil { return err } + if startTime.IsZero() { + startTime = human.Time(manifest.StartTime) + } + timeRange := timemachine.TimeRange{ + Start: time.Time(startTime), + End: time.Time(startTime).Add(time.Duration(duration)), + } + process, err := registry.LookupProcess(ctx, manifest.Process.Digest) if err != nil { return err @@ -122,18 +133,12 @@ func profile(ctx context.Context, args []string) error { records := &recordProfiler{ records: timemachine.NewLogRecordReader(logReader), - startTime: time.Time(startTime), - endTime: time.Time(startTime).Add(time.Duration(duration)), - sampleRate: float64(sampleRate), + startTime: timeRange.Start, + endTime: timeRange.End, + sampleRate: 1.0, } - records.cpu = wzprof.NewCPUProfiler(wzprof.TimeFunc(records.now)) records.mem = wzprof.NewMemoryProfiler() - defer func() { - records.stop() - writeProfile("cpu", string(cpuProfile), records.cpuProfile) - writeProfile("memory", string(memProfile), records.memProfile) - }() ctx = context.WithValue(ctx, experimental.FunctionListenerFactoryKey{}, @@ -162,7 +167,47 @@ func profile(ctx context.Context, args []string) error { hostModuleInstance := wazergo.MustInstantiate(ctx, runtime, hostModule, wasi_snapshot_preview1.WithWASI(system)) ctx = wazergo.WithModuleInstance(ctx, hostModuleInstance) - return exec(ctx, runtime, compiledModule) + if err := exec(ctx, runtime, compiledModule); err != nil { + return err + } + + records.stop() + + desc, err := createProfiles(registry, processID, records.cpuProfile, records.memProfile) + if err != nil { + return err + } + + for _, typ := range exportedProfileTypes { + var p *pprof.Profile + switch typ { + case "cpu": + p = records.cpuProfile + case "memory": + p = records.memProfile + } + if p != nil { + path := exports[typ] + fmt.Fprintf(os.Stderr, "==> writing %s profile to %s\n", typ, path) + if err := wzprof.WriteProfile(path, p); err != nil { + return err + } + } + } + + var writer stream.WriteCloser[*format.Descriptor] + switch output { + case "json": + writer = jsonprint.NewWriter[*format.Descriptor](os.Stdout) + case "yaml": + writer = yamlprint.NewWriter[*format.Descriptor](os.Stdout) + default: + writer = getProfiles(ctx, os.Stdout, registry) + } + defer writer.Close() + + _, err = stream.Copy[*format.Descriptor](writer, stream.NewReader(desc...)) + return err } type recordProfiler struct { @@ -174,13 +219,15 @@ type recordProfiler struct { cpuProfile *pprof.Profile memProfile *pprof.Profile - currentTime int64 - startTime time.Time - endTime time.Time - started bool - stopped bool - sampleRate float64 - symbols wzprof.Symbolizer + firstTimestamp int64 + lastTimestamp int64 + + startTime time.Time + endTime time.Time + started bool + stopped bool + sampleRate float64 + symbols wzprof.Symbolizer } func (r *recordProfiler) Read(records []timemachine.Record) (int, error) { @@ -192,8 +239,9 @@ func (r *recordProfiler) Read(records []timemachine.Record) (int, error) { } n, err := r.records.Read(records[:1]) if n > 0 { - r.currentTime = records[0].Timestamp() + r.lastTimestamp = records[0].Timestamp() if !r.started && !records[0].Time().Before(r.startTime) { + r.firstTimestamp = r.lastTimestamp r.start() } if !r.stopped && !records[0].Time().Before(r.endTime) { @@ -204,7 +252,7 @@ func (r *recordProfiler) Read(records []timemachine.Record) (int, error) { } func (r *recordProfiler) now() int64 { - return r.currentTime + return r.lastTimestamp } func (r *recordProfiler) start() { @@ -219,22 +267,46 @@ func (r *recordProfiler) stop() { r.stopped = true r.cpuProfile = r.cpu.StopProfile(r.sampleRate, r.symbols) r.memProfile = r.mem.NewProfile(r.sampleRate, r.symbols) + r.cpuProfile.TimeNanos = r.startTime.UnixNano() + r.memProfile.TimeNanos = r.startTime.UnixNano() + duration := r.lastTimestamp - r.firstTimestamp + r.cpuProfile.DurationNanos = duration + r.memProfile.DurationNanos = duration } } -func writeProfile(profileName, path string, prof *pprof.Profile) { - if prof == nil { - return - } - - prof.Mapping = []*pprof.Mapping{{ +func createProfiles(reg *timemachine.Registry, processID format.UUID, profiles ...*pprof.Profile) ([]*format.Descriptor, error) { + mapping := []*pprof.Mapping{{ ID: 1, File: "module.wasm", }} - fmt.Printf("writing %s profile:\t%s\n", profileName, path) + ch := make(chan stream.Optional[*format.Descriptor]) + for _, p := range profiles { + p.Mapping = mapping + + profileType := "memory" + for _, sample := range p.SampleType { + if sample.Type == "cpu" || sample.Type == "samples" { + profileType = "cpu" + break + } + } + + go func(profile *pprof.Profile) { + ch <- stream.Opt(reg.CreateProfile(context.TODO(), processID, profileType, profile)) + }(p) + } - if err := wzprof.WriteProfile(path, prof); err != nil { - fmt.Printf("ERR: %s\n", err) + var descriptors = make([]*format.Descriptor, 0, len(profiles)) + var lastErr error + for range profiles { + d, err := (<-ch).Value() + if err != nil { + lastErr = err + } else { + descriptors = append(descriptors, d) + } } + return descriptors, lastErr } diff --git a/internal/cmd/replay.go b/internal/cmd/replay.go index 4476cd3b..edb1aec1 100644 --- a/internal/cmd/replay.go +++ b/internal/cmd/replay.go @@ -35,9 +35,8 @@ func replay(ctx context.Context, args []string) error { flagSet := newFlagSet("timecraft replay", replayUsage) customVar(flagSet, ®istryPath, "r", "registry") boolVar(flagSet, &trace, "T", "trace") - parseFlags(flagSet, args) + args = parseFlags(flagSet, args) - args = flagSet.Args() if len(args) != 1 { return errors.New(`expected exactly one process id as argument`) } diff --git a/internal/cmd/root.go b/internal/cmd/root.go index bcba8826..6741e3ef 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -31,6 +31,8 @@ import ( "github.com/stealthrocket/timecraft/internal/object" "github.com/stealthrocket/timecraft/internal/print/human" "github.com/stealthrocket/timecraft/internal/timemachine" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" ) // ExitCode is an error type returned from Root to indicate the exit code that @@ -66,7 +68,7 @@ func init() { // Root is the timecraft entrypoint. func Root(ctx context.Context, args ...string) int { flagSet := newFlagSet("timecraft", helpUsage) - parseFlags(flagSet, args) + _ = flagSet.Parse(args) if args = flagSet.Args(); len(args) == 0 { fmt.Println(rootUsage) @@ -78,6 +80,8 @@ func Root(ctx context.Context, args ...string) int { switch cmd { case "describe": err = describe(ctx, args) + case "export": + err = export(ctx, args) case "get": err = get(ctx, args) case "help": @@ -156,6 +160,31 @@ func (s *stringList) Set(value string) error { return nil } +type stringMap map[string]string + +func (m stringMap) String() string { + b := new(strings.Builder) + keys := maps.Keys(m) + slices.Sort(keys) + for i, k := range keys { + if i != 0 { + b.WriteByte(',') + } + b.WriteString(k) + b.WriteByte(':') + b.WriteString(m[k]) + } + return b.String() +} + +func (m stringMap) Set(value string) error { + k, v, _ := strings.Cut(value, ":") + k = strings.TrimSpace(k) + v = strings.TrimSpace(v) + m[k] = v + return nil +} + func createRegistry(path human.Path) (*timemachine.Registry, error) { p, err := path.Resolve() if err != nil { @@ -190,10 +219,29 @@ func newFlagSet(cmd, usage string) *flag.FlagSet { return flagSet } -func parseFlags(f *flag.FlagSet, args []string) { - // The flag set is consutrcted with ExitOnError, it should never error. - if err := f.Parse(args); err != nil { - panic(err) +// parseFlags is a greedy parser which consumes all options known to f and +// returns the remaining arguments. +func parseFlags(f *flag.FlagSet, args []string) []string { + var unknownArgs []string + for { + // The flag set is constructed with ExitOnError, it should never error. + if err := f.Parse(args); err != nil { + panic(err) + } + if args = f.Args(); len(args) == 0 { + return unknownArgs + } + i := slices.IndexFunc(args, func(s string) bool { + return strings.HasPrefix(s, "-") + }) + if i < 0 { + i = len(args) + } + if i == 0 { + panic("parsing command line arguments did not error on " + args[0]) + } + unknownArgs = append(unknownArgs, args[:i]...) + args = args[i:] } } diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 0cf1750f..3690bf9b 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -63,7 +63,7 @@ func run(ctx context.Context, args []string) error { boolVar(flagSet, &record, "R", "record") customVar(flagSet, &batchSize, "record-batch-size") customVar(flagSet, &compression, "record-compression") - parseFlags(flagSet, args) + _ = flagSet.Parse(args) envs = append(os.Environ(), envs...) args = flagSet.Args() @@ -229,7 +229,10 @@ func exec(ctx context.Context, runtime wazero.Runtime, compiledModule wazero.Com switch e := err.(type) { case *sys.ExitError: - return ExitCode(e.ExitCode()) + if exitCode := e.ExitCode(); exitCode != 0 { + return ExitCode(e.ExitCode()) + } + err = nil } return err diff --git a/internal/stream/chan.go b/internal/stream/chan.go index 9defdc40..6f5be2b6 100644 --- a/internal/stream/chan.go +++ b/internal/stream/chan.go @@ -11,7 +11,7 @@ func Opt[T any](val T, err error) Optional[T] { return Optional[T]{val: val, err: err} } -func (opt *Optional[T]) Value() (T, error) { +func (opt Optional[T]) Value() (T, error) { return opt.val, opt.err } diff --git a/internal/timemachine/hash.go b/internal/timemachine/hash.go index 2347190c..6c3041e3 100644 --- a/internal/timemachine/hash.go +++ b/internal/timemachine/hash.go @@ -1,23 +1,16 @@ package timemachine import ( - "fmt" - "io" + "encoding/binary" "github.com/stealthrocket/timecraft/format" ) -type Hash = format.Hash - -func SHA256(b []byte) Hash { return format.SHA256(b) } - -func UUIDv4(r io.Reader) Hash { - var uuid [16]byte - if _, err := io.ReadFull(r, uuid[:]); err != nil { - panic("readfull") - } - uuid[6] = (uuid[6] & 0x0f) | 0x40 // version 4 - uuid[8] = (uuid[8] & 0x3f) | 0x80 // variant 1 - s := fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", uuid[:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]) - return Hash{Algorithm: "uuidv4", Digest: s} +func HashProfile(processID format.UUID, profileType string, timeRange TimeRange) format.Hash { + b := make([]byte, 32, 64) + copy(b, processID[:]) + binary.LittleEndian.PutUint64(b[16:], uint64(timeRange.Start.UnixNano())) + binary.LittleEndian.PutUint64(b[24:], uint64(timeRange.End.UnixNano())) + b = append(b, profileType...) + return format.SHA256(b) } diff --git a/internal/timemachine/registry.go b/internal/timemachine/registry.go index e850549e..875650c0 100644 --- a/internal/timemachine/registry.go +++ b/internal/timemachine/registry.go @@ -15,6 +15,7 @@ import ( "sync" "time" + "github.com/google/pprof/profile" "github.com/google/uuid" "github.com/stealthrocket/timecraft/format" "github.com/stealthrocket/timecraft/internal/object" @@ -74,6 +75,51 @@ func (reg *Registry) CreateProcess(ctx context.Context, process *format.Process, return reg.createObject(ctx, process, tags) } +func (reg *Registry) CreateProfile(ctx context.Context, processID format.UUID, profileType string, prof *profile.Profile) (*format.Descriptor, error) { + buffer := new(bytes.Buffer) + if err := prof.Write(buffer); err != nil { + return nil, err + } + + timeRange := TimeRange{ + Start: time.Unix(0, prof.TimeNanos), + End: time.Unix(0, prof.TimeNanos+prof.DurationNanos), + } + + annotations := make(map[string]string, 2+len(reg.CreateTags)) + assignTags(annotations, reg.CreateTags) + assignTags(annotations, []object.Tag{{ + Name: "timecraft.object.mediatype", + Value: format.TypeTimecraftProfile.String(), + }, { + Name: "timecraft.process.id", + Value: processID.String(), + }, { + Name: "timecraft.profile.type", + Value: profileType, + }, { + Name: "timecraft.profile.start", + Value: timeRange.Start.Format(time.RFC3339Nano), + }, { + Name: "timecraft.profile.end", + Value: timeRange.End.Format(time.RFC3339Nano), + }}) + + tags := makeTags(annotations) + hash := HashProfile(processID, profileType, timeRange) + name := reg.objectKey(hash) + desc := &format.Descriptor{ + MediaType: format.TypeTimecraftProfile, + Digest: hash, + Size: int64(buffer.Len()), + Annotations: annotations, + } + if err := reg.Store.CreateObject(ctx, name, buffer, tags...); err != nil { + return nil, err + } + return desc, nil +} + func (reg *Registry) LookupModule(ctx context.Context, hash format.Hash) (*format.Module, error) { module := new(format.Module) return module, reg.lookupObject(ctx, hash, module) @@ -94,10 +140,23 @@ func (reg *Registry) LookupProcess(ctx context.Context, hash format.Hash) (*form return process, reg.lookupObject(ctx, hash, process) } +func (reg *Registry) LookupProfile(ctx context.Context, hash format.Hash) (*profile.Profile, error) { + r, err := reg.Store.ReadObject(ctx, reg.objectKey(hash)) + if err != nil { + return nil, err + } + defer r.Close() + return profile.Parse(r) +} + func (reg *Registry) LookupDescriptor(ctx context.Context, hash format.Hash) (*format.Descriptor, error) { return reg.lookupDescriptor(ctx, hash) } +func (reg *Registry) LookupResource(ctx context.Context, hash format.Hash) (io.ReadCloser, error) { + return reg.Store.ReadObject(ctx, reg.objectKey(hash)) +} + func (reg *Registry) ListModules(ctx context.Context, timeRange TimeRange, tags ...object.Tag) stream.ReadCloser[*format.Descriptor] { return reg.listObjects(ctx, format.TypeTimecraftModule, timeRange, tags) } @@ -151,13 +210,17 @@ func makeTags(annotations map[string]string) []object.Tag { Value: value, }) } + sortTags(tags) + return tags +} + +func sortTags(tags []object.Tag) { slices.SortFunc(tags, func(t1, t2 object.Tag) bool { return t1.Name < t2.Name }) - return tags } -func sha256Hash(data []byte, tags []object.Tag) format.Hash { +func hashTags(data []byte, tags []object.Tag) format.Hash { buf := object.AppendTags(make([]byte, 0, 256), tags...) sha := sha256.New() sha.Write(data) @@ -169,7 +232,7 @@ func sha256Hash(data []byte, tags []object.Tag) format.Hash { } func (reg *Registry) createObject(ctx context.Context, value format.ResourceMarshaler, extraTags []object.Tag) (*format.Descriptor, error) { - b, err := value.MarshalResource() + data, err := value.MarshalResource() if err != nil { return nil, err } @@ -179,17 +242,17 @@ func (reg *Registry) createObject(ctx context.Context, value format.ResourceMars assignTags(annotations, reg.CreateTags) assignTags(annotations, extraTags) assignTags(annotations, []object.Tag{{ - Name: "timecraft.object.media-type", + Name: "timecraft.object.mediatype", Value: mediaType.String(), }}) tags := makeTags(annotations) - hash := sha256Hash(b, tags) + hash := hashTags(data, tags) name := reg.objectKey(hash) desc := &format.Descriptor{ MediaType: mediaType, Digest: hash, - Size: int64(len(b)), + Size: int64(len(data)), Annotations: annotations, } @@ -201,7 +264,7 @@ func (reg *Registry) createObject(ctx context.Context, value format.ResourceMars return desc, nil } - if err := reg.Store.CreateObject(ctx, name, bytes.NewReader(b), tags...); err != nil { + if err := reg.Store.CreateObject(ctx, name, bytes.NewReader(data), tags...); err != nil { return nil, errorCreateObject(hash, value, err) } return desc, nil @@ -277,8 +340,8 @@ func newDescriptor(info object.Info) *format.Descriptor { annotations := make(map[string]string, len(info.Tags)) assignTags(annotations, info.Tags) - mediaType := annotations["timecraft.object.media-type"] - delete(annotations, "timecraft.object.media-type") + mediaType := annotations["timecraft.object.mediatype"] + delete(annotations, "timecraft.object.mediatype") return &format.Descriptor{ MediaType: format.MediaType(mediaType), @@ -294,7 +357,7 @@ func (reg *Registry) listObjects(ctx context.Context, mediaType format.MediaType } filters := []object.Filter{ - object.MATCH("timecraft.object.media-type", mediaType.String()), + object.MATCH("timecraft.object.mediatype", mediaType.String()), object.AFTER(timeRange.Start), object.BEFORE(timeRange.End), }