Skip to content

Commit

Permalink
Merge pull request #82 from warptools/normalize-regen
Browse files Browse the repository at this point in the history
Normalize regen
  • Loading branch information
TripleDogDare committed Sep 13, 2023
2 parents 9d44a58 + 91ca5d1 commit 8751fe7
Show file tree
Hide file tree
Showing 5 changed files with 119 additions and 18 deletions.
72 changes: 72 additions & 0 deletions app/testutil/guidmapper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package testutil

import (
"log"
"math/rand"
"regexp"
"strings"
"sync"

"github.com/google/uuid"
"github.com/icholy/replace"
)

var reGuidJson = regexp.MustCompile(`"guid": "(?P<GUID>[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"`)

type GuidMapper struct {
mapping map[string]string
randSource rand.Source
mappingMutex sync.Mutex
}

func NewGuidMapper(r rand.Source) *GuidMapper {
// Create a new source with a constant seed
if r == nil {
r = rand.NewSource(424242)
}
return &GuidMapper{
mapping: make(map[string]string),
randSource: r,
}
}

// Transformer returns a transform.Transformer for mapping GUIDs on streams
func (gm *GuidMapper) Transformer() *replace.RegexpTransformer {
return replace.RegexpStringSubmatchFunc(reGuidJson, gm.replacer)
}

// get will return the mapping for the requested guid.
// If the guid is unmapped, a new guid will be created.
// GuidMapper uses a known random source so mapped guids should be deterministic
// by order of requested guids.
func (gm *GuidMapper) get(guid string) string {
gm.mappingMutex.Lock()
defer gm.mappingMutex.Unlock()
if newGuid, exists := gm.mapping[guid]; exists {
return newGuid
}

// If the GUID is not in the mapping, generate a new one
// Uses rand with a deterministic source so that a deterministic order is preserved.
r := rand.New(gm.randSource)
randBuf := make([]byte, 16)
r.Read(randBuf)
newGuid := uuid.NewSHA1(uuid.Nil, randBuf).String()
gm.mapping[guid] = newGuid
return newGuid
}

// replacer implements the function for replace.RegexpStringSubmatchFunc.
func (gm *GuidMapper) replacer(match []string) string {
result := match[0]
log.Println(match)
for _, guid := range match[1:] {
if _, err := uuid.Parse(guid); err != nil {
continue
}
newGuid := gm.get(guid)
result = strings.ReplaceAll(result, guid, newGuid)
}
log.Println(result)
return result
}
41 changes: 31 additions & 10 deletions app/testutil/testutil.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,19 @@ import (
"bytes"
"errors"
"io"
"math/rand"
"os"
"path/filepath"
"regexp"
"strings"
"testing"

qt "github.com/frankban/quicktest"
"github.com/icholy/replace"
"github.com/serum-errors/go-serum"
"github.com/warpfork/go-testmark"
"github.com/warpfork/go-testmark/testexec"
"golang.org/x/text/transform"

wfapp "github.com/warptools/warpforge/app"
"github.com/warptools/warpforge/pkg/testutil"
Expand Down Expand Up @@ -132,16 +135,6 @@ func cleanOutput(str string) string {
// Also, it uses `os.Chdir` on this process (because we're "emulating a shell" rather than making subprocesses, whee).
func buildExecFn(t *testing.T, projPath string) func([]string, io.Reader, io.Writer, io.Writer) (int, error) {
return func(args []string, stdin io.Reader, stdout, stderr io.Writer) (int, error) {
bufout, buferr := &bytes.Buffer{}, &bytes.Buffer{}
var testout io.Writer = bufout
if stdout != nil {
testout = io.MultiWriter(stdout, bufout)
}
var testerr io.Writer = bufout
if stderr != nil {
testerr = io.MultiWriter(stderr, buferr)
}

// override the path to required binaries
pluginPath := filepath.Join(projPath, "plugins")
err := os.Setenv("WARPFORGE_PATH", pluginPath)
Expand Down Expand Up @@ -171,10 +164,38 @@ func buildExecFn(t *testing.T, projPath string) func([]string, io.Reader, io.Wri
// return nil
// })

bufout, buferr := &bytes.Buffer{}, &bytes.Buffer{}
var testout io.Writer = bufout

if stdout != nil {
testout = io.MultiWriter(stdout, bufout)
}
// transform.NewWriter must call Close() to flush all output
testout = transform.NewWriter(testout,
transform.Chain(
NewGuidMapper(rand.NewSource(65)).Transformer(),
replace.RegexpString(regexp.MustCompile(`"time": [0-9]+`), `"time": 169455678`),
),
)

var testerr io.Writer = bufout
if stderr != nil {
testerr = io.MultiWriter(stderr, buferr)
}
// transform.NewWriter must call Close() to flush all output
testerr = transform.NewWriter(testerr,
transform.Chain(
NewGuidMapper(rand.NewSource(66)).Transformer(),
replace.RegexpString(regexp.MustCompile(`"time": [0-9]+`), `"time": 169455699`),
),
)

wfapp.App.Reader = stdin
wfapp.App.Writer = testout
wfapp.App.ErrWriter = testerr
err = wfapp.App.Run(args)
testout.(io.Closer).Close()
testerr.(io.Closer).Close()

exitCode := 0
if err != nil {
Expand Down
14 changes: 7 additions & 7 deletions examples/500-cli/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ The result of this will be a `RunRecord` object printed to stdout:

[testmark]:# (runformula/tags=net/stdout)
```
{ "runrecord": { "guid": "857b9774-e251-4bbc-9268-1883919fab23", "time": 1674646900, "formulaID": "zM5K3Zz8R3ioVVWZ6o6GocxPKvubAJfv4iQmDH3GCq9UjtDjHtRWrry4DRoEBPvfUEYFx1D", "exitcode": 0, "results": {} } }
{ "runrecord": { "guid": "4a1d0896-f161-5fe8-87e5-d8fbb6d87368", "time": 169455678, "formulaID": "zM5K3Zz8R3ioVVWZ6o6GocxPKvubAJfv4iQmDH3GCq9UjtDjHtRWrry4DRoEBPvfUEYFx1D", "exitcode": 0, "results": {} } }
```

(Note that we've normalized some of the values in this object for testing purposes.
Expand Down Expand Up @@ -358,10 +358,10 @@ there's only one record in this map.

[testmark]:# (base-workspace/then-runmodule/stdout)
```
{ "runrecord": { "guid": "c7fedecf-79b7-434b-9521-bf79a210cb2d", "time": 1692910078, "formulaID": "zM5K3RvfmKy9zLfHk1T6kPafmvzAGt9Ls1QYFS4BvWTaCBgxYoJLDkkqP7SD7QWuoRTYw3j", "exitcode": 0, "results": { "test": "ware:tar:2En3zD1ho1qNeLpPryZVM1UTGnqPvnt48WY36TzCGJwSCudxPXkDtN3UuS4J3AYWAM" } } }
{ "runrecord": { "guid": "25f4a58b-2d45-4c01-b3ca-3d15165535a4", "time": 1692910079, "formulaID": "zM5K3Rqj146W38bBjgU8yeJ4i37YtydoZGvpsqaHbNE2akLWfDYp8vi2KAh7vvU3XdUoy12", "exitcode": 0, "results": { "test": "ware:tar:4tvpCNb1XJ3gkH25MREMPBHRWa7gLUiYt7pF6AHNbqgwBrs3btvvmijebyZrYsi6Y9" } } }
{ "runrecord": { "guid": "4a1d0896-f161-5fe8-87e5-d8fbb6d87368", "time": 169455678, "formulaID": "zM5K3RvfmKy9zLfHk1T6kPafmvzAGt9Ls1QYFS4BvWTaCBgxYoJLDkkqP7SD7QWuoRTYw3j", "exitcode": 0, "results": { "test": "ware:tar:2En3zD1ho1qNeLpPryZVM1UTGnqPvnt48WY36TzCGJwSCudxPXkDtN3UuS4J3AYWAM" } } }
{ "runrecord": { "guid": "423d9dfb-ea0b-51eb-8cbd-7bf6f198e25b", "time": 169455678, "formulaID": "zM5K3Rqj146W38bBjgU8yeJ4i37YtydoZGvpsqaHbNE2akLWfDYp8vi2KAh7vvU3XdUoy12", "exitcode": 0, "results": { "test": "ware:tar:4tvpCNb1XJ3gkH25MREMPBHRWa7gLUiYt7pF6AHNbqgwBrs3btvvmijebyZrYsi6Y9" } } }
{ "plotresults": { "test": "tar:4tvpCNb1XJ3gkH25MREMPBHRWa7gLUiYt7pF6AHNbqgwBrs3btvvmijebyZrYsi6Y9" } }
{ "runrecord": { "guid": "df7923fd-1ff6-4600-8984-f896d79610fd", "time": 1692910079, "formulaID": "zM5K3T8946y1A7Z4ZEuoCizPdDuneUQMqXqyfxXSh93CtK3n6gzgJgz9PMTUzJiexPErUqM", "exitcode": 0, "results": {} } }
{ "runrecord": { "guid": "24658cf3-ee69-588f-a86a-40d1ef2b9a34", "time": 169455678, "formulaID": "zM5K3T8946y1A7Z4ZEuoCizPdDuneUQMqXqyfxXSh93CtK3n6gzgJgz9PMTUzJiexPErUqM", "exitcode": 0, "results": {} } }
{ "plotresults": { "test": "tar:4tvpCNb1XJ3gkH25MREMPBHRWa7gLUiYt7pF6AHNbqgwBrs3btvvmijebyZrYsi6Y9" } }
```

Expand Down Expand Up @@ -627,7 +627,7 @@ Check that `ferk` ran successfully, no outputs are expected.

[testmark]:# (base-workspace/then-ferk/stdout)
```
{ "runrecord": { "guid": "61940b67-024a-476e-996e-740ff80356c7", "time": 1692910079, "formulaID": "zM5K3V1fXVjExjfVd8d7ByUQ7HP16QAcZcoRd1bh3X4uvms1Xbpb87c1a7WNaw8Hw2B3uF6", "exitcode": 0, "results": { "out": "ware:tar:-" } } }
{ "runrecord": { "guid": "4a1d0896-f161-5fe8-87e5-d8fbb6d87368", "time": 169455678, "formulaID": "zM5K3V1fXVjExjfVd8d7ByUQ7HP16QAcZcoRd1bh3X4uvms1Xbpb87c1a7WNaw8Hw2B3uF6", "exitcode": 0, "results": { "out": "ware:tar:-" } } }
{ "plotresults": { "out": "tar:-" } }
```

Expand Down Expand Up @@ -671,7 +671,7 @@ warpforge --json --quiet ferk --plot ./plot.wf --cmd /bin/echo --no-interactive

[testmark]:# (base-workspace/then-ferk-with-plot/stdout)
```
{ "runrecord": { "guid": "b964aa2f-bb7a-44c4-b4fa-bfc9d15d0ace", "time": 1692910079, "formulaID": "zM5K3YWRYqSgvxgMkAA9KbzPpqtRPbufF2z397SNJ1mKTkp9SpmxA8jD3YmTPu3EWvijMSv", "exitcode": 0, "results": {} } }
{ "runrecord": { "guid": "4a1d0896-f161-5fe8-87e5-d8fbb6d87368", "time": 169455678, "formulaID": "zM5K3YWRYqSgvxgMkAA9KbzPpqtRPbufF2z397SNJ1mKTkp9SpmxA8jD3YmTPu3EWvijMSv", "exitcode": 0, "results": {} } }
{ "plotresults": {} }
```

Expand Down Expand Up @@ -710,7 +710,7 @@ warpforge --json run
{ "log": { "Msg": "ware mount: wareId = tar:4z9DCTxoKkStqXQRwtf9nimpfQQ36dbndDsAPCQgECfbXt3edanUrsVKCjE9TkX2v9 destPath = /" } }
{ "log": { "Msg": "executing script interpreter = /bin/sh" } }
{ "log": { "Msg": "packed \"out\": path = /output wareId=tar:6U2WhgnXRCLsNjZLyvLzG6Eer5MH4MpguDeimPrEafHytjmXjbvxjm1STCuqHV5AQA" } }
{ "runrecord": { "guid": "6785b641-6e41-4ecb-9207-4ecd60b85bd6", "time": 1692910079, "formulaID": "zM5K3ZMzLiBwQB93yZ4nFUsVSSgVtNPjpY72hKHxDjc9FRk9KnJSoCvkHFEPWfxARdjaguZ", "exitcode": 0, "results": { "out": "ware:tar:6U2WhgnXRCLsNjZLyvLzG6Eer5MH4MpguDeimPrEafHytjmXjbvxjm1STCuqHV5AQA" } } }
{ "runrecord": { "guid": "4a1d0896-f161-5fe8-87e5-d8fbb6d87368", "time": 169455678, "formulaID": "zM5K3ZMzLiBwQB93yZ4nFUsVSSgVtNPjpY72hKHxDjc9FRk9KnJSoCvkHFEPWfxARdjaguZ", "exitcode": 0, "results": { "out": "ware:tar:6U2WhgnXRCLsNjZLyvLzG6Eer5MH4MpguDeimPrEafHytjmXjbvxjm1STCuqHV5AQA" } } }
{ "log": { "Msg": "(hello-world) collected output hello-world:out" } }
{ "log": { "Msg": "(hello-world) complete" } }
{ "plotresults": { "output": "tar:6U2WhgnXRCLsNjZLyvLzG6Eer5MH4MpguDeimPrEafHytjmXjbvxjm1STCuqHV5AQA" } }
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ require (
github.com/frankban/quicktest v1.14.5
github.com/go-git/go-git/v5 v5.6.1
github.com/google/uuid v1.3.0
github.com/icholy/replace v0.6.0
github.com/ipfs/go-cid v0.4.1
github.com/ipld/go-ipld-prime v0.20.0
github.com/opencontainers/runtime-spec v1.0.2
Expand Down Expand Up @@ -96,7 +97,7 @@ require (
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/term v0.8.0
golang.org/x/text v0.9.0 // indirect
golang.org/x/text v0.9.0
golang.org/x/tools v0.8.0 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
google.golang.org/grpc v1.54.0 // indirect
Expand Down
7 changes: 7 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAsz
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/icholy/replace v0.6.0 h1:EBiD2pGqZIOJAbEaf/5GVRaD/Pmbb4n+K3LrBdXd4dw=
github.com/icholy/replace v0.6.0/go.mod h1:zzi8pxElj2t/5wHHHYmH45D+KxytX/t4w3ClY5nlK+g=
github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
Expand Down Expand Up @@ -291,6 +293,7 @@ github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand Down Expand Up @@ -325,6 +328,7 @@ github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
Expand Down Expand Up @@ -560,6 +564,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
Expand Down Expand Up @@ -705,6 +710,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
Expand Down

0 comments on commit 8751fe7

Please sign in to comment.