Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extracting filesystem accesses; 'workspace inspect' command #13

Draft
wants to merge 9 commits into
base: master
Choose a base branch
from
45 changes: 31 additions & 14 deletions cmd/warpforge/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"bytes"
"fmt"
"io/fs"
"os"
"os/exec"
"path/filepath"
Expand All @@ -13,10 +14,12 @@ import (
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/urfave/cli/v2"
"go.opentelemetry.io/otel"

"github.com/warpfork/warpforge/pkg/dab"
"github.com/warpfork/warpforge/pkg/logging"
"github.com/warpfork/warpforge/pkg/plotexec"
"github.com/warpfork/warpforge/wfapi"
"go.opentelemetry.io/otel"
)

const defaultCatalogUrl = "https://github.com/warpsys/mincatalog.git"
Expand Down Expand Up @@ -108,9 +111,10 @@ func cmdCatalogInit(c *cli.Context) error {
return fmt.Errorf("no catalog name provided")
}
catalogName := c.Args().First()
fsys := os.DirFS("/")

// open the workspace set and get the catalog path
wsSet, err := openWorkspaceSet()
wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand Down Expand Up @@ -145,8 +149,10 @@ func cmdCatalogAdd(c *cli.Context) error {
catalogRefStr := c.Args().Get(1)
url := c.Args().Get(2)

fsys := os.DirFS("/")

// open the workspace set
wsSet, err := openWorkspaceSet()
wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand Down Expand Up @@ -255,7 +261,9 @@ func cmdCatalogAdd(c *cli.Context) error {
}

func cmdCatalogLs(c *cli.Context) error {
wsSet, err := openWorkspaceSet()
fsys := os.DirFS("/")

wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand Down Expand Up @@ -311,7 +319,9 @@ func gatherCatalogRefs(plot wfapi.Plot) []wfapi.CatalogRef {
}

func cmdCatalogBundle(c *cli.Context) error {
wsSet, err := openWorkspaceSet()
fsys := os.DirFS("/")

wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand All @@ -320,8 +330,9 @@ func cmdCatalogBundle(c *cli.Context) error {
if err != nil {
return fmt.Errorf("failed to get pwd: %s", err)
}
pwd = pwd[1:] // Drop leading slash, for use with fs package.

plot, err := plotFromFile(filepath.Join(pwd, PLOT_FILE_NAME))
plot, err := dab.PlotFromFile(fsys, filepath.Join(pwd, dab.MagicFilename_Plot))
if err != nil {
return err
}
Expand All @@ -330,14 +341,14 @@ func cmdCatalogBundle(c *cli.Context) error {

catalogPath := filepath.Join(pwd, ".warpforge", "catalog")
// create a catalog if it does not exist
if _, err = os.Stat(catalogPath); os.IsNotExist(err) {
err = os.MkdirAll(catalogPath, 0755)
if _, err = fs.Stat(fsys, catalogPath); os.IsNotExist(err) {
err = os.MkdirAll("/"+catalogPath, 0755)
if err != nil {
return fmt.Errorf("failed to create catalog directory: %s", err)
}

// we need to reopen the workspace set after creating the directory
wsSet, err = openWorkspaceSet()
wsSet, err = openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand Down Expand Up @@ -397,7 +408,9 @@ func installDefaultRemoteCatalog(c *cli.Context, path string) error {
}

func cmdCatalogUpdate(c *cli.Context) error {
wss, err := openWorkspaceSet()
fsys := os.DirFS("/")

wss, err := openWorkspaceSet(fsys)
if err != nil {
return fmt.Errorf("failed to open workspace set: %s", err)
}
Expand Down Expand Up @@ -480,8 +493,10 @@ func cmdCatalogRelease(c *cli.Context) error {
}
catalogName := c.String("name")

fsys := os.DirFS("/")

// open the workspace set
wsSet, err := openWorkspaceSet()
wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand All @@ -499,15 +514,15 @@ func cmdCatalogRelease(c *cli.Context) error {
}

// get the module, release, and item values (in format `module:release:item`)
module, err := moduleFromFile("module.wf")
module, err := dab.ModuleFromFile(fsys, "module.wf")
if err != nil {
return err
}

releaseName := c.Args().Get(0)

fmt.Printf("building replay for module = %q, release = %q, executing plot...\n", module.Name, releaseName)
plot, err := plotFromFile(PLOT_FILE_NAME)
plot, err := dab.PlotFromFile(fsys, dab.MagicFilename_Plot)
if err != nil {
return err
}
Expand Down Expand Up @@ -561,6 +576,8 @@ func cmdIngestGitTags(c *cli.Context) error {
url := c.Args().Get(1)
itemName := c.Args().Get(2)

fsys := os.DirFS("/")

// open the remote and list all references
remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
Name: "origin",
Expand All @@ -573,7 +590,7 @@ func cmdIngestGitTags(c *cli.Context) error {

// open the workspace set and catalog
catalogName := c.String("name")
wsSet, err := openWorkspaceSet()
wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand Down
1 change: 1 addition & 0 deletions cmd/warpforge/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/json"
"github.com/urfave/cli/v2"

"github.com/warpfork/warpforge/pkg/plotexec"
"github.com/warpfork/warpforge/wfapi"
)
Expand Down
10 changes: 7 additions & 3 deletions cmd/warpforge/ferk.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@ import (
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/json"
"github.com/urfave/cli/v2"
"go.opentelemetry.io/otel"

"github.com/warpfork/warpforge/pkg/dab"
"github.com/warpfork/warpforge/pkg/logging"
"github.com/warpfork/warpforge/pkg/plotexec"
"github.com/warpfork/warpforge/wfapi"
"go.opentelemetry.io/otel"
)

var ferkCmdDef = cli.Command{
Expand Down Expand Up @@ -89,15 +91,17 @@ func cmdFerk(c *cli.Context) error {
ctx, span := tr.Start(ctx, c.Command.FullName())
defer span.End()

wss, err := openWorkspaceSet()
fsys := os.DirFS("/")

wss, err := openWorkspaceSet(fsys)
if err != nil {
return err
}

plot := wfapi.Plot{}
if c.String("plot") != "" {
// plot was provided, load from file
plot, err = plotFromFile(c.String("plot"))
plot, err = dab.PlotFromFile(fsys, c.String("plot"))
if err != nil {
return fmt.Errorf("error loading plot from file %q: %s", c.String("plot"), err)
}
Expand Down
7 changes: 7 additions & 0 deletions cmd/warpforge/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ func makeApp(stdin io.Reader, stdout, stderr io.Writer) *cli.App {
&statusCmdDef,
&quickstartCmdDef,
&ferkCmdDef,
&cli.Command{
Name: "workspace",
Usage: "Grouping for subcommands that inspect or affect a whole workspace.",
Subcommands: []*cli.Command{
&cmdDefWorkspaceInspect,
},
},
}
return app
}
Expand Down
1 change: 1 addition & 0 deletions cmd/warpforge/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
qt "github.com/frankban/quicktest"
"github.com/warpfork/go-testmark"
"github.com/warpfork/go-testmark/testexec"

"github.com/warpfork/warpforge/pkg/workspace"
)

Expand Down
20 changes: 11 additions & 9 deletions cmd/warpforge/quickstart.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/json"
"github.com/urfave/cli/v2"

"github.com/warpfork/warpforge/pkg/dab"
"github.com/warpfork/warpforge/wfapi"
)

Expand Down Expand Up @@ -59,13 +61,13 @@ func cmdQuickstart(c *cli.Context) error {
return fmt.Errorf("no module name provided")
}

_, err := os.Stat(MODULE_FILE_NAME)
_, err := os.Stat(dab.MagicFilename_Module)
if !os.IsNotExist(err) {
return fmt.Errorf("%s file already exists", MODULE_FILE_NAME)
return fmt.Errorf("%s file already exists", dab.MagicFilename_Module)
}
_, err = os.Stat(PLOT_FILE_NAME)
_, err = os.Stat(dab.MagicFilename_Plot)
if !os.IsNotExist(err) {
return fmt.Errorf("%s file already exists", PLOT_FILE_NAME)
return fmt.Errorf("%s file already exists", dab.MagicFilename_Plot)
}

moduleName := c.Args().First()
Expand All @@ -79,7 +81,7 @@ func cmdQuickstart(c *cli.Context) error {
if err != nil {
return fmt.Errorf("failed to serialize module")
}
err = os.WriteFile(MODULE_FILE_NAME, moduleSerial, 0644)
err = os.WriteFile(dab.MagicFilename_Module, moduleSerial, 0644)
if err != nil {
return fmt.Errorf("failed to write module.json file: %s", err)
}
Expand All @@ -94,17 +96,17 @@ func cmdQuickstart(c *cli.Context) error {
return fmt.Errorf("failed to serialize plot")
}

err = os.WriteFile(PLOT_FILE_NAME, plotSerial, 0644)
err = os.WriteFile(dab.MagicFilename_Plot, plotSerial, 0644)
if err != nil {
return fmt.Errorf("failed to write %s: %s", PLOT_FILE_NAME, err)
return fmt.Errorf("failed to write %s: %s", dab.MagicFilename_Plot, err)
}

if !c.Bool("quiet") {
fmt.Fprintf(c.App.Writer, "Successfully created %s and %s for module %q.\n", MODULE_FILE_NAME, PLOT_FILE_NAME, moduleName)
fmt.Fprintf(c.App.Writer, "Successfully created %s and %s for module %q.\n", dab.MagicFilename_Module, dab.MagicFilename_Plot, moduleName)
fmt.Fprintf(c.App.Writer, "Ensure your catalogs are up to date by running `%s catalog update.`.\n", os.Args[0])
fmt.Fprintf(c.App.Writer, "You can check status of this module with `%s status`.\n", os.Args[0])
fmt.Fprintf(c.App.Writer, "You can run this module with `%s run`.\n", os.Args[0])
fmt.Fprintf(c.App.Writer, "Once you've run the Hello World example, edit the 'script' section of %s to customize what happens.\n", PLOT_FILE_NAME)
fmt.Fprintf(c.App.Writer, "Once you've run the Hello World example, edit the 'script' section of %s to customize what happens.\n", dab.MagicFilename_Plot)
}

return nil
Expand Down
29 changes: 17 additions & 12 deletions cmd/warpforge/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,21 @@ package main
import (
"context"
"fmt"
"io/ioutil"
"io/fs"
"os"
"path/filepath"

"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/json"
"github.com/urfave/cli/v2"
"go.opentelemetry.io/otel"

"github.com/warpfork/warpforge/pkg/dab"
"github.com/warpfork/warpforge/pkg/formulaexec"
"github.com/warpfork/warpforge/pkg/logging"
"github.com/warpfork/warpforge/pkg/plotexec"
"github.com/warpfork/warpforge/pkg/workspace"
"github.com/warpfork/warpforge/wfapi"
"go.opentelemetry.io/otel"
)

var runCmdDef = cli.Command{
Expand All @@ -36,16 +38,16 @@ var runCmdDef = cli.Command{
},
}

func execModule(ctx context.Context, config wfapi.PlotExecConfig, fileName string) (wfapi.PlotResults, error) {
func execModule(ctx context.Context, fsys fs.FS, config wfapi.PlotExecConfig, fileName string) (wfapi.PlotResults, error) {
result := wfapi.PlotResults{}

// parse the module, even though it is not currently used
_, err := moduleFromFile(fileName)
_, err := dab.ModuleFromFile(fsys, fileName)
if err != nil {
return result, err
}

plot, err := plotFromFile(filepath.Join(filepath.Dir(fileName), PLOT_FILE_NAME))
plot, err := dab.PlotFromFile(fsys, filepath.Join(filepath.Dir(fileName), dab.MagicFilename_Plot))
if err != nil {
return result, err
}
Expand All @@ -55,7 +57,7 @@ func execModule(ctx context.Context, config wfapi.PlotExecConfig, fileName strin
return result, err
}

wss, err := openWorkspaceSet()
wss, err := openWorkspaceSet(fsys)
if err != nil {
return result, err
}
Expand Down Expand Up @@ -97,13 +99,16 @@ func cmdRun(c *cli.Context) error {
},
}

fsys := os.DirFS("/")

if !c.Args().Present() {
// execute the module in the current directory
pwd, err := os.Getwd()
if err != nil {
return fmt.Errorf("could not get current directory")
}
_, err = execModule(ctx, config, filepath.Join(pwd, MODULE_FILE_NAME))
pwd = pwd[1:] // Drop leading slash, for use with fs package.
_, err = execModule(ctx, fsys, config, filepath.Join(pwd, dab.MagicFilename_Module))
if err != nil {
return err
}
Expand All @@ -114,11 +119,11 @@ func cmdRun(c *cli.Context) error {
if err != nil {
return err
}
if filepath.Base(path) == MODULE_FILE_NAME {
if filepath.Base(path) == dab.MagicFilename_Module {
if c.Bool("verbose") {
logger.Debug("executing %q", path)
}
_, err = execModule(ctx, config, path)
_, err = execModule(ctx, fsys, config, path)
if err != nil {
return err
}
Expand All @@ -134,13 +139,13 @@ func cmdRun(c *cli.Context) error {
}
if info.IsDir() {
// directory provided, execute module if it exists
_, err := execModule(ctx, config, filepath.Join(fileName, "module.wf"))
_, err := execModule(ctx, fsys, config, filepath.Join(fileName, "module.wf"))
if err != nil {
return err
}
} else {
// formula or module file provided
f, err := ioutil.ReadFile(fileName)
f, err := fs.ReadFile(fsys, fileName)
if err != nil {
return err
}
Expand Down Expand Up @@ -169,7 +174,7 @@ func cmdRun(c *cli.Context) error {
return err
}
case "module":
_, err := execModule(ctx, config, fileName)
_, err := execModule(ctx, fsys, config, fileName)
if err != nil {
return err
}
Expand Down
Loading