Increase limit for file upload

.vscode/tasks.json

@@ -4,12 +4,12 @@
  {
  "type": "shell",
  "label": "prepare-debug",
- "command": "make -C packages/devbookd run-debug",
+ "command": "make -C packages/envd run-debug",
  },
  {
  "type": "shell",
  "label": "stop-debug-docker",
- "command": "make -C packages/devbookd stop-debug-docker",
+ "command": "make -C packages/envd stop-debug-docker",
  },
  ]
 }

packages/envd/Makefile

@@ -45,6 +45,7 @@ start-debug-docker:
  -p 49982:49982 \
  -p 2345:2345 \
  -p 9999:9999 \
+ --rm \
  -d envd-debug sh -l -c "/go/bin/dlv --listen=:2345 --headless=true --log=true --log-output=debugger,debuglineerr,gdbwire,lldbout,rpc --accept-multiclient --api-version=2 exec /usr/bin/envd"
 
 stop-debug-docker:

packages/envd/debug.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM golang:1.21
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -9,6 +9,8 @@ RUN update-ca-certificates
 
 RUN useradd -ms /bin/bash user
 
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
 WORKDIR /
 
 RUN mkdir -p /etc/systemd/system/multi-user.target.wants

packages/envd/internal/file/upload.go

@@ -1,57 +1,130 @@
 package file
 
 import (
+ "bytes"
  "fmt"
  "io"
  "net/http"
  "os"
  "path"
+ "syscall"
 
  "github.com/e2b-dev/infra/packages/envd/internal/user"
 
  "go.uber.org/zap"
 )
 
-const maxFileInMemory = 512 * 1024 * 1024 // 512MB
+func getFreeDiskSpace(path string) (uint64, error) {
+ var stat syscall.Statfs_t
+
+ err := syscall.Statfs(path, &stat)
+ if err != nil {
+ return 0, fmt.Errorf("error getting free disk space: %w", err)
+ }
+
+ // Available blocks * size per block = available space in bytes
+ freeSpace := stat.Bavail * uint64(stat.Bsize)
+
+ return freeSpace, nil
+}
 
 func Upload(logger *zap.SugaredLogger, w http.ResponseWriter, r *http.Request) {
  logger.Debug(
  "Starting file upload",
  )
 
- if err := r.ParseMultipartForm(maxFileInMemory); err != nil {
+ f, err := r.MultipartReader()
+ if err != nil {
  logger.Error("Error parsing multipart form:", err)
- http.Error(w, fmt.Sprintf("The uploaded file is too big. Please choose an file that's less than 100MB in size: %s", err.Error()), http.StatusBadRequest)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+
+ return
+ }
+ defer r.Body.Close()
+
+ tmpFreeSpace, err := getFreeDiskSpace(os.TempDir())
+ if err != nil {
+ logger.Error("Error getting free disk space:", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
 
  return
  }
 
- logger.Debug("Multipart form parsed successfully")
+ if tmpFreeSpace < uint64(r.ContentLength) {
+ logger.Error("Not enough free disk space")
+ http.Error(w, "Not enough free disk space", http.StatusInternalServerError)
+
+ return
+ }
 
- // The argument to FormFile must match the name attribute
- // of the file input on the frontend
- file, fileHeader, err := r.FormFile("file")
+ tmpFile, err := os.CreateTemp(os.TempDir(), "envd-upload")
  if err != nil {
- logger.Error("Error retrieving the file from form-data:", err)
- http.Error(w, err.Error(), http.StatusBadRequest)
+ logger.Error("Error creating temp file:", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
 
  return
  }
+ defer os.Remove(tmpFile.Name())
+ defer tmpFile.Close()
+
+ var filepath string
+
+ var filename string
 
- logger.Debug("File retrieved successfully")
+ for {
+ // Get the next part.
+ part, partErr := f.NextPart()
 
- defer func() {
- closeErr := file.Close()
- if closeErr != nil {
- logger.Error("Error closing file:", closeErr)
+ logger.Debugw("Part", "part", partErr)
+
+ if partErr == io.EOF {
+ // We're done reading the parts.
+ break
+ } else if partErr != nil {
+ logger.Error("Error reading form:", partErr)
+ http.Error(w, partErr.Error(), http.StatusInternalServerError)
+
+ return
+ }
+
+ // Get the key of the part.
+ key := part.FormName()
+ logger.Debugw("Part", "part key", key)
+
+ if key == "file" {
+ filename = part.FileName()
+
+ _, readErr := tmpFile.ReadFrom(part)
+ if readErr != nil {
+ part.Close()
+ logger.Error("Error reading file:", readErr)
+ http.Error(w, readErr.Error(), http.StatusInternalServerError)
+
+ return
+ }
+ } else if key == "path" {
+ buf := new(bytes.Buffer)
+ _, err = buf.ReadFrom(part)
+ if err != nil {
+ part.Close()
+ logger.Error("Error reading file path:", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+
+ return
+ }
+ filepath = buf.String()
+ } else {
+ fmt.Printf("key not found: %s", key)
  }
- }()
 
- filepath := r.Form.Get("path")
+ part.Close()
+ }
 
  var newFilePath string
 
- if filepath == "" {
+ if filepath != "" {
+ newFilePath = filepath
+ } else if filename != "" {
  // Create a new file in the user's homedir if no path in the form is specified
  _, _, homedir, _, userErr := user.GetUser(user.DefaultUser)
  if userErr != nil {
@@ -60,51 +133,31 @@ func Upload(logger *zap.SugaredLogger, w http.ResponseWriter, r *http.Request) {
 
  return
  }
-
- newFilePath = path.Join(homedir, fileHeader.Filename)
+ newFilePath = path.Join(homedir, filename)
  } else {
- newFilePath = filepath
+ logger.Error("No file or path provided")
+ http.Error(w, "No file or path provided", http.StatusBadRequest)
+
+ return
  }
 
- logger.Debugw(
- "Starting file upload",
- "path", newFilePath,
- )
+ logger.Debugw("New file path", "path", newFilePath, "filepath", filepath)
 
- dst, err := os.Create(newFilePath)
+ err = os.Rename(tmpFile.Name(), newFilePath)
  if err != nil {
- logger.Error("Error creating the file:", err)
+ logger.Error("Error renaming file:", err)
  http.Error(w, err.Error(), http.StatusInternalServerError)
 
  return
  }
 
- logger.Debugw("File created successfully",
- "path", newFilePath,
- )
-
- logger.Debugw("File created successfully")
-
- defer func() {
- closeErr := dst.Close()
- if closeErr != nil {
- logger.Error("Error closing file:", closeErr)
- }
- }()
-
- pr := &Progress{
- TotalSize: fileHeader.Size,
- }
-
- // Copy the uploaded file to the filesystem
- // at the specified destination
- _, err = io.Copy(dst, io.TeeReader(file, pr))
+ err = os.Chmod(newFilePath, 0o666)
  if err != nil {
- logger.Error("Error saving file to filesystem:", err)
+ logger.Error("Error setting file permissions:", err)
  http.Error(w, err.Error(), http.StatusInternalServerError)
 
  return
  }
 
- logger.Info("Upload complete ", "path", newFilePath)
+ logger.Infow("Upload complete", "path", newFilePath)
 }

packages/envd/main.go

@@ -31,6 +31,8 @@ const (
  Version = "dev"
 
  startCmdID = "_startCmd"
+
+ serverTimeout = 1 * time.Hour
 )
 
 var (
@@ -67,6 +69,19 @@ func pingHandler(w http.ResponseWriter, r *http.Request) {
  }
 }
 
+func createFileHandler(logger *zap.SugaredLogger) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case http.MethodGet:
+ file.Download(logger, w, r)
+ case http.MethodPost:
+ file.Upload(logger, w, r)
+ default:
+ http.Error(w, "Invalid request method", http.StatusMethodNotAllowed)
+ }
+ }
+}
+
 func fileHandler(w http.ResponseWriter, r *http.Request) {
  switch r.Method {
  case http.MethodGet:
@@ -200,11 +215,11 @@ func main() {
  // Register the profiling handlers that were added in default mux with the `net/http/pprof` import.
  router.PathPrefix("/debug/pprof").Handler(http.DefaultServeMux)
  // The /file route used for downloading and uploading files via SDK.
- router.HandleFunc("/file", fileHandler)
+ router.HandleFunc("/file", createFileHandler(logger.Named("file")))
 
  server := &http.Server{
- ReadTimeout: 300 * time.Second,
- WriteTimeout: 300 * time.Second,
+ ReadTimeout: serverTimeout,
+ WriteTimeout: serverTimeout,
  Addr: fmt.Sprintf("0.0.0.0:%d", serverPort),
  Handler: handlers.CORS(handlers.AllowedMethods([]string{"GET", "POST", "PUT"}), handlers.AllowedOrigins([]string{"*"}))(router),
  }

packages/nomad/client-proxy.hcl

@@ -140,7 +140,9 @@ server {
  proxy_cache_bypass 1;
  proxy_no_cache 1;
 
- client_max_body_size 1024M;
+ client_max_body_size 0;
+ proxy_buffering off;
+ proxy_request_buffering off;
 
  location / {
  proxy_pass $scheme://[[ .Address ]]:[[ .Port ]]$request_uri;

packages/nomad/session-proxy.hcl

@@ -125,7 +125,9 @@ server {
  # See https://stackoverflow.com/questions/29980884/proxy-pass-does-not-resolve-dns-using-etc-hosts
  resolver 127.0.0.53;
 
- client_max_body_size 1024M;
+ client_max_body_size 0;
+ proxy_buffering off;
+ proxy_request_buffering off;
 
  proxy_set_header Host $host;
  proxy_set_header X-Real-IP $remote_addr;

packages/template-manager/internal/build/provision.sh

@@ -40,6 +40,8 @@ LimitCORE=infinity
 ExecStart=/bin/bash -l -c "/usr/bin/envd"
 OOMPolicy=continue
 OOMScoreAdjust=-1000
+Environment="GOMEMLIMIT={{ .MemoryLimit }}MiB"
+
 
 ExecStartPre=/bin/bash -c 'echo 0 > /proc/sys/vm/swappiness && swapon /swap/swapfile'