Added Terraform lock file generation with provider caching (#3108)

* feat: generating .terraform.lock.hcl

* chore: code improvements

* chore: test improvements

* fix: unit test

* chore: UpdateLockfile unit test

* chore: update TestTerragruntProviderCache

* fix: unit test

* chore: test data

* fix: test

* fix: linter tip

* fix: TestTerragruntProviderCache test

* chore: update docs

* chore: update comment

* chore: comments update

* chore: comment update

* chore: comment update

* chore: update docs

.circleci/config.yml

@@ -161,6 +161,11 @@ jobs:
  export GOPATH=~/go/bin && export PATH=$PATH:$GOPATH
  pre-commit install
  pre-commit run --all-files
+ - run:
+ name: generate mocks
+ command: |
+ go install github.com/vektra/mockery/[email protected]
+ go generate ./...
  - run:
  name: run lint
  command: |

.gitignore

@@ -13,3 +13,4 @@ vendor
 .terraform.lock.hcl
 terragrunt
 .DS_Store
+mocks/

cli/commands/flags.go

@@ -50,20 +50,18 @@ const (
  TerragruntJsonOutDirFlagName = "terragrunt-json-out-dir"
 
  // Terragrunt Provider Cache flags/envs
- TerragruntProviderCacheFlagName = "terragrunt-provider-cache"
- TerragruntProviderCacheEnvVarName = "TERRAGRUNT_PROVIDER_CACHE"
- TerragruntProviderCacheDirFlagName = "terragrunt-provider-cache-dir"
- TerragruntProviderCacheDirEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_DIR"
- TerragruntProviderCacheDisablePartialLockFileFlagName = "terragrunt-provider-cache-disable-partial-lock-file"
- TerragruntProviderCacheDisablePartialLockFileEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_DISABLE_PARTIAL_LOCK_FILE"
- TerragruntProviderCacheHostnameFlagName = "terragrunt-provider-cache-hostname"
- TerragruntProviderCacheHostnameEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_HOSTNAME"
- TerragruntProviderCachePortFlagName = "terragrunt-provider-cache-port"
- TerragruntProviderCachePortEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_PORT"
- TerragruntProviderCacheTokenFlagName = "terragrunt-provider-cache-token"
- TerragruntProviderCacheTokenEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_TOKEN"
- TerragruntProviderCacheRegistryNamesFlagName = "terragrunt-provider-cache-registry-names"
- TerragruntProviderCacheRegistryNamesEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_REGISTRY_NAMES"
+ TerragruntProviderCacheFlagName = "terragrunt-provider-cache"
+ TerragruntProviderCacheEnvVarName = "TERRAGRUNT_PROVIDER_CACHE"
+ TerragruntProviderCacheDirFlagName = "terragrunt-provider-cache-dir"
+ TerragruntProviderCacheDirEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_DIR"
+ TerragruntProviderCacheHostnameFlagName = "terragrunt-provider-cache-hostname"
+ TerragruntProviderCacheHostnameEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_HOSTNAME"
+ TerragruntProviderCachePortFlagName = "terragrunt-provider-cache-port"
+ TerragruntProviderCachePortEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_PORT"
+ TerragruntProviderCacheTokenFlagName = "terragrunt-provider-cache-token"
+ TerragruntProviderCacheTokenEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_TOKEN"
+ TerragruntProviderCacheRegistryNamesFlagName = "terragrunt-provider-cache-registry-names"
+ TerragruntProviderCacheRegistryNamesEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_REGISTRY_NAMES"
 
  HelpFlagName = "help"
  VersionFlagName = "version"
@@ -287,12 +285,6 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags {
  EnvVar: TerragruntProviderCacheDirEnvVarName,
  Usage: "The path to the Terragrunt provider cache directory. By default, 'terragrunt/providers' folder in the user cache directory.",
  },
- &cli.BoolFlag{
- Name: TerragruntProviderCacheDisablePartialLockFileFlagName,
- Destination: &opts.ProviderCacheDisablePartialLockFile,
- EnvVar: TerragruntProviderCacheDisablePartialLockFileEnvVarName,
- Usage: "Don't use 'plugin_cache_may_break_dependency_lock_file' with Terragrunt provider caching. Provider downloads for modules without lock files will be much slower.",
- },
  &cli.GenericFlag[string]{
  Name: TerragruntProviderCacheTokenFlagName,
  Destination: &opts.ProviderCacheToken,

cli/provider_cache.go

@@ -21,6 +21,7 @@ import (
  "github.com/gruntwork-io/terragrunt/terraform/cache/controllers"
  "github.com/gruntwork-io/terragrunt/terraform/cache/handlers"
  "github.com/gruntwork-io/terragrunt/terraform/cliconfig"
+ "github.com/gruntwork-io/terragrunt/terraform/getproviders"
  "github.com/gruntwork-io/terragrunt/util"
  "golang.org/x/exp/maps"
 )
@@ -31,7 +32,7 @@ const (
 )
 
 var (
- // HTTPStatusCacheProviderReg is regular expression to determine the success result of the command `terraform lock providers -platform=cache provider`.
+ // HTTPStatusCacheProviderReg is regular expression to determine the success result of the command `terraform init`.
  // The reg matches if the text contains "423 Locked", for example:
  //
  // - registry.terraform.io/hashicorp/template: could not query provider registry for registry.terraform.io/hashicorp/template: 423 Locked.
@@ -68,14 +69,6 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e
  return nil, errors.WithStackTrace(err)
  }
 
- if opts.ProviderCacheArchiveDir == "" {
- opts.ProviderCacheArchiveDir = filepath.Join(cacheDir, "archives")
- }
-
- if opts.ProviderCacheArchiveDir, err = filepath.Abs(opts.ProviderCacheArchiveDir); err != nil {
- return nil, errors.WithStackTrace(err)
- }
-
  if opts.ProviderCacheToken == "" {
  opts.ProviderCacheToken = uuid.New().String()
  }
@@ -95,7 +88,6 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e
  cache.WithToken(opts.ProviderCacheToken),
  cache.WithUserProviderDir(userProviderDir),
  cache.WithProviderCacheDir(opts.ProviderCacheDir),
- cache.WithProviderArchiveDir(opts.ProviderCacheArchiveDir),
  )
 
  return &ProviderCache{Server: cache}, nil
@@ -111,10 +103,9 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
  }
 
  var (
- cliConfigFilename = filepath.Join(opts.WorkingDir, localCLIFilename)
- terraformLockFilename = filepath.Join(opts.WorkingDir, terraform.TerraformLockFile)
- cacheRequestID = uuid.New().String()
- env = providerCacheEnvironment(opts, cliConfigFilename)
+ cliConfigFilename = filepath.Join(opts.WorkingDir, localCLIFilename)
+ cacheRequestID = uuid.New().String()
+ env = providerCacheEnvironment(opts, cliConfigFilename)
  )
 
  // Create terraform cli config file that enables provider caching and does not use provider cache dir
@@ -130,29 +121,16 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
  return nil, err
  }
 
- cache.Provider.WaitForCacheReady(cacheRequestID)
+ caches := cache.Provider.WaitForCacheReady(cacheRequestID)
+ if err := getproviders.UpdateLockfile(ctx, opts.WorkingDir, caches); err != nil {
+ return nil, err
+ }
 
  // Create terraform cli config file that uses provider cache dir
  if err := cache.createLocalCLIConfig(opts, cliConfigFilename, ""); err != nil {
  return nil, err
  }
 
- if opts.ProviderCacheDisablePartialLockFile && !util.FileExists(terraformLockFilename) {
- log.Infof("Getting terraform modules for %s", opts.WorkingDir)
- if err := runTerraformCommand(ctx, opts, []string{terraform.CommandNameGet}, env); err != nil {
- return nil, err
- }
-
- log.Infof("Generating Terraform lock file for %s", opts.WorkingDir)
- // Create complete terraform lock files. By default this feature is disabled, since it's not superfast.
- // Instead we use Terraform `TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE` feature, that creates hashes from the local cache.
- // And since the Terraform developers warn that this feature will be removed soon, it's good to have a workaround.
- if err := runTerraformCommand(ctx, opts, []string{terraform.CommandNameProviders, terraform.CommandNameLock}, env); err != nil {
- return nil, err
- }
-
- }
-
  cloneOpts := opts.Clone(opts.TerragruntConfigPath)
  cloneOpts.WorkingDir = opts.WorkingDir
  maps.Copy(cloneOpts.Env, env)
@@ -252,6 +230,7 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a
  if err := shell.RunTerraformCommand(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && len(errWriter.Msgs()) == 0 {
  return err
  }
+
  return nil
 }
 
@@ -266,12 +245,6 @@ func providerCacheEnvironment(opts *options.TerragruntOptions, cliConfigFile str
  envs[envName] = opts.ProviderCacheToken
  }
 
- if !opts.ProviderCacheDisablePartialLockFile {
- // By using `TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE` we force terraform to generate `.terraform.lock.hcl` only based on cached files, otherwise it downloads three files (provider zip archive, SHA256SUMS, sig) from the original registry to calculate hashes.
- // https://developer.hashicorp.com/terraform/cli/config/config-file#allowing-the-provider-plugin-cache-to-break-the-dependency-lock-file
- envs[terraform.EnvNameTFPluginCacheMayBreakDependencyLockFile] = "1"
- }
-
  // By using `TF_CLI_CONFIG_FILE` we force terraform to use our auto-generated cli configuration file.
  // https://developer.hashicorp.com/terraform/cli/config/environment-variables#tf_cli_config_file
  envs[terraform.EnvNameTFCLIConfigFile] = cliConfigFile

docs/_docs/02_features/provider-cache.md

@@ -81,17 +81,16 @@ terragrunt apply
 * Configure Terraform instances to use the Terragrunt Provider Cache server as a remote registry:
  * Create local CLI config file `.terraformrc` for each module that concatenates the user configuration from the Terraform [CLI config file](https://developer.hashicorp.com/terraform/cli/config/config-file) with additional sections:
  * [provider-installation](https://developer.hashicorp.com/terraform/cli/config/config-file#provider-installation) forces Terraform to look for for the required providers in the cache directory and create symbolic links to them, if not found, then request them from the remote registry.
- * [host](https://github.com/hashicorp/terraform/issues/28309) forces Terraform to [forward](#how-forwarding-terraform-requests-through-the-terragrunt-Provider-cache-works) all provider requests through the Terragrunt Provider Cache server.
+ * [host](https://github.com/hashicorp/terraform/issues/28309) forces Terraform to [forward](#how-forwarding-terraform-requests-through-the-terragrunt-Provider-cache-works) all provider requests through the Terragrunt Provider Cache server. The address link contains [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier) and is unique for each module, used by Terragrunt Provider Cache server to associate modules with the requested providers.
  * Set environment variables:
- * [TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE](https://developer.hashicorp.com/terraform/cli/config/config-file#allowing-the-provider-plugin-cache-to-break-the-dependency-lock-file) allows to generate `.terraform.lock.hcl` files based only on provider hashes from the cache directory.
  * [TF_CLI_CONFIG_FILE](https://developer.hashicorp.com/terraform/cli/config/environment-variables#tf_plugin_cache_dir) sets to use just created local CLI config `.terragrunt-cache/.terraformrc`
  * [TF_TOKEN_*](https://developer.hashicorp.com/terraform/cli/config/config-file#environment-variable-credentials) sets per-remote-registry tokens for authentication to Terragrunt Provider Cache server.
 * Any time Terragrunt is going to run `init`:
  * Call `terraform init`. This gets Terraform to request all the providers it needs from the Terragrunt Provider Cache server.
  * The Terragrunt Provider Cache server will download the provider from the remote registry, unpack and store it into the cache directory or [create a symlink](#reusing-providers-from-the-user-plugins-directory) if the required provider exists in the user plugins directory. Note that the Terragrunt Provider Cache server will ensure that each unique provider is only ever downloaded and stored on disk once, handling concurrency (from multiple Terraform and Terragrunt instances) correctly. Along with the provider, the cache server downloads hashes and signatures of the providers to check that the files are not corrupted.
  * The Terragrunt Provider Cache server returns the HTTP status _429 Locked_ to Terraform. This is because we do _not_ want Terraform to actually download any providers as a result of calling `terraform init`; we only use that command to request the Terragrunt Provider Cache Server to start caching providers.
  * At this point, all providers are downloaded and cached, so finally, we run `terragrunt init` a second time, which will find all the providers it needs in the cache, and it'll create symlinks to them nearly instantly, with no additional downloading.
- * Note that if a Terraform module doesn't have a lock file, Terraform does _not_ use the cache, so it would end up downloading all the providers from scratch. To work around this, by default, we use the `plugin_cache_may_break_dependency_lock_file` feature, which, for modules without lock files, will allow Terraform to automatically generate a partial lock file if it finds the providers it needs in the cache, with no additional downloading. This ensures that no additional downloads are necessary, but at the cost of partial lock files. If you wish to disable this feature, set the [`terragrunt-provider-cache-disable-partial-lock-file`](https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-provider-cache-disable-partial-lock-file) flag, and for modules without a lock file, Terragrunt will call `terraform providers lock`, which will give you a complete lock file, at the cost of more downloading and bandwidth.
+ * Note that if a Terraform module doesn't have a lock file, Terraform does _not_ use the cache, so it would end up downloading all the providers from scratch. To work around this, we generate `.terraform.lock.hcl` based on the request made by `terrafrom init` to the Terragrunt Provider Cache server. Since `terraform init` only requestes the providers that need to be added/updated, we can keep track of them using the Terragrunt Provider Cache server and update the Terraform lock file with the appropriate hashes without having to parse `tf` configs.
 
 #### Reusing providers from the user plugins directory
 

go.mod

@@ -69,6 +69,7 @@ require (
  github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
  github.com/pkg/errors v0.9.1
  github.com/posener/complete v1.2.3
+ github.com/rogpeppe/go-internal v1.11.0
  github.com/urfave/cli/v2 v2.26.0
  go.opentelemetry.io/otel v1.23.1
  go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.23.1
@@ -215,6 +216,7 @@ require (
  github.com/sourcegraph/jsonrpc2 v0.2.0 // indirect
  github.com/spf13/afero v1.9.5 // indirect
  github.com/spf13/cast v1.3.1 // indirect
+ github.com/stretchr/objx v0.5.2 // indirect
  github.com/terraform-linters/tflint-plugin-sdk v0.17.0 // indirect
  github.com/terraform-linters/tflint-ruleset-terraform v0.4.0 // indirect
  github.com/urfave/cli v1.22.14 // indirect

go.sum

@@ -1010,6 +1010,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=