Merge pull request #693 from mozilla/develop

v3.6.0 RC (develop -> master)

.circleci/config.yml

@@ -1,9 +1,21 @@
-version: 2
+version: 2.1
+
+workflows:
+ build-and-deploy:
+ jobs:
+ - build
+ - push: 
+ filters:
+ tags:
+ only: /^v.*/ 
+ branches:
+ ignore: /.*/
 jobs:
  build:
  working_directory: /go/src/go.mozilla.org/sops
  docker:
  - image: circleci/golang:1.13
+ resource_class: large
  steps:
  - checkout
  - setup_remote_docker
@@ -12,10 +24,79 @@ jobs:
  command: |
  docker build -t mozilla/sops .
  docker tag mozilla/sops "mozilla/sops:$CIRCLE_SHA1"
+
+ push:
+ machine: true
+ resource_class: large
+ steps:
+ - checkout
+ - run: 
+ name: semver check
+ command: |
+ MAJOR=$(echo ${CIRCLE_TAG#v} | cut -d"." -f1)
+ MINOR=$(echo ${CIRCLE_TAG#v} | cut -d"." -f2)
+ PATCH=$(echo ${CIRCLE_TAG#v} | cut -d"." -f3)
+ echo "export MAJOR=${MAJOR}" >> $BASH_ENV
+ echo "export MINOR=${MINOR}" >> $BASH_ENV
+ echo "export PATCH=${PATCH}" >> $BASH_ENV
+
+ if [ -z $MAJOR ];then
+ cat \<< EOF
+ Failure Info:
+
+ This job uses the semver from the git TAG as the public version to publish.
+
+ - This should only run on workflows triggered by a tag. 
+ - The tag name should be a semver like 'v1.2.3' 
+ - The version should follow conventions documented at https://github.com/fsaintjacques/semver-tool
+ EOF
+ exit 1
+ fi
+ - run:
+ name: Build containers
+ command: |
+ docker build -t mozilla/sops .
+ docker build -f Dockerfile.alpine -t mozilla/sops:alpine .
  - run:
- name: Push containers
+ name: Tag & Push containers
  command: |
- if [ "${CIRCLE_BRANCH}" == "master" ]; then
- ${GOPATH}/src/go.mozilla.org/sops/bin/ci/deploy_dockerhub.sh "latest"
- ${GOPATH}/src/go.mozilla.org/sops/bin/ci/deploy_dockerhub.sh "$CIRCLE_SHA1"
+ #latest
+ bin/ci/deploy_dockerhub.sh "latest"
+ bin/ci/deploy_dockerhub.sh "alpine"
+
+ # by sha
+ echo "Tag and push mozilla/sops:$CIRCLE_SHA1"
+ docker tag mozilla/sops "mozilla/sops:$CIRCLE_SHA1"
+ bin/ci/deploy_dockerhub.sh "$CIRCLE_SHA1"
+
+ # no sha for alpine
+
+ # by semver
+ # v1.2.3
+ if [ ! -z $PATCH ];then
+ echo "Tag and Push mozilla/sops:v$MAJOR.$MINOR.$PATCH"
+ docker tag mozilla/sops "mozilla/sops:v$MAJOR.$MINOR.$PATCH"
+ bin/ci/deploy_dockerhub.sh "v$MAJOR.$MINOR.$PATCH"
+
+ echo "Tag and Push mozilla/sops:v$MAJOR.$MINOR.$PATCH-alpine"
+ docker tag mozilla/sops:alpine "mozilla/sops:v$MAJOR.$MINOR.$PATCH-alpine"
+ bin/ci/deploy_dockerhub.sh "v$MAJOR.$MINOR.$PATCH-alpine"
+ fi
+ # v1.2
+ if [ ! -z $MINOR ];then
+ echo "Tag and Push mozilla/sops:v$MAJOR.$MINOR"
+ docker tag mozilla/sops "mozilla/sops:v$MAJOR.$MINOR"
+ bin/ci/deploy_dockerhub.sh "v$MAJOR.$MINOR"
+
+ echo "Tag and Push mozilla/sops:v$MAJOR.$MINOR-alpine"
+ docker tag mozilla/sops:alpine "mozilla/sops:v$MAJOR.$MINOR-alpine"
+ bin/ci/deploy_dockerhub.sh "v$MAJOR.$MINOR-alpine"
  fi
+ # v1
+ echo "Tag and Push mozilla/sops:v$MAJOR"
+ docker tag mozilla/sops "mozilla/sops:v$MAJOR"
+ bin/ci/deploy_dockerhub.sh "v$MAJOR"
+
+ echo "Tag and Push mozilla/sops:v$MAJOR-alpine"
+ docker tag mozilla/sops:alpine "mozilla/sops:v$MAJOR-alpine"
+ bin/ci/deploy_dockerhub.sh "v$MAJOR-alpine"

.dockerignore

@@ -0,0 +1,3 @@
+/.git
+/Dockerfile
+/Dockerfile.alpine

.gitignore

@@ -1,3 +1,5 @@
 target
 Cargo.lock
 vendor/
+coverage.txt
+profile.out

.sops.yaml

@@ -1,2 +1,4 @@
 creation_rules:
- - pgp: 1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A
+ - pgp: >-
+ FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4,
+ D7229043384BCC60326C6FB9D8720D957C3D3074

CHANGELOG.rst

@@ -1,6 +1,37 @@
 Changelog
 =========
 
+3.6.0
+-----
+Features:
+
+ * Support for encrypting data through the use of Hashicorp Vault (#655)
+ * `sops publish` now supports `--recursive` flag for publishing all files in a directory (#602)
+ * `sops publish` now supports `--omit-extensions` flag for omitting the extension in the destination path (#602)
+ * sops now supports JSON arrays of arrays (#642)
+
+Improvements:
+
+ * Updates and standardization for the dotenv store (#612, #622)
+ * Close temp files after using them for edit command (#685)
+
+Bug fixes:
+
+ * AWS SDK usage now correctly resolves the `~/.aws/config` file (#680)
+ * `sops updatekeys` now correctly matches config rules (#682)
+ * `sops updatekeys` now correctly uses the config path cli flag (#672)
+ * Partially empty sops config files don't break the use of sops anymore (#662)
+ * Fix possible infinite loop in PGP's passphrase prompt call (#690)
+
+Project changes:
+
+ * Dockerfile now based off of golang version 1.14 (#649)
+ * Push alpine version of docker image to Dockerhub (#609)
+ * Push major, major.minor, and major.minor.patch tagged docker images to Dockerhub (#607)
+ * Removed out of date contact information (#668)
+ * Update authors in the cli help text (#645)
+
+
 3.5.0
 -----
 Features:

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.12
+FROM golang:1.14
 
 COPY . /go/src/go.mozilla.org/sops
 WORKDIR /go/src/go.mozilla.org/sops

Dockerfile.alpine

@@ -0,0 +1,17 @@
+FROM golang:1.12-alpine3.10 AS builder
+
+RUN apk --no-cache add make
+
+COPY . /go/src/go.mozilla.org/sops
+WORKDIR /go/src/go.mozilla.org/sops
+
+RUN CGO_ENABLED=1 make install
+
+
+FROM alpine:3.10
+
+RUN apk --no-cache add \
+ vim ca-certificates
+ENV EDITOR vim
+COPY --from=builder /go/bin/sops /usr/local/bin/sops
+ENTRYPOINT ["/usr/local/bin/sops"]

README.rst

@@ -48,9 +48,6 @@ Or whatever variation of the above fits your system and shell.
 
 To use **sops** as a library, take a look at the `decrypt package <https://godoc.org/go.mozilla.org/sops/decrypt>`_.
 
-**Questions?** ping "ulfr" and "autrilla" in ``#security`` on `irc.mozilla.org <https://wiki.mozilla.org/IRC>`_
-(use a web client like `mibbit <https://chat.mibbit.com>`_ ).
-
 **What happened to Python Sops?** We rewrote Sops in Go to solve a number of
 deployment issues, but the Python branch still exists under ``python-sops``. We
 will keep maintaining it for a while, and you can still ``pip install sops``,
@@ -290,6 +287,66 @@ And decrypt it using::
  $ sops --decrypt test.enc.yaml
 
 
+Encrypting using Hashicorp Vault
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We assume you have an instance (or more) of Vault running and you have privileged access to it. For instructions on how to deploy a secure instance of Vault, refer to Hashicorp's official documentation.
+
+To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) 
+
+.. code:: bash
+
+ $ docker run -d -p8200:8200 vault:1.2.0 server -dev -dev-root-token-id=toor
+
+
+.. code:: bash
+
+ $ # Substitute this with the address Vault is running on
+ $ export VAULT_ADDR=http://127.0.0.1:8200 
+
+ $ # this may not be necessary in case you previously used `vault login` for production use
+ $ export VAULT_TOKEN=toor 
+ 
+ $ # to check if Vault started and is configured correctly
+ $ vault status
+ Key Value
+ --- -----
+ Seal Type shamir
+ Initialized true
+ Sealed false
+ Total Shares 1
+ Threshold 1
+ Version 1.2.0
+ Cluster Name vault-cluster-618cc902
+ Cluster ID e532e461-e8f0-1352-8a41-fc7c11096908
+ HA Enabled false
+
+ $ # It is required to enable a transit engine if not already done (It is suggested to create a transit engine specifically for sops, in which it is possible to have multiple keys with various permission levels)
+ $ vault secrets enable -path=sops transit
+ Success! Enabled the transit secrets engine at: sops/
+
+ $ # Then create one or more keys
+ $ vault write sops/keys/firstkey type=rsa-4096
+ Success! Data written to: sops/keys/firstkey
+
+ $ vault write sops/keys/secondkey type=rsa-2048
+ Success! Data written to: sops/keys/secondkey
+
+ $ vault write sops/keys/thirdkey type=chacha20-poly1305
+ Success! Data written to: sops/keys/thirdkey
+
+ $ sops --hc-vault-transit $VAULT_ADDR/v1/sops/keys/firstkey vault_example.yml
+
+ $ cat <<EOF > .sops.yaml
+ creation_rules:
+ - path_regex: \.dev\.yaml$
+ hc_vault_transit_uri: "$VAULT_ADDR/v1/sops/keys/secondkey"
+ - path_regex: \.prod\.yaml$
+ hc_vault_transit_uri: "$VAULT_ADDR/v1/sops/keys/thirdkey"
+ EOF
+
+ $ sops --verbose -e prod/raw.yaml > prod/encrypted.yaml
+
 Adding and removing keys
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -546,6 +603,7 @@ can manage the three sets of configurations for the three types of files:
  - path_regex: \.prod\.yaml$
  kms: 'arn:aws:kms:us-west-2:361527076523:key/5052f06a-5d3f-489e-b86c-57201e06f31e+arn:aws:iam::361527076523:role/hiera-sops-prod,arn:aws:kms:eu-central-1:361527076523:key/cb1fab90-8d17-42a1-a9d8-334968904f94+arn:aws:iam::361527076523:role/hiera-sops-prod'
  pgp: 'FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4'
+ hc_vault_uris: "http://localhost:8200/v1/sops/keys/thirdkey"
 
  # gcp files using GCP KMS
  - path_regex: \.gcp\.yaml$
@@ -865,21 +923,21 @@ written to disk.
  "AWS_ACCESS_KEY_ID": "AKIAIOSFODNN7EXAMPLE",
  "AWS_SECRET_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
  }
- 
+
  # decrypt out.json and run a command
  # the command prints the environment variable and runs a script that uses it
  $ sops exec-env out.json 'echo secret: $database_password; ./database-import'
  secret: jf48t9wfw094gf4nhdf023r
- 
+
  # launch a shell with the secrets available in its environment
  $ sops exec-env out.json 'sh'
  sh-3.2# echo $database_password
  jf48t9wfw094gf4nhdf023r
- 
+
  # the secret is not accessible anywhere else
  sh-3.2$ exit
  $ echo your password: $database_password
- your password: 
+ your password:
 
 
 If the command you want to run only operates on files, you can use ``exec-file``
@@ -904,7 +962,7 @@ substituted with the temporary file path (whether a FIFO or an actual file).
  "AWS_ACCESS_KEY_ID": "AKIAIOSFODNN7EXAMPLE",
  "AWS_SECRET_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
  }
- 
+
  # launch a shell with a variable TMPFILE pointing to the temporary file
  $ sops exec-file --no-fifo out.json 'TMPFILE={} sh'
  sh-3.2$ echo $TMPFILE
@@ -934,7 +992,7 @@ for added security.
  # the encrypted file can't be read by the current user
  $ cat out.json
  cat: out.json: Permission denied
- 
+
  # execute sops as root, decrypt secrets, then drop privileges
  $ sudo sops exec-env --user nobody out.json 'sh'
  sh-3.2$ echo $database_password
@@ -968,6 +1026,7 @@ This command requires a ``.sops.yaml`` configuration file. Below is an example:
  vault_kv_mount_name: "secret/" # default
  vault_kv_version: 2 # default
  path_regex: vault/*
+ omit_extensions: true
 
 The above configuration will place all files under ``s3/*`` into the S3 bucket ``sops-secrets``,
 all files under ``gcs/*`` into the GCS bucket ``sops-secrets``, and the contents of all files under
@@ -977,6 +1036,11 @@ published to S3 and GCS, it will decrypt them and re-encrypt them using the
 
 You would deploy a file to S3 with a command like: ``sops publish s3/app.yaml``
 
+To publish all files in selected directory recursively, you need to specify ``--recursive`` flag.
+
+If you don't want file extension to appear in destination secret path, use ``--omit-extensions``
+flag or ``omit_extensions: true`` in the destination rule in ``.sops.yaml``.
+
 Publishing to Vault
 *******************
 
@@ -991,6 +1055,9 @@ configuring the client.
 ``vault_kv_mount_name`` is used if your Vault KV is mounted somewhere other than ``secret/``.
 ``vault_kv_version`` supports ``1`` and ``2``, with ``2`` being the default.
 
+If destination secret path already exists in Vault and contains same data as the source file, it
+will be skipped.
+
 Below is an example of publishing to Vault (using token auth with a local dev instance of Vault).
 
 .. code:: bash
@@ -1293,9 +1360,10 @@ You can import sops as a module and use it in your python program.
  tree = sops.walk_and_decrypt(tree, sops_key)
  sops.write_file(tree, path=path, filetype=pathtype)
 
-Note: this uses the previous implemenation of `sops` written in python,
+Note: this uses the previous implementation of `sops` written in python,
+
 and so doesn't support newer features such as GCP-KMS.
-To use the current version, call out to `sops` using `subprocess.check_output`
+To use the current version, call out to ``sops`` using ``subprocess.run``
 
 Showing diffs in cleartext in git
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

cmd/sops/edit.go

@@ -132,6 +132,9 @@ func editTree(opts editOpts, tree *sops.Tree, dataKey []byte) ([]byte, error) {
  if err != nil {
  return nil, common.NewExitError(fmt.Sprintf("Could not write output file: %s", err), codes.CouldNotWriteOutputFile)
  }
+
+ // Close temporary file, since Windows won't delete the file unless it's closed beforehand
+ defer tmpfile.Close()
 
  // Compute file hash to detect if the file has been edited
  origHash, err := hashFile(tmpfile.Name())