Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support KMS encryption context for S3 transfer commands #8585

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
+49 −17
Open

Support KMS encryption context for S3 transfer commands #8585

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
16 changes: 15 additions & 1 deletion awscli/customizations/s3/subcommands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,19 @@
}


SSE_KMS_ENCRYPTION_CONTEXT = {
'name': 'sse-kms-encryption-context',
'help_text': (
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I copied the help text for the --ssekms-encryption-context arg for the s3api put-object command

'Specifies the Amazon Web Services KMS Encryption Context to use for'
'object encryption. The value of this header is a base64-encoded'
'UTF-8 string holding JSON with the encryption context key-value'
'pairs. This value is stored as object metadata and automatically'
'gets passed on to Amazon Web Services KMS for future GetObject or'
'CopyObject operations on this object.'
)
}


SSE_C_COPY_SOURCE = {
'name': 'sse-c-copy-source', 'nargs': '?',
'const': 'AES256', 'choices': ['AES256'],
Expand Down Expand Up @@ -432,7 +445,8 @@

TRANSFER_ARGS = [DRYRUN, QUIET, INCLUDE, EXCLUDE, ACL,
FOLLOW_SYMLINKS, NO_FOLLOW_SYMLINKS, NO_GUESS_MIME_TYPE,
SSE, SSE_C, SSE_C_KEY, SSE_KMS_KEY_ID, SSE_C_COPY_SOURCE,
SSE, SSE_C, SSE_C_KEY, SSE_KMS_KEY_ID,
SSE_KMS_ENCRYPTION_CONTEXT, SSE_C_COPY_SOURCE,
SSE_C_COPY_SOURCE_KEY, STORAGE_CLASS, GRANTS,
WEBSITE_REDIRECT, CONTENT_TYPE, CACHE_CONTROL,
CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE,
Expand Down
4 changes: 3 additions & 1 deletion awscli/customizations/s3/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -601,8 +601,10 @@ def _set_metadata_directive_param(cls, request_params, cli_params):
def _set_sse_request_params(cls, request_params, cli_params):
if cli_params.get('sse'):
request_params['ServerSideEncryption'] = cli_params['sse']
if cli_params.get('sse_kms_key_id'):
if cli_params.get('sse_kms_key_id'):
request_params['SSEKMSKeyId'] = cli_params['sse_kms_key_id']
if cli_params.get('sse_kms_encryption_context'):
request_params['SSEKMSEncryptionContext'] = cli_params['sse_kms_encryption_context']
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/boto/s3transfer/blob/2885dabd397799fc096f87f7a3aa0a64e805aaf3/s3transfer/manager.py#L185


@classmethod
def _set_sse_c_request_params(cls, request_params, cli_params):
Expand Down
41 changes: 26 additions & 15 deletions tests/functional/s3/test_cp_command.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import base64
import os

from awscli.testutils import BaseAWSCommandParamsTest
from awscli.testutils import capture_input
from awscli.testutils import mock
from awscli.testutils import mock
from awscli.compat import six
from tests.functional.s3 import BaseS3TransferCommandTest
from tests import requires_crt
Expand Down Expand Up @@ -552,32 +553,35 @@ def test_cp_with_sse_c_copy_source_fileb(self):
# Note ideally the kms sse with a key id would be integration tests
# However, you cannot delete kms keys so there would be no way to clean
# up the tests
def test_cp_upload_with_sse_kms_and_key_id(self):
def test_cp_upload_with_sse_kms_and_key_id_and_encryption_context(self):
full_path = self.files.create_file('foo.txt', 'contents')
encryption_context = base64.standard_b64encode(b'{"key":"value"}').decode('ascii')
cmdline = (
'%s %s s3://bucket/key.txt --sse aws:kms --sse-kms-key-id foo' % (
self.prefix, full_path))
'%s %s s3://bucket/key.txt --sse aws:kms --sse-kms-key-id foo --sse-kms-encryption-context %s' % (
self.prefix, full_path, encryption_context))
self.run_cmd(cmdline, expected_rc=0)
self.assertEqual(len(self.operations_called), 1)
self.assertEqual(self.operations_called[0][0].name, 'PutObject')
self.assertDictEqual(
self.operations_called[0][1],
{'Key': 'key.txt', 'Bucket': 'bucket',
'ContentType': 'text/plain', 'Body': mock.ANY,
'SSEKMSKeyId': 'foo', 'ServerSideEncryption': 'aws:kms'}
'SSEKMSKeyId': 'foo', 'ServerSideEncryption': 'aws:kms',
'SSEKMSEncryptionContext': encryption_context}
)

def test_cp_upload_large_file_with_sse_kms_and_key_id(self):
def test_cp_upload_large_file_with_sse_kms_and_key_id_and_encryption_context(self):
self.parsed_responses = [
{'UploadId': 'foo'}, # CreateMultipartUpload
{'ETag': '"foo"'}, # UploadPart
{'ETag': '"foo"'}, # UploadPart
{} # CompleteMultipartUpload
]
full_path = self.files.create_file('foo.txt', 'a' * 10 * (1024 ** 2))
encryption_context = base64.standard_b64encode(b'{"key":"value"}').decode('ascii')
cmdline = (
'%s %s s3://bucket/key.txt --sse aws:kms --sse-kms-key-id foo' % (
self.prefix, full_path))
'%s %s s3://bucket/key.txt --sse aws:kms --sse-kms-key-id foo --sse-kms-encryption-context %s' % (
self.prefix, full_path, encryption_context))
self.run_cmd(cmdline, expected_rc=0)
self.assertEqual(len(self.operations_called), 4)

Expand All @@ -589,17 +593,20 @@ def test_cp_upload_large_file_with_sse_kms_and_key_id(self):
self.operations_called[0][1],
{'Key': 'key.txt', 'Bucket': 'bucket',
'ContentType': 'text/plain',
'SSEKMSKeyId': 'foo', 'ServerSideEncryption': 'aws:kms'}
'SSEKMSKeyId': 'foo', 'ServerSideEncryption': 'aws:kms',
'SSEKMSEncryptionContext': encryption_context}
)

def test_cp_copy_with_sse_kms_and_key_id(self):
def test_cp_copy_with_sse_kms_and_key_id_and_encryption_context(self):
self.parsed_responses = [
{'ContentLength': 5, 'LastModified': '00:00:00Z'}, # HeadObject
{} # CopyObject
]
encryption_context = base64.standard_b64encode(b'{"key":"value"}').decode('ascii')
cmdline = (
'%s s3://bucket/key1.txt s3://bucket/key2.txt '
'--sse aws:kms --sse-kms-key-id foo' % self.prefix)
'--sse aws:kms --sse-kms-key-id foo --sse-kms-encryption-context %s' % (
self.prefix, encryption_context))
self.run_cmd(cmdline, expected_rc=0)
self.assertEqual(len(self.operations_called), 2)
self.assertEqual(self.operations_called[1][0].name, 'CopyObject')
Expand All @@ -614,11 +621,12 @@ def test_cp_copy_with_sse_kms_and_key_id(self):
'Key': 'key1.txt'
},
'SSEKMSKeyId': 'foo',
'ServerSideEncryption': 'aws:kms'
'ServerSideEncryption': 'aws:kms',
'SSEKMSEncryptionContext': encryption_context
}
)

def test_cp_copy_large_file_with_sse_kms_and_key_id(self):
def test_cp_copy_large_file_with_sse_kms_and_key_id_and_encryption_context(self):
self.parsed_responses = [
{'ContentLength': 10 * (1024 ** 2),
'LastModified': '00:00:00Z'}, # HeadObject
Expand All @@ -627,9 +635,11 @@ def test_cp_copy_large_file_with_sse_kms_and_key_id(self):
{'CopyPartResult': {'ETag': '"foo"'}}, # UploadPartCopy
{} # CompleteMultipartUpload
]
encryption_context = base64.standard_b64encode(b'{"key":"value"}').decode('ascii')
cmdline = (
'%s s3://bucket/key1.txt s3://bucket/key2.txt '
'--sse aws:kms --sse-kms-key-id foo' % self.prefix)
'--sse aws:kms --sse-kms-key-id foo --sse-kms-encryption-context %s' % (
self.prefix, encryption_context))
self.run_cmd(cmdline, expected_rc=0)
self.assertEqual(len(self.operations_called), 5)

Expand All @@ -641,7 +651,8 @@ def test_cp_copy_large_file_with_sse_kms_and_key_id(self):
self.operations_called[1][1],
{'Key': 'key2.txt', 'Bucket': 'bucket',
'ContentType': 'text/plain',
'SSEKMSKeyId': 'foo', 'ServerSideEncryption': 'aws:kms'}
'SSEKMSKeyId': 'foo', 'ServerSideEncryption': 'aws:kms',
'SSEKMSEncryptionContext': encryption_context}
)

def test_cannot_use_recursive_with_stream(self):
Expand Down
5 changes: 5 additions & 0 deletions tests/unit/customizations/s3/test_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# language governing permissions and limitations under the License.
from awscli.testutils import mock, unittest, temporary_file
import argparse
import base64
import errno
import os
import tempfile
Expand Down Expand Up @@ -554,6 +555,7 @@ def setUp(self):
self.cli_params = {
'sse': 'AES256',
'sse_kms_key_id': 'my-kms-key',
'sse_kms_encryption_context': base64.standard_b64encode(b'{"key":"value"}').decode('ascii'),
'sse_c': 'AES256',
'sse_c_key': 'my-sse-c-key',
'sse_c_copy_source': 'AES256',
Expand All @@ -577,6 +579,7 @@ def test_put_object(self):
{'SSECustomerAlgorithm': 'AES256',
'SSECustomerKey': 'my-sse-c-key',
'SSEKMSKeyId': 'my-kms-key',
'SSEKMSEncryptionContext': base64.standard_b64encode(b'{"key":"value"}').decode('ascii'),
'ServerSideEncryption': 'AES256'}
)

Expand All @@ -599,6 +602,7 @@ def test_copy_object(self):
'SSECustomerAlgorithm': 'AES256',
'SSECustomerKey': 'my-sse-c-key',
'SSEKMSKeyId': 'my-kms-key',
'SSEKMSEncryptionContext': base64.standard_b64encode(b'{"key":"value"}').decode('ascii'),
'ServerSideEncryption': 'AES256'}
)

Expand All @@ -611,6 +615,7 @@ def test_create_multipart_upload(self):
{'SSECustomerAlgorithm': 'AES256',
'SSECustomerKey': 'my-sse-c-key',
'SSEKMSKeyId': 'my-kms-key',
'SSEKMSEncryptionContext': base64.standard_b64encode(b'{"key":"value"}').decode('ascii'),
'ServerSideEncryption': 'AES256'}
)

Expand Down