Allow Python dynamic providers to capture secrets

changelog/pending/20240404--sdk-python--allow-python-dynamic-providers-to-capture-secrets.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: feat
+ scope: sdk/python
+ description: Allow Python dynamic providers to capture secrets

sdk/python/lib/pulumi/dynamic/__main__.py

@@ -23,6 +23,7 @@
 import dill
 import grpc
 from google.protobuf import empty_pb2
+from pulumi.runtime._serialization import _deserialize
 from pulumi.runtime import proto, rpc
 from pulumi.runtime.proto import provider_pb2_grpc, ResourceProviderServicer
 from pulumi.dynamic import ResourceProvider
@@ -50,9 +51,14 @@ def get_provider(props) -> ResourceProvider:
  with _PROVIDER_LOCK:
  provider = _PROVIDER_CACHE.get(providerStr)
  if provider is None:
- byts = base64.b64decode(providerStr)
- provider = dill.loads(byts)
+
+ def deserialize():
+ byts = base64.b64decode(providerStr)
+ return dill.loads(byts)
+
+ provider = _deserialize(deserialize)
  _PROVIDER_CACHE[providerStr] = provider
+
  return provider
 
 

sdk/python/lib/pulumi/dynamic/dynamic.py

@@ -22,6 +22,7 @@
 import pulumi
 
 from .. import CustomResource, ResourceOptions
+from ..runtime._serialization import _serialize
 
 if TYPE_CHECKING:
  from ..output import Inputs, Output
@@ -168,6 +169,14 @@ class ResourceProvider:
  whose CRUD operations are implemented inside your Python program.
  """
 
+ auto_secret: bool = False
+ """
+ auto_secret controls whether the serialized provider is a secret.
+ By default this is False which makes the serialized provider a secret always.
+ Set to True in a subclass to make the serialized provider a secret only
+ if any secret Outputs were captured during serialization of the provider.
+ """
+
  def check(self, _olds: Any, news: Any) -> CheckResult:
  """
  Check validates that the given property bag is valid for a resource of the given type.
@@ -281,6 +290,15 @@ def __init__(
  raise Exception("A dynamic resource must not define the __provider key")
 
  props = cast(dict, props)
- props[PROVIDER_KEY] = pulumi.Output.secret(serialize_provider(provider))
+
+ serialized_provider, contains_secrets = _serialize(
+ True, serialize_provider, provider
+ )
+
+ auto_secret: bool = getattr(provider, "auto_secret", False)
+ if not auto_secret or contains_secrets:
+ serialized_provider = pulumi.Output.secret(serialized_provider)
+
+ props[PROVIDER_KEY] = serialized_provider
 
  super().__init__(f"pulumi-python:{self._resource_type_name}", name, props, opts)

sdk/python/lib/pulumi/output.py

@@ -40,6 +40,11 @@
 from .runtime import rpc
 from .runtime.sync_await import _sync_await
 from .runtime.settings import SETTINGS
+from .runtime._serialization import (
+ _serialization_enabled,
+ _secrets_allowed,
+ _set_contained_secrets,
+)
 
 if TYPE_CHECKING:
  from .resource import Resource
@@ -139,6 +144,68 @@ def is_known(self) -> Awaitable[bool]:
 
  # End private implementation details.
 
+ def __getstate__(self):
+ """
+ Serialize this Output into a dictionary for pickling, only when serialization is enabled.
+ """
+
+ if not _serialization_enabled():
+ raise Exception("__getstate__ can only be called during serialization")
+
+ value, is_secret = _sync_await(asyncio.gather(self.future(), self.is_secret()))
+
+ if is_secret:
+ if _secrets_allowed():
+ _set_contained_secrets(True)
+ else:
+ raise Exception("Secret outputs cannot be captured")
+
+ return {"value": value}
+
+ def __setstate__(self, state):
+ """
+ Deserialize this Output from a dictionary, only when serialization is enabled.
+ """
+
+ if not _serialization_enabled():
+ raise Exception("__setstate__ can only be called during deserialization")
+
+ value = state["value"]
+
+ # Replace '.get' with a function that returns the value without raising an error.
+ self.get = lambda: value
+
+ def error(name: str):
+ def f(*args: Any, **kwargs: Any):
+ raise Exception(
+ f"'{name}' is not allowed from inside a cloud-callback. "
+ + "Use 'get' to retrieve the value of this Output directly."
+ )
+
+ return f
+
+ # Replace '.apply' and other methods on Output with implementations that raise an error.
+ self.apply = error("apply")
+ self.resources = error("resources")
+ self.future = error("future")
+ self.is_known = error("is_known")
+ self.is_secret = error("is_secret")
+
+ def get(self) -> T_co:
+ """
+ Retrieves the underlying value of this Output.
+
+ This function is only callable in code that runs in the cloud post-deployment. At this
+ point all Output values will be known and can be safely retrieved. During pulumi deployment
+ or preview execution this must not be called (and will raise an error). This is because doing
+ so would allow Output values to flow into Resources while losing hte data that would allow the
+ dependency graph to be changed.
+ """
+ raise Exception(
+ "Cannot call '.get' during update or preview. To manipulate the value of this Output, "
+ + "use '.apply' instead."
+ )
+
  def is_secret(self) -> Awaitable[bool]:
  return self._is_secret
 

sdk/python/lib/pulumi/runtime/_serialization.py

@@ -0,0 +1,90 @@
+# Copyright 2016-2024, Pulumi Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from contextvars import ContextVar, copy_context
+from typing import Callable, ParamSpec, TypeVar, Tuple
+
+
+__all__ = [
+ "_serialize",
+ "_deserialize",
+ "_serialization_enabled",
+ "_secrets_allowed",
+ "_set_contained_secrets",
+]
+
+
+_T = TypeVar("_T")
+_P = ParamSpec("_P")
+
+# ContextVars that control whether serialization is enabled, whether secrets are allowed to be serialized,
+# and whether the serialized data contains secrets.
+_var_serialization_enabled = ContextVar("serialization_enabled", default=False)
+_var_serialization_allow_secrets = ContextVar(
+ "serialization_allow_secrets", default=False
+)
+_var_serialization_contained_secrets = ContextVar(
+ "serialization_contained_secrets", default=False
+)
+
+
+def _serialize(
+ allow_secrets: bool, f: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
+) -> Tuple[_T, bool]:
+ """
+ Run the given function with serialization enabled.
+ """
+
+ def serialize():
+ _var_serialization_enabled.set(True)
+ _var_serialization_allow_secrets.set(allow_secrets)
+ result = f(*args, **kwargs)
+ return result, _var_serialization_contained_secrets.get()
+
+ ctx = copy_context()
+ return ctx.run(serialize)
+
+
+def _deserialize(f: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> _T:
+ """
+ Run the given function with serialization enabled.
+ """
+
+ def deserialize():
+ _var_serialization_enabled.set(True)
+ return f(*args, **kwargs)
+
+ ctx = copy_context()
+ return ctx.run(deserialize)
+
+
+def _serialization_enabled() -> bool:
+ """
+ Returns whether serialization is enabled.
+ """
+ return _var_serialization_enabled.get()
+
+
+def _secrets_allowed() -> bool:
+ """
+ Returns whether secrets are allowed to be serialized.
+ """
+ return _var_serialization_allow_secrets.get()
+
+
+def _set_contained_secrets(value: bool) -> None:
+ """
+ Set that the serialized data contains secrets.
+ """
+ _var_serialization_contained_secrets.set(value)

sdk/python/lib/test/test_output.py

@@ -18,6 +18,10 @@
 from typing import Mapping, Optional, Sequence, cast
 
 from pulumi.runtime import rpc, rpc_manager, settings
+from pulumi.runtime._serialization import (
+ _deserialize,
+ _serialize,
+)
 
 import pulumi
 from pulumi import Output
@@ -426,3 +430,73 @@ async def test_basic(self):
  self.assertEqual(await x.future(), [0, 1])
  self.assertEqual(await x.is_secret(), False)
  self.assertEqual(await x.is_known(), True)
+
+
+class OutputSerializationTests(unittest.TestCase):
+ @pulumi_test
+ async def test_get_raises(self):
+ i = Output.from_input("hello")
+ with self.assertRaisesRegex(
+ Exception,
+ "Cannot call '.get' during update or preview. To manipulate the value of this Output, use '.apply' instead."
+ ):
+ i.get()
+
+ @pulumi_test
+ async def test_get_state_raises(self):
+ i = Output.from_input("hello")
+ with self.assertRaisesRegex(Exception, "__getstate__ can only be called during serialization"):
+ i.__getstate__()
+
+ @pulumi_test
+ async def test_get_state_allow_secrets(self):
+ i = Output.from_input("hello")
+ result, contains_secrets = _serialize(True, lambda: i.__getstate__())
+ self.assertEqual(result, {"value": "hello"})
+ self.assertFalse(contains_secrets)
+
+ @pulumi_test
+ async def test_get_state_disallow_secrets(self):
+ i = Output.from_input("hello")
+ result, contains_secrets = _serialize(False, lambda: i.__getstate__())
+ self.assertEqual(result, {"value": "hello"})
+ self.assertFalse(contains_secrets)
+
+ @pulumi_test
+ async def test_get_state_allow_secrets_secret(self):
+ i = Output.secret("shh")
+ result, contains_secrets = _serialize(True, lambda: i.__getstate__())
+ self.assertEqual(result, {"value": "shh"})
+ self.assertTrue(contains_secrets)
+
+ @pulumi_test
+ async def test_get_state_disallow_secrets_secret_raises(self):
+ i = Output.secret("shh")
+ with self.assertRaisesRegex(Exception, "Secret outputs cannot be captured"):
+ _serialize(False, lambda: i.__getstate__())
+
+ @pulumi_test
+ async def test_get_after_set_state(self):
+ i = Output.from_input("hello")
+ _deserialize(lambda: i.__setstate__({"value": "world"}))
+ self.assertEqual(i.get(), "world")
+
+ @pulumi_test
+ async def test_raises_after_set_state(self):
+ i = Output.from_input("hello")
+ _deserialize(lambda: i.__setstate__({"value": "world"}))
+
+ def expected_msg(name: str):
+ return f"'{name}' is not allowed from inside a cloud-callback. " \
+ + "Use 'get' to retrieve the value of this Output directly."
+
+ with self.assertRaisesRegex(Exception, expected_msg("apply")):
+ i.apply(lambda x: x)
+ with self.assertRaisesRegex(Exception, expected_msg("resources")):
+ i.resources()
+ with self.assertRaisesRegex(Exception, expected_msg("future")):
+ i.future()
+ with self.assertRaisesRegex(Exception, expected_msg("is_known")):
+ i.is_known()
+ with self.assertRaisesRegex(Exception, expected_msg("is_secret")):
+ i.is_secret()

tests/integration/dynamic/python-auto-secret/.gitignore

@@ -0,0 +1,4 @@
+*.pyc
+/.pulumi/
+/dist/
+/*.egg-info

tests/integration/dynamic/python-auto-secret/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: dynamic_py_auto_secret
+description: A simple Python program that uses dynamic providers with auto secret capture.
+runtime: python

tests/integration/dynamic/python-auto-secret/__main__.py

@@ -0,0 +1,23 @@
+# Copyright 2016-2024, Pulumi Corporation. All rights reserved.
+
+import binascii
+import os
+from pulumi import export
+from pulumi.dynamic import Resource, ResourceProvider, CreateResult
+
+class RandomResourceProvider(ResourceProvider):
+ auto_secret = True
+
+ def create(self, props):
+ val = binascii.b2a_hex(os.urandom(15)).decode("ascii")
+ return CreateResult(val, { "val": val })
+
+class Random(Resource):
+ val: str
+ def __init__(self, name, opts = None):
+ super().__init__(RandomResourceProvider(), name, {"val": ""}, opts)
+
+r = Random("foo")
+
+export("random_id", r.id)
+export("random_val", r.val)

tests/integration/dynamic/python-auto-secret/step1/README.md

@@ -0,0 +1 @@
+Intentionally make no changes.

tests/integration/dynamic/python-secrets/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: dynamic-provider-secret-py
+description: test for dynamic provider secret capture
+runtime: python

tests/integration/dynamic/python-secrets/__main__.py

@@ -0,0 +1,28 @@
+# Copyright 2016-2024, Pulumi Corporation. All rights reserved.
+
+import pulumi
+from pulumi.dynamic import CreateResult, Resource, ResourceProvider
+
+
+config = pulumi.Config()
+password = config.require_secret("password")
+
+
+class SimpleProvider(ResourceProvider):
+ auto_secret = True
+
+ def create(self, props):
+ # Need to use `password.get()` to get the underlying value of the secret from within the serialized code.
+ # This simulates using this as a credential to talk to an external system.
+ return CreateResult("0", { "authenticated": "200" if password.get() == "s3cret" else "401" })
+
+
+class SimpleResource(Resource):
+ authenticated: pulumi.Output[str]
+
+ def __init__(self, name):
+ super().__init__(SimpleProvider(), name, { "authenticated": None })
+
+
+r = SimpleResource("foo")
+pulumi.export("out", r.authenticated)