Skip to content

Commit

Permalink
Resolve deprecation warnings in Azure FileShare-to-GCS tests
Browse files Browse the repository at this point in the history
Related: apache#39485

AzureFileShareToGCSOperator has a deprecated parameter, `directory_name`, which was replaced by `directory_path`. This PR updates system and unit tests as well as operator docs to reflect this deprecated change.
  • Loading branch information
josh-fell committed May 14, 2024
1 parent 339ea50 commit 3422b0b
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,10 @@ class AzureFileShareToGCSOperator(BaseOperator):
Does not include subdirectories. May be filtered by prefix.
:param share_name: The Azure FileShare share where to find the objects. (templated)
:param directory_name: (Optional) Path to Azure FileShare directory which content is to be transferred.
:param directory_name: (Deprecated) Path to Azure FileShare directory which content is to be transferred.
Defaults to root directory (templated)
:param directory_path: (Optional) Path to Azure FileShare directory which content is to be transferred.
Defaults to root directory. Use this instead of ``directory_name``. (templated)
:param prefix: Prefix string which filters objects whose name begin with
such prefix. (templated)
:param azure_fileshare_conn_id: The source WASB connection
Expand All @@ -63,13 +65,14 @@ class AzureFileShareToGCSOperator(BaseOperator):
Service Account Token Creator IAM role to the directly preceding identity, with first
account from the list granting this role to the originating account (templated).
Note that ``share_name``, ``directory_name``, ``prefix``, ``delimiter`` and ``dest_gcs`` are
Note that ``share_name``, ``directory_path``, ``prefix``, and ``dest_gcs`` are
templated, so you can use variables in them if you wish.
"""

template_fields: Sequence[str] = (
"share_name",
"directory_name",
"directory_path",
"prefix",
"dest_gcs",
)
Expand All @@ -94,8 +97,8 @@ def __init__(
self.share_name = share_name
self.directory_path = directory_path
self.directory_name = directory_name
if self.directory_path is None:
self.directory_path = directory_name
if self.directory_path is None and self.directory_name is not None:
self.directory_path = self.directory_name
warnings.warn(
"Use 'directory_path' instead of 'directory_name'.",
AirflowProviderDeprecationWarning,
Expand Down
1 change: 0 additions & 1 deletion tests/always/test_example_dags.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@
"tests/system/providers/amazon/aws/example_eks_with_nodegroups.py",
"tests/system/providers/amazon/aws/example_emr.py",
"tests/system/providers/amazon/aws/example_emr_notebook_execution.py",
"tests/system/providers/google/cloud/azure/example_azure_fileshare_to_gcs.py",
"tests/system/providers/google/cloud/bigquery/example_bigquery_operations.py",
"tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py",
"tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

TASK_ID = "test-azure-fileshare-to-gcs"
AZURE_FILESHARE_SHARE = "test-share"
AZURE_FILESHARE_DIRECTORY_NAME = "/path/to/dir"
AZURE_FILESHARE_DIRECTORY_PATH = "/path/to/dir"
GCS_PATH_PREFIX = "gs://gcs-bucket/data/"
MOCK_FILES = ["TEST1.csv", "TEST2.csv", "TEST3.csv"]
AZURE_FILESHARE_CONN_ID = "azure_fileshare_default"
Expand All @@ -37,7 +37,7 @@ def test_init(self):
operator = AzureFileShareToGCSOperator(
task_id=TASK_ID,
share_name=AZURE_FILESHARE_SHARE,
directory_name=AZURE_FILESHARE_DIRECTORY_NAME,
directory_path=AZURE_FILESHARE_DIRECTORY_PATH,
azure_fileshare_conn_id=AZURE_FILESHARE_CONN_ID,
gcp_conn_id=GCS_CONN_ID,
dest_gcs=GCS_PATH_PREFIX,
Expand All @@ -46,7 +46,7 @@ def test_init(self):

assert operator.task_id == TASK_ID
assert operator.share_name == AZURE_FILESHARE_SHARE
assert operator.directory_name == AZURE_FILESHARE_DIRECTORY_NAME
assert operator.directory_path == AZURE_FILESHARE_DIRECTORY_PATH
assert operator.azure_fileshare_conn_id == AZURE_FILESHARE_CONN_ID
assert operator.gcp_conn_id == GCS_CONN_ID
assert operator.dest_gcs == GCS_PATH_PREFIX
Expand All @@ -60,7 +60,7 @@ def test_execute(self, gcs_mock_hook, azure_fileshare_mock_hook):
operator = AzureFileShareToGCSOperator(
task_id=TASK_ID,
share_name=AZURE_FILESHARE_SHARE,
directory_name=AZURE_FILESHARE_DIRECTORY_NAME,
directory_path=AZURE_FILESHARE_DIRECTORY_PATH,
azure_fileshare_conn_id=AZURE_FILESHARE_CONN_ID,
gcp_conn_id=GCS_CONN_ID,
dest_gcs=GCS_PATH_PREFIX,
Expand Down Expand Up @@ -97,7 +97,7 @@ def test_execute_with_gzip(self, gcs_mock_hook, azure_fileshare_mock_hook):
operator = AzureFileShareToGCSOperator(
task_id=TASK_ID,
share_name=AZURE_FILESHARE_SHARE,
directory_name=AZURE_FILESHARE_DIRECTORY_NAME,
directory_path=AZURE_FILESHARE_DIRECTORY_PATH,
azure_fileshare_conn_id=AZURE_FILESHARE_CONN_ID,
gcp_conn_id=GCS_CONN_ID,
dest_gcs=GCS_PATH_PREFIX,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@

BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"
AZURE_SHARE_NAME = os.environ.get("AZURE_SHARE_NAME", "test-azure-share")
AZURE_DIRECTORY_NAME = "test-azure-dir"
AZURE_DIRECTORY_PATH = "test-azure-dir"

with DAG(
dag_id=DAG_ID,
Expand All @@ -49,15 +49,17 @@
tags=["example", "azure"],
) as dag:
create_bucket = GCSCreateBucketOperator(
task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID
task_id="create_bucket",
bucket_name=BUCKET_NAME,
project_id=PROJECT_ID, # type: ignore[arg-type]
)

# [START howto_operator_azure_fileshare_to_gcs_basic]
sync_azure_files_with_gcs = AzureFileShareToGCSOperator(
task_id="sync_azure_files_with_gcs",
share_name=AZURE_SHARE_NAME,
dest_gcs=BUCKET_NAME,
directory_name=AZURE_DIRECTORY_NAME,
directory_path=AZURE_DIRECTORY_PATH,
replace=False,
gzip=True,
google_impersonation_chain=None,
Expand Down

0 comments on commit 3422b0b

Please sign in to comment.