Add missing permissions for regression tests (#54)

* Add missing permissions for regression tests

* Verify that blob storage Gen2 is *not* enabled

* Increment version

* Fix compile error

* Fix references

* Mark test as itest

* Upgrade ci-isolation

* Update release date

* Rename TestContainer class to avoid running it in unit tests

* Update dependency

* Apply suggestions from code review

Co-authored-by: Sebastian Bär <[email protected]>

* Implement review finding

---------

Co-authored-by: Sebastian Bär <[email protected]>

.vscode/settings.json

@@ -15,6 +15,9 @@
  "-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"
  ]
  },
+ "[json]": {
+ "editor.indentSize": 2
+ },
  "sonarlint.connectedMode.project": {
  "connectionId": "exasol",
  "projectKey": "com.exasol:azure-blob-storage-document-files-virtual-schema"

ci-isolation/pom.xml

@@ -38,7 +38,7 @@
  <dependency>
  <groupId>com.exasol</groupId>
  <artifactId>ci-isolation-aws</artifactId>
- <version>2.0.2</version>
+ <version>2.0.3</version>
  </dependency>
  </dependencies>
 </project>

ci-isolation/src/main/resources/exasol-permissions.json

@@ -46,6 +46,7 @@
  "iam:GetRole",
  "iam:CreateRole",
  "iam:DeleteRole",
+ "iam:TagRole",
  "iam:PutRolePolicy",
  "iam:DetachRolePolicy",
  "iam:DeleteRolePolicy",
@@ -75,4 +76,4 @@
  "Resource": "*"
  }
  ]
-}
+}

doc/changes/changes_2.0.4.md

@@ -0,0 +1,12 @@
+# Virtual Schema for Document Data in Files on Azure Blob Storage 2.0.4, released 2024-04-10
+
+Code name: Verify storage generation in integration tests
+
+## Summary
+
+This release adds an integration test that verifies, that "Data Lake Gen2 upgrade" is not enabled for the Azure storage account.
+## Dependency Updates
+
+### Test Dependency Updates
+
+* Updated `com.exasol:udf-debugging-java:0.6.12` to `0.6.13`

doc/user_guide/user_guide.md

@@ -17,7 +17,7 @@ Next create the Adapter Script:
  ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.AZURE_BLOB_STORAGE_FILES_ADAPTER AS
  %scriptclass com.exasol.adapter.RequestDispatcher;
- %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.3.jar;
+ %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.4.jar;
 /
 ```
 
@@ -30,7 +30,7 @@ CREATE OR REPLACE JAVA SET SCRIPT ADAPTER.IMPORT_FROM_AZURE_BLOB_STORAGE_DOCUMEN
  CONNECTION_NAME VARCHAR(500))
  EMITS(...) AS
  %scriptclass com.exasol.adapter.document.UdfEntryPoint;
- %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.3.jar;
+ %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.4.jar;
 /
 ```
 

pom.xml

@@ -2,7 +2,7 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <artifactId>azure-blob-storage-document-files-virtual-schema</artifactId>
- <version>2.0.3</version>
+ <version>2.0.4</version>
  <name>Virtual Schema for document data in files on Azure Blob Storage</name>
  <description>Adapter for document data access from files from Azure Blob Storage.</description>
  <url>https://github.com/exasol/azure-blob-storage-document-files-virtual-schema/</url>
@@ -90,7 +90,7 @@
  <dependency>
  <groupId>com.exasol</groupId>
  <artifactId>udf-debugging-java</artifactId>
- <version>0.6.12</version>
+ <version>0.6.13</version>
  <scope>test</scope>
  </dependency>
  <dependency>
@@ -188,7 +188,7 @@
  <parent>
  <artifactId>azure-blob-storage-document-files-virtual-schema-generated-parent</artifactId>
  <groupId>com.exasol</groupId>
- <version>2.0.3</version>
+ <version>2.0.4</version>
  <relativePath>pk_generated_parent.pom</relativePath>
  </parent>
 </project>

src/test/java/com/exasol/adapter/document/files/AbsDocumentFilesAdapterIT.java

@@ -29,15 +29,15 @@
 @Testcontainers
 class AbsDocumentFilesAdapterIT extends AbstractDocumentFilesAdapterIT {
  private static IntegrationTestSetup setup;
- private static TestContainer testContainer;
+ private static AbsTestContainer testContainer;
  private static AbsTestSetup absTestSetup;
 
  @BeforeAll
  static void beforeAll() throws Exception {
  final ExasolTestSetup exasolTestSetup = new ExasolTestSetupFactory(
  Path.of("cloudSetup/generated/testConfig.json")).getTestSetup();
  absTestSetup = getAbsTestSetup(exasolTestSetup);
- testContainer = new TestContainer(absTestSetup);
+ testContainer = AbsTestContainer.create(absTestSetup);
  setup = new IntegrationTestSetup(exasolTestSetup, absTestSetup, testContainer.getBlobContainerClient());
  }
 

src/test/java/com/exasol/adapter/document/files/AbsRandomAccessInputStreamIT.java

@@ -11,13 +11,13 @@ class AbsRandomAccessInputStreamIT extends RandomAccessInputStreamTestBase {
  private static final String TEST_DATA_KEY = "TEST_DATA";
  long testDataLength;
  private AbsTestSetup testSetup;
- private TestContainer testContainer;
+ private AbsTestContainer testContainer;
  private BlobClient blob;
 
  @Override
  protected void prepareTestSetup(final byte[] bytes) {
  this.testSetup = new LocalAbsTestSetup();
- this.testContainer = new TestContainer(this.testSetup);
+ this.testContainer = AbsTestContainer.create(this.testSetup);
  this.testDataLength = bytes.length;
  this.blob = this.testContainer.getBlobContainerClient().getBlobClient(TEST_DATA_KEY);
  this.blob.upload(BinaryData.fromBytes(bytes));
@@ -33,4 +33,4 @@ protected void cleanupTestSetup() {
  protected RandomAccessInputStream getSeekableInputStream() {
  return new AbsRandomAccessInputStream(this.blob, this.testDataLength);
  }
-}
+}

src/test/java/com/exasol/adapter/document/files/AbsRemoteFileContentIT.java

@@ -20,13 +20,13 @@ class AbsRemoteFileContentIT {
  private static final String TEST_DATA_KEY = "TEST_DATA";
  private static AbsTestSetup testSetup;
  private static RemoteFileContent remoteFileContent;
- private static TestContainer testBucket;
+ private static AbsTestContainer testBucket;
  private static ExecutorServiceFactory executorServiceFactory;
 
  @BeforeAll
  static void beforeAll() {
  testSetup = new LocalAbsTestSetup();
- testBucket = new TestContainer(testSetup);
+ testBucket = AbsTestContainer.create(testSetup);
  final var blobContainerClient = testBucket.getBlobContainerClient();
  blobContainerClient.getBlobClient(TEST_DATA_KEY).upload(BinaryData.fromBytes(TEST_DATA_VALUE.getBytes()));
  executorServiceFactory = new ExecutorServiceFactory();
@@ -71,4 +71,4 @@ void testCancelAsyncLoading() {
  future.cancel(true);
  assertTrue(future.isCancelled());
  }
-}
+}

src/test/java/com/exasol/adapter/document/files/AbsRemoteFileFinderIT.java

@@ -7,14 +7,14 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import com.azure.core.util.BinaryData;
 import org.junit.jupiter.api.*;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.CsvSource;
 
-import com.exasol.adapter.document.files.connection.AbsConnectionProperties;
+import com.azure.core.util.BinaryData;
 import com.exasol.adapter.document.files.abstestsetup.AbsTestSetup;
 import com.exasol.adapter.document.files.abstestsetup.LocalAbsTestSetup;
+import com.exasol.adapter.document.files.connection.AbsConnectionProperties;
 import com.exasol.adapter.document.files.stringfilter.wildcardexpression.WildcardExpression;
 
 @Tag("integration")
@@ -24,19 +24,20 @@ class AbsRemoteFileFinderIT {
  private static final String CONTENT_2 = "content-2";
  private static final String CONTENT_OTHER = "other";
  private static AbsConnectionProperties connectionInformation;
- private static TestContainer testContainer;
+ private static AbsTestContainer testContainer;
 
  @BeforeAll
  static void beforeAll() {
- testContainer = new TestContainer(TEST_SETUP);
- var blobContainerClient = testContainer.getBlobContainerClient();
- var file1= blobContainerClient.getBlobClient("file-1.json");
+ testContainer = AbsTestContainer.create(TEST_SETUP);
+ final var blobContainerClient = testContainer.getBlobContainerClient();
+ final var file1 = blobContainerClient.getBlobClient("file-1.json");
  file1.upload(BinaryData.fromBytes(CONTENT_1.getBytes()));
- var file2 = blobContainerClient.getBlobClient("file-2.json");
+ final var file2 = blobContainerClient.getBlobClient("file-2.json");
  file2.upload(BinaryData.fromBytes(CONTENT_2.getBytes()));
- var file3 = blobContainerClient.getBlobClient("other.json");
+ final var file3 = blobContainerClient.getBlobClient("other.json");
  file3.upload(BinaryData.fromBytes(CONTENT_OTHER.getBytes()));
- connectionInformation = AbsConnectionProperties.builder().absContainerName(testContainer.getBlobContainerClient().getBlobContainerName())
+ connectionInformation = AbsConnectionProperties.builder()
+ .absContainerName(testContainer.getBlobContainerClient().getBlobContainerName())
  .absStorageAccountConnectionString(TEST_SETUP.getStorageAccountConnectionString()).build();
  }
 
@@ -85,4 +86,4 @@ private String readFirstLine(final InputStream stream) {
  throw new IllegalArgumentException("", exception);
  }
  }
-}
+}

src/test/java/com/exasol/adapter/document/files/AbsRemoteFileFinderOnlineIT.java

@@ -7,28 +7,29 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import com.azure.core.util.BinaryData;
 import org.junit.jupiter.api.*;
 
-import com.exasol.adapter.document.files.connection.AbsConnectionProperties;
+import com.azure.core.util.BinaryData;
 import com.exasol.adapter.document.files.abstestsetup.AbsTestSetup;
 import com.exasol.adapter.document.files.abstestsetup.OnlineAbsTestSetup;
+import com.exasol.adapter.document.files.connection.AbsConnectionProperties;
 import com.exasol.adapter.document.files.stringfilter.wildcardexpression.WildcardExpression;
 
 @Tag("integration")
 class AbsRemoteFileFinderOnlineIT {
  private static final AbsTestSetup TEST_SETUP = new OnlineAbsTestSetup();
  private static final String CONTENT_1 = "content-1";
  private static AbsConnectionProperties connectionInformation;
- private static TestContainer testContainer;
+ private static AbsTestContainer testContainer;
 
  @BeforeAll
  static void beforeAll() {
- testContainer = new TestContainer(TEST_SETUP);
- var blobContainerClient = testContainer.getBlobContainerClient();
- var file1= blobContainerClient.getBlobClient("file-1.json");
+ testContainer = AbsTestContainer.create(TEST_SETUP);
+ final var blobContainerClient = testContainer.getBlobContainerClient();
+ final var file1 = blobContainerClient.getBlobClient("file-1.json");
  file1.upload(BinaryData.fromBytes(CONTENT_1.getBytes()));
- connectionInformation = AbsConnectionProperties.builder().absContainerName(testContainer.getBlobContainerClient().getBlobContainerName())
+ connectionInformation = AbsConnectionProperties.builder()
+ .absContainerName(testContainer.getBlobContainerClient().getBlobContainerName())
  .absStorageAccountConnectionString(TEST_SETUP.getStorageAccountConnectionString()).build();
  }
 
@@ -58,4 +59,4 @@ private String readFirstLine(final InputStream stream) {
  throw new IllegalArgumentException("", exception);
  }
  }
-}
+}

src/test/java/com/exasol/adapter/document/files/TestContainer.java → src/test/java/com/exasol/adapter/document/files/AbsTestContainer.java

@@ -5,15 +5,20 @@
 import com.azure.storage.blob.models.BlobItem;
 import com.exasol.adapter.document.files.abstestsetup.AbsTestSetup;
 
-public class TestContainer implements AutoCloseable {
+public class AbsTestContainer implements AutoCloseable {
  private final BlobContainerClient blobContainerClient;
  private final String blobContainerName;
 
- public TestContainer(final AbsTestSetup testSetup) {
+ private AbsTestContainer(final BlobContainerClient containerClient, final String containerName) {
+ this.blobContainerClient = containerClient;
+ this.blobContainerName = containerName;
+ }
+
+ public static AbsTestContainer create(final AbsTestSetup testSetup) {
  final String containerName = "abs-document-vs-test-" + System.currentTimeMillis();
  final BlobServiceClient absClient = testSetup.getAbsClient();
- this.blobContainerClient = absClient.createBlobContainer(containerName);
- this.blobContainerName = containerName;
+ final BlobContainerClient containerClient = absClient.createBlobContainer(containerName);
+ return new AbsTestContainer(containerClient, containerName);
  }
 
  // https://github.com/Azure/azure-sdk-for-java/issues/10180

src/test/java/com/exasol/adapter/document/files/AbsTestContainerIT.java

@@ -0,0 +1,64 @@
+package com.exasol.adapter.document.files;
+
+import static java.util.stream.Collectors.toList;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.empty;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+import com.azure.core.util.BinaryData;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.models.BlobItem;
+import com.exasol.adapter.document.files.abstestsetup.AbsTestSetup;
+import com.exasol.adapter.document.files.abstestsetup.OnlineAbsTestSetup;
+
+@Tag("integration")
+class AbsTestContainerIT {
+ static AbsTestSetup absTestSetup = new OnlineAbsTestSetup();
+
+ @Test
+ void emptyFilledContainer() {
+ try (final AbsTestContainer testContainer = AbsTestContainer.create(absTestSetup)) {
+ final BlobContainerClient client = testContainer.getBlobContainerClient();
+ client.getBlobClient("blobName").upload(BinaryData.fromString("blob content"));
+ assertThat(listBlobNames(client), contains("blobName"));
+ testContainer.empty();
+ assertThat(listBlobNames(client), empty());
+ }
+ }
+
+ // Note: This test is expected to fail when Azure Blob Storage Gen2 is enabled.
+ @Test
+ void emptyFilledContainerWithDirectory() {
+ try (final AbsTestContainer testContainer = AbsTestContainer.create(absTestSetup)) {
+ final BlobContainerClient client = testContainer.getBlobContainerClient();
+ client.getBlobClient("dir/blobName").upload(BinaryData.fromString("blob content"));
+ assertThat(listBlobNames(client), contains("dir/blobName"));
+ testContainer.empty();
+ assertThat(listBlobNames(client), empty());
+ }
+ }
+
+ @Test
+ void checkBlobStorageVersion() {
+ try (final AbsTestContainer testContainer = AbsTestContainer.create(absTestSetup)) {
+ final BlobContainerClient client = testContainer.getBlobContainerClient();
+ client.getBlobClient("dir/blobName").upload(BinaryData.fromString("blob content"));
+ final List<String> blobNames = listBlobNames(client);
+ if (blobNames.size() > 1) {
+ fail("Expected exactly one blob, but found " + blobNames.size() + " blobs.: " + blobNames
+ + ". This usually means that Azure Blob Storage Gen2 is enabled. Please create a new storage account without Gen2 enabled.");
+ }
+ assertThat(listBlobNames(client), contains("dir/blobName"));
+ }
+ }
+
+ private List<String> listBlobNames(final BlobContainerClient client) {
+ return client.listBlobs().stream().map(BlobItem::getName).collect(toList());
+ }
+}

src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java

@@ -26,7 +26,7 @@
 import jakarta.json.*;
 
 public class IntegrationTestSetup implements AutoCloseable {
- private static final String ADAPTER_JAR = "document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.3.jar";
+ private static final String ADAPTER_JAR = "document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.4.jar";
  private final ExasolTestSetup exasolTestSetup;
  private final Connection exasolConnection;
  private final Statement exasolStatement;