-
Notifications
You must be signed in to change notification settings - Fork 9.2k
HADOOP-18960: [ABFS] Making Contract tests run in sequential and Other Test Fixes #7104
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 3 commits
19e9aa2
acca3fb
d755255
7ba0a41
0964df6
1630291
9419e36
0dc4ce0
5a20358
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -30,11 +30,14 @@ | |
| import org.apache.hadoop.fs.FileSystem; | ||
| import org.apache.hadoop.fs.Path; | ||
|
|
||
| import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; | ||
| import static org.apache.hadoop.test.LambdaTestUtils.intercept; | ||
|
|
||
| /** | ||
|
|
@@ -140,17 +143,7 @@ public void testCustomTokenFetchRetryCount() throws Exception { | |
|
|
||
| public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exception { | ||
| AzureBlobFileSystem fs = this.getFileSystem(); | ||
|
|
||
| Configuration config = new Configuration(this.getRawConfiguration()); | ||
| String accountName = config.get("fs.azure.abfs.account.name"); | ||
| // Setup to configure custom token provider. | ||
| config.set("fs.azure.account.auth.type." + accountName, "Custom"); | ||
| config.set("fs.azure.account.oauth.provider.type." + accountName, "org.apache.hadoop.fs" | ||
| + ".azurebfs.oauth2.RetryTestTokenProvider"); | ||
| config.set("fs.azure.custom.token.fetch.retry.count", Integer.toString(numOfRetries)); | ||
| // Stop filesystem creation as it will lead to calls to store. | ||
| config.set("fs.azure.createRemoteFileSystemDuringInitialization", "false"); | ||
|
|
||
| Configuration config = getEntries(numOfRetries); | ||
| try (final AzureBlobFileSystem fs1 = | ||
| (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), | ||
| config)) { | ||
|
|
@@ -174,18 +167,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce | |
|
|
||
| @Test | ||
| public void testAuthFailException() throws Exception { | ||
| Configuration config = new Configuration(getRawConfiguration()); | ||
| String accountName = config | ||
| .get(FS_AZURE_ABFS_ACCOUNT_NAME); | ||
| // Setup to configure custom token provider | ||
| config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT | ||
| + accountName, "Custom"); | ||
| config.set( | ||
| FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName, | ||
| RETRY_TEST_TOKEN_PROVIDER); | ||
| // Stop filesystem creation as it will lead to calls to store. | ||
| config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); | ||
|
|
||
| Configuration config = getEntries(0); | ||
| final AzureBlobFileSystem fs = getFileSystem(config); | ||
| AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> { | ||
| fs.getFileStatus(new Path("/")); | ||
|
|
@@ -201,4 +183,20 @@ public void testAuthFailException() throws Exception { | |
| .describedAs("Incorrect error message: " + errorDesc) | ||
| .contains("Auth failure: "); | ||
| } | ||
|
|
||
| private Configuration getEntries(final int numOfRetries) { | ||
| Configuration config = new Configuration(this.getRawConfiguration()); | ||
| String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME); | ||
| // Setup to configure custom token provider. | ||
| config.set(accountProperty(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, accountName), "Custom"); | ||
| config.set(accountProperty(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, accountName), | ||
| RETRY_TEST_TOKEN_PROVIDER); | ||
| config.set(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, Integer.toString( | ||
| numOfRetries)); | ||
| // Stop filesystem creation as it will lead to calls to store. | ||
| config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); | ||
|
||
| config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean( | ||
|
||
| FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + ""); | ||
| return config; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -28,14 +28,16 @@ | |
| import org.apache.hadoop.conf.Configuration; | ||
|
|
||
| import org.apache.hadoop.fs.Path; | ||
| import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; | ||
| import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; | ||
| import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException; | ||
| import org.apache.hadoop.fs.azurebfs.extensions.MockSASTokenProvider; | ||
| import org.apache.hadoop.fs.azurebfs.services.AuthType; | ||
| import org.apache.hadoop.fs.permission.FsAction; | ||
| import org.apache.hadoop.fs.permission.FsPermission; | ||
|
|
||
| import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.MOCK_SASTOKENPROVIDER_FAIL_INIT; | ||
| import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN; | ||
| import static org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers.aclEntry; | ||
|
|
@@ -65,8 +67,8 @@ public void setup() throws Exception { | |
| TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); | ||
| Assume.assumeTrue(isHNSEnabled); | ||
| loadConfiguredFileSystem(); | ||
| this.getConfiguration().set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS); | ||
| this.getConfiguration().set(ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, "SAS"); | ||
| this.getConfiguration().set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS); | ||
|
||
| this.getConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, "SAS"); | ||
| super.setup(); | ||
| } | ||
|
|
||
|
|
@@ -75,13 +77,15 @@ public void testSASTokenProviderInitializeException() throws Exception { | |
| final AzureBlobFileSystem fs = this.getFileSystem(); | ||
|
|
||
| final AzureBlobFileSystem testFs = new AzureBlobFileSystem(); | ||
| Configuration testConfig = this.getConfiguration().getRawConfiguration(); | ||
| testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); | ||
| Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); | ||
| testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); | ||
| testConfig.set(MOCK_SASTOKENPROVIDER_FAIL_INIT, "true"); | ||
| // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check. | ||
| testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); | ||
|
|
||
| intercept(SASTokenProviderException.class, | ||
| ()-> { | ||
| testFs.initialize(fs.getUri(), this.getConfiguration().getRawConfiguration()); | ||
| testFs.initialize(fs.getUri(), testConfig); | ||
| }); | ||
| } | ||
|
|
||
|
|
@@ -90,12 +94,13 @@ public void testSASTokenProviderEmptySASToken() throws Exception { | |
| final AzureBlobFileSystem fs = this.getFileSystem(); | ||
|
|
||
| final AzureBlobFileSystem testFs = new AzureBlobFileSystem(); | ||
| Configuration testConfig = this.getConfiguration().getRawConfiguration(); | ||
| testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); | ||
| Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); | ||
| testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); | ||
| testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); | ||
| // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check. | ||
| testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); | ||
|
|
||
| testFs.initialize(fs.getUri(), | ||
| this.getConfiguration().getRawConfiguration()); | ||
| testFs.initialize(fs.getUri(), testConfig); | ||
| intercept(SASTokenProviderException.class, | ||
| () -> { | ||
| testFs.create(new org.apache.hadoop.fs.Path("/testFile")).close(); | ||
|
|
@@ -107,10 +112,13 @@ public void testSASTokenProviderNullSASToken() throws Exception { | |
| final AzureBlobFileSystem fs = this.getFileSystem(); | ||
|
|
||
| final AzureBlobFileSystem testFs = new AzureBlobFileSystem(); | ||
| Configuration testConfig = this.getConfiguration().getRawConfiguration(); | ||
| testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); | ||
| Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); | ||
| testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); | ||
| testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); | ||
|
||
| // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check. | ||
| testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); | ||
|
||
|
|
||
| testFs.initialize(fs.getUri(), this.getConfiguration().getRawConfiguration()); | ||
| testFs.initialize(fs.getUri(), testConfig); | ||
| intercept(SASTokenProviderException.class, | ||
| ()-> { | ||
| testFs.create(new org.apache.hadoop.fs.Path("/testFile")).close(); | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -68,6 +68,9 @@ protected Configuration createConfiguration() { | |
| protected AbstractFSContract createContract(final Configuration conf) { | ||
| conf.setInt(AZURE_READ_AHEAD_RANGE, MIN_BUFFER_SIZE); | ||
| conf.setInt(AZURE_READ_BUFFER_SIZE, MIN_BUFFER_SIZE); | ||
| // Disabling cache to make sure new configs are picked up. | ||
|
||
| conf.set("fs.abfss.impl.disable.cache", "true"); | ||
| conf.set("fs.abfs.impl.disable.cache", "true"); | ||
|
||
| return new AbfsFileSystemContract(conf, isSecure); | ||
| } | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: javadoc
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Added