Skip to content

HADOOP-19146 noaa-cors-pds bucket access with global endpoint fails #6723

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
steveloughran merged 3 commits into from
Apr 30, 2024
Merged

HADOOP-19146 noaa-cors-pds bucket access with global endpoint fails #6723

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f3b15ae
HADOOP-19146 noaa-cors-pds bucket access with global endpoint fails
virajjasani Apr 11, 2024
dcb8853
isUsingDefaultExternalDataFile
virajjasani Apr 21, 2024
60bd00b
keep prefetch removal
virajjasani Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions ...ols/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING;
import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.CONSTRUCTOR_EXCEPTION;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.*;

Expand Down Expand Up @@ -207,9 +208,11 @@ public void testBadCredentialsWithRemap() throws Exception {
@Test
public void testAnonymousProvider() throws Exception {
Configuration conf = createConf(AnonymousAWSCredentialsProvider.class);
removeBaseAndBucketOverrides(conf,
ENDPOINT);
conf.set(ENDPOINT, CENTRAL_ENDPOINT);
if (isUsingDefaultExternalDataFile(conf)) {
removeBaseAndBucketOverrides(conf,
ENDPOINT);
conf.set(ENDPOINT, CENTRAL_ENDPOINT);
}
Path testFile = getExternalData(conf);
try (FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf)) {
Assertions.assertThat(fs)
Expand Down
10 changes: 4 additions & 6 deletions hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,6 @@
import org.apache.hadoop.fs.store.audit.AuditSpan;

import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


import java.io.IOException;
import java.util.ArrayList;
Expand All @@ -47,6 +44,7 @@
import static org.apache.hadoop.fs.s3a.S3ATestUtils.isBulkDeleteEnabled;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.failIf;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireDefaultExternalData;
import static org.apache.hadoop.test.LambdaTestUtils.*;
import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator;
Expand All @@ -56,15 +54,15 @@
* ITest for failure handling, primarily multipart deletion.
*/
public class ITestS3AFailureHandling extends AbstractS3ATestBase {
private static final Logger LOG =
LoggerFactory.getLogger(ITestS3AFailureHandling.class);

@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
S3ATestUtils.disableFilesystemCaching(conf);
conf.setBoolean(Constants.ENABLE_MULTI_DELETE, true);
removeBaseAndBucketOverrides(conf, Constants.ENDPOINT);
if (isUsingDefaultExternalDataFile(conf)) {
removeBaseAndBucketOverrides(conf, Constants.ENDPOINT);
}
return conf;
}

Expand Down
9 changes: 6 additions & 3 deletions ...ools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_SIZE_KEY;
import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;

/**
Expand Down Expand Up @@ -85,9 +86,11 @@ public void setUp() throws Exception {
@Override
public Configuration createConfiguration() {
Configuration configuration = super.createConfiguration();
S3ATestUtils.removeBaseAndBucketOverrides(configuration,
PREFETCH_ENABLED_KEY,
ENDPOINT);
if (isUsingDefaultExternalDataFile(configuration)) {
S3ATestUtils.removeBaseAndBucketOverrides(configuration,
PREFETCH_ENABLED_KEY,
ENDPOINT);
}
configuration.setBoolean(PREFETCH_ENABLED_KEY, true);
// use a small block size unless explicitly set in the test config.
configuration.setInt(PREFETCH_BLOCK_SIZE_KEY, BLOCK_SIZE);
Expand Down
7 changes: 5 additions & 2 deletions ...adoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import static org.apache.hadoop.fs.s3a.auth.delegation.MiniKerberizedHadoopCluster.assertSecurityEnabled;
import static org.apache.hadoop.fs.s3a.auth.delegation.MiniKerberizedHadoopCluster.closeUserFileSystems;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getOrcData;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireAnonymousDataPath;

/**
Expand Down Expand Up @@ -253,8 +254,10 @@ public void testCommonCrawlLookup() throws Throwable {
public void testJobSubmissionCollectsTokens() throws Exception {
describe("Mock Job test");
JobConf conf = new JobConf(getConfiguration());
removeBaseAndBucketOverrides(conf,
Constants.ENDPOINT);
if (isUsingDefaultExternalDataFile(conf)) {
removeBaseAndBucketOverrides(conf,
Constants.ENDPOINT);
}

// the input here is the external file; which lets
// us differentiate source URI from dest URI
Expand Down
15 changes: 10 additions & 5 deletions hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Uploads;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;

/**
* Test S3Guard Tool CLI commands.
Expand All @@ -62,9 +63,11 @@ public class ITestS3GuardTool extends AbstractS3GuardToolTestBase {
@Test
public void testExternalBucketRequireUnencrypted() throws Throwable {
Configuration conf = getConfiguration();
removeBaseAndBucketOverrides(conf,
S3_ENCRYPTION_ALGORITHM,
ENDPOINT);
if (isUsingDefaultExternalDataFile(conf)) {
removeBaseAndBucketOverrides(conf,
S3_ENCRYPTION_ALGORITHM,
ENDPOINT);
}
run(conf, BucketInfo.NAME,
"-" + BucketInfo.ENCRYPTION_FLAG, "none",
externalBucket());
Expand All @@ -86,8 +89,10 @@ private String externalBucket() {
@Test
public void testExternalBucketRequireEncrypted() throws Throwable {
Configuration conf = getConfiguration();
removeBaseAndBucketOverrides(conf,
ENDPOINT);
if (isUsingDefaultExternalDataFile(conf)) {
removeBaseAndBucketOverrides(conf,
ENDPOINT);
}
runToFailure(E_BAD_STATE,
conf,
BucketInfo.NAME,
Expand Down
10 changes: 7 additions & 3 deletions ...doop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getInputStreamStatistics;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getS3AInputStream;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticMinimum;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMaximumStatistic;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMeanStatistic;
Expand Down Expand Up @@ -99,9 +100,12 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
@Override
protected Configuration createScaleConfiguration() {
Configuration conf = super.createScaleConfiguration();
S3ATestUtils.removeBaseAndBucketOverrides(conf,
PREFETCH_ENABLED_KEY,
ENDPOINT);
if (isUsingDefaultExternalDataFile(conf)) {
S3ATestUtils.removeBaseAndBucketOverrides(
conf,
PREFETCH_ENABLED_KEY,
Copy link
Contributor

@steveloughran steveloughran Apr 22, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

prefetch key must always be removed

Copy link
Contributor Author

@virajjasani virajjasani Apr 23, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

trivial oversight from my side, fixed it

ENDPOINT);
}
conf.setBoolean(PREFETCH_ENABLED_KEY, false);
return conf;
}
Expand Down
12 changes: 12 additions & 0 deletions ...-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,18 @@ public static String requireDefaultExternalDataFile(Configuration conf) {
return filename;
}

/**
* To determine whether {@value S3ATestConstants#KEY_CSVTEST_FILE} is configured to be
* different from the default external file.
*
* @param conf Configuration object.
* @return True if the default external data file is being used.
*/
public static boolean isUsingDefaultExternalDataFile(final Configuration conf) {
final String filename = getExternalData(conf).toUri().toString();
return DEFAULT_EXTERNAL_FILE.equals(filename);
}

/**
* Get the test external file; assume() that it is not modified (i.e. we haven't
* switched to a new storage infrastructure where the bucket is no longer
Expand Down