Skip to content

Commit a2610e2

Browse files
bilaharithbilaharith
authored
HADOOP-17183. ABFS: Enabling checkaccess on ABFS
- Contributed by Bilahari T H
1 parent 1d5ccc7 commit a2610e2

2 files changed

Lines changed: 72 additions & 40 deletions

File tree

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public final class FileSystemConfigurations {
8484
public static final boolean DEFAULT_ENABLE_HTTPS = true;
8585

8686
public static final boolean DEFAULT_USE_UPN = false;
87-
public static final boolean DEFAULT_ENABLE_CHECK_ACCESS = false;
87+
public static final boolean DEFAULT_ENABLE_CHECK_ACCESS = true;
8888
public static final boolean DEFAULT_ABFS_LATENCY_TRACK = false;
8989
public static final long DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS = 120;
9090

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java‎

Lines changed: 71 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,18 @@
1717
*/
1818
package org.apache.hadoop.fs.azurebfs;
1919

20-
import com.google.common.collect.Lists;
21-
2220
import java.io.FileNotFoundException;
2321
import java.io.IOException;
22+
import java.lang.reflect.Field;
2423
import java.util.List;
2524

25+
import com.google.common.collect.Lists;
2626
import org.junit.Assume;
2727
import org.junit.Test;
28+
import org.mockito.Mockito;
2829

2930
import org.apache.hadoop.conf.Configuration;
31+
import org.apache.hadoop.fs.azurebfs.services.AuthType;
3032
import org.apache.hadoop.fs.FileSystem;
3133
import org.apache.hadoop.fs.Path;
3234
import org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers;
@@ -37,16 +39,27 @@
3739
import org.apache.hadoop.security.AccessControlException;
3840

3941
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
42+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
4043
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_CHECK_ACCESS;
4144
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID;
4245
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET;
4346
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID;
4447
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_ID;
4548
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_SECRET;
4649
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT;
50+
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
4751

4852
/**
4953
* Test cases for AzureBlobFileSystem.access()
54+
*
55+
* Some of the tests in this class requires the following 3 configs set in the
56+
* test config file.
57+
* fs.azure.account.test.oauth2.client.id
58+
* fs.azure.account.test.oauth2.client.secret
59+
* fs.azure.check.access.testuser.guid
60+
* Set the above client id, secret and guid of a service principal which has no
61+
* RBAC on the account.
62+
*
5063
*/
5164
public class ITestAzureBlobFileSystemCheckAccess
5265
extends AbstractAbfsIntegrationTest {
@@ -66,31 +79,29 @@ public ITestAzureBlobFileSystemCheckAccess() throws Exception {
6679
this.isCheckAccessEnabled = getConfiguration().isCheckAccessEnabled();
6780
this.isHNSEnabled = getConfiguration()
6881
.getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
82+
setTestUserFs();
6983
}
7084

7185
private void setTestUserFs() throws Exception {
7286
if (this.testUserFs != null) {
7387
return;
7488
}
75-
String orgClientId = getConfiguration().get(FS_AZURE_BLOB_FS_CLIENT_ID);
76-
String orgClientSecret = getConfiguration()
77-
.get(FS_AZURE_BLOB_FS_CLIENT_SECRET);
78-
Boolean orgCreateFileSystemDurungInit = getConfiguration()
79-
.getBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
80-
getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_ID,
81-
getConfiguration().get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID));
82-
getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_SECRET, getConfiguration()
83-
.get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET));
89+
final String testClientIdConfKey =
90+
FS_AZURE_BLOB_FS_CLIENT_ID + "." + getAccountName();
91+
final String testClientId = getConfiguration()
92+
.getString(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID, "");
93+
getRawConfiguration().set(testClientIdConfKey, testClientId);
94+
final String clientSecretConfKey =
95+
FS_AZURE_BLOB_FS_CLIENT_SECRET + "." + getAccountName();
96+
final String testClientSecret = getConfiguration()
97+
.getString(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET, "");
98+
getRawConfiguration().set(clientSecretConfKey, testClientSecret);
8499
getRawConfiguration()
85100
.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
86101
false);
87-
FileSystem fs = FileSystem.newInstance(getRawConfiguration());
88-
getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_ID, orgClientId);
89-
getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_SECRET, orgClientSecret);
90-
getRawConfiguration()
91-
.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
92-
orgCreateFileSystemDurungInit);
93-
this.testUserFs = fs;
102+
getRawConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME,
103+
AuthType.OAuth.name());
104+
this.testUserFs = FileSystem.newInstance(getRawConfiguration());
94105
}
95106

96107
@Test(expected = IllegalArgumentException.class)
@@ -100,15 +111,15 @@ public void testCheckAccessWithNullPath() throws IOException {
100111

101112
@Test(expected = NullPointerException.class)
102113
public void testCheckAccessForFileWithNullFsAction() throws Exception {
103-
assumeHNSAndCheckAccessEnabled();
114+
Assume.assumeTrue(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT + " is false",
115+
isHNSEnabled);
104116
// NPE when trying to convert null FsAction enum
105117
superUserFs.access(new Path("test.txt"), null);
106118
}
107119

108120
@Test(expected = FileNotFoundException.class)
109121
public void testCheckAccessForNonExistentFile() throws Exception {
110-
assumeHNSAndCheckAccessEnabled();
111-
setTestUserFs();
122+
checkPrerequisites();
112123
Path nonExistentFile = setupTestDirectoryAndUserAccess(
113124
"/nonExistentFile1.txt", FsAction.ALL);
114125
superUserFs.delete(nonExistentFile, true);
@@ -153,15 +164,36 @@ public void testCheckAccessForAccountWithoutNS() throws Exception {
153164
getConfiguration()
154165
.getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true));
155166
Assume.assumeTrue(FS_AZURE_ENABLE_CHECK_ACCESS + " is false",
156-
isCheckAccessEnabled);
157-
setTestUserFs();
167+
isCheckAccessEnabled);
168+
checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID);
169+
checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET);
170+
checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID);
171+
172+
// When the driver does not know if the account is HNS enabled or not it
173+
// makes a server call and fails
174+
intercept(AccessControlException.class,
175+
"\"This request is not authorized to perform this operation using "
176+
+ "this permission.\", 403",
177+
() -> testUserFs.access(new Path("/"), FsAction.READ));
178+
179+
// When the driver has already determined if the account is HNS enabled
180+
// or not, and as the account is non HNS the AzureBlobFileSystem#access
181+
// acts as noop
182+
AzureBlobFileSystemStore mockAbfsStore =
183+
Mockito.mock(AzureBlobFileSystemStore.class);
184+
Mockito.when(mockAbfsStore.getIsNamespaceEnabled()).thenReturn(true);
185+
Field abfsStoreField = AzureBlobFileSystem.class.getDeclaredField(
186+
"abfsStore");
187+
abfsStoreField.setAccessible(true);
188+
abfsStoreField.set(testUserFs, mockAbfsStore);
158189
testUserFs.access(new Path("/"), FsAction.READ);
190+
191+
superUserFs.access(new Path("/"), FsAction.READ);
159192
}
160193

161194
@Test
162195
public void testFsActionNONE() throws Exception {
163-
assumeHNSAndCheckAccessEnabled();
164-
setTestUserFs();
196+
checkPrerequisites();
165197
Path testFilePath = setupTestDirectoryAndUserAccess("/test2.txt",
166198
FsAction.NONE);
167199
assertInaccessible(testFilePath, FsAction.EXECUTE);
@@ -175,8 +207,7 @@ public void testFsActionNONE() throws Exception {
175207

176208
@Test
177209
public void testFsActionEXECUTE() throws Exception {
178-
assumeHNSAndCheckAccessEnabled();
179-
setTestUserFs();
210+
checkPrerequisites();
180211
Path testFilePath = setupTestDirectoryAndUserAccess("/test3.txt",
181212
FsAction.EXECUTE);
182213
assertAccessible(testFilePath, FsAction.EXECUTE);
@@ -191,8 +222,7 @@ public void testFsActionEXECUTE() throws Exception {
191222

192223
@Test
193224
public void testFsActionREAD() throws Exception {
194-
assumeHNSAndCheckAccessEnabled();
195-
setTestUserFs();
225+
checkPrerequisites();
196226
Path testFilePath = setupTestDirectoryAndUserAccess("/test4.txt",
197227
FsAction.READ);
198228
assertAccessible(testFilePath, FsAction.READ);
@@ -207,8 +237,7 @@ public void testFsActionREAD() throws Exception {
207237

208238
@Test
209239
public void testFsActionWRITE() throws Exception {
210-
assumeHNSAndCheckAccessEnabled();
211-
setTestUserFs();
240+
checkPrerequisites();
212241
Path testFilePath = setupTestDirectoryAndUserAccess("/test5.txt",
213242
FsAction.WRITE);
214243
assertAccessible(testFilePath, FsAction.WRITE);
@@ -223,8 +252,7 @@ public void testFsActionWRITE() throws Exception {
223252

224253
@Test
225254
public void testFsActionREADEXECUTE() throws Exception {
226-
assumeHNSAndCheckAccessEnabled();
227-
setTestUserFs();
255+
checkPrerequisites();
228256
Path testFilePath = setupTestDirectoryAndUserAccess("/test6.txt",
229257
FsAction.READ_EXECUTE);
230258
assertAccessible(testFilePath, FsAction.EXECUTE);
@@ -239,8 +267,7 @@ public void testFsActionREADEXECUTE() throws Exception {
239267

240268
@Test
241269
public void testFsActionWRITEEXECUTE() throws Exception {
242-
assumeHNSAndCheckAccessEnabled();
243-
setTestUserFs();
270+
checkPrerequisites();
244271
Path testFilePath = setupTestDirectoryAndUserAccess("/test7.txt",
245272
FsAction.WRITE_EXECUTE);
246273
assertAccessible(testFilePath, FsAction.EXECUTE);
@@ -255,8 +282,7 @@ public void testFsActionWRITEEXECUTE() throws Exception {
255282

256283
@Test
257284
public void testFsActionALL() throws Exception {
258-
assumeHNSAndCheckAccessEnabled();
259-
setTestUserFs();
285+
checkPrerequisites();
260286
Path testFilePath = setupTestDirectoryAndUserAccess("/test8.txt",
261287
FsAction.ALL);
262288
assertAccessible(testFilePath, FsAction.EXECUTE);
@@ -268,13 +294,19 @@ public void testFsActionALL() throws Exception {
268294
assertAccessible(testFilePath, FsAction.ALL);
269295
}
270296

271-
private void assumeHNSAndCheckAccessEnabled() {
297+
private void checkPrerequisites() {
272298
Assume.assumeTrue(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT + " is false",
273299
isHNSEnabled);
274300
Assume.assumeTrue(FS_AZURE_ENABLE_CHECK_ACCESS + " is false",
275301
isCheckAccessEnabled);
302+
checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID);
303+
checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET);
304+
checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID);
305+
}
276306

277-
Assume.assumeNotNull(getRawConfiguration().get(FS_AZURE_BLOB_FS_CLIENT_ID));
307+
private void checkIfConfigIsSet(String configKey){
308+
AbfsConfiguration conf = getConfiguration();
309+
Assume.assumeNotNull(configKey + " config missing", conf.get(configKey));
278310
}
279311

280312
private void assertAccessible(Path testFilePath, FsAction fsAction)

0 commit comments

Comments
 (0)