Skip to content

Commit af594f4

Browse files
Merge branch 'trunk' into HADOOP-19280_AbfsClientTimer
2 parents efb4eef + d8ca2db commit af594f4

411 files changed

Lines changed: 14609 additions & 3005 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

LICENSE-binary

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -233,22 +233,22 @@ com.google:guice:5.1.0
233233
com.google:guice-servlet:5.1.0
234234
com.google.api.grpc:proto-google-common-protos:1.0.0
235235
com.google.code.gson:2.9.0
236-
com.google.errorprone:error_prone_annotations:2.2.0
237-
com.google.j2objc:j2objc-annotations:1.1
236+
com.google.errorprone:error_prone_annotations:2.5.1
237+
com.google.j2objc:j2objc-annotations:1.3
238238
com.google.json-simple:json-simple:1.1.1
239239
com.google.guava:failureaccess:1.0
240240
com.google.guava:guava:20.0
241-
com.google.guava:guava:27.0-jre
241+
com.google.guava:guava:32.0.1-jre
242242
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
243243
com.microsoft.azure:azure-storage:7.0.0
244244
com.nimbusds:nimbus-jose-jwt:9.37.2
245245
com.zaxxer:HikariCP:4.0.3
246246
commons-beanutils:commons-beanutils:1.9.4
247247
commons-cli:commons-cli:1.5.0
248-
commons-codec:commons-codec:1.11
248+
commons-codec:commons-codec:1.15
249249
commons-collections:commons-collections:3.2.2
250250
commons-daemon:commons-daemon:1.0.13
251-
commons-io:commons-io:2.14.0
251+
commons-io:commons-io:2.16.1
252252
commons-net:commons-net:3.9.0
253253
de.ruedigermoeller:fst:2.50
254254
io.grpc:grpc-api:1.53.0
@@ -298,6 +298,7 @@ javax.inject:javax.inject:1
298298
net.java.dev.jna:jna:5.2.0
299299
net.minidev:accessors-smart:1.2
300300
org.apache.avro:avro:1.9.2
301+
org.apache.avro:avro:1.11.3
301302
org.apache.commons:commons-collections4:4.2
302303
org.apache.commons:commons-compress:1.26.1
303304
org.apache.commons:commons-configuration2:2.10.1
@@ -361,8 +362,8 @@ org.lz4:lz4-java:1.7.1
361362
org.objenesis:objenesis:2.6
362363
org.xerial.snappy:snappy-java:1.1.10.4
363364
org.yaml:snakeyaml:2.0
364-
org.wildfly.openssl:wildfly-openssl:1.1.3.Final
365-
software.amazon.awssdk:bundle:jar:2.25.53
365+
org.wildfly.openssl:wildfly-openssl:2.1.4.Final
366+
software.amazon.awssdk:bundle:2.25.53
366367

367368

368369
--------------------------------------------------------------------------------
@@ -379,7 +380,7 @@ hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree
379380
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
380381

381382
com.github.luben:zstd-jni:1.5.2-1
382-
dnsjava:dnsjava:2.1.7
383+
dnsjava:dnsjava:3.6.1
383384
org.codehaus.woodstox:stax2-api:4.2.1
384385

385386

@@ -395,7 +396,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
395396
leveldb v1.13
396397

397398
com.google.protobuf:protobuf-java:2.5.0
398-
com.google.protobuf:protobuf-java:3.21.12
399+
com.google.protobuf:protobuf-java:3.25.3
399400
com.google.re2j:re2j:1.1
400401
com.jcraft:jsch:0.1.55
401402
com.thoughtworks.paranamer:paranamer:2.3
@@ -485,7 +486,7 @@ com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
485486
org.bouncycastle:bcpkix-jdk18on:1.78.1
486487
org.bouncycastle:bcprov-jdk18on:1.78.1
487488
org.bouncycastle:bcutil-jdk18on:1.78.1
488-
org.checkerframework:checker-qual:2.5.2
489+
org.checkerframework:checker-qual:3.8.0
489490
org.codehaus.mojo:animal-sniffer-annotations:1.21
490491
org.jruby.jcodings:jcodings:1.0.13
491492
org.jruby.joni:joni:2.1.2

dev-support/docker/pkg-resolver/install-yasm.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ fi
4040

4141
if [ "$version_to_install" == "1.2.0-4" ]; then
4242
mkdir -p /tmp/yasm &&
43-
curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
43+
curl -L -s -S https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
4444
-o /tmp/yasm-1.2.0-4.el7.x86_64.rpm &&
4545
rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm
4646
else

hadoop-client-modules/hadoop-client-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ allowed_expr+="|^[^-]*-default.xml$"
5151
allowed_expr+="|^[^-]*-version-info.properties$"
5252
# * Hadoop's application classloader properties file.
5353
allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
54+
# Comes from dnsjava, not sure if relocatable.
55+
allowed_expr+="|^messages.properties$"
5456
# public suffix list used by httpcomponents
5557
allowed_expr+="|^mozilla/$"
5658
allowed_expr+="|^mozilla/public-suffix-list.txt$"

hadoop-client-modules/hadoop-client-runtime/pom.xml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,8 @@
229229
<exclude>jnamed*</exclude>
230230
<exclude>lookup*</exclude>
231231
<exclude>update*</exclude>
232+
<exclude>META-INF/versions/21/*</exclude>
233+
<exclude>META-INF/versions/21/**/*</exclude>
232234
</excludes>
233235
</filter>
234236
<filter>
@@ -243,6 +245,7 @@
243245
<excludes>
244246
<exclude>META-INF/versions/9/module-info.class</exclude>
245247
<exclude>META-INF/versions/11/module-info.class</exclude>
248+
<exclude>META-INF/versions/21/module-info.class</exclude>
246249
</excludes>
247250
</filter>
248251

hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -454,4 +454,10 @@
454454
<Class name="org.apache.hadoop.ipc.internal.ShadedProtobufHelper" />
455455
<Bug pattern="AT_OPERATION_SEQUENCE_ON_CONCURRENT_ABSTRACTION" />
456456
</Match>
457+
458+
<!-- class cast after an assignableFrom check. -->
459+
<Match>
460+
<Class name="org.apache.hadoop.util.dynamic.DynMethods" />
461+
<Bug pattern="BC_UNCONFIRMED_CAST" />
462+
</Match>
457463
</FindBugsFilter>

hadoop-common-project/hadoop-common/pom.xml

Lines changed: 11 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -330,25 +330,6 @@
330330
<dependency>
331331
<groupId>org.apache.zookeeper</groupId>
332332
<artifactId>zookeeper</artifactId>
333-
<exclusions>
334-
<exclusion>
335-
<groupId>org.jboss.netty</groupId>
336-
<artifactId>netty</artifactId>
337-
</exclusion>
338-
<exclusion>
339-
<!-- otherwise seems to drag in junit 3.8.1 via jline -->
340-
<groupId>junit</groupId>
341-
<artifactId>junit</artifactId>
342-
</exclusion>
343-
<exclusion>
344-
<groupId>com.sun.jdmk</groupId>
345-
<artifactId>jmxtools</artifactId>
346-
</exclusion>
347-
<exclusion>
348-
<groupId>com.sun.jmx</groupId>
349-
<artifactId>jmxri</artifactId>
350-
</exclusion>
351-
</exclusions>
352333
</dependency>
353334
<dependency>
354335
<groupId>io.netty</groupId>
@@ -708,6 +689,17 @@
708689
</filesets>
709690
</configuration>
710691
</plugin>
692+
693+
<plugin>
694+
<groupId>org.apache.maven.plugins</groupId>
695+
<artifactId>maven-javadoc-plugin</artifactId>
696+
<configuration>
697+
<sourceFileExcludes>
698+
<sourceFileExclude>**/FSProtos.java</sourceFileExclude>
699+
</sourceFileExcludes>
700+
<excludePackageNames>*.proto:*.tracing:*.protobuf</excludePackageNames>
701+
</configuration>
702+
</plugin>
711703
</plugins>
712704
</build>
713705

@@ -1279,16 +1271,6 @@
12791271
</execution>
12801272
</executions>
12811273
</plugin>
1282-
<plugin>
1283-
<groupId>org.apache.maven.plugins</groupId>
1284-
<artifactId>maven-javadoc-plugin</artifactId>
1285-
<configuration>
1286-
<sourceFileExcludes>
1287-
<sourceFileExclude>**/FSProtos.java</sourceFileExclude>
1288-
</sourceFileExcludes>
1289-
<excludePackageNames>*.proto:*.tracing:*.protobuf</excludePackageNames>
1290-
</configuration>
1291-
</plugin>
12921274
</plugins>
12931275
</build>
12941276
</profile>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@
119119
/**
120120
* Provides access to configuration parameters.
121121
*
122-
* <h3 id="Resources">Resources</h3>
122+
* <h2 id="Resources">Resources</h2>
123123
*
124124
* <p>Configurations are specified by resources. A resource contains a set of
125125
* name/value pairs as XML data. Each resource is named by either a
@@ -130,16 +130,16 @@
130130
*
131131
* <p>Unless explicitly turned off, Hadoop by default specifies two
132132
* resources, loaded in-order from the classpath: <ol>
133-
* <li><tt>
133+
* <li><code>
134134
* <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
135-
* core-default.xml</a></tt>: Read-only defaults for hadoop.</li>
136-
* <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
135+
* core-default.xml</a></code>: Read-only defaults for hadoop.</li>
136+
* <li><code>core-site.xml</code>: Site-specific configuration for a given hadoop
137137
* installation.</li>
138138
* </ol>
139139
* Applications may add additional resources, which are loaded
140140
* subsequent to these resources in the order they are added.
141141
*
142-
* <h4 id="FinalParams">Final Parameters</h4>
142+
* <h3 id="FinalParams">Final Parameters</h3>
143143
*
144144
* <p>Configuration parameters may be declared <i>final</i>.
145145
* Once a resource declares a value final, no subsequently-loaded
@@ -153,9 +153,9 @@
153153
* &lt;/property&gt;</code></pre>
154154
*
155155
* Administrators typically define parameters as final in
156-
* <tt>core-site.xml</tt> for values that user applications may not alter.
156+
* <code>core-site.xml</code> for values that user applications may not alter.
157157
*
158-
* <h4 id="VariableExpansion">Variable Expansion</h4>
158+
* <h3 id="VariableExpansion">Variable Expansion</h3>
159159
*
160160
* <p>Value strings are first processed for <i>variable expansion</i>. The
161161
* available properties are:<ol>
@@ -185,22 +185,22 @@
185185
* &lt;/property&gt;
186186
* </code></pre>
187187
*
188-
* <p>When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
188+
* <p>When <code>conf.get("tempdir")</code> is called, then <code>${<i>basedir</i>}</code>
189189
* will be resolved to another property in this Configuration, while
190-
* <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
190+
* <code>${<i>user.name</i>}</code> would then ordinarily be resolved to the value
191191
* of the System property with that name.
192-
* <p>When <tt>conf.get("otherdir")</tt> is called, then <tt>${<i>env.BASE_DIR</i>}</tt>
193-
* will be resolved to the value of the <tt>${<i>BASE_DIR</i>}</tt> environment variable.
194-
* It supports <tt>${<i>env.NAME:-default</i>}</tt> and <tt>${<i>env.NAME-default</i>}</tt> notations.
195-
* The former is resolved to "default" if <tt>${<i>NAME</i>}</tt> environment variable is undefined
192+
* <p>When <code>conf.get("otherdir")</code> is called, then <code>${<i>env.BASE_DIR</i>}</code>
193+
* will be resolved to the value of the <code>${<i>BASE_DIR</i>}</code> environment variable.
194+
* It supports <code>${<i>env.NAME:-default</i>}</code> and <code>${<i>env.NAME-default</i>}</code> notations.
195+
* The former is resolved to "default" if <code>${<i>NAME</i>}</code> environment variable is undefined
196196
* or its value is empty.
197-
* The latter behaves the same way only if <tt>${<i>NAME</i>}</tt> is undefined.
197+
* The latter behaves the same way only if <code>${<i>NAME</i>}</code> is undefined.
198198
* <p>By default, warnings will be given to any deprecated configuration
199199
* parameters and these are suppressible by configuring
200-
* <tt>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</tt> in
200+
* <code>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</code> in
201201
* log4j.properties file.
202202
*
203-
* <h4 id="Tags">Tags</h4>
203+
* <h3 id="Tags">Tags</h3>
204204
*
205205
* <p>Optionally we can tag related properties together by using tag
206206
* attributes. System tags are defined by hadoop.tags.system property. Users
@@ -220,9 +220,9 @@
220220
* &lt;tag&gt;HDFS,SECURITY&lt;/tag&gt;
221221
* &lt;/property&gt;
222222
* </code></pre>
223-
* <p> Properties marked with tags can be retrieved with <tt>conf
224-
* .getAllPropertiesByTag("HDFS")</tt> or <tt>conf.getAllPropertiesByTags
225-
* (Arrays.asList("YARN","SECURITY"))</tt>.</p>
223+
* <p> Properties marked with tags can be retrieved with <code>conf
224+
* .getAllPropertiesByTag("HDFS")</code> or <code>conf.getAllPropertiesByTags
225+
* (Arrays.asList("YARN","SECURITY"))</code>.</p>
226226
*/
227227
@InterfaceAudience.Public
228228
@InterfaceStability.Stable
@@ -576,7 +576,7 @@ public static void addDeprecations(DeprecationDelta[] deltas) {
576576
* It does not override any existing entries in the deprecation map.
577577
* This is to be used only by the developers in order to add deprecation of
578578
* keys, and attempts to call this method after loading resources once,
579-
* would lead to <tt>UnsupportedOperationException</tt>
579+
* would lead to <code>UnsupportedOperationException</code>
580580
*
581581
* If a key is deprecated in favor of multiple keys, they are all treated as
582582
* aliases of each other, and setting any one of them resets all the others
@@ -604,7 +604,7 @@ public static void addDeprecation(String key, String[] newKeys,
604604
* It does not override any existing entries in the deprecation map.
605605
* This is to be used only by the developers in order to add deprecation of
606606
* keys, and attempts to call this method after loading resources once,
607-
* would lead to <tt>UnsupportedOperationException</tt>
607+
* would lead to <code>UnsupportedOperationException</code>
608608
*
609609
* If you have multiple deprecation entries to add, it is more efficient to
610610
* use #addDeprecations(DeprecationDelta[] deltas) instead.
@@ -624,7 +624,7 @@ public static void addDeprecation(String key, String newKey,
624624
* It does not override any existing entries in the deprecation map.
625625
* This is to be used only by the developers in order to add deprecation of
626626
* keys, and attempts to call this method after loading resources once,
627-
* would lead to <tt>UnsupportedOperationException</tt>
627+
* would lead to <code>UnsupportedOperationException</code>
628628
*
629629
* If a key is deprecated in favor of multiple keys, they are all treated as
630630
* aliases of each other, and setting any one of them resets all the others
@@ -648,7 +648,7 @@ public static void addDeprecation(String key, String[] newKeys) {
648648
* It does not override any existing entries in the deprecation map.
649649
* This is to be used only by the developers in order to add deprecation of
650650
* keys, and attempts to call this method after loading resources once,
651-
* would lead to <tt>UnsupportedOperationException</tt>
651+
* would lead to <code>UnsupportedOperationException</code>
652652
*
653653
* If you have multiple deprecation entries to add, it is more efficient to
654654
* use #addDeprecations(DeprecationDelta[] deltas) instead.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,20 @@ private static Transform tokenizeTransformation(String transformation)
177177
}
178178
return new Transform(parts[0], parts[1], parts[2]);
179179
}
180+
181+
public static boolean isSupported(CipherSuite suite) {
182+
Transform transform;
183+
int algMode;
184+
int padding;
185+
try {
186+
transform = tokenizeTransformation(suite.getName());
187+
algMode = AlgMode.get(transform.alg, transform.mode);
188+
padding = Padding.get(transform.padding);
189+
} catch (NoSuchAlgorithmException|NoSuchPaddingException e) {
190+
return false;
191+
}
192+
return isSupportedSuite(algMode, padding);
193+
}
180194

181195
/**
182196
* Initialize this cipher with a key and IV.
@@ -298,5 +312,7 @@ private native int doFinal(long context, ByteBuffer output, int offset,
298312

299313
private native void clean(long ctx, long engineNum);
300314

315+
private native static boolean isSupportedSuite(int alg, int padding);
316+
301317
public native static String getLibraryName();
302318
}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslSm4CtrCryptoCodec.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,10 @@ public OpensslSm4CtrCryptoCodec() {
4141
if (loadingFailureReason != null) {
4242
throw new RuntimeException(loadingFailureReason);
4343
}
44+
45+
if (!OpensslCipher.isSupported(CipherSuite.SM4_CTR_NOPADDING)) {
46+
throw new RuntimeException("The OpenSSL native library is built without SM4 CTR support");
47+
}
4448
}
4549

4650
@Override

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ protected int init(String[] args) throws IOException {
169169

170170
@Override
171171
public String getCommandUsage() {
172-
StringBuffer sbuf = new StringBuffer(USAGE_PREFIX + COMMANDS);
172+
StringBuilder sbuf = new StringBuilder(USAGE_PREFIX + COMMANDS);
173173
String banner = StringUtils.repeat("=", 66);
174174
sbuf.append(banner + "\n");
175175
sbuf.append(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC + "\n");

0 commit comments

Comments
 (0)