Skip to content

Commit 19f8e4f

Browse files
steveloughransteveloughran
authored
YARN-11330. use secure XML parsers (#4981)
 Move construction of XML parsers in YARN modules to using the locked-down parser factory of HADOOP-18469. One exception: GpuDeviceInformationParser still supports DTD resolution; all other features are disabled. Contributed by P J Fanning
1 parent 237814a commit 19f8e4f

19 files changed

Lines changed: 80 additions & 60 deletions

File tree

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/XMLUtils.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,18 @@
4141
@InterfaceStability.Unstable
4242
public class XMLUtils {
4343

44-
private static final String DISALLOW_DOCTYPE_DECL =
44+
public static final String DISALLOW_DOCTYPE_DECL =
4545
"http://apache.org/xml/features/disallow-doctype-decl";
46-
private static final String LOAD_EXTERNAL_DECL =
46+
public static final String LOAD_EXTERNAL_DECL =
4747
"http://apache.org/xml/features/nonvalidating/load-external-dtd";
48-
private static final String EXTERNAL_GENERAL_ENTITIES =
48+
public static final String EXTERNAL_GENERAL_ENTITIES =
4949
"http://xml.org/sax/features/external-general-entities";
50-
private static final String EXTERNAL_PARAMETER_ENTITIES =
50+
public static final String EXTERNAL_PARAMETER_ENTITIES =
5151
"http://xml.org/sax/features/external-parameter-entities";
52-
private static final String CREATE_ENTITY_REF_NODES =
52+
public static final String CREATE_ENTITY_REF_NODES =
5353
"http://apache.org/xml/features/dom/create-entity-ref-nodes";
54-
54+
public static final String VALIDATION =
55+
"http://xml.org/sax/features/validation";
5556

5657
/**
5758
* Transform input xml given a stylesheet.

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.apache.hadoop.security.ssl.SSLFactory;
3838
import org.apache.hadoop.security.UserGroupInformation;
3939
import org.apache.hadoop.util.Tool;
40+
import org.apache.hadoop.util.XMLUtils;
4041
import org.apache.hadoop.yarn.conf.YarnConfiguration;
4142
import org.apache.hadoop.yarn.webapp.dao.ConfInfo;
4243
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
@@ -190,7 +191,7 @@ private static void prettyFormatWithIndent(String input, int indent)
190191
Source xmlInput = new StreamSource(new StringReader(input));
191192
StringWriter sw = new StringWriter();
192193
StreamResult xmlOutput = new StreamResult(sw);
193-
TransformerFactory transformerFactory = TransformerFactory.newInstance();
194+
TransformerFactory transformerFactory = XMLUtils.newSecureTransformerFactory();
194195
transformerFactory.setAttribute("indent-number", indent);
195196
Transformer transformer = transformerFactory.newTransformer();
196197
transformer.setOutputProperty(OutputKeys.INDENT, "yes");

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,27 @@
1818

1919
package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
2020

21+
import java.io.StringReader;
22+
import javax.xml.XMLConstants;
23+
import javax.xml.bind.JAXBContext;
24+
import javax.xml.bind.JAXBException;
25+
import javax.xml.bind.Unmarshaller;
26+
import javax.xml.parsers.SAXParserFactory;
27+
import javax.xml.transform.sax.SAXSource;
28+
2129
import org.apache.hadoop.classification.InterfaceAudience;
2230
import org.apache.hadoop.classification.InterfaceStability;
2331
import org.apache.hadoop.yarn.exceptions.YarnException;
32+
2433
import org.slf4j.Logger;
2534
import org.slf4j.LoggerFactory;
2635
import org.xml.sax.InputSource;
2736
import org.xml.sax.XMLReader;
2837

29-
import javax.xml.bind.JAXBContext;
30-
import javax.xml.bind.JAXBException;
31-
import javax.xml.bind.Unmarshaller;
32-
import javax.xml.parsers.SAXParserFactory;
33-
import javax.xml.transform.sax.SAXSource;
34-
import java.io.StringReader;
38+
import static org.apache.hadoop.util.XMLUtils.EXTERNAL_GENERAL_ENTITIES;
39+
import static org.apache.hadoop.util.XMLUtils.EXTERNAL_PARAMETER_ENTITIES;
40+
import static org.apache.hadoop.util.XMLUtils.LOAD_EXTERNAL_DECL;
41+
import static org.apache.hadoop.util.XMLUtils.VALIDATION;
3542

3643
/**
3744
* Parse XML and get GPU device information
@@ -68,10 +75,11 @@ public GpuDeviceInformationParser() throws YarnException {
6875
*/
6976
private SAXParserFactory initSaxParserFactory() throws Exception {
7077
SAXParserFactory spf = SAXParserFactory.newInstance();
71-
spf.setFeature(
72-
"http://apache.org/xml/features/nonvalidating/load-external-dtd",
73-
false);
74-
spf.setFeature("http://xml.org/sax/features/validation", false);
78+
spf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
79+
spf.setFeature(LOAD_EXTERNAL_DECL, false);
80+
spf.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
81+
spf.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
82+
spf.setFeature(VALIDATION, false);
7583
return spf;
7684
}
7785

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.hadoop.fs.Path;
3636
import org.apache.hadoop.http.JettyUtils;
3737
import org.apache.hadoop.util.VersionInfo;
38+
import org.apache.hadoop.util.XMLUtils;
3839
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
3940
import org.apache.hadoop.yarn.api.records.ApplicationId;
4041
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -432,10 +433,9 @@ public void testSingleNodesXML() throws JSONException, Exception {
432433
assertEquals(MediaType.APPLICATION_XML+ "; " + JettyUtils.UTF_8,
433434
response.getType().toString());
434435
String xml = response.getEntity(String.class);
435-
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
436+
DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
436437
DocumentBuilder db = dbf.newDocumentBuilder();
437-
InputSource is = new InputSource();
438-
is.setCharacterStream(new StringReader(xml));
438+
InputSource is = new InputSource(new StringReader(xml));
439439
Document dom = db.parse(is);
440440
NodeList nodes = dom.getElementsByTagName("nodeInfo");
441441
assertEquals("incorrect number of elements", 1, nodes.getLength());

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.hadoop.conf.Configuration;
4141
import org.apache.hadoop.fs.FileUtil;
4242
import org.apache.hadoop.http.JettyUtils;
43+
import org.apache.hadoop.util.XMLUtils;
4344
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
4445
import org.apache.hadoop.yarn.api.records.NodeId;
4546
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -486,7 +487,7 @@ public void testNodeAppsStateInvalidXML() throws JSONException, Exception {
486487
response.getType().toString());
487488
String msg = response.getEntity(String.class);
488489

489-
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
490+
DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
490491
DocumentBuilder db = dbf.newDocumentBuilder();
491492
InputSource is = new InputSource();
492493
is.setCharacterStream(new StringReader(msg));
@@ -651,7 +652,7 @@ public void testNodeAppsXML() throws JSONException, Exception {
651652
assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
652653
response.getType().toString());
653654
String xml = response.getEntity(String.class);
654-
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
655+
DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
655656
DocumentBuilder db = dbf.newDocumentBuilder();
656657
InputSource is = new InputSource();
657658
is.setCharacterStream(new StringReader(xml));
@@ -676,7 +677,7 @@ public void testNodeSingleAppsXML() throws JSONException, Exception {
676677
assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
677678
response.getType().toString());
678679
String xml = response.getEntity(String.class);
679-
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
680+
DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
680681
DocumentBuilder db = dbf.newDocumentBuilder();
681682
InputSource is = new InputSource();
682683
is.setCharacterStream(new StringReader(xml));

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.hadoop.conf.Configuration;
4141
import org.apache.hadoop.fs.FileUtil;
4242
import org.apache.hadoop.http.JettyUtils;
43+
import org.apache.hadoop.util.XMLUtils;
4344
import org.apache.hadoop.yarn.api.records.NodeId;
4445
import org.apache.hadoop.yarn.conf.YarnConfiguration;
4546
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -257,7 +258,7 @@ public void testNodeAuxServicesXML() throws Exception {
257258
assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
258259
response.getType().toString());
259260
String xml = response.getEntity(String.class);
260-
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
261+
DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
261262
DocumentBuilder db = dbf.newDocumentBuilder();
262263
InputSource is = new InputSource();
263264
is.setCharacterStream(new StringReader(xml));

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.apache.hadoop.conf.Configuration;
4040
import org.apache.hadoop.fs.FileUtil;
4141
import org.apache.hadoop.http.JettyUtils;
42+
import org.apache.hadoop.util.XMLUtils;
4243
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
4344
import org.apache.hadoop.yarn.api.records.ContainerId;
4445
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -447,7 +448,7 @@ public void testNodeSingleContainerXML() throws JSONException, Exception {
447448
assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
448449
response.getType().toString());
449450
String xml = response.getEntity(String.class);
450-
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
451+
DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
451452
DocumentBuilder db = dbf.newDocumentBuilder();
452453
InputSource is = new InputSource();
453454
is.setCharacterStream(new StringReader(xml));
@@ -476,7 +477,7 @@ public void testNodeContainerXML() throws JSONException, Exception {
476477
assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
477478
response.getType().toString());
478479
String xml = response.getEntity(String.class);
479-
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
480+
DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
480481
DocumentBuilder db = dbf.newDocumentBuilder();
481482
InputSource is = new InputSource();
482483
is.setCharacterStream(new StringReader(xml));

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.hadoop.fs.UnsupportedFileSystemException;
2929
import org.apache.hadoop.security.authorize.AccessControlList;
3030
import org.apache.hadoop.service.AbstractService;
31+
import org.apache.hadoop.util.XMLUtils;
3132
import org.apache.hadoop.yarn.api.records.QueueACL;
3233
import org.apache.hadoop.yarn.security.AccessType;
3334
import org.apache.hadoop.yarn.security.Permission;
@@ -236,8 +237,7 @@ public synchronized void reloadAllocations()
236237
LOG.info("Loading allocation file " + allocFile);
237238

238239
// Read and parse the allocations file.
239-
DocumentBuilderFactory docBuilderFactory =
240-
DocumentBuilderFactory.newInstance();
240+
DocumentBuilderFactory docBuilderFactory = XMLUtils.newSecureDocumentBuilderFactory();
241241
docBuilderFactory.setIgnoringComments(true);
242242
DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
243243
Document doc = builder.parse(fs.open(allocFile));

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.hadoop.security.Credentials;
2525
import org.apache.hadoop.security.token.Token;
2626
import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
27+
import org.apache.hadoop.util.XMLUtils;
2728
import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore;
2829
import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
2930
import org.apache.hadoop.yarn.server.api.ResourceTracker;
@@ -2601,7 +2602,7 @@ private void writeToHostsFile(File file, String... hosts)
26012602
private void writeToHostsXmlFile(
26022603
File file, Pair<String, Integer>... hostsAndTimeouts) throws Exception {
26032604
ensureFileExists(file);
2604-
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
2605+
DocumentBuilderFactory dbFactory = XMLUtils.newSecureDocumentBuilderFactory();
26052606
Document doc = dbFactory.newDocumentBuilder().newDocument();
26062607
Element hosts = doc.createElement("hosts");
26072608
doc.appendChild(hosts);
@@ -2619,7 +2620,7 @@ private void writeToHostsXmlFile(
26192620
);
26202621
}
26212622
}
2622-
TransformerFactory transformerFactory = TransformerFactory.newInstance();
2623+
TransformerFactory transformerFactory = XMLUtils.newSecureTransformerFactory();
26232624
Transformer transformer = transformerFactory.newTransformer();
26242625
transformer.setOutputProperty(OutputKeys.INDENT, "yes");
26252626
transformer.transform(new DOMSource(doc), new StreamResult(file));

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementRuleFS.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.hadoop.yarn.server.resourcemanager.placement;
2020

2121
import org.apache.commons.io.IOUtils;
22+
import org.apache.hadoop.util.XMLUtils;
2223
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
2324
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
2425
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
@@ -188,11 +189,10 @@ private void ruleInit(Class <? extends PlacementRule> ruleClass) {
188189

189190
private Element createConf(String str) {
190191
// Create a simple rule element to use in the rule create
191-
DocumentBuilderFactory docBuilderFactory =
192-
DocumentBuilderFactory.newInstance();
193-
docBuilderFactory.setIgnoringComments(true);
194192
Document doc = null;
195193
try {
194+
DocumentBuilderFactory docBuilderFactory = XMLUtils.newSecureDocumentBuilderFactory();
195+
docBuilderFactory.setIgnoringComments(true);
196196
DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
197197
doc = builder.parse(IOUtils.toInputStream(str, StandardCharsets.UTF_8));
198198
} catch (Exception ex) {

0 commit comments

Comments
 (0)