NUTCH-3206 XML parsers should not be vulnerable to XXE attacks

src/java/org/apache/nutch/exchange/Exchanges.java

@@ -22,6 +22,7 @@
 import org.apache.nutch.plugin.ExtensionPoint;
 import org.apache.nutch.plugin.PluginRepository;
 import org.apache.nutch.plugin.PluginRuntimeException;
+import org.apache.nutch.util.XmlUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.Element;
@@ -104,7 +105,7 @@ private ExchangeConfig[] loadConfigurations(Configuration conf) {
     final List<ExchangeConfig> configList = new LinkedList<>();
 
     try {
-      DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory();
       DocumentBuilder builder = factory.newDocumentBuilder();
       Element rootElement = builder.parse(inputSource).getDocumentElement();
       NodeList exchangeList = rootElement.getElementsByTagName("exchange");

src/java/org/apache/nutch/indexer/IndexWriters.java

@@ -26,6 +26,7 @@
 import org.apache.nutch.plugin.PluginRepository;
 import org.apache.nutch.plugin.PluginRuntimeException;
 import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.XmlUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.Document;
@@ -121,7 +122,7 @@ private IndexWriterConfig[] loadWritersConfiguration(Configuration conf) {
     InputSource inputSource = new InputSource(ssInputStream);
 
     try {
-      DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory();
       DocumentBuilder builder = factory.newDocumentBuilder();
       Document document = builder.parse(inputSource);
       Element rootElement = document.getDocumentElement();

src/java/org/apache/nutch/parse/ParsePluginsReader.java

@@ -38,6 +38,7 @@
 import org.apache.hadoop.conf.Configuration;
 
 import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.XmlUtil;
 
 /**
  * A reader to load the information stored in the
@@ -102,7 +103,7 @@ public ParsePluginList parse(Configuration conf) {
     inputSource = new InputSource(ppInputStream);
 
     try {
-      factory = DocumentBuilderFactory.newInstance();
+      factory = XmlUtil.newSecureDocumentBuilderFactory();
       parser = factory.newDocumentBuilder();
       document = parser.parse(inputSource);
     } catch (Exception e) {

src/java/org/apache/nutch/plugin/PluginManifestParser.java

@@ -31,6 +31,7 @@
 import javax.xml.parsers.ParserConfigurationException;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.util.XmlUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.Document;
@@ -158,7 +159,7 @@ private PluginDescriptor parseManifestFile(String pManifestPath)
    */
   private Document parseXML(URL url)
           throws SAXException, IOException, ParserConfigurationException {
-    DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory();
     DocumentBuilder builder = factory.newDocumentBuilder();
     return builder.parse(url.openStream());
   }

src/java/org/apache/nutch/tools/DmozParser.java

@@ -38,6 +38,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.io.MD5Hash;
 import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.XmlUtil;
 import org.apache.xerces.util.XMLChar;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -296,9 +297,7 @@ public void parseDmozFile(File dmozFile, int subsetDenom,
       boolean includeAdult, int skew, Pattern topicPattern)
               throws IOException, SAXException, ParserConfigurationException {
 
-    SAXParserFactory parserFactory = SAXParserFactory.newInstance();
-    parserFactory.setFeature("http://xml.org/sax/features/external-general-entities", false);
-    parserFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+    SAXParserFactory parserFactory = XmlUtil.newSecureSAXParserFactory();
     SAXParser parser = parserFactory.newSAXParser();
     XMLReader reader = parser.getXMLReader();
     reader.setFeature("http://xml.org/sax/features/external-general-entities", false);

src/java/org/apache/nutch/util/DomUtil.java

@@ -60,6 +60,7 @@ public static Element getDom(InputStream is) {
 
     InputSource input;
     try {
+      XmlUtil.configureSecure(parser);
       input = new InputSource(is);
       input.setEncoding("UTF-8");
       parser.parse(input);
@@ -87,9 +88,10 @@ public static Element getDom(InputStream is) {
   public static void saveDom(OutputStream os, Element e) {
 
     DOMSource source = new DOMSource(e);
-    TransformerFactory transFactory = TransformerFactory.newInstance();
+    TransformerFactory transFactory;
     Transformer transformer;
     try {
+      transFactory = XmlUtil.newSecureTransformerFactory();
       transformer = transFactory.newTransformer();
       transformer.setOutputProperty("indent", "yes");
       transformer.setOutputProperty(OutputKeys.ENCODING,

src/java/org/apache/nutch/util/XmlUtil.java

@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.util;
+
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerFactory;
+
+import org.apache.xerces.parsers.DOMParser;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXNotRecognizedException;
+import org.xml.sax.SAXNotSupportedException;
+
+/**
+ * Helpers that return XML parser/transformer factories configured to reject
+ * XXE (external entity) attacks. Prefer these over bare
+ * {@link DocumentBuilderFactory#newInstance()} /
+ * {@link SAXParserFactory#newInstance()} /
+ * {@link TransformerFactory#newInstance()}.
+ */
+public final class XmlUtil {
+
+  private static final String DISALLOW_DOCTYPE_DECL =
+      "http://apache.org/xml/features/disallow-doctype-decl";
+  private static final String EXTERNAL_GENERAL_ENTITIES =
+      "http://xml.org/sax/features/external-general-entities";
+  private static final String EXTERNAL_PARAMETER_ENTITIES =
+      "http://xml.org/sax/features/external-parameter-entities";
+  private static final String LOAD_EXTERNAL_DTD =
+      "http://apache.org/xml/features/nonvalidating/load-external-dtd";
+
+  private XmlUtil() {
+  }
+
+  /**
+   * @return a {@link DocumentBuilderFactory} hardened against XXE
+   * @throws ParserConfigurationException if a required secure feature is
+   *           unsupported
+   */
+  public static DocumentBuilderFactory newSecureDocumentBuilderFactory()
+      throws ParserConfigurationException {
+    return newSecureDocumentBuilderFactory(false);
+  }
+
+  /**
+   * @param namespaceAware whether the factory should be namespace-aware
+   * @return a {@link DocumentBuilderFactory} hardened against XXE
+   * @throws ParserConfigurationException if a required secure feature is
+   *           unsupported
+   */
+  public static DocumentBuilderFactory newSecureDocumentBuilderFactory(
+      boolean namespaceAware) throws ParserConfigurationException {
+    DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+    factory.setNamespaceAware(namespaceAware);
+    factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+    factory.setFeature(DISALLOW_DOCTYPE_DECL, true);
+    factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
+    factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
+    factory.setFeature(LOAD_EXTERNAL_DTD, false);
+    factory.setXIncludeAware(false);
+    // Sonar java:S2755 "Going the extra mile" — does not alone block XXE
+    factory.setExpandEntityReferences(false);
+    return factory;
+  }
+
+  /**
+   * @return a {@link SAXParserFactory} hardened against XXE
+   * @throws ParserConfigurationException if a required secure feature is
+   *           unsupported
+   * @throws SAXNotRecognizedException if a feature name is not recognized
+   * @throws SAXNotSupportedException if a feature value is not supported
+   */
+  public static SAXParserFactory newSecureSAXParserFactory()
+      throws ParserConfigurationException, SAXNotRecognizedException,
+      SAXNotSupportedException {
+    SAXParserFactory factory = SAXParserFactory.newInstance();
+    factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+    factory.setFeature(DISALLOW_DOCTYPE_DECL, true);
+    factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
+    factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
+    factory.setXIncludeAware(false);
+    return factory;
+  }
+
+  /**
+   * @return a {@link TransformerFactory} that cannot load external DTDs or
+   *         stylesheets
+   * @throws TransformerConfigurationException if secure attributes cannot be
+   *           applied
+   */
+  public static TransformerFactory newSecureTransformerFactory()
+      throws TransformerConfigurationException {
+    TransformerFactory factory = TransformerFactory.newInstance();
+    factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+    factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+    factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+    return factory;
+  }
+
+  /**
+   * Apply XXE-hardening features to an Xerces {@link DOMParser}.
+   *
+   * @param parser parser to configure
+   * @throws SAXException if a secure feature cannot be set
+   */
+  public static void configureSecure(DOMParser parser) throws SAXException {
+    parser.setFeature(DISALLOW_DOCTYPE_DECL, true);
+    parser.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
+    parser.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
+    parser.setFeature(LOAD_EXTERNAL_DTD, false);
+    try {
+      parser.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+    } catch (SAXNotRecognizedException | SAXNotSupportedException e) {
+      // Xerces' DOMParser does not expose the JAXP secure-processing feature.
+      // DTDs and external entities are already disabled above.
+    }
+  }
+}

src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java

@@ -26,6 +26,7 @@
 import org.apache.nutch.parse.ParseResult;
 import org.apache.nutch.parse.ParseStatus;
 import org.apache.nutch.parse.ParseText;
+import org.apache.nutch.util.XmlUtil;
 import org.apache.hadoop.conf.Configuration;
 
 import org.slf4j.Logger;
@@ -45,6 +46,7 @@
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
 
 import org.xml.sax.InputSource;
 
@@ -167,11 +169,14 @@ private void findLicenseUrl(Element element) {
       }
     }
 
-    /** Configure a namespace aware XML parser. */
-    private static final DocumentBuilderFactory FACTORY = DocumentBuilderFactory
-        .newInstance();
+    /** Configure a namespace aware XML parser hardened against XXE. */
+    private static final DocumentBuilderFactory FACTORY;
     static {
-      FACTORY.setNamespaceAware(true);
+      try {
+        FACTORY = XmlUtil.newSecureDocumentBuilderFactory(true);
+      } catch (ParserConfigurationException e) {
+        throw new ExceptionInInitializerError(e);
+      }
     }
 
     /** Creative Commons' namespace URI. */

src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java

@@ -26,7 +26,6 @@
 import java.util.Map;
 import java.util.Set;
 
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
 import org.xml.sax.SAXException;
@@ -55,6 +54,7 @@
 import org.apache.nutch.protocol.http.api.HttpBase;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.XmlUtil;
 
 /**
  * <p>
@@ -275,7 +275,7 @@ private static synchronized void setCredentials()
 
     InputStream is = conf.getConfResourceAsInputStream(authFile);
     if (is != null) {
-      Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+      Document doc = XmlUtil.newSecureDocumentBuilderFactory().newDocumentBuilder()
           .parse(is);
 
       Element rootElement = doc.getDocumentElement();

src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java

@@ -32,13 +32,12 @@
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 
-import javax.xml.parsers.DocumentBuilderFactory;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.nutch.net.URLNormalizer;
 import org.apache.nutch.net.URLNormalizers;
 import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.XmlUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.Document;
@@ -224,7 +223,7 @@ private List<Rule> readConfiguration(Reader reader) {
     try {
 
       // borrowed heavily from code in Configuration.java
-      Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+      Document doc = XmlUtil.newSecureDocumentBuilderFactory().newDocumentBuilder()
           .parse(new InputSource(reader));
       Element root = doc.getDocumentElement();
       if (!"regex-normalize".equals(root.getTagName())) {