From 1c181f4e5ffd396e49fa0f90ffb691d449339644 Mon Sep 17 00:00:00 2001 From: lewismc Date: Sat, 15 Aug 2026 12:01:23 -0700 Subject: [PATCH 1/2] NUTCH-3206 XML parsers should not be vulnerable to XXE attacks --- .../org/apache/nutch/exchange/Exchanges.java | 3 +- .../apache/nutch/indexer/IndexWriters.java | 3 +- .../nutch/parse/ParsePluginsReader.java | 3 +- .../nutch/plugin/PluginManifestParser.java | 3 +- .../org/apache/nutch/tools/DmozParser.java | 5 +- src/java/org/apache/nutch/util/DomUtil.java | 4 +- src/java/org/apache/nutch/util/XmlUtil.java | 130 ++++++++++++++++++ .../creativecommons/nutch/CCParseFilter.java | 13 +- .../nutch/protocol/httpclient/Http.java | 4 +- .../regex/RegexURLNormalizer.java | 5 +- .../org/apache/nutch/util/TestXmlUtil.java | 77 +++++++++++ 11 files changed, 233 insertions(+), 17 deletions(-) create mode 100644 src/java/org/apache/nutch/util/XmlUtil.java create mode 100644 src/test/org/apache/nutch/util/TestXmlUtil.java diff --git a/src/java/org/apache/nutch/exchange/Exchanges.java b/src/java/org/apache/nutch/exchange/Exchanges.java index 1e0518b6bc..37b2406bf5 100644 --- a/src/java/org/apache/nutch/exchange/Exchanges.java +++ b/src/java/org/apache/nutch/exchange/Exchanges.java @@ -22,6 +22,7 @@ import org.apache.nutch.plugin.ExtensionPoint; import org.apache.nutch.plugin.PluginRepository; import org.apache.nutch.plugin.PluginRuntimeException; +import org.apache.nutch.util.XmlUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Element; @@ -104,7 +105,7 @@ private ExchangeConfig[] loadConfigurations(Configuration conf) { final List configList = new LinkedList<>(); try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory(); DocumentBuilder builder = factory.newDocumentBuilder(); Element rootElement = builder.parse(inputSource).getDocumentElement(); NodeList exchangeList = rootElement.getElementsByTagName("exchange"); diff --git a/src/java/org/apache/nutch/indexer/IndexWriters.java b/src/java/org/apache/nutch/indexer/IndexWriters.java index f8ae8ee866..4b809d1e44 100644 --- a/src/java/org/apache/nutch/indexer/IndexWriters.java +++ b/src/java/org/apache/nutch/indexer/IndexWriters.java @@ -26,6 +26,7 @@ import org.apache.nutch.plugin.PluginRepository; import org.apache.nutch.plugin.PluginRuntimeException; import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.XmlUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -121,7 +122,7 @@ private IndexWriterConfig[] loadWritersConfiguration(Configuration conf) { InputSource inputSource = new InputSource(ssInputStream); try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory(); DocumentBuilder builder = factory.newDocumentBuilder(); Document document = builder.parse(inputSource); Element rootElement = document.getDocumentElement(); diff --git a/src/java/org/apache/nutch/parse/ParsePluginsReader.java b/src/java/org/apache/nutch/parse/ParsePluginsReader.java index 978e3c0972..8b84d44797 100644 --- a/src/java/org/apache/nutch/parse/ParsePluginsReader.java +++ b/src/java/org/apache/nutch/parse/ParsePluginsReader.java @@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.XmlUtil; /** * A reader to load the information stored in the @@ -102,7 +103,7 @@ public ParsePluginList parse(Configuration conf) { inputSource = new InputSource(ppInputStream); try { - factory = DocumentBuilderFactory.newInstance(); + factory = XmlUtil.newSecureDocumentBuilderFactory(); parser = factory.newDocumentBuilder(); document = parser.parse(inputSource); } catch (Exception e) { diff --git a/src/java/org/apache/nutch/plugin/PluginManifestParser.java b/src/java/org/apache/nutch/plugin/PluginManifestParser.java index 95208fa433..dca2029102 100644 --- a/src/java/org/apache/nutch/plugin/PluginManifestParser.java +++ b/src/java/org/apache/nutch/plugin/PluginManifestParser.java @@ -31,6 +31,7 @@ import javax.xml.parsers.ParserConfigurationException; import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.util.XmlUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -158,7 +159,7 @@ private PluginDescriptor parseManifestFile(String pManifestPath) */ private Document parseXML(URL url) throws SAXException, IOException, ParserConfigurationException { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory(); DocumentBuilder builder = factory.newDocumentBuilder(); return builder.parse(url.openStream()); } diff --git a/src/java/org/apache/nutch/tools/DmozParser.java b/src/java/org/apache/nutch/tools/DmozParser.java index 4924e3b1b0..5de457fb3e 100644 --- a/src/java/org/apache/nutch/tools/DmozParser.java +++ b/src/java/org/apache/nutch/tools/DmozParser.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.io.MD5Hash; import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.XmlUtil; import org.apache.xerces.util.XMLChar; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -296,9 +297,7 @@ public void parseDmozFile(File dmozFile, int subsetDenom, boolean includeAdult, int skew, Pattern topicPattern) throws IOException, SAXException, ParserConfigurationException { - SAXParserFactory parserFactory = SAXParserFactory.newInstance(); - parserFactory.setFeature("http://xml.org/sax/features/external-general-entities", false); - parserFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + SAXParserFactory parserFactory = XmlUtil.newSecureSAXParserFactory(); SAXParser parser = parserFactory.newSAXParser(); XMLReader reader = parser.getXMLReader(); reader.setFeature("http://xml.org/sax/features/external-general-entities", false); diff --git a/src/java/org/apache/nutch/util/DomUtil.java b/src/java/org/apache/nutch/util/DomUtil.java index 50cc43c98b..c31c2a1f0d 100644 --- a/src/java/org/apache/nutch/util/DomUtil.java +++ b/src/java/org/apache/nutch/util/DomUtil.java @@ -60,6 +60,7 @@ public static Element getDom(InputStream is) { InputSource input; try { + XmlUtil.configureSecure(parser); input = new InputSource(is); input.setEncoding("UTF-8"); parser.parse(input); @@ -87,9 +88,10 @@ public static Element getDom(InputStream is) { public static void saveDom(OutputStream os, Element e) { DOMSource source = new DOMSource(e); - TransformerFactory transFactory = TransformerFactory.newInstance(); + TransformerFactory transFactory; Transformer transformer; try { + transFactory = XmlUtil.newSecureTransformerFactory(); transformer = transFactory.newTransformer(); transformer.setOutputProperty("indent", "yes"); transformer.setOutputProperty(OutputKeys.ENCODING, diff --git a/src/java/org/apache/nutch/util/XmlUtil.java b/src/java/org/apache/nutch/util/XmlUtil.java new file mode 100644 index 0000000000..85ec8981b1 --- /dev/null +++ b/src/java/org/apache/nutch/util/XmlUtil.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.util; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParserFactory; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerFactory; + +import org.apache.xerces.parsers.DOMParser; +import org.xml.sax.SAXException; +import org.xml.sax.SAXNotRecognizedException; +import org.xml.sax.SAXNotSupportedException; + +/** + * Helpers that return XML parser/transformer factories configured to reject + * XXE (external entity) attacks. Prefer these over bare + * {@link DocumentBuilderFactory#newInstance()} / + * {@link SAXParserFactory#newInstance()} / + * {@link TransformerFactory#newInstance()}. + */ +public final class XmlUtil { + + private static final String DISALLOW_DOCTYPE_DECL = + "http://apache.org/xml/features/disallow-doctype-decl"; + private static final String EXTERNAL_GENERAL_ENTITIES = + "http://xml.org/sax/features/external-general-entities"; + private static final String EXTERNAL_PARAMETER_ENTITIES = + "http://xml.org/sax/features/external-parameter-entities"; + private static final String LOAD_EXTERNAL_DTD = + "http://apache.org/xml/features/nonvalidating/load-external-dtd"; + + private XmlUtil() { + } + + /** + * @return a {@link DocumentBuilderFactory} hardened against XXE + * @throws ParserConfigurationException if a required secure feature is + * unsupported + */ + public static DocumentBuilderFactory newSecureDocumentBuilderFactory() + throws ParserConfigurationException { + return newSecureDocumentBuilderFactory(false); + } + + /** + * @param namespaceAware whether the factory should be namespace-aware + * @return a {@link DocumentBuilderFactory} hardened against XXE + * @throws ParserConfigurationException if a required secure feature is + * unsupported + */ + public static DocumentBuilderFactory newSecureDocumentBuilderFactory( + boolean namespaceAware) throws ParserConfigurationException { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(namespaceAware); + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setFeature(DISALLOW_DOCTYPE_DECL, true); + factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false); + factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); + factory.setFeature(LOAD_EXTERNAL_DTD, false); + factory.setXIncludeAware(false); + // Sonar java:S2755 "Going the extra mile" — does not alone block XXE + factory.setExpandEntityReferences(false); + return factory; + } + + /** + * @return a {@link SAXParserFactory} hardened against XXE + * @throws ParserConfigurationException if a required secure feature is + * unsupported + * @throws SAXNotRecognizedException if a feature name is not recognized + * @throws SAXNotSupportedException if a feature value is not supported + */ + public static SAXParserFactory newSecureSAXParserFactory() + throws ParserConfigurationException, SAXNotRecognizedException, + SAXNotSupportedException { + SAXParserFactory factory = SAXParserFactory.newInstance(); + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setFeature(DISALLOW_DOCTYPE_DECL, true); + factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false); + factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); + factory.setXIncludeAware(false); + return factory; + } + + /** + * @return a {@link TransformerFactory} that cannot load external DTDs or + * stylesheets + * @throws TransformerConfigurationException if secure attributes cannot be + * applied + */ + public static TransformerFactory newSecureTransformerFactory() + throws TransformerConfigurationException { + TransformerFactory factory = TransformerFactory.newInstance(); + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, ""); + return factory; + } + + /** + * Apply XXE-hardening features to an Xerces {@link DOMParser}. + * + * @param parser parser to configure + * @throws SAXException if a secure feature cannot be set + */ + public static void configureSecure(DOMParser parser) throws SAXException { + parser.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + parser.setFeature(DISALLOW_DOCTYPE_DECL, true); + parser.setFeature(EXTERNAL_GENERAL_ENTITIES, false); + parser.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); + parser.setFeature(LOAD_EXTERNAL_DTD, false); + } +} diff --git a/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java b/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java index 030c6bf66d..ee4e22ec2a 100644 --- a/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java +++ b/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java @@ -26,6 +26,7 @@ import org.apache.nutch.parse.ParseResult; import org.apache.nutch.parse.ParseStatus; import org.apache.nutch.parse.ParseText; +import org.apache.nutch.util.XmlUtil; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; @@ -45,6 +46,7 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; import org.xml.sax.InputSource; @@ -167,11 +169,14 @@ private void findLicenseUrl(Element element) { } } - /** Configure a namespace aware XML parser. */ - private static final DocumentBuilderFactory FACTORY = DocumentBuilderFactory - .newInstance(); + /** Configure a namespace aware XML parser hardened against XXE. */ + private static final DocumentBuilderFactory FACTORY; static { - FACTORY.setNamespaceAware(true); + try { + FACTORY = XmlUtil.newSecureDocumentBuilderFactory(true); + } catch (ParserConfigurationException e) { + throw new ExceptionInInitializerError(e); + } } /** Creative Commons' namespace URI. */ diff --git a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java index ebeb652dda..6d66b9b9d5 100644 --- a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java +++ b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Set; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import org.xml.sax.SAXException; @@ -55,6 +54,7 @@ import org.apache.nutch.protocol.http.api.HttpBase; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.XmlUtil; /** *

@@ -275,7 +275,7 @@ private static synchronized void setCredentials() InputStream is = conf.getConfResourceAsInputStream(authFile); if (is != null) { - Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() + Document doc = XmlUtil.newSecureDocumentBuilderFactory().newDocumentBuilder() .parse(is); Element rootElement = doc.getDocumentElement(); diff --git a/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java b/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java index 2aeb722b25..2714e3e194 100644 --- a/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java +++ b/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java @@ -32,13 +32,12 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -import javax.xml.parsers.DocumentBuilderFactory; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.nutch.net.URLNormalizer; import org.apache.nutch.net.URLNormalizers; import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.XmlUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -224,7 +223,7 @@ private List readConfiguration(Reader reader) { try { // borrowed heavily from code in Configuration.java - Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() + Document doc = XmlUtil.newSecureDocumentBuilderFactory().newDocumentBuilder() .parse(new InputSource(reader)); Element root = doc.getDocumentElement(); if (!"regex-normalize".equals(root.getTagName())) { diff --git a/src/test/org/apache/nutch/util/TestXmlUtil.java b/src/test/org/apache/nutch/util/TestXmlUtil.java new file mode 100644 index 0000000000..8b16d3ead0 --- /dev/null +++ b/src/test/org/apache/nutch/util/TestXmlUtil.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.util; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.junit.jupiter.api.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Unit tests for {@link XmlUtil} XXE hardening. + */ +public class TestXmlUtil { + + @Test + public void testSecureFactoryParsesSimpleDocument() throws Exception { + DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream( + "".getBytes(StandardCharsets.UTF_8))); + assertEquals("root", doc.getDocumentElement().getTagName()); + assertEquals("ok", doc.getDocumentElement().getAttribute("attr")); + } + + @Test + public void testSecureFactoryRejectsDoctype() throws Exception { + DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory(); + DocumentBuilder builder = factory.newDocumentBuilder(); + String xxe = "" + + "]>" + + "&xxe;"; + assertThrows(SAXException.class, () -> builder + .parse(new ByteArrayInputStream(xxe.getBytes(StandardCharsets.UTF_8)))); + } + + @Test + public void testDomUtilRejectsDoctype() { + String xxe = "" + + "]>" + + "&xxe;"; + Element element = DomUtil.getDom( + new ByteArrayInputStream(xxe.getBytes(StandardCharsets.UTF_8))); + // DomUtil swallows parse errors and returns null + assertEquals(null, element); + } + + @Test + public void testNamespaceAwareFactory() throws Exception { + DocumentBuilderFactory factory = XmlUtil + .newSecureDocumentBuilderFactory(true); + assertNotNull(factory.newDocumentBuilder()); + } +} From 56cb0c299643bea57ead5c96648c9607564f8a7b Mon Sep 17 00:00:00 2001 From: lewismc Date: Sat, 15 Aug 2026 13:02:57 -0700 Subject: [PATCH 2/2] NUTCH-3206 XML parsers should not be vulnerable to XXE attacks --- src/java/org/apache/nutch/util/XmlUtil.java | 7 ++++++- src/test/org/apache/nutch/util/TestXmlUtil.java | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/nutch/util/XmlUtil.java b/src/java/org/apache/nutch/util/XmlUtil.java index 85ec8981b1..a8ce9b64f2 100644 --- a/src/java/org/apache/nutch/util/XmlUtil.java +++ b/src/java/org/apache/nutch/util/XmlUtil.java @@ -121,10 +121,15 @@ public static TransformerFactory newSecureTransformerFactory() * @throws SAXException if a secure feature cannot be set */ public static void configureSecure(DOMParser parser) throws SAXException { - parser.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); parser.setFeature(DISALLOW_DOCTYPE_DECL, true); parser.setFeature(EXTERNAL_GENERAL_ENTITIES, false); parser.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); parser.setFeature(LOAD_EXTERNAL_DTD, false); + try { + parser.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + } catch (SAXNotRecognizedException | SAXNotSupportedException e) { + // Xerces' DOMParser does not expose the JAXP secure-processing feature. + // DTDs and external entities are already disabled above. + } } } diff --git a/src/test/org/apache/nutch/util/TestXmlUtil.java b/src/test/org/apache/nutch/util/TestXmlUtil.java index 8b16d3ead0..d5c9d7daec 100644 --- a/src/test/org/apache/nutch/util/TestXmlUtil.java +++ b/src/test/org/apache/nutch/util/TestXmlUtil.java @@ -68,6 +68,19 @@ public void testDomUtilRejectsDoctype() { assertEquals(null, element); } + @Test + public void testDomUtilParsesDocumentWithLeadingComment() { + String xml = "" + + "" + + "nutch" + + ""; + Element element = DomUtil.getDom( + new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))); + assertNotNull(element); + assertEquals("subcollections", element.getTagName()); + assertEquals(1, element.getElementsByTagName("subcollection").getLength()); + } + @Test public void testNamespaceAwareFactory() throws Exception { DocumentBuilderFactory factory = XmlUtil