TIKA-5014: write dupes up to a point

CHANGES.txt

@@ -56,6 +56,13 @@ Release 4.3.0 - ???
      and split GFM table cells on |. Math is emitted as <span class="math inline"> or
      <span class="math display">, and the Markdown writer outputs it unescaped (TIKA-5011).
 
+   * The OpenDocument parser writes repeated cells and rows (table:number-columns-repeated,
+     table:number-rows-repeated) once per repeat; a cell was written once with that
+     colspan and a row once, so values repeated across columns or rows were lost. Empty
+     cells and rows at the end of a row or table, such as the filler out to a sheet's
+     last column and row, are dropped. Past 10 million characters of repeated content
+     per document the rest are written once, with a tk:exception:warn entry (TIKA-5014).
+
    * VLM text recognizers keep superscripts and subscripts: inline <sup>/<sub> in model
      output (x<sup>2</sup>, H<sub>2</sub>O, footnote markers) reach the XHTML as sup/sub
      instead of plain text ("x2"), and the Markdown writer outputs them as inline HTML

tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java

@@ -91,7 +91,8 @@ public void testMacroODS() throws Exception {
         assertEquals(4, metadataList.size());
         Metadata parent = metadataList.get(0);
 
-        assertContains("<tr>", parent.get(TikaCoreProperties.TIKA_CONTENT));
+        // the sheet is an image shape and one empty row, which is dropped (TIKA-5014)
+        assertContains("<table><img", parent.get(TikaCoreProperties.TIKA_CONTENT));
         assertEquals("application/vnd.oasis.opendocument.spreadsheet",
                 parent.get(HttpHeaders.CONTENT_TYPE));
 
@@ -169,7 +170,8 @@ public void testMacroFODS() throws Exception {
         assertEquals(3, metadataList.size());
         Metadata parent = metadataList.get(0);
 
-        assertContains("<tr>", parent.get(TikaCoreProperties.TIKA_CONTENT));
+        // the sheet is an image shape and one empty row, which is dropped (TIKA-5014)
+        assertContains("<table><img", parent.get(TikaCoreProperties.TIKA_CONTENT));
         assertEquals("application/vnd.oasis.opendocument.flat.spreadsheet",
                 parent.get(HttpHeaders.CONTENT_TYPE));
 

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java

@@ -110,13 +110,14 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
         // and masks the real well-formedness error.
         final XHTMLBalancingHandler balancer = new XHTMLBalancingHandler(xhtml);
         tis.setCloseShield();
+        FlatOpenDocumentParserHandler fodHandler = null;
         try {
-            ContentHandler fodHandler = getContentHandler(balancer, metadata, context);
+            fodHandler = getContentHandler(balancer, metadata, context);
             XMLReaderUtils.parseSAX(tis,
                     new EmbeddedContentHandler(fodHandler), context);
             //can only detect subtype (text/pres/sheet) during parse.
             //update it here.
-            MediaType detected = ((FlatOpenDocumentParserHandler) fodHandler).getDetectedType();
+            MediaType detected = fodHandler.getDetectedType();
             if (detected != null) {
                 metadata.set(HttpHeaders.CONTENT_TYPE, detected.toString());
             }
@@ -125,6 +126,9 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
             // loop. Without this, the finally's xhtml.endDocument() throws
             // on the unbalanced stack and the masked </body> vs <p>/<b>
             // hides the real failure (truncated input, malformed XML, etc.).
+            if (fodHandler != null) {
+                fodHandler.bodyHandler.flushBuffered(e);
+            }
             balancer.drainOpenElements();
             throw e;
         } finally {
@@ -140,8 +144,9 @@ public void setExtractMacros(boolean extractMacros) {
     public boolean isExtractMacros() {
         return extractMacros;
     }
-    private ContentHandler getContentHandler(ContentHandler handler, Metadata metadata,
-                                             ParseContext context) {
+    private FlatOpenDocumentParserHandler getContentHandler(ContentHandler handler,
+                                                            Metadata metadata,
+                                                            ParseContext context) {
         return new FlatOpenDocumentParserHandler(handler, metadata, context, extractMacros);
     }
 
@@ -154,7 +159,7 @@ private static class FlatOpenDocumentParserHandler extends ContentHandlerDecorat
 
         private final ContentHandler defaultHandler = new DefaultHandler();
 
-        private final ContentHandler bodyHandler;
+        private final OpenDocumentBodyHandler bodyHandler;
         private final ContentHandler metadataHandler;
         private final ContentHandler macroHandler;
         private final boolean extractMacros;
@@ -180,7 +185,7 @@ private FlatOpenDocumentParserHandler(ContentHandler baseHandler, Metadata metad
             this.extractMacros = extractMacros;
 
             this.bodyHandler = new OpenDocumentBodyHandler(new NSNormalizerContentHandler(baseHandler),
-                            parseContext);
+                            parseContext, metadata);
 
             this.metadataHandler = new NSNormalizerContentHandler(
                     OpenDocumentMetaParser.getContentHandler(metadata, parseContext));

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java

@@ -72,6 +72,9 @@ class OpenDocumentBodyHandler extends ElementMappingContentHandler {
             new HashMap<>();
     private static final char[] SPACE = new char[]{' '};
     private static final String CLASS = "class";
+    // ODF 1.2+ sheet limits
+    private static final int MAX_COLUMNS = 16384;
+    private static final int MAX_ROWS = 1048576;
     private static final Attributes ANNOTATION_ATTRIBUTES = buildAttributes(CLASS, "annotation");
     private static final Attributes NOTE_ATTRIBUTES = buildAttributes(CLASS, "note");
     private static final Attributes NOTES_ATTRIBUTES = buildAttributes(CLASS, "notes");
@@ -104,27 +107,21 @@ class OpenDocumentBodyHandler extends ElementMappingContentHandler {
 
         // create HTML tables from table:-tags
         MAPPINGS.put(new QName(TABLE_NS, "table"), new TargetElement(XHTML, "table"));
-        // repeating of rows is ignored; for columns, see below!
+        // number-rows-repeated: see RepeatedElementHandler
         MAPPINGS.put(new QName(TABLE_NS, "table-row"), new TargetElement(XHTML, "tr"));
         // special mapping for rowspan/colspan attributes
         final HashMap<QName, QName> tableCellAttsMapping = new HashMap<>();
         tableCellAttsMapping
                 .put(new QName(TABLE_NS, "number-columns-spanned"), new QName("colspan"));
         tableCellAttsMapping.put(new QName(TABLE_NS, "number-rows-spanned"), new QName("rowspan"));
-        /* TODO: The following is not correct, the cell should be repeated not spanned!
-         * Code generates a HTML cell, spanning all repeated columns, to make the cell look correct.
-         * Problems may occur when both spanning and repeating is given, which is not allowed by
-         *  spec.
-         * Cell spanning instead of repeating  is not a problem, because OpenOffice uses it
-         * only for empty cells.
-         */
-        tableCellAttsMapping
-                .put(new QName(TABLE_NS, "number-columns-repeated"), new QName("colspan"));
+        // number-columns-repeated: see RepeatedElementHandler
         MAPPINGS.put(new QName(TABLE_NS, "table-cell"),
                 new TargetElement(XHTML, "td", tableCellAttsMapping));
     }
 
     private final ContentHandler handler;
+    private final RepeatedElementHandler repeatedCells;
+    private final RepeatedElementHandler repeatedRows;
     private final ParseContext parseContext;
     private final BitSet textNodeStack = new BitSet();
     //have we written the start style tags
@@ -156,12 +153,49 @@ class OpenDocumentBodyHandler extends ElementMappingContentHandler {
     // is that bold/italic/underline runs that begin or end inside a link lose
     // their formatting in the captured XHTML.
     private int anchorDepth = 0;
-    OpenDocumentBodyHandler(ContentHandler handler, ParseContext parseContext) {
-        super(handler, MAPPINGS);
-        this.handler = handler;
+    OpenDocumentBodyHandler(ContentHandler handler, ParseContext parseContext, Metadata metadata) {
+        this(new RepeatedElementHandler(handler, "tr", "table", MAX_ROWS,
+                new RepeatedElementHandler.Budget(metadata)), parseContext);
+    }
+
+    private OpenDocumentBodyHandler(RepeatedElementHandler rows, ParseContext parseContext) {
+        this(new RepeatedElementHandler(rows, "td", "tr", MAX_COLUMNS, rows.budget()), rows,
+                parseContext);
+    }
+
+    private OpenDocumentBodyHandler(RepeatedElementHandler cells, RepeatedElementHandler rows,
+                                    ParseContext parseContext) {
+        super(cells, MAPPINGS);
+        this.handler = cells;
+        this.repeatedCells = cells;
+        this.repeatedRows = rows;
         this.parseContext = parseContext;
     }
 
+    /**
+     * Writes table content still held for repeat handling. Call when the parse fails,
+     * so the partly read row is not lost.
+     */
+    void flushBuffered(Exception cause) {
+        try {
+            repeatedCells.flushBuffered();
+            repeatedRows.flushBuffered();
+        } catch (SAXException e) {
+            cause.addSuppressed(e);
+        }
+    }
+
+    private static int parseRepeat(String value) {
+        if (value == null) {
+            return 1;
+        }
+        try {
+            return Math.max(1, Integer.parseInt(value.trim()));
+        } catch (NumberFormatException e) {
+            return 1;
+        }
+    }
+
     private static Attributes buildAttributes(String key, String value) {
         AttributesImpl attrs = new AttributesImpl();
         attrs.addAttribute("", key, key, "CDATA", value);
@@ -478,6 +512,13 @@ public void startElement(String namespaceURI, String localName, String qName, At
         }
         // call next handler if no filtering
         if (completelyFiltered == 0) {
+            if (TABLE_NS.equals(namespaceURI) && "table-cell".equals(localName)) {
+                repeatedCells.repeatNext(
+                        parseRepeat(attrs.getValue(TABLE_NS, "number-columns-repeated")));
+            } else if (TABLE_NS.equals(namespaceURI) && "table-row".equals(localName)) {
+                repeatedRows.repeatNext(
+                        parseRepeat(attrs.getValue(TABLE_NS, "number-rows-repeated")));
+            }
             // special handling of text:h, that are directly passed
             // to incoming handler
             if (TEXT_NS.equals(namespaceURI) && "h".equals(localName)) {

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java

@@ -22,7 +22,6 @@
 
 import org.xml.sax.ContentHandler;
 import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
 
 import org.apache.tika.exception.TikaException;
 import org.apache.tika.io.TikaInputStream;
@@ -50,12 +49,15 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
     void parseInternal(TikaInputStream tis, final ContentHandler handler, Metadata metadata,
                        ParseContext context) throws IOException, SAXException, TikaException {
 
-        DefaultHandler dh = new OpenDocumentBodyHandler(handler, context);
+        OpenDocumentBodyHandler dh = new OpenDocumentBodyHandler(handler, context, metadata);
 
         tis.setCloseShield();
         try {
             XMLReaderUtils.parseSAX(tis,
                     new NSNormalizerContentHandler(dh), context);
+        } catch (SAXException | IOException | TikaException e) {
+            dh.flushBuffered(e);
+            throw e;
         } finally {
             tis.removeCloseShield();
         }