diff --git a/data/custom.xml b/data/custom.xml
index 767b538..1df40ef 100644
--- a/data/custom.xml
+++ b/data/custom.xml
@@ -243,20 +243,4 @@
-
-
-
-
-
-
-
-
diff --git a/lib/marcel/tables.rb b/lib/marcel/tables.rb
index dd69e2c..ba053e7 100644
--- a/lib/marcel/tables.rb
+++ b/lib/marcel/tables.rb
@@ -2777,7 +2777,7 @@ module Marcel
['video/webm', [[0, b["\032E\337\243"], [[4..4096, b["B\202"], [[4..4096, b['webm'], [[4..4096, b['V_VP8']], [4..4096, b['V_VP9']], [4..4096, b['V_AV1']]]]]]]]]],
['video/x-matroska', [[0, b["\032E\337\243\223B\202\210matroska"]]]],
['video/x-flv', [[0, b['FLV']]]],
- ['audio/mpeg', [[0, b["\377\362"]], [0, b["\377\363"]], [0, b["\377\364"]], [0, b["\377\365"]], [0, b["\377\366"]], [0, b["\377\367"]], [0, b["\377\372"]], [0, b["\377\373"]], [0, b["\377\374"]], [0, b["\377\375"]], [0, b["\377\343"]], [0, b['ID3']]]],
+ ['audio/mpeg', [[0, b["\377\362"]], [0, b["\377\363"]], [0, b["\377\364"]], [0, b["\377\365"]], [0, b["\377\366"]], [0, b["\377\367"]], [0, b["\377\372"]], [0, b["\377\373"]], [0, b["\377\374"]], [0, b["\377\375"]], [0, b["\377\343"]]]],
['application/pdf', [[0, b['%PDF-']], [0, b["\357\273\277%PDF-"]]]],
['application/msword', [[2080, b['Microsoft Word 6.0 Document']], [2080, b['Documento Microsoft Word 6']], [2112, b['MSWordDoc']], [0, b["1\276\000\000"]], [0, b['PO^Q`']], [0, b["\3767\000#"]], [0, b["\333\245-\000\000\000"]], [0, b["\224\246."]], [0..8, b["\320\317\021\340\241\261\032\341"], [[1152..4096, b["W\000o\000r\000d\000D\000o\000c\000u\000m\000e\000n\000t"]]]]]],
['application/vnd.openxmlformats-officedocument.wordprocessingml.document', [[0, b["PK\003\004"], [[30..65536, b['[Content_Types].xml'], [[0..4096, b['word/']]]], [30, b['_rels/.rels'], [[0..4096, b['word/']]]]]]]],
@@ -2786,7 +2786,6 @@ module Marcel
['application/x-tika-msworks-spreadsheet', [[0..8, b["\320\317\021\340\241\261\032\341"], [[1152..4096, b["W\000k\000s\000S\000S\000W\000o\000r\000k\000B\000o\000o\000k"]]]]]],
['application/vnd.ms-excel', [[2080, b['Microsoft Excel 5.0 Worksheet']], [2080, b['Foglio di lavoro Microsoft Exce']], [2114, b['Biff5']], [2121, b['Biff5']], [0..8, b["\320\317\021\340\241\261\032\341"], [[1152..4096, b["W\000o\000r\000k\000b\000o\000o\000k"]]]]]],
['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', [[0, b["PK\003\004"], [[30..65536, b['[Content_Types].xml'], [[0..4096, b['xl/']]]], [30, b['_rels/.rels'], [[0..4096, b['xl/']]]]]]]],
- ['image/x-tga', [[1, b["\001\001\000\000"]], [1, b["\000\002\000\000"]], [1, b["\000\003\000\000"]]]],
['application/x-endnote-refer', [[0..50, b['%A '], [[0..1000, b["\n%D "], [[0..1000, b["\n%T "]]]]]]]],
['application/x-tmx', [[0..256, b['']], [0, b["!\n"]]]],
['application/x-arj', [[0, b["`\352"]]]],
['application/x-asprs', [[0, b['LASF'], [[24, b["\001\001"]], [24, b["\001\002"]]]]]],
- ['application/x-atari-floppy-disk-image', [[0, b['0x9602'], [[4, b['0x8000'], [[11, b['0x00000000']]]], [4, b['0x0001'], [[11, b['0x00000000']]]]]]]],
+ ['application/x-atari-floppy-disk-image', [[0, b["\226\002"], [[4, b["\200\000"], [[11, b["\000\000\000\000"]]]], [4, b["\000\001"], [[11, b["\000\000\000\000"]]]]]]]],
['application/x-bat', [[0, b['@echo off']], [0, b['rem ']]]],
['application/x-berkeley-db;format=btree', [[0, b["b1\005\000"]], [0, b["\000\0051b"]], [0, b["b1\005\000"]], [12, b["b1\005\000"]], [12, b["\000\0051b"]], [12, b["b1\005\000"]]]],
['application/x-berkeley-db;format=hash', [[0, b["a\025\006\000"]], [0, b["\000\006\025a"]], [0, b["a\025\006\000"]], [12, b["a\025\006\000"]], [12, b["\000\006\025a"]], [12, b["a\025\006\000"]]]],
@@ -2995,7 +2990,6 @@ module Marcel
['application/x-endnote-style', [[0, b["\000\b"], [[4, b["\000\000"], [[8, b['RSFTSTYL']], [8, b['ENDNENFT']]]]]]]],
['application/x-erdas-hfa', [[0, b['EHFA_HEADER_TAG']]]],
['application/x-executable', [[0, b["\177ELF"], [[16, b["\002\000"]], [16, b["\000\002"]]]]]],
- ['application/x-fat-diskimage', [[0, b["\\xEB"], [[2, b["\\x90"]]]]]],
['application/x-filemaker', [[14, b["\300HBAM7"], [[525, b["HBAM2101OCT99\301\002H\aPro 7.0\300\300"]]]]]],
['application/x-foxmail', [[0, b["\020\020\020\020\020\020\020\021\021\021\021\021\021S"]]]],
['application/x-gnumeric', [[39, b['= 1
+ unsupported_rules.skip "mask", mime_type, match,
+ "#{mime_type}: unsupported multi-segment mask at range offset #{match.to_s}"
+ next nil
+ end
+
chain = children
segments.reverse_each do |(mask_offset, mask_length)|
masked_value = value[mask_offset, mask_length]
+ segment_offset = if Range === offset
+ (offset.begin + mask_offset)..(offset.end + mask_offset)
+ else
+ offset + mask_offset
+ end
+
+ # The match's own offset was validated, but shifting by the mask position can
+ # push a segment past the bound the offset validation enforces.
+ segment_bound = Range === segment_offset ? segment_offset.end : segment_offset
+ if segment_bound > MimeData::MAX_MAGIC_OFFSET
+ raise ArgumentError,
+ "Masked segment offset exceeds #{MimeData::MAX_MAGIC_OFFSET} in #{mime_type}: #{match.to_s}"
+ end
if chain.empty?
- chain = [[mask_offset, masked_value]]
+ chain = [[segment_offset, masked_value]]
else
- chain = [[mask_offset, masked_value, chain]]
+ chain = [[segment_offset, masked_value, chain]]
end
end
next chain[0]
diff --git a/test/generate_tables_test.rb b/test/generate_tables_test.rb
index 202a3e3..2d956fa 100644
--- a/test/generate_tables_test.rb
+++ b/test/generate_tables_test.rb
@@ -238,8 +238,8 @@ class Marcel::GenerateTablesTest < Marcel::TestCase
assert status.success?, errors
warning_lines = errors.lines
- assert_equal "Skipped 60 unsupported magic rules\n", warning_lines.pop
- assert_equal 114, warning_lines.size
+ assert_equal "Skipped 77 unsupported magic rules\n", warning_lines.pop
+ assert_equal 189, warning_lines.size
assert File.exist?(tables_path)
end
end
@@ -501,4 +501,207 @@ class Marcel::GenerateTablesTest < Marcel::TestCase
assert status.success?, errors
end
end
+
+ test "decodes typeless matches as Tika's default string type" do
+ xml = <<-'XML'
+
+
+
+
+
+
+
+
+ XML
+
+ verification = <<~'RUBY'
+ load ARGV.fetch(0)
+ matches = Marcel::MAGIC.to_h.fetch("application/x-typeless")
+ expected = [[0, "ZIM\x04".b], [3, "\x90plain".b]]
+ abort matches.inspect unless matches == expected
+ RUBY
+
+ assert_generates xml, verification
+ end
+
+ test "passes structural minShouldMatch containers through with no value" do
+ xml = <<-'XML'
+
+
+
+
+
+
+
+
+
+
+ XML
+
+ verification = <<~'RUBY'
+ load ARGV.fetch(0)
+ matches = Marcel::MAGIC.to_h.fetch("application/x-structural")
+ expected = [[0, nil, [[0, "alpha".b], [4, "beta".b]]]]
+ abort matches.inspect unless matches == expected
+ RUBY
+
+ assert_generates xml, verification
+ end
+
+ test "anchors mask segments at the match's own offset" do
+ xml = <<-'XML'
+
+
+
+
+
+
+
+
+ XML
+
+ verification = <<~'RUBY'
+ load ARGV.fetch(0)
+ matches = Marcel::MAGIC.to_h.fetch("application/x-masked")
+ expected = [
+ [16, "ABCD".b, [[24, "WXYZ".b]]],
+ [2..6, "\x06\x0B\x2A\x86\x48\x86\xF7\x0D\x01\x09\x10\x01".b],
+ ]
+ abort matches.inspect unless matches == expected
+ RUBY
+
+ assert_generates xml, verification
+ end
+
+ test "skips multi-segment masks at range offsets and their orphaned parents, recursively" do
+ xml = <<-'XML'
+
+
+
+
+
+
+
+
+
+
+
+ XML
+
+ Dir.mktmpdir("marcel-generator-test") do |directory|
+ xml_path = File.join(directory, "input.xml")
+ File.binwrite(xml_path, xml)
+
+ generated, errors, status = Open3.capture3(
+ RbConfig.ruby, File.expand_path("../script/generate_tables.rb", __dir__), xml_path
+ )
+
+ # The synthetic unsupported rules fail the pinned-manifest check, and by then the
+ # generator has warned about the whole skipped chain: the inexpressible masked leaf,
+ # its parent, and its grandparent in turn.
+ refute status.success?
+ assert_empty generated
+ assert_includes errors, "unsupported multi-segment mask at range offset"
+ assert_includes errors, %(match with no supported children: )
+ assert_includes errors, %(match with no supported children: )
+ end
+ end
+
+ test "rejects mask segments shifted past the maximum magic offset" do
+ fixed = <<-'XML'
+
+
+
+
+
+
+
+ XML
+ range = <<-'XML'
+
+
+
+
+
+
+
+ XML
+
+ [fixed, range].each do |xml|
+ Dir.mktmpdir("marcel-generator-test") do |directory|
+ xml_path = File.join(directory, "input.xml")
+ File.binwrite(xml_path, xml)
+
+ generated, errors, status = Open3.capture3(
+ RbConfig.ruby, File.expand_path("../script/generate_tables.rb", __dir__), xml_path
+ )
+
+ refute status.success?
+ assert_empty generated
+ assert_includes errors, "Masked segment offset exceeds 65536"
+ end
+ end
+ end
+
+ test "skips all-zero masks and their orphaned parents" do
+ xml = <<-'XML'
+
+
+
+
+
+
+
+
+
+ XML
+
+ Dir.mktmpdir("marcel-generator-test") do |directory|
+ xml_path = File.join(directory, "input.xml")
+ File.binwrite(xml_path, xml)
+
+ generated, errors, status = Open3.capture3(
+ RbConfig.ruby, File.expand_path("../script/generate_tables.rb", __dir__), xml_path
+ )
+
+ # Fails the pinned-manifest check after warning about the whole skipped chain.
+ refute status.success?
+ assert_empty generated
+ assert_includes errors, "unsupported all-zero mask"
+ assert_includes errors, %(match with no supported children: )
+ end
+ end
+
+ test "retains supported siblings when a parent loses only some children" do
+ # From the shipped data: Tika's DER-encoded pkcs7-signature rule needs a two-segment
+ # mask at a range offset, which is inexpressible here, so the 0x30 parent is skipped.
+ # Its supported PEM sibling must survive alone.
+ signature_magics = Marcel::MAGIC.select { |type, _| type == "application/pkcs7-signature" }.map(&:last)
+ assert_equal [[[0, "-----BEGIN PKCS7".b]]], signature_magics
+
+ # audio/mpeg's priority-50 rule loses its ID3 parent (regex child) but keeps the frame
+ # sync siblings, while Tika's deliberate priority-10 bare ID3 fallback survives whole.
+ mpeg_magics = Marcel::MAGIC.select { |type, _| type == "audio/mpeg" }.map(&:last)
+ assert_includes mpeg_magics, [[0, "ID3".b]]
+ assert mpeg_magics.any? { |matches| matches.include?([0, "\xFF\xFB".b]) && matches.none? { |match| match[1] == "ID3".b } },
+ "expected audio/mpeg frame-sync rule without a bare ID3 alternative"
+ end
+
+ private
+ def assert_generates(xml, verification)
+ Dir.mktmpdir("marcel-generator-test") do |directory|
+ xml_path = File.join(directory, "input.xml")
+ tables_path = File.join(directory, "tables.rb")
+ File.binwrite(xml_path, xml)
+
+ generated, errors, status = Open3.capture3(
+ RbConfig.ruby, File.expand_path("../script/generate_tables.rb", __dir__), xml_path
+ )
+ assert status.success?, errors
+ File.binwrite(tables_path, generated)
+
+ _output, errors, status = Open3.capture3(RbConfig.ruby, "-e", verification, tables_path)
+ assert status.success?, errors
+ end
+ end
end
diff --git a/test/magic_test.rb b/test/magic_test.rb
index 7babe00..0938626 100644
--- a/test/magic_test.rb
+++ b/test/magic_test.rb
@@ -40,12 +40,73 @@ def read(*)
# Before Tika 4.0.0, the timestamped-data magic matched any 11-byte OID under the
# 1.2.840.113549 arc, so sibling CMS content types (compressedData, authData, ...) in
# 1.2.840.113549.1.9.16.1.* were misdetected as application/timestamped-data. The 4.0.0
- # rules (carried in data/custom.xml with explicit match types) require the full
- # id-ct-timestampedData OID ending in .31.
+ # rules require the full id-ct-timestampedData OID ending in .31; the sibling arc gets
+ # Tika's deliberately coarse pkcs7-mime family label instead.
test "other CMS content types are not misdetected as timestamped-data" do
compressed_data = "\x30\x80\x06\x0B\x2A\x86\x48\x86\xF7\x0D\x01\x09\x10\x01\x09\xA0\x80".b
- assert_equal "application/octet-stream", Marcel::MimeType.for(compressed_data)
+ assert_equal "application/pkcs7-mime", Marcel::MimeType.for(compressed_data)
+ end
+
+ test "timestamped-data still beats the coarser pkcs7-mime family label" do
+ timestamped_data = "\x30\x80\x06\x0B\x2A\x86\x48\x86\xF7\x0D\x01\x09\x10\x01\x1F\xA0\x80".b
+
+ assert_equal "application/timestamped-data", Marcel::MimeType.for(timestamped_data)
+ end
+
+ # Tika's DER pkcs7-signature rule needs a mask marcel can't express, so it is skipped
+ # entirely rather than emitted as its bare SEQUENCE-tag parent, which would have
+ # classified nearly every DER structure (certificates, keys, ASN.1 blobs) as a signature.
+ test "unrelated DER structures are not misdetected as pkcs7-signature" do
+ bare_sequence = ("\x30\x80" + "\x00" * 30).b
+ der_certificate_shape = "\x30\x82\x03\x00\x30\x82\x02\x00\xA0\x03\x02\x01\x02\x02\x01\x01".b
+
+ assert_equal "application/octet-stream", Marcel::MimeType.for(bare_sequence)
+ assert_equal "application/octet-stream", Marcel::MimeType.for(der_certificate_shape)
+ end
+
+ test "typeless Tika magics are hex-decoded rather than matched as literal text" do
+ probes = {
+ "application/x-zim" => "ZIM\x04" + "\x00" * 12,
+ "application/x-ms-compress-szdd" => "SZDD\x88\xF0\x27\x33\x41\x00\x00\x00",
+ # OneNote section file: GUID {7B5C52E4-D88C-4DA7-AEB1-5378D02996D3} in its
+ # mixed-endian on-disk layout
+ "application/onenote;format=one" => "\xE4\x52\x5C\x7B\x8C\xD8\xA7\x4D\xAE\xB1\x53\x78\xD0\x29\x96\xD3",
+ "application/onenote;format=onetoc2" => "\xA1\x2F\xFF\x43\xD9\xEF\x76\x4C\x9E\xE2\x10\xEA\x57\x22\x76\x5F",
+ # 64-bit little-endian Mach-O with MH_OBJECT filetype
+ "application/x-mach-o-object" => "\xCF\xFA\xED\xFE\x07\x00\x00\x01\x03\x00\x00\x00\x01\x00\x00\x00",
+ # Atari ST floppy image: bootable checksum magic, executable flag, zeroed serial
+ "application/x-atari-floppy-disk-image" => "\x96\x02\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
+ # PKCS#12: SEQUENCE, INTEGER version 3, nested SEQUENCE, then the id-data OID
+ "application/x-pkcs12" => "\x30\x82\x03\x50\x02\x01\x03\x30\x82\x03\x46\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x07\x01",
+ "text/vtt" => "\xEF\xBB\xBFWEBVTT\n\nsubtitles",
+ }
+
+ probes.each do |expected_type, content|
+ assert_equal expected_type, Marcel::MimeType.for(content.b)
+ end
+ end
+
+ # These formerly matched via parents whose discriminating children marcel can't express:
+ # the bare parents (2-4 literal bytes) matched far more than the format they named.
+ test "over-broad bare parents of unsupported rules are no longer emitted" do
+ assert_equal "application/octet-stream", Marcel::MimeType.for("t1 is not a Touhou replay")
+ # Truecolor-TGA-shaped header bytes at offset 1 previously matched image/x-tga at priority 90
+ assert_equal "application/octet-stream", Marcel::MimeType.for("\x00\x00\x02\x00\x00\x00\x00\x00\x01\x02".b)
+
+ # AC-3 detection survives via Tika's deliberate bare syncword fallback rule
+ assert_equal "audio/ac3", Marcel::MimeType.for("\x0B\x77\x10\x40\x2F\x84\x29\x00".b)
+ end
+
+ # Sereal's version lives in the low nibble of byte 4, which needs a mask marcel can't
+ # express; the bare magic mislabeled every v2 stream as version=1. The deliberate
+ # retirement of content detection is pinned here so it can't quietly return; extension
+ # lookup still resolves the unversioned type.
+ test "sereal streams are no longer identified by their over-broad bare magic" do
+ assert_equal "application/octet-stream", Marcel::MimeType.for("=srl\x01\x00\x00\x00".b)
+ assert_equal "application/octet-stream", Marcel::MimeType.for("=srl\x02\x00\x00\x00".b)
+ assert_equal "application/octet-stream", Marcel::MimeType.for("=\xF3rl\x03\x00\x00\x00".b)
+ assert_equal "application/sereal", Marcel::MimeType.for(name: "data.srl")
end
test "add and remove type" do