From d74ea14ffa3e94a96abfea50bb2509e4212b1fb2 Mon Sep 17 00:00:00 2001 From: Norbert Hartl Date: Fri, 11 Sep 2026 13:53:12 +0200 Subject: [PATCH] Add raw HTML blocks to the markdown parser A line starting with '<' followed by a letter, '/' or '!' now opens a raw HTML block that runs verbatim until the next blank line (or end of input). The lines are neither inline-parsed nor escaped when rendered, so authors can embed elements the markdown syntax has no notation for, e.g. an iframe. Previously such a line fell through to consumeParagraph: and came out as escaped tag soup. Follows the existing four-part shape: #htmlBlock classification in FOLineClassifier (checked after the indented-code rule, so indented tags stay code), consumeHTMLBlock: in FOMarkdownParser, the FORawHTMLBlock node, and visitRawHTMLBlock: on both visitors (HTML: unchanged string, plain text: ''). No tag whitelist, no tag balancing, no inline HTML inside paragraphs; a '<' followed by a space or digit (as in '5 < 3') is still paragraph text. Co-Authored-By: Claude Fable 5.1 --- .../FOHTMLRenderVisitorTest.class.st | 26 ++++++++++ .../FOLineClassifierTest.class.st | 51 ++++++++++++++++++ .../FOMarkdownBlockParserTest.class.st | 52 +++++++++++++++++++ .../FOMarkdownIntegrationTest.class.st | 18 +++++++ .../FOPlainTextVisitorTest.class.st | 6 +++ .../FOHTMLRenderVisitor.class.st | 6 +++ .../FOLineClassifier.class.st | 14 +++++ .../FOMarkdownParser.class.st | 12 +++++ .../FOPlainTextVisitor.class.st | 6 +++ src/Foliage-Markdown/FORawHTMLBlock.class.st | 29 +++++++++++ src/Foliage-Markdown/FOVisitor.class.st | 5 ++ 11 files changed, 225 insertions(+) create mode 100644 src/Foliage-Markdown/FORawHTMLBlock.class.st diff --git a/src/Foliage-Markdown-Tests/FOHTMLRenderVisitorTest.class.st b/src/Foliage-Markdown-Tests/FOHTMLRenderVisitorTest.class.st index c313f41..c6163ad 100644 --- a/src/Foliage-Markdown-Tests/FOHTMLRenderVisitorTest.class.st +++ b/src/Foliage-Markdown-Tests/FOHTMLRenderVisitorTest.class.st @@ -171,3 +171,29 @@ FOHTMLRenderVisitorTest >> testRenderParagraphWithMixedInlineContent [ html := FOHTMLRenderVisitor render: doc. self assert: html equals: '

Text can also contain some bold.

' ] + +{ #category : #tests } +FOHTMLRenderVisitorTest >> testRenderRawHtmlBlockIsVerbatim [ + | html source | + source := ''. + html := FOHTMLRenderVisitor render: (FORawHTMLBlock new html: source; yourself). + self assert: html equals: source +] + +{ #category : #tests } +FOHTMLRenderVisitorTest >> testRenderDocumentWithRawHtmlBlock [ + | doc html iframe | + iframe := ''. + doc := FOMarkdownParser parse: 'Some text.', self nl, self nl, iframe. + html := FOHTMLRenderVisitor render: doc. + self assert: html equals: '

Some text.

', self nl, iframe +] + +{ #category : #tests } +FOHTMLRenderVisitorTest >> testRenderParagraphStillEscapesLessThan [ + "Only a line that starts like a tag becomes raw HTML; a '<' inside prose is still escaped." + | doc html | + doc := FOMarkdownParser parse: '5 < 3 and not a block'. + html := FOHTMLRenderVisitor render: doc. + self assert: html equals: '

5 < 3 and <b>not a block</b>

' +] diff --git a/src/Foliage-Markdown-Tests/FOLineClassifierTest.class.st b/src/Foliage-Markdown-Tests/FOLineClassifierTest.class.st index 35a996d..8100aca 100644 --- a/src/Foliage-Markdown-Tests/FOLineClassifierTest.class.st +++ b/src/Foliage-Markdown-Tests/FOLineClassifierTest.class.st @@ -127,3 +127,54 @@ FOLineClassifierTest >> testParagraphText [ c := FOLineClassifier classify: 'Lorem ipsum dolor sit amet.'. self assert: c kind equals: #paragraphText ] + +{ #category : #tests } +FOLineClassifierTest >> testHtmlBlockOpeningTag [ + | c | + c := FOLineClassifier classify: ''. + self assert: c kind equals: #htmlBlock +] + +{ #category : #tests } +FOLineClassifierTest >> testHtmlBlockClosingTag [ + | c | + c := FOLineClassifier classify: ''. + self assert: c kind equals: #htmlBlock +] + +{ #category : #tests } +FOLineClassifierTest >> testHtmlBlockComment [ + | c | + c := FOLineClassifier classify: ''. + self assert: c kind equals: #htmlBlock +] + +{ #category : #tests } +FOLineClassifierTest >> testLessThanFollowedBySpaceIsParagraphText [ + | c | + c := FOLineClassifier classify: '5 < 3 and more text'. + self assert: c kind equals: #paragraphText. + c := FOLineClassifier classify: '< not a tag'. + self assert: c kind equals: #paragraphText +] + +{ #category : #tests } +FOLineClassifierTest >> testLessThanFollowedByDigitIsParagraphText [ + | c | + c := FOLineClassifier classify: '<3 is a heart'. + self assert: c kind equals: #paragraphText +] + +{ #category : #tests } +FOLineClassifierTest >> testLoneLessThanIsParagraphText [ + | c | + c := FOLineClassifier classify: '<'. + self assert: c kind equals: #paragraphText +] + +{ #category : #tests } +FOLineClassifierTest >> testIndentedHtmlStaysIndentedCode [ + | c | + c := FOLineClassifier classify: '
code sample
'. + self assert: c kind equals: #indentedCode +] diff --git a/src/Foliage-Markdown-Tests/FOMarkdownBlockParserTest.class.st b/src/Foliage-Markdown-Tests/FOMarkdownBlockParserTest.class.st index 7a17ded..d4087cc 100644 --- a/src/Foliage-Markdown-Tests/FOMarkdownBlockParserTest.class.st +++ b/src/Foliage-Markdown-Tests/FOMarkdownBlockParserTest.class.st @@ -331,3 +331,55 @@ FOMarkdownBlockParserTest >> testInlineImageWithoutAttributes [ self assert: image src equals: '/images/x.png'. self assert: image attributes isEmpty ] + +{ #category : #tests } +FOMarkdownBlockParserTest >> testHtmlBlockBetweenParagraphs [ + | doc | + doc := FOMarkdownParser parse: + 'Some text.', self nl, self nl, + '', self nl, self nl, + 'More text.'. + self assert: (doc children collect: [ :c | c class ]) asArray equals: { FOParagraph. FORawHTMLBlock. FOParagraph }. + self assert: doc children second html equals: ''. + self assert: (self plainTextOf: doc children first) equals: 'Some text.'. + self assert: (self plainTextOf: doc children third) equals: 'More text.' +] + +{ #category : #tests } +FOMarkdownBlockParserTest >> testHtmlBlockKeepsLinesVerbatimWithoutInlineParsing [ + | doc block | + doc := FOMarkdownParser parse: + '
', self nl, + ' **not parsed** and [not a link](http://x)', self nl, + '
'. + self assert: doc children size equals: 1. + block := doc children first. + self assert: (block isKindOf: FORawHTMLBlock). + self assert: block html equals: '
', self nl, ' **not parsed** and [not a link](http://x)', self nl, '
' +] + +{ #category : #tests } +FOMarkdownBlockParserTest >> testHtmlBlockEndsAtBlankLine [ + | doc | + doc := FOMarkdownParser parse: '
', self nl, self nl, '
'. + self assert: doc children size equals: 2. + self assert: doc children first html equals: '
'. + self assert: doc children second html equals: '
' +] + +{ #category : #tests } +FOMarkdownBlockParserTest >> testHtmlBlockEndsAtEndOfInput [ + | doc | + doc := FOMarkdownParser parse: 'Intro', self nl, self nl, '

', self nl, 'inner', self nl, '

'. + self assert: doc children size equals: 2. + self assert: doc children last html equals: '

', self nl, 'inner', self nl, '

' +] + +{ #category : #tests } +FOMarkdownBlockParserTest >> testLessThanInParagraphIsNotAnHtmlBlock [ + | doc | + doc := FOMarkdownParser parse: '5 < 3 and more text'. + self assert: doc children size equals: 1. + self assert: (doc children first isKindOf: FOParagraph). + self assert: (self plainTextOf: doc children first) equals: '5 < 3 and more text' +] diff --git a/src/Foliage-Markdown-Tests/FOMarkdownIntegrationTest.class.st b/src/Foliage-Markdown-Tests/FOMarkdownIntegrationTest.class.st index 0bf3cc4..c261a34 100644 --- a/src/Foliage-Markdown-Tests/FOMarkdownIntegrationTest.class.st +++ b/src/Foliage-Markdown-Tests/FOMarkdownIntegrationTest.class.st @@ -168,3 +168,21 @@ FOMarkdownIntegrationTest >> testFullDocumentRendersWithoutError [ self assert: (html includesSubstring: '

H2 header

'). self assert: (html includesSubstring: 'u-max-full-width') ] + +{ #category : #tests } +FOMarkdownIntegrationTest >> testRawHtmlBlockAppendedToRealisticDocument [ + "The syntax.md fixture itself contains no raw HTML (see testTopLevelBlockCountOrderAndTypes); + appending an iframe after a blank line must add exactly one FORawHTMLBlock and leave + every other block untouched." + | iframe extended html | + iframe := ''. + extended := FOMarkdownParser parse: + (FOFrontmatterParser parse: self syntaxMdFixture) body, self nl, self nl, iframe. + self assert: extended children size equals: doc children size + 1. + self assert: (extended children allButLast collect: [ :c | c class ]) asArray + equals: (doc children collect: [ :c | c class ]) asArray. + self assert: (extended children last isKindOf: FORawHTMLBlock). + html := FOHTMLRenderVisitor render: extended. + self assert: (html endsWith: self nl, iframe). + self assert: (html includesSubstring: '<iframe') not +] diff --git a/src/Foliage-Markdown-Tests/FOPlainTextVisitorTest.class.st b/src/Foliage-Markdown-Tests/FOPlainTextVisitorTest.class.st index 4a6caaa..780c729 100644 --- a/src/Foliage-Markdown-Tests/FOPlainTextVisitorTest.class.st +++ b/src/Foliage-Markdown-Tests/FOPlainTextVisitorTest.class.st @@ -68,3 +68,9 @@ FOPlainTextVisitorTest >> testComposedDocumentJoinsBlocksWithSpace [ '- B'. self assert: text equals: 'Picking it up What happened so far? A B' ] + +{ #category : #tests } +FOPlainTextVisitorTest >> testRawHtmlBlockContributesNothing [ + self assert: (self plainTextOf: 'Before.', self nl, self nl, '', self nl, self nl, 'After.') + equals: 'Before. After.' +] diff --git a/src/Foliage-Markdown/FOHTMLRenderVisitor.class.st b/src/Foliage-Markdown/FOHTMLRenderVisitor.class.st index e0448c4..25c8d6b 100644 --- a/src/Foliage-Markdown/FOHTMLRenderVisitor.class.st +++ b/src/Foliage-Markdown/FOHTMLRenderVisitor.class.st @@ -38,6 +38,12 @@ FOHTMLRenderVisitor >> visitCodeBlock: aCodeBlock [ ^ '
', (self escapeHtml: aCodeBlock code), '
' ] +{ #category : #visiting } +FOHTMLRenderVisitor >> visitRawHTMLBlock: aRawHTMLBlock [ + "Deliberately not escaped and not wrapped: the author wrote HTML on purpose." + ^ aRawHTMLBlock html +] + { #category : #visiting } FOHTMLRenderVisitor >> visitBlockquote: aBlockquote [ | rendered | diff --git a/src/Foliage-Markdown/FOLineClassifier.class.st b/src/Foliage-Markdown/FOLineClassifier.class.st index 1c669c1..c656e92 100644 --- a/src/Foliage-Markdown/FOLineClassifier.class.st +++ b/src/Foliage-Markdown/FOLineClassifier.class.st @@ -25,6 +25,8 @@ FOLineClassifier class >> classify: aLine [ ^ self classifyImage: trimmed indent: indent ]. indent >= 4 ifTrue: [ ^ FOLineClassification new kind: #indentedCode; indent: indent; yourself ]. + (self isHTMLBlockStart: trimmed) ifTrue: [ + ^ FOLineClassification new kind: #htmlBlock; indent: indent; yourself ]. ^ FOLineClassification new kind: #paragraphText; indent: indent; yourself ] @@ -119,6 +121,18 @@ FOLineClassifier class >> isImageLine: aLine [ ^ (aLine beginsWith: '![') and: [ aLine notEmpty and: [ (aLine indexOf: $)) = aLine size ] ] ] +{ #category : #testing } +FOLineClassifier class >> isHTMLBlockStart: aLine [ + "Simplified CommonMark detection: an opening tag, a closing tag or a + comment/doctype. A lone '<' or one followed by a space or digit + (as in '5 < 3') stays ordinary paragraph text." + | second | + aLine size < 2 ifTrue: [ ^ false ]. + aLine first = $< ifFalse: [ ^ false ]. + second := aLine at: 2. + ^ second isLetter or: [ second = $/ or: [ second = $! ] ] +] + { #category : #classifying } FOLineClassifier class >> classifyImage: aLine indent: anIndent [ | closeBracket inner spaceIndex src attributes attrString | diff --git a/src/Foliage-Markdown/FOMarkdownParser.class.st b/src/Foliage-Markdown/FOMarkdownParser.class.st index 7c3dd0f..ee86d37 100644 --- a/src/Foliage-Markdown/FOMarkdownParser.class.st +++ b/src/Foliage-Markdown/FOMarkdownParser.class.st @@ -30,6 +30,7 @@ FOMarkdownParser >> consumeBlock: aReader classification: aClassification [ aClassification kind = #blockquote ifTrue: [ ^ self consumeBlockquote: aReader ]. (aClassification kind = #unorderedListItem or: [ aClassification kind = #orderedListItem ]) ifTrue: [ ^ FOListParser parseFrom: aReader ]. + aClassification kind = #htmlBlock ifTrue: [ ^ self consumeHTMLBlock: aReader ]. ^ self consumeParagraph: aReader ] @@ -87,6 +88,17 @@ FOMarkdownParser >> consumeFencedCode: aReader classification: aClassification [ yourself ] +{ #category : #private } +FOMarkdownParser >> consumeHTMLBlock: aReader [ + "Raw lines are kept verbatim (no trimming, no inline parsing) up to the next + blank line or the end of input. No attempt is made to balance tags." + | lines | + lines := OrderedCollection new. + [ aReader atEnd not and: [ (FOLineClassifier classify: aReader peek) kind ~= #blank ] ] + whileTrue: [ lines add: aReader next ]. + ^ FORawHTMLBlock new html: (self joinWithNewline: lines); yourself +] + { #category : #private } FOMarkdownParser >> consumeIndentedCode: aReader [ | lines line | diff --git a/src/Foliage-Markdown/FOPlainTextVisitor.class.st b/src/Foliage-Markdown/FOPlainTextVisitor.class.st index 699af0e..8df1831 100644 --- a/src/Foliage-Markdown/FOPlainTextVisitor.class.st +++ b/src/Foliage-Markdown/FOPlainTextVisitor.class.st @@ -32,6 +32,12 @@ FOPlainTextVisitor >> visitCodeBlock: aCodeBlock [ ^ aCodeBlock code ] +{ #category : #visiting } +FOPlainTextVisitor >> visitRawHTMLBlock: aRawHTMLBlock [ + "Markup has no textual content worth including in a plain-text abstract." + ^ '' +] + { #category : #visiting } FOPlainTextVisitor >> visitBlockquote: aBlockquote [ ^ self renderChildren: aBlockquote diff --git a/src/Foliage-Markdown/FORawHTMLBlock.class.st b/src/Foliage-Markdown/FORawHTMLBlock.class.st new file mode 100644 index 0000000..87a5ce1 --- /dev/null +++ b/src/Foliage-Markdown/FORawHTMLBlock.class.st @@ -0,0 +1,29 @@ +" +A block of raw HTML copied verbatim from the markdown source. It starts at a line +that looks like an HTML tag or comment and runs until the next blank line. Its +content is neither inline-parsed nor escaped when rendered, so authors can embed +elements the markdown syntax has no notation for (iframes, custom markup, ...). +" +Class { + #name : #FORawHTMLBlock, + #superclass : #FOBlockNode, + #instVars : [ + 'html' + ], + #category : #'Foliage-Markdown' +} + +{ #category : #accessing } +FORawHTMLBlock >> html [ + ^ html +] + +{ #category : #accessing } +FORawHTMLBlock >> html: aString [ + html := aString +] + +{ #category : #visiting } +FORawHTMLBlock >> acceptVisitor: aVisitor [ + ^ aVisitor visitRawHTMLBlock: self +] diff --git a/src/Foliage-Markdown/FOVisitor.class.st b/src/Foliage-Markdown/FOVisitor.class.st index e652748..e5ee163 100644 --- a/src/Foliage-Markdown/FOVisitor.class.st +++ b/src/Foliage-Markdown/FOVisitor.class.st @@ -24,6 +24,11 @@ FOVisitor >> visitCodeBlock: aCodeBlock [ ^ self subclassResponsibility ] +{ #category : #visiting } +FOVisitor >> visitRawHTMLBlock: aRawHTMLBlock [ + ^ self subclassResponsibility +] + { #category : #visiting } FOVisitor >> visitBlockquote: aBlockquote [ ^ self subclassResponsibility