Add raw HTML blocks to the markdown parser

src/Foliage-Markdown-Tests/FOHTMLRenderVisitorTest.class.st

@@ -171,3 +171,29 @@ FOHTMLRenderVisitorTest >> testRenderParagraphWithMixedInlineContent [
 	html := FOHTMLRenderVisitor render: doc.
 	self assert: html equals: '<p>Text can also contain some <strong>bold</strong>.</p>'
 ]
+
+{ #category : #tests }
+FOHTMLRenderVisitorTest >> testRenderRawHtmlBlockIsVerbatim [
+	| html source |
+	source := '<iframe src="/crowdsec.html" style="width:100%; height:800px; border:1px solid #ddd;"></iframe>'.
+	html := FOHTMLRenderVisitor render: (FORawHTMLBlock new html: source; yourself).
+	self assert: html equals: source
+]
+
+{ #category : #tests }
+FOHTMLRenderVisitorTest >> testRenderDocumentWithRawHtmlBlock [
+	| doc html iframe |
+	iframe := '<iframe src="/crowdsec.html" style="width:100%; height:800px; border:1px solid #ddd;"></iframe>'.
+	doc := FOMarkdownParser parse: 'Some text.', self nl, self nl, iframe.
+	html := FOHTMLRenderVisitor render: doc.
+	self assert: html equals: '<p>Some text.</p>', self nl, iframe
+]
+
+{ #category : #tests }
+FOHTMLRenderVisitorTest >> testRenderParagraphStillEscapesLessThan [
+	"Only a line that starts like a tag becomes raw HTML; a '<' inside prose is still escaped."
+	| doc html |
+	doc := FOMarkdownParser parse: '5 < 3 and <b>not a block</b>'.
+	html := FOHTMLRenderVisitor render: doc.
+	self assert: html equals: '<p>5 &lt; 3 and &lt;b&gt;not a block&lt;/b&gt;</p>'
+]

src/Foliage-Markdown-Tests/FOLineClassifierTest.class.st

@@ -127,3 +127,54 @@ FOLineClassifierTest >> testParagraphText [
 	c := FOLineClassifier classify: 'Lorem ipsum dolor sit amet.'.
 	self assert: c kind equals: #paragraphText
 ]
+
+{ #category : #tests }
+FOLineClassifierTest >> testHtmlBlockOpeningTag [
+	| c |
+	c := FOLineClassifier classify: '<iframe src="x"></iframe>'.
+	self assert: c kind equals: #htmlBlock
+]
+
+{ #category : #tests }
+FOLineClassifierTest >> testHtmlBlockClosingTag [
+	| c |
+	c := FOLineClassifier classify: '</div>'.
+	self assert: c kind equals: #htmlBlock
+]
+
+{ #category : #tests }
+FOLineClassifierTest >> testHtmlBlockComment [
+	| c |
+	c := FOLineClassifier classify: '<!-- a comment -->'.
+	self assert: c kind equals: #htmlBlock
+]
+
+{ #category : #tests }
+FOLineClassifierTest >> testLessThanFollowedBySpaceIsParagraphText [
+	| c |
+	c := FOLineClassifier classify: '5 < 3 and more text'.
+	self assert: c kind equals: #paragraphText.
+	c := FOLineClassifier classify: '< not a tag'.
+	self assert: c kind equals: #paragraphText
+]
+
+{ #category : #tests }
+FOLineClassifierTest >> testLessThanFollowedByDigitIsParagraphText [
+	| c |
+	c := FOLineClassifier classify: '<3 is a heart'.
+	self assert: c kind equals: #paragraphText
+]
+
+{ #category : #tests }
+FOLineClassifierTest >> testLoneLessThanIsParagraphText [
+	| c |
+	c := FOLineClassifier classify: '<'.
+	self assert: c kind equals: #paragraphText
+]
+
+{ #category : #tests }
+FOLineClassifierTest >> testIndentedHtmlStaysIndentedCode [
+	| c |
+	c := FOLineClassifier classify: '    <div>code sample</div>'.
+	self assert: c kind equals: #indentedCode
+]

src/Foliage-Markdown-Tests/FOMarkdownBlockParserTest.class.st

@@ -331,3 +331,55 @@ FOMarkdownBlockParserTest >> testInlineImageWithoutAttributes [
 	self assert: image src equals: '/images/x.png'.
 	self assert: image attributes isEmpty
 ]
+
+{ #category : #tests }
+FOMarkdownBlockParserTest >> testHtmlBlockBetweenParagraphs [
+	| doc |
+	doc := FOMarkdownParser parse:
+		'Some text.', self nl, self nl,
+		'<iframe src="/crowdsec.html"></iframe>', self nl, self nl,
+		'More text.'.
+	self assert: (doc children collect: [ :c | c class ]) asArray equals: { FOParagraph. FORawHTMLBlock. FOParagraph }.
+	self assert: doc children second html equals: '<iframe src="/crowdsec.html"></iframe>'.
+	self assert: (self plainTextOf: doc children first) equals: 'Some text.'.
+	self assert: (self plainTextOf: doc children third) equals: 'More text.'
+]
+
+{ #category : #tests }
+FOMarkdownBlockParserTest >> testHtmlBlockKeepsLinesVerbatimWithoutInlineParsing [
+	| doc block |
+	doc := FOMarkdownParser parse:
+		'<div class="note">', self nl,
+		'  **not parsed** and [not a link](http://x)', self nl,
+		'</div>'.
+	self assert: doc children size equals: 1.
+	block := doc children first.
+	self assert: (block isKindOf: FORawHTMLBlock).
+	self assert: block html equals: '<div class="note">', self nl, '  **not parsed** and [not a link](http://x)', self nl, '</div>'
+]
+
+{ #category : #tests }
+FOMarkdownBlockParserTest >> testHtmlBlockEndsAtBlankLine [
+	| doc |
+	doc := FOMarkdownParser parse: '<hr>', self nl, self nl, '<hr>'.
+	self assert: doc children size equals: 2.
+	self assert: doc children first html equals: '<hr>'.
+	self assert: doc children second html equals: '<hr>'
+]
+
+{ #category : #tests }
+FOMarkdownBlockParserTest >> testHtmlBlockEndsAtEndOfInput [
+	| doc |
+	doc := FOMarkdownParser parse: 'Intro', self nl, self nl, '<p class="x">', self nl, 'inner', self nl, '</p>'.
+	self assert: doc children size equals: 2.
+	self assert: doc children last html equals: '<p class="x">', self nl, 'inner', self nl, '</p>'
+]
+
+{ #category : #tests }
+FOMarkdownBlockParserTest >> testLessThanInParagraphIsNotAnHtmlBlock [
+	| doc |
+	doc := FOMarkdownParser parse: '5 < 3 and more text'.
+	self assert: doc children size equals: 1.
+	self assert: (doc children first isKindOf: FOParagraph).
+	self assert: (self plainTextOf: doc children first) equals: '5 < 3 and more text'
+]

src/Foliage-Markdown-Tests/FOMarkdownIntegrationTest.class.st

@@ -168,3 +168,21 @@ FOMarkdownIntegrationTest >> testFullDocumentRendersWithoutError [
 	self assert: (html includesSubstring: '<h2>H2 header</h2>').
 	self assert: (html includesSubstring: 'u-max-full-width')
 ]
+
+{ #category : #tests }
+FOMarkdownIntegrationTest >> testRawHtmlBlockAppendedToRealisticDocument [
+	"The syntax.md fixture itself contains no raw HTML (see testTopLevelBlockCountOrderAndTypes);
+	 appending an iframe after a blank line must add exactly one FORawHTMLBlock and leave
+	 every other block untouched."
+	| iframe extended html |
+	iframe := '<iframe src="/crowdsec.html" style="width:100%; height:800px; border:1px solid #ddd;"></iframe>'.
+	extended := FOMarkdownParser parse:
+		(FOFrontmatterParser parse: self syntaxMdFixture) body, self nl, self nl, iframe.
+	self assert: extended children size equals: doc children size + 1.
+	self assert: (extended children allButLast collect: [ :c | c class ]) asArray
+		equals: (doc children collect: [ :c | c class ]) asArray.
+	self assert: (extended children last isKindOf: FORawHTMLBlock).
+	html := FOHTMLRenderVisitor render: extended.
+	self assert: (html endsWith: self nl, iframe).
+	self assert: (html includesSubstring: '&lt;iframe') not
+]

src/Foliage-Markdown-Tests/FOPlainTextVisitorTest.class.st

@@ -68,3 +68,9 @@ FOPlainTextVisitorTest >> testComposedDocumentJoinsBlocksWithSpace [
 		'- B'.
 	self assert: text equals: 'Picking it up What happened so far? A B'
 ]
+
+{ #category : #tests }
+FOPlainTextVisitorTest >> testRawHtmlBlockContributesNothing [
+	self assert: (self plainTextOf: 'Before.', self nl, self nl, '<iframe src="/x.html"></iframe>', self nl, self nl, 'After.')
+		equals: 'Before.  After.'
+]

src/Foliage-Markdown/FOHTMLRenderVisitor.class.st

@@ -38,6 +38,12 @@ FOHTMLRenderVisitor >> visitCodeBlock: aCodeBlock [
 	^ '<pre><code', classAttr, '>', (self escapeHtml: aCodeBlock code), '</code></pre>'
 ]
 
+{ #category : #visiting }
+FOHTMLRenderVisitor >> visitRawHTMLBlock: aRawHTMLBlock [
+	"Deliberately not escaped and not wrapped: the author wrote HTML on purpose."
+	^ aRawHTMLBlock html
+]
+
 { #category : #visiting }
 FOHTMLRenderVisitor >> visitBlockquote: aBlockquote [
 	| rendered |

src/Foliage-Markdown/FOLineClassifier.class.st

@@ -25,6 +25,8 @@ FOLineClassifier class >> classify: aLine [
 		^ self classifyImage: trimmed indent: indent ].
 	indent >= 4 ifTrue: [
 		^ FOLineClassification new kind: #indentedCode; indent: indent; yourself ].
+	(self isHTMLBlockStart: trimmed) ifTrue: [
+		^ FOLineClassification new kind: #htmlBlock; indent: indent; yourself ].
 	^ FOLineClassification new kind: #paragraphText; indent: indent; yourself
 ]
 
@@ -119,6 +121,18 @@ FOLineClassifier class >> isImageLine: aLine [
 	^ (aLine beginsWith: '![') and: [ aLine notEmpty and: [ (aLine indexOf: $)) = aLine size ] ]
 ]
 
+{ #category : #testing }
+FOLineClassifier class >> isHTMLBlockStart: aLine [
+	"Simplified CommonMark detection: an opening tag, a closing tag or a
+	 comment/doctype. A lone '<' or one followed by a space or digit
+	 (as in '5 < 3') stays ordinary paragraph text."
+	| second |
+	aLine size < 2 ifTrue: [ ^ false ].
+	aLine first = $< ifFalse: [ ^ false ].
+	second := aLine at: 2.
+	^ second isLetter or: [ second = $/ or: [ second = $! ] ]
+]
+
 { #category : #classifying }
 FOLineClassifier class >> classifyImage: aLine indent: anIndent [
 	| closeBracket inner spaceIndex src attributes attrString |

src/Foliage-Markdown/FOMarkdownParser.class.st

@@ -30,6 +30,7 @@ FOMarkdownParser >> consumeBlock: aReader classification: aClassification [
 	aClassification kind = #blockquote ifTrue: [ ^ self consumeBlockquote: aReader ].
 	(aClassification kind = #unorderedListItem or: [ aClassification kind = #orderedListItem ])
 		ifTrue: [ ^ FOListParser parseFrom: aReader ].
+	aClassification kind = #htmlBlock ifTrue: [ ^ self consumeHTMLBlock: aReader ].
 	^ self consumeParagraph: aReader
 ]
 
@@ -87,6 +88,17 @@ FOMarkdownParser >> consumeFencedCode: aReader classification: aClassification [
 		yourself
 ]
 
+{ #category : #private }
+FOMarkdownParser >> consumeHTMLBlock: aReader [
+	"Raw lines are kept verbatim (no trimming, no inline parsing) up to the next
+	 blank line or the end of input. No attempt is made to balance tags."
+	| lines |
+	lines := OrderedCollection new.
+	[ aReader atEnd not and: [ (FOLineClassifier classify: aReader peek) kind ~= #blank ] ]
+		whileTrue: [ lines add: aReader next ].
+	^ FORawHTMLBlock new html: (self joinWithNewline: lines); yourself
+]
+
 { #category : #private }
 FOMarkdownParser >> consumeIndentedCode: aReader [
 	| lines line |

src/Foliage-Markdown/FOPlainTextVisitor.class.st

@@ -32,6 +32,12 @@ FOPlainTextVisitor >> visitCodeBlock: aCodeBlock [
 	^ aCodeBlock code
 ]
 
+{ #category : #visiting }
+FOPlainTextVisitor >> visitRawHTMLBlock: aRawHTMLBlock [
+	"Markup has no textual content worth including in a plain-text abstract."
+	^ ''
+]
+
 { #category : #visiting }
 FOPlainTextVisitor >> visitBlockquote: aBlockquote [
 	^ self renderChildren: aBlockquote

src/Foliage-Markdown/FORawHTMLBlock.class.st

@@ -0,0 +1,29 @@
+"
+A block of raw HTML copied verbatim from the markdown source. It starts at a line
+that looks like an HTML tag or comment and runs until the next blank line. Its
+content is neither inline-parsed nor escaped when rendered, so authors can embed
+elements the markdown syntax has no notation for (iframes, custom markup, ...).
+"
+Class {
+	#name : #FORawHTMLBlock,
+	#superclass : #FOBlockNode,
+	#instVars : [
+		'html'
+	],
+	#category : #'Foliage-Markdown'
+}
+
+{ #category : #accessing }
+FORawHTMLBlock >> html [
+	^ html
+]
+
+{ #category : #accessing }
+FORawHTMLBlock >> html: aString [
+	html := aString
+]
+
+{ #category : #visiting }
+FORawHTMLBlock >> acceptVisitor: aVisitor [
+	^ aVisitor visitRawHTMLBlock: self
+]

src/Foliage-Markdown/FOVisitor.class.st

@@ -24,6 +24,11 @@ FOVisitor >> visitCodeBlock: aCodeBlock [
 	^ self subclassResponsibility
 ]
 
+{ #category : #visiting }
+FOVisitor >> visitRawHTMLBlock: aRawHTMLBlock [
+	^ self subclassResponsibility
+]
+
 { #category : #visiting }
 FOVisitor >> visitBlockquote: aBlockquote [
 	^ self subclassResponsibility