diff --git a/Cargo.toml b/Cargo.toml index 233268b4..5d0b11ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ default = [ "serde" ] serde = ["dep:serde", "dep:serde_json"] docs = [] external-jets = [] +fmt = [] [dependencies] base64 = "0.21.2" diff --git a/src/ast.rs b/src/ast.rs index 8d4a0c93..971e36e2 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -343,10 +343,11 @@ impl PartialEq for CallName { } /// Definition of a custom function. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct CustomFunction { params: Arc<[FunctionParam]>, body: Arc, + span: Span, } impl CustomFunction { @@ -360,6 +361,11 @@ impl CustomFunction { &self.body } + /// Access the span of the complete function declaration. + pub fn span(&self) -> &Span { + &self.span + } + /// Return a pattern for the parameters of the function. pub fn params_pattern(&self) -> Pattern { Pattern::tuple( @@ -372,11 +378,14 @@ impl CustomFunction { } } +impl_eq_hash!(CustomFunction; params, body); + /// Parameter of a function. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct FunctionParam { identifier: Identifier, ty: ResolvedType, + span: Span, } impl FunctionParam { @@ -389,8 +398,15 @@ impl FunctionParam { pub fn ty(&self) -> &ResolvedType { &self.ty } + + /// Access the span of the complete parameter declaration. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(FunctionParam; identifier, ty); + /// Match expression. #[derive(Clone, Debug)] pub struct Match { @@ -461,6 +477,7 @@ pub struct EnumMatchArm { /// variants. pattern: Pattern, body: Arc, + span: Span, } impl EnumMatchArm { @@ -473,6 +490,11 @@ impl EnumMatchArm { pub fn body(&self) -> &Expression { &self.body } + + /// Access the span of the complete enum match arm. + pub fn span(&self) -> &Span { + &self.span + } } impl_eq_hash!(EnumMatchArm; pattern, body); @@ -514,10 +536,11 @@ impl AsRef for EnumMatch { } /// Arm of a [`Match`] expression. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct MatchArm { pattern: MatchPattern, expression: Arc, + span: Span, } impl MatchArm { @@ -530,8 +553,15 @@ impl MatchArm { pub fn expression(&self) -> &Expression { &self.expression } + + /// Access the span of the complete match arm. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(MatchArm; pattern, expression); + #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] pub enum ExprTree<'a> { Expression(&'a Expression), @@ -1379,7 +1409,11 @@ impl AbstractSyntaxTree for Function { .map(|param| { let identifier = param.identifier().clone(); let ty = scope.resolve(param.ty())?; - Ok(FunctionParam { identifier, ty }) + Ok(FunctionParam { + identifier, + ty, + span: *param.span(), + }) }) .collect::, Error>>() .with_span(from)?; @@ -1398,7 +1432,11 @@ impl AbstractSyntaxTree for Function { scope.exit_block(); debug_assert!(scope.is_outside_function()); - let function = CustomFunction { params, body }; + let function = CustomFunction { + params, + body, + span: *from.span(), + }; scope .insert_function(from.name().clone(), from.visibility().clone(), function) .with_span(from)?; @@ -1945,10 +1983,11 @@ impl AbstractSyntaxTree for EnumMatch { .into_iter() .zip(info.variants()) .map(|(arm, variant)| { - let pattern = analyze_enum_arm_bindings(arm, variant, scope, span)?; + let arm_span = *arm.span(); + let pattern = analyze_enum_arm_bindings(arm, variant, scope, arm_span)?; scope.enter_block(); let payload_ty = variant.payload_type(); - let typed_variables = pattern.is_of_type(payload_ty).with_span(span)?; + let typed_variables = pattern.is_of_type(payload_ty).with_span(arm_span)?; for (identifier, variable_ty) in typed_variables { scope.insert_variable(identifier, variable_ty); } @@ -1957,6 +1996,7 @@ impl AbstractSyntaxTree for EnumMatch { Ok(EnumMatchArm { pattern, body: body?, + span: arm_span, }) }) .collect::, Diagnostic>>()?; @@ -2347,8 +2387,8 @@ impl AbstractSyntaxTree for Match { scope.enter_block(); if let Some((pat_l, ty_l)) = from.left().pattern().as_typed_pattern() { - let ty_l = scope.resolve(ty_l).with_span(from)?; - let typed_variables = pat_l.is_of_type(&ty_l).with_span(from)?; + let ty_l = scope.resolve(ty_l).with_span(from.left())?; + let typed_variables = pat_l.is_of_type(&ty_l).with_span(from.left())?; for (identifier, ty) in typed_variables { scope.insert_variable(identifier, ty); } @@ -2357,8 +2397,8 @@ impl AbstractSyntaxTree for Match { scope.exit_block(); scope.enter_block(); if let Some((pat_r, ty_r)) = from.right().pattern().as_typed_pattern() { - let ty_r = scope.resolve(ty_r).with_span(from)?; - let typed_variables = pat_r.is_of_type(&ty_r).with_span(from)?; + let ty_r = scope.resolve(ty_r).with_span(from.right())?; + let typed_variables = pat_r.is_of_type(&ty_r).with_span(from.right())?; for (identifier, ty) in typed_variables { scope.insert_variable(identifier, ty); } @@ -2371,10 +2411,12 @@ impl AbstractSyntaxTree for Match { left: MatchArm { pattern: from.left().pattern().clone(), expression: ast_l, + span: *from.left().span(), }, right: MatchArm { pattern: from.right().pattern().clone(), expression: ast_r, + span: *from.right().span(), }, span: *from.as_ref(), }) @@ -2387,6 +2429,18 @@ impl AsRef for Assignment { } } +impl AsRef for FunctionParam { + fn as_ref(&self) -> &Span { + &self.span + } +} + +impl AsRef for CustomFunction { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for Expression { fn as_ref(&self) -> &Span { &self.span @@ -2411,6 +2465,110 @@ impl AsRef for Match { } } +impl AsRef for MatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + +impl AsRef for EnumMatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + +#[cfg(test)] +mod span_tests { + use crate::parse::ParseFromStr; + + use super::*; + + #[test] + fn analyzed_custom_function_preserves_declaration_and_parameter_spans() { + let source = "fn helper(value: u8) -> u8 { value }"; + let parsed = parse::Function::parse_from_str(source).expect("function parses"); + let mut scope = Scope::new(Box::new(ElementsJetHinter)); + + Function::analyze(&parsed, &ResolvedType::unit(), &mut scope).expect("function analyzes"); + let function = scope + .get_function(parsed.name()) + .expect("function is registered in scope"); + + assert_eq!(function.span().to_slice(source), Some(source)); + assert_eq!( + function.params()[0].span().to_slice(source), + Some("value: u8") + ); + } + + #[test] + fn analyzed_match_arms_preserve_their_parsed_spans() { + let source = r#"fn main() { + let input: Either = Left(1); + match input { + Left(left: u8) => {}, + Right(right: u8) => {}, + } +}"#; + let parsed = parse::Program::parse_from_str(source).expect("program parses"); + let program = + Program::analyze(&parsed, Box::new(ElementsJetHinter)).expect("program analyzes"); + + let ExpressionInner::Block(_, Some(last)) = program.main().inner() else { + panic!("main body should end in a match"); + }; + let ExpressionInner::Single(single) = last.inner() else { + panic!("match should be a single expression"); + }; + let SingleExpressionInner::Match(match_) = single.inner() else { + panic!("expected a binary match"); + }; + + assert_eq!( + match_.left().span().to_slice(source), + Some("Left(left: u8) => {},") + ); + assert_eq!( + match_.right().span().to_slice(source), + Some("Right(right: u8) => {},") + ); + } + + #[test] + fn analyzed_enum_match_arms_preserve_their_parsed_spans() { + let source = r#"enum Choice { First, Second, } +fn main() { + let input: Choice = Choice::First; + match input { + Choice::First => {}, + Choice::Second => {}, + } +}"#; + let parsed = parse::Program::parse_from_str(source).expect("program parses"); + let program = + Program::analyze(&parsed, Box::new(ElementsJetHinter)).expect("program analyzes"); + + let ExpressionInner::Block(_, Some(last)) = program.main().inner() else { + panic!("main body should end in an enum match"); + }; + let ExpressionInner::Single(single) = last.inner() else { + panic!("enum match should be a single expression"); + }; + let SingleExpressionInner::EnumMatch(match_) = single.inner() else { + panic!("expected an enum match"); + }; + + assert_eq!( + match_.arms()[0].span().to_slice(source), + Some("Choice::First => {},") + ); + assert_eq!( + match_.arms()[1].span().to_slice(source), + Some("Choice::Second => {},") + ); + } +} + #[cfg(test)] mod scope_resolution_tests { use super::{ElementsJetHinter, Program}; diff --git a/src/lexer.rs b/src/lexer.rs index 862195e1..116e112a 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -11,7 +11,11 @@ use crate::version::SIMC_STR; pub type Spanned = (T, SimpleSpan); pub type Tokens<'src> = Vec<(Token<'src>, crate::error::Span)>; +#[cfg(feature = "fmt")] +pub type FmtTokens<'src> = Vec<(FmtToken<'src>, crate::error::Span)>; + #[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] pub enum Token<'src> { // Keywords Pub, @@ -67,12 +71,81 @@ pub enum Token<'src> { // Built-in functions Macro(&'src str), +} - // Comments and block comments - // - // We would discard them for the compiler, but they are needed, for example, for the formatter. - Comment, +#[cfg(feature = "fmt")] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum TriviaKind { + LineComment, BlockComment, + Newline, + Whitespace, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LineEnding { + /// Windows carriage-return/line-feed (`\r\n`). + CrLf, + /// Unix line-feed (`\n`). + Lf, + /// Classic Mac OS carriage-return (`\r`). + Cr, +} + +impl LineEnding { + pub const fn as_str(self) -> &'static str { + match self { + Self::CrLf => "\r\n", + Self::Lf => "\n", + Self::Cr => "\r", + } + } +} + +#[cfg(feature = "fmt")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Trivia<'src> { + LineComment(&'src str), + BlockComment(&'src str), + Newline(LineEnding), + Whitespace(&'src str), +} + +#[cfg(feature = "fmt")] +impl<'src> Trivia<'src> { + pub const fn line_comment(text: &'src str) -> Self { + Self::LineComment(text) + } + + pub const fn block_comment(text: &'src str) -> Self { + Self::BlockComment(text) + } + + pub const fn newline(line_ending: LineEnding) -> Self { + Self::Newline(line_ending) + } + + pub const fn whitespace(text: &'src str) -> Self { + Self::Whitespace(text) + } + + pub const fn kind(&self) -> TriviaKind { + match self { + Self::LineComment(_) => TriviaKind::LineComment, + Self::BlockComment(_) => TriviaKind::BlockComment, + Self::Newline(_) => TriviaKind::Newline, + Self::Whitespace(_) => TriviaKind::Whitespace, + } + } +} + +#[cfg(feature = "fmt")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum FmtToken<'src> { + Token(Token<'src>), + Trivia(Trivia<'src>), } impl<'src> fmt::Display for Token<'src> { @@ -120,9 +193,21 @@ impl<'src> fmt::Display for Token<'src> { Token::Param(s) => write!(f, "param::{}", s), Token::Bool(b) => write!(f, "{}", b), + } + } +} - Token::Comment => write!(f, "comment"), - Token::BlockComment => write!(f, "block_comment"), +#[cfg(feature = "fmt")] +impl<'src> fmt::Display for FmtToken<'src> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FmtToken::Token(t) => { + write!(f, "{}", t) + } + FmtToken::Trivia( + Trivia::LineComment(text) | Trivia::BlockComment(text) | Trivia::Whitespace(text), + ) => write!(f, "{text}"), + FmtToken::Trivia(Trivia::Newline(line_ending)) => write!(f, "{}", line_ending.as_str()), } } } @@ -130,11 +215,40 @@ impl<'src> fmt::Display for Token<'src> { /// Recognizer for a `// ...` line comment. fn line_comment<'src>( ) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + let newline = line_ending(); + just("//") - .then(any().and_is(just('\n').not()).repeated()) + .ignore_then(any().and_is(newline.not()).repeated()) .ignored() } +/// Recognizer for different newline encodings (`Windows`: `\r\n`, `Unix`: `\n`, `Mac`: `\r`). +fn line_ending<'src>( +) -> impl Parser<'src, &'src str, LineEnding, extra::Err>> + Clone { + choice(( + just("\r\n").to(LineEnding::CrLf), + just("\n").to(LineEnding::Lf), + just("\r").to(LineEnding::Cr), + )) +} + +/// Recognizer for whitespace. +#[cfg(feature = "fmt")] +fn whitespace<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + any() + .filter(|c: &char| c.is_whitespace() && *c != '\n' && *c != '\r') + .repeated() + .at_least(1) + .ignored() +} + +/// Recognizer for whitespace or a newline. +fn whitespace_or_newline<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + any().filter(|c: &char| c.is_whitespace()).ignored() +} + /// Recognizer for a (possibly nested) `/* ... */` block comment; an unterminated /// comment is reported and swallows the rest of the input. fn block_comment<'src>( @@ -152,23 +266,23 @@ fn block_comment<'src>( }) } +/// One non-empty trivia item. Keeping this separate from [`trivia`] lets the +/// ordinary lexer include trivia in the same recovery boundary as tokens. +fn trivia_item<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + choice((line_comment(), block_comment(), whitespace_or_newline())) +} + /// Trivia — whitespace and comments — shared with the version-directive scanner /// (`version::SimcDirective::scan`) so the lexer and the scanner agree on comment /// syntax. pub(crate) fn trivia<'src>( ) -> impl Parser<'src, &'src str, (), extra::Err>> { - choice(( - line_comment(), - block_comment(), - any().filter(|c: &char| c.is_whitespace()).ignored(), - )) - .repeated() - .ignored() + trivia_item().repeated().ignored() } -pub fn lexer<'src>( -) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> -{ +fn to_token<'src>( +) -> impl Parser<'src, &'src str, Token<'src>, extra::Err>> { let digits_with_underscore = |radix: u32| { any() .filter(move |c: &char| c.is_digit(radix)) @@ -243,25 +357,46 @@ pub fn lexer<'src>( just(">").to(Token::RAngle), )); - let comment = line_comment().to(Token::Comment); - let block_comment = block_comment().to(Token::BlockComment); - let token = choice(( - comment, - block_comment, - jet, - witness, - param, - macros, - keyword, - hex, - bin, - num, - op, - )); + choice((jet, witness, param, macros, keyword, hex, bin, num, op)) +} - token - .map_with(|tok, e| (tok, e.span())) - .padded() +pub fn lexer<'src>( +) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> +{ + let lexeme = choice((trivia_item().to(None), to_token().map(Some))) + .map_with(|token, e| (token, e.span())) + .recover_with(skip_then_retry_until(any().ignored(), end())); + + lexeme.repeated().collect::>().map(|lexemes| { + lexemes + .into_iter() + .filter_map(|(token, span)| token.map(|token| (token, span))) + .collect() + }) +} + +#[cfg(feature = "fmt")] +pub fn lexer_lossless<'src>( +) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> +{ + let token = to_token().map(FmtToken::Token); + + let newline = line_ending().map(Trivia::newline).map(FmtToken::Trivia); + let whitespace = whitespace() + .to_slice() + .map(Trivia::whitespace) + .map(FmtToken::Trivia); + let line_comment = line_comment() + .to_slice() + .map(Trivia::line_comment) + .map(FmtToken::Trivia); + let block_comment = block_comment() + .to_slice() + .map(Trivia::block_comment) + .map(FmtToken::Trivia); + + choice((line_comment, block_comment, newline, whitespace, token)) + .map_with(|lexeme, e| (lexeme, e.span())) .recover_with(skip_then_retry_until(any().ignored(), end())) .repeated() .collect() @@ -271,7 +406,7 @@ pub fn lexer<'src>( /// offset `start` — the end of the version directive per `SimcDirective::prescan`, /// or `0`. Spans are reported relative to the full input. /// -/// All comments in the input code are discarded. +/// All comments, newlines, and spaces in the input code are discarded. pub fn lex( file_id: usize, input: &str, @@ -280,26 +415,59 @@ pub fn lex( let (tokens, lex_errors) = lexer().parse(&input[start..]).into_output_errors(); let shift = |span| Span::from_chumsky(file_id, span, start); - let mut diagnostics: Vec = Vec::new(); + let mut diagnostics: Vec = lex_errors + .into_iter() + .map(|err| { + Diagnostic::new( + Error::CannotParse { + msg: err.reason().to_string(), + }, + shift(*err.span()), + ) + }) + .collect(); + + let tokens = tokens.map(|vec| { + vec.into_iter() + .filter_map(|(tok, span)| filter_token(tok, span, &mut diagnostics, shift)) + .collect() + }); + + (tokens, diagnostics) +} - diagnostics.extend(lex_errors.into_iter().map(|err| { - Diagnostic::new( - Error::CannotParse { - msg: err.reason().to_string(), - }, - shift(*err.span()), - ) - })); +/// Lexes an input string into a lossles stream of tokens with spans, beginning at byte +/// offset `start` — the end of the version directive per `SimcDirective::prescan`, +/// or `0`. Spans are reported relative to the full input. +/// +/// All comments, newlines, and spaces in the input code are remained. +#[cfg(feature = "fmt")] +pub fn lex_lossless( + file_id: usize, + input: &str, + start: usize, +) -> (Option>, Vec) { + let (tokens, lex_errors) = lexer_lossless().parse(&input[start..]).into_output_errors(); + let shift = |span: SimpleSpan| Span::from_chumsky(file_id, span, start); + + let mut diagnostics: Vec = lex_errors + .into_iter() + .map(|err| { + Diagnostic::new( + Error::CannotParse { + msg: err.reason().to_string(), + }, + shift(*err.span()), + ) + }) + .collect(); let tokens = tokens.map(|vec| { vec.into_iter() - .filter_map(|(tok, span)| match tok { - Token::Comment | Token::BlockComment => None, - Token::Simc => { - diagnostics.push(Diagnostic::new(Error::ReservedSimcKeyword, shift(span))); - None - } - tok => Some((tok, shift(span))), + .filter_map(|(fmt_tok, span)| match fmt_tok { + FmtToken::Token(tok) => filter_token(tok, span, &mut diagnostics, shift) + .map(|(t, s)| (FmtToken::Token(t), s)), + FmtToken::Trivia(t) => Some((FmtToken::Trivia(t), shift(span))), }) .collect() }); @@ -307,6 +475,26 @@ pub fn lex( (tokens, diagnostics) } +fn filter_token<'src, F: Fn(SimpleSpan) -> Span>( + tok: Token<'src>, + span: SimpleSpan, + errors: &mut Vec, + convert_span: F, +) -> Option<(Token<'src>, Span)> { + match tok { + // The reserved keyword is a sentinel: the prescan consumed the one + // legitimate directive before lexing, so any occurrence is misplaced. + Token::Simc => { + errors.push(Diagnostic::new( + Error::ReservedSimcKeyword, + convert_span(span), + )); + None + } + tok => Some((tok, convert_span(span))), + } +} + /// A list of all reserved keywords. pub const KEYWORDS: &[&str] = &[ "pub", "use", "as", "fn", "let", "type", "mod", "const", "match", "enum", CRATE_STR, SIMC_STR, @@ -324,148 +512,525 @@ mod tests { use super::*; - fn lex<'src>( - input: &'src str, - ) -> (Option>>, Vec>) { - let (tokens, errors) = lexer().parse(input).into_output_errors(); - let tokens = tokens.map(|vec| vec.iter().map(|(tok, _)| tok.clone()).collect::>()); - (tokens, errors) - } + mod lexer { + use super::*; + + fn lex<'src>( + input: &'src str, + ) -> (Option>>, Vec>) { + let (tokens, errors) = lexer().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(tok, _)| tok.clone()) + .collect::>() + }); + (tokens, errors) + } + #[test] + fn test_block_comment_simple() { + let input = "/* hello world */"; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![]), + "Should produce a single block comment token" + ); + } - #[test] - fn test_block_comment_simple() { - let input = "/* hello world */"; - let (tokens, errors) = lex(input); - - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!( - tokens, - Some(vec![Token::BlockComment]), - "Should produce a single block comment token" - ); - } + #[test] + fn test_block_comment_nested() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_nested() { - let input = "/* outer /* inner */ outer */"; - let (tokens, errors) = lex(input); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_deeply_nested() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_deeply_nested() { - let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; - let (tokens, errors) = lex(input); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_multiline() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_multiline() { - let input = "/* \n line 1 \n /* inner \n line */ \n */"; - let (tokens, errors) = lex(input); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_unclosed() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_unclosed() { - let input = "/* unclosed comment start"; - let (tokens, errors) = lex(input); + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); - assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); - let err = &errors[0]; - assert_eq!(err.span().start, 0); - assert_eq!(err.span().end, 2); - assert_eq!(err.to_string(), "Unclosed block comment"); + assert_eq!(tokens, Some(vec![])); + } - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_partial_nesting_unclosed() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_partial_nesting_unclosed() { - let input = "/* outer /* inner */"; - let (tokens, errors) = lex(input); + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!(tokens, Some(vec![])); + } - assert_eq!(errors.len(), 1); - assert_eq!(errors[0].span().start, 0); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_double_unclosed() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex(input); + + assert_eq!(errors.len(), 2); + + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); + + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); + + assert_eq!(tokens, Some(vec![])); + } + + #[test] + fn test_spaces_resolution() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } + + #[test] + fn test_ignoring_tokens_after_comment_with_incorrect_symbol() { + let input = "fn main() {} @// fn hello(){} simc"; + let (tokens, errors) = lex(input); + + assert_eq!( + errors.len(), + 1, + "comment contents must not be retried as code" + ); + assert!(errors[0].to_string().contains("found '@' expected")); + assert_eq!( + tokens, + Some(vec![ + Token::Fn, + Token::Ident("main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + ]) + ); - #[test] - fn test_block_comment_double_unclosed() { - let input = "/* outer /* inner"; - let (tokens, errors) = lex(input); + let (_tokens, diagnostics) = super::lex(0, input, 0); + assert_eq!(diagnostics.len(), 1); + assert!(matches!(diagnostics[0].error(), Error::CannotParse { .. })); + } - assert_eq!(errors.len(), 2); + #[test] + fn test_ignoring_tokens_after_comment() { + let input = "fn main() {} /* fn hello(){} \n simc 0.6.0; */ \n\ + // enum Name {} match true {} \n fn other_main() {} "; + let (tokens, errors) = lex(input); + + assert!(dbg!(errors).is_empty()); + assert_eq!( + tokens, + Some(vec![ + Token::Fn, + Token::Ident("main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + Token::Fn, + Token::Ident("other_main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + ]) + ); + } - assert_eq!(errors[0].span().start, 9); - assert_eq!(errors[0].to_string(), "Unclosed block comment"); + #[test] + fn simc_is_reserved() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); + assert!( + tokens + .expect("recovery keeps the stream") + .iter() + .all(|(tok, _)| !matches!(tok, Token::Simc)), + "the sentinel must not reach the token stream for {src:?}" + ); + } + + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex("simcfoo"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); + } - assert_eq!(errors[1].span().start, 0); - assert_eq!(errors[1].to_string(), "Unclosed block comment"); + #[test] + fn test_enum_token() { + let (tokens, errors) = lex("enum Path { Inherit, ColdSpend }"); + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![ + Token::Enum, + Token::Ident("Path"), + Token::LBrace, + Token::Ident("Inherit"), + Token::Comma, + Token::Ident("ColdSpend"), + Token::RBrace, + ]) + ); + } + + #[test] + fn lexer_test() { + use chumsky::prelude::*; + + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + let (tokens, lex_errs) = lexer().parse(src).into_output_errors(); + let _ = tokens.unwrap(); + + assert!(lex_errs.is_empty()); + } } - #[test] - fn simc_is_reserved() { - // The prescan consumes the one legitimate leading directive before lexing, - // so `lex` reports any `simc` it sees and drops the sentinel token. - for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { - let (tokens, errors) = super::lex(0, src, 0); - assert!( - errors.iter().any(|e| e.to_string().contains("reserved")), - "expected a reserved-keyword error for {src:?}, got: {errors:?}" + #[cfg(feature = "fmt")] + mod fmt_lexer { + use super::*; + + fn lex_lossless<'src>( + input: &'src str, + ) -> ( + Option>>, + Vec>, + ) { + let (tokens, errors) = lexer_lossless().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(fmt_tok, _)| fmt_tok) + .collect::>() + }); + (tokens, errors) + } + + fn fmt_trivia<'src>(kind: TriviaKind, text: &'src str) -> FmtToken<'src> { + let trivia = match kind { + TriviaKind::LineComment => Trivia::line_comment(text), + TriviaKind::BlockComment => Trivia::block_comment(text), + TriviaKind::Newline => Trivia::newline(match text { + "\r\n" => LineEnding::CrLf, + "\n" => LineEnding::Lf, + "\r" => LineEnding::Cr, + _ => panic!("invalid newline spelling: {text:?}"), + }), + TriviaKind::Whitespace => Trivia::whitespace(text), + }; + + FmtToken::Trivia(trivia) + } + + #[test] + fn test_block_comment_simple_fmt() { + let input = "/* hello world */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]), + "Should produce a single block comment token" + ); + } + + #[test] + fn lossless_trivia_display_preserves_source_text() { + let input = "// comment\r\n\t"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + let rendered: String = tokens + .expect("lossless lexing succeeds") + .iter() + .map(ToString::to_string) + .collect(); + + assert_eq!(rendered, input); + } + + #[test] + fn lossless_lexer_keeps_each_newline_kind_with_its_span() { + let input = "first\r\nsecond\rthird\nfourth"; + let (tokens, errors) = super::lex_lossless(0, input, 0); + + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + let tokens = tokens.expect("lossless lexing succeeds"); + let line_endings: Vec<_> = tokens + .iter() + .filter_map(|(token, _)| match token { + FmtToken::Trivia(Trivia::Newline(line_ending)) => Some(*line_ending), + _ => None, + }) + .collect(); + let newlines: Vec<_> = tokens + .iter() + .filter(|(token, _)| { + matches!(token, FmtToken::Trivia(trivia) if trivia.kind() == TriviaKind::Newline) + }) + .map(|(_, span)| span.to_slice(input)) + .collect(); + + assert_eq!( + line_endings, + vec![LineEnding::CrLf, LineEnding::Cr, LineEnding::Lf] + ); + assert_eq!(newlines, vec![Some("\r\n"), Some("\r"), Some("\n")]); + } + + #[test] + fn test_block_comment_nested_fmt() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_deeply_nested_fmt() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_multiline_fmt() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_unclosed_fmt() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); + + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_partial_nesting_unclosed_fmt() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_double_unclosed_fmt() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 2); + + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); + + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); + + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_spaces_resolution_fmt() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![ + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r"), + ]) ); + } + + #[test] + fn test_ignoring_tokens_after_comment_with_incorrect_symbol() { + let input = "fn main() {} @// fn hello(){} simc"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1, "only the invalid character is an error"); + assert!(errors[0].to_string().contains("found '@' expected")); + + let tokens = tokens.expect("recovery keeps the lossless stream"); + assert!(matches!( + tokens.last(), + Some(FmtToken::Trivia(Trivia::LineComment(_))) + )); assert!( tokens - .expect("recovery keeps the stream") .iter() - .all(|(tok, _)| !matches!(tok, Token::Simc)), - "the sentinel must not reach the token stream for {src:?}" + .all(|token| !matches!(token, FmtToken::Token(Token::Simc))), + "comment contents must not become semantic tokens" ); } - // Identifiers merely starting with `simc` are ordinary identifiers. - let (tokens, errors) = lex("simcfoo"); - assert!(errors.is_empty(), "unexpected: {errors:?}"); - assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); - } + #[test] + fn test_not_ignoring_tokens_after_comment() { + let input = "fn main() {} /* fn hello(){} \n simc 0.6.0; */ \n\ + // enum Name {} match true {} \n fn other_main() {} "; + let (tokens, errors) = lex_lossless(input); + + assert!(dbg!(errors).is_empty()); + assert_eq!( + tokens, + Some(vec![ + FmtToken::Token(Token::Fn), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Ident("main")), + FmtToken::Token(Token::LParen), + FmtToken::Token(Token::RParen), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::LBrace), + FmtToken::Token(Token::RBrace), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia( + TriviaKind::BlockComment, + "/* fn hello(){} \n simc 0.6.0; */" + ), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::LineComment, "// enum Name {} match true {} "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Fn), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Ident("other_main")), + FmtToken::Token(Token::LParen), + FmtToken::Token(Token::RParen), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::LBrace), + FmtToken::Token(Token::RBrace), + fmt_trivia(TriviaKind::Whitespace, " "), + ]) + ); + } - #[test] - fn test_enum_token() { - let (tokens, errors) = lex("enum Path { Inherit, ColdSpend }"); - assert!(errors.is_empty()); - assert_eq!( - tokens, - Some(vec![ - Token::Enum, - Token::Ident("Path"), - Token::LBrace, - Token::Ident("Inherit"), - Token::Comma, - Token::Ident("ColdSpend"), - Token::RBrace, - ]) - ); - } + #[test] + fn simc_is_reserved_fmt() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex_lossless(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); + + assert!( + tokens + .expect("recovery keeps the stream") + .iter() + .all(|(tok, _)| !matches!(tok, FmtToken::Token(Token::Simc))), + "the sentinel must not reach the token stream for {src:?}" + ); + } + + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex_lossless("simcfoo"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![FmtToken::Token(Token::Ident("simcfoo"))])); + } - #[test] - fn lexer_test() { - use chumsky::prelude::*; + #[test] + fn lossless_lexer_test() { + use chumsky::prelude::*; - // Check if the lexer parses the example file without errors. - let src = include_str!("../examples/last_will.simf"); + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); - let (tokens, lex_errs) = lexer().parse(src).into_output_errors(); - let _ = tokens.unwrap(); + let (tokens, lex_errs) = lexer_lossless().parse(src).into_output_errors(); + let _ = tokens.unwrap(); - assert!(lex_errs.is_empty()); + assert!(lex_errs.is_empty()); + } } } diff --git a/src/parse.rs b/src/parse.rs index a627adeb..60c522f1 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -22,7 +22,7 @@ use crate::driver::{CRATE_STR, MAIN_MODULE}; use crate::error::DiagnosticManager; use crate::error::{Diagnostic, Error, Span}; use crate::impl_eq_hash; -use crate::lexer::Token; +use crate::lexer::{Token, Tokens}; use crate::num::NonZeroPow2Usize; use crate::pattern::Pattern; use crate::str::{ @@ -30,9 +30,12 @@ use crate::str::{ SymbolName, WitnessName, }; use crate::types::{AliasedType, BuiltinAlias, TypeConstructible, UIntType}; -use crate::unstable::{impl_require_feature, UnstableFeature, UnstableFeatures}; +use crate::unstable::{impl_require_feature, RequireFeature, UnstableFeature, UnstableFeatures}; use crate::version::SimcDirective; +#[cfg(feature = "fmt")] +use crate::lexer::{FmtToken, FmtTokens}; + /// A program is a sequence of items. #[derive(Clone, Debug)] pub struct Program { @@ -40,6 +43,40 @@ pub struct Program { span: Span, } +/// Source-aware parse result used by formatters. +/// +/// The compiler keeps using [`Program`] directly. Formatters need the same +/// semantic tree plus the lossless trivia stream that the grammar intentionally +/// does not consume. +#[cfg(feature = "fmt")] +#[derive(Clone, Debug)] +pub struct ParsedSource<'src> { + program: Program, + tokens: FmtTokens<'src>, + prefix: Span, +} + +#[cfg(feature = "fmt")] +impl<'src> ParsedSource<'src> { + /// Access the semantic program used for formatting. + pub fn program(&self) -> &Program { + &self.program + } + + /// Access the lossless token stream in source order. + pub fn tokens(&self) -> &FmtTokens<'src> { + &self.tokens + } + + /// Access the source prefix skipped by the version-directive prescan. + /// + /// A formatter should preserve this range verbatim until the directive gains + /// its own formatting grammar. + pub fn prefix(&self) -> Span { + self.prefix + } +} + impl Program { // Need for driver usage pub(crate) fn new(items: &[Item], span: Span) -> Self { @@ -53,6 +90,50 @@ impl Program { pub fn items(&self) -> &[Item] { &self.items } + + /// Parse source for formatting while retaining all comments and whitespace. + #[cfg(feature = "fmt")] + pub fn parse_with_errors_for_fmt<'src>( + file_id: usize, + source: &'src str, + unstable_features: &UnstableFeatures, + diagnostics: &mut DiagnosticManager, + ) -> Option> { + let before = diagnostics.error_count(); + + let start = pipeline::directive_prescan(source, file_id, diagnostics)?; + + let (tokens, lex_errors) = crate::lexer::lex_lossless(file_id, source, start); + let lex_ok = pipeline::is_lex_ok(lex_errors, diagnostics)?; + + let tokens = tokens?; + + let semantic_tokens = tokens + .iter() + .filter_map(|(token, span)| match token { + FmtToken::Token(token) => Some((token.clone(), *span)), + FmtToken::Trivia(_) => None, + }) + .collect::>(); + + let (program, parse_ok) = + pipeline::parse_ast(file_id, source, semantic_tokens, diagnostics); + + if parse_ok && lex_ok { + pipeline::post_check(unstable_features, program.as_ref(), diagnostics); + } + + if diagnostics.error_count() > before { + None + } else { + let program = program?; + Some(ParsedSource { + program, + tokens, + prefix: Span::new(file_id, 0..start), + }) + } + } } impl_eq_hash!(Program; items); @@ -80,6 +161,22 @@ pub enum Item { Ignored, } +impl Item { + /// Access the source span when this item was parsed successfully. + /// + /// Error-recovery placeholders have no source node to decorate. + pub fn span(&self) -> Option<&Span> { + match self { + Self::TypeAlias(alias) => Some(alias.span()), + Self::Function(function) => Some(function.span()), + Self::Use(use_decl) => Some(use_decl.span()), + Self::EnumDeclaration(declaration) => Some(declaration.span()), + Self::Module(module) => Some(module.span()), + Self::Ignored => None, + } + } +} + impl_require_feature!(Item { variants: TypeAlias(alias), @@ -277,11 +374,12 @@ impl_eq_hash!(Function; visibility, name, params, ret, body); impl_require_feature!(Function { recurse: params, ret, body; }); /// Parameter of a function. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] pub struct FunctionParam { identifier: Identifier, ty: AliasedType, + span: Span, } impl FunctionParam { @@ -294,8 +392,15 @@ impl FunctionParam { pub fn ty(&self) -> &AliasedType { &self.ty } + + /// Access the source span of the complete parameter. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(FunctionParam; identifier, ty); + impl_require_feature!(FunctionParam { recurse: ty; }); /// A statement is a component of a block expression. @@ -307,6 +412,19 @@ pub enum Statement { Expression(Expression), } +impl Statement { + /// Access the span of the statement contents. + /// + /// The terminating semicolon is deliberately not part of this span: it is a + /// separator owned by the containing block's layout. + pub fn span(&self) -> &Span { + match self { + Self::Assignment(assignment) => assignment.span(), + Self::Expression(expression) => expression.span(), + } + } +} + impl_require_feature!(Statement { variants: Assignment(assignment), @@ -502,6 +620,11 @@ impl EnumVariant { pub fn payload(&self) -> &[AliasedType] { &self.payload } + + /// Access the source span of the complete variant. + pub fn span(&self) -> &Span { + &self.span + } } impl_eq_hash!(EnumVariant; name, payload); @@ -533,6 +656,11 @@ impl EnumDeclaration { pub fn variants(&self) -> &[EnumVariant] { &self.variants } + + /// Access the source span of the complete declaration. + pub fn span(&self) -> &Span { + &self.span + } } impl_require_feature!(EnumVariant { recurse: payload; }); @@ -829,6 +957,7 @@ pub struct EnumMatchArm { variant: Identifier, bindings: Arc<[(Pattern, AliasedType)]>, expression: Arc, + span: Span, } impl EnumMatchArm { @@ -860,6 +989,11 @@ impl EnumMatchArm { pub fn expression(&self) -> &Expression { &self.expression } + + /// Access the source span of the complete arm, including its optional comma. + pub fn span(&self) -> &Span { + &self.span + } } impl_eq_hash!(EnumMatchArm; enum_path, variant, bindings, expression); @@ -942,10 +1076,11 @@ impl AsRef for EnumConstruction { } /// Arm of a match expression. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct MatchArm { pattern: MatchPattern, expression: Arc, + span: Span, } impl MatchArm { @@ -958,8 +1093,15 @@ impl MatchArm { pub fn expression(&self) -> &Expression { &self.expression } + + /// Access the source span of the complete arm, including its optional comma. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(MatchArm; pattern, expression); + impl_require_feature!(MatchArm {recurse: pattern, expression; }); /// Pattern of a match arm. @@ -1520,6 +1662,10 @@ impl_parse_wrapped_string!(WitnessName, "witness name"); impl_parse_wrapped_string!(AliasName, "alias name"); impl_parse_wrapped_string!(ModuleName, "module name"); +trait AstNode: ChumskyParse + crate::unstable::RequireFeature + std::fmt::Debug {} + +impl AstNode for T where T: ChumskyParse + crate::unstable::RequireFeature + std::fmt::Debug {} + /// Copy of [`FromStr`] that internally uses the `chumsky` parser. pub trait ParseFromStr: Sized { /// Parse a value from the string `s`. @@ -1540,6 +1686,73 @@ pub trait ParseFromStrWithErrors: Sized { ) -> Option; } +mod pipeline { + use super::*; + /// Handle the `simc` directive before lexing: an incompatible or malformed + /// directive is reported as the only diagnostic (the rest is noise), and + /// lexing starts right after a valid one, so the lexer and grammar never + /// see it. + pub fn directive_prescan( + content: &str, + file_id: usize, + diagnostics: &mut DiagnosticManager, + ) -> Option { + match SimcDirective::prescan(content, file_id) { + Ok(start) => Some(start), + Err((err, span)) => { + diagnostics.push(Diagnostic::new(err, span)); + None + } + } + } + + pub fn is_lex_ok( + mut lex_errs: Vec, + diagnostics: &mut DiagnosticManager, + ) -> Option { + // A stray `simc` makes every other diagnostic noise — its `"";` remnant + // does not lex — so the reserved-keyword errors are reported alone. + if lex_errs + .iter() + .any(|e| matches!(e.error(), Error::ReservedSimcKeyword)) + { + lex_errs.retain(|e| matches!(e.error(), Error::ReservedSimcKeyword)); + diagnostics.extend(lex_errs); + None + } else { + let lex_ok = lex_errs.is_empty(); + diagnostics.extend(lex_errs); + Some(lex_ok) + } + } + + pub fn parse_ast( + file_id: usize, + src: &str, + tokens: Tokens<'_>, + diagnostics: &mut DiagnosticManager, + ) -> (Option, bool) { + let eoi = Span::eof(file_id, src.len()); + let (ast, parse_errs) = T::parser() + .parse(tokens.as_slice().map(eoi, |(t, s)| (t, s))) + .into_output_errors(); + + let parse_ok = parse_errs.is_empty(); + diagnostics.extend(parse_errs); + (ast, parse_ok) + } + + pub fn post_check( + unstable_features: &UnstableFeatures, + program: Option<&T>, + diagnostics: &mut DiagnosticManager, + ) { + if let Some(ast) = program { + unstable_features.check_program(ast, diagnostics); + } + } +} + /// Trait for generating parsers of themselves. /// /// Replacement for previous `PestParse` trait. @@ -1594,9 +1807,7 @@ impl ParseFromStr for A { } } -impl ParseFromStrWithErrors - for A -{ +impl ParseFromStrWithErrors for A { fn parse_from_str_with_errors( file_id: usize, content: &str, @@ -1605,42 +1816,18 @@ impl ParseF ) -> Option { let before = diagnostics.error_count(); - // Handle the `simc` directive before lexing: an incompatible or malformed - // directive is reported as the only diagnostic (the rest is noise), and - // lexing starts right after a valid one, so the lexer and grammar never - // see it. - let start = match SimcDirective::prescan(content, file_id) { - Ok(start) => start, - Err((err, span)) => { - diagnostics.push(Diagnostic::new(err, span)); - return None; - } - }; + let start = pipeline::directive_prescan(content, file_id, diagnostics)?; + + let (tokens, lex_errs) = crate::lexer::lex(file_id, content, start); + + let lex_ok = pipeline::is_lex_ok(lex_errs, diagnostics)?; - let (tokens, mut lex_errs) = crate::lexer::lex(file_id, content, start); - // A stray `simc` makes every other diagnostic noise — its `"";` remnant - // does not lex — so the reserved-keyword errors are reported alone. - if lex_errs - .iter() - .any(|e| matches!(e.error(), Error::ReservedSimcKeyword)) - { - lex_errs.retain(|e| matches!(e.error(), Error::ReservedSimcKeyword)); - diagnostics.extend(lex_errs); - return None; - } - let lex_ok = lex_errs.is_empty(); - diagnostics.extend(lex_errs); let tokens = tokens?; - let eoi = Span::eof(file_id, content.len()); - let (ast, parse_errs) = A::parser() - .parse(tokens.as_slice().map(eoi, |(t, s)| (t, s))) - .into_output_errors(); - let parse_ok = parse_errs.is_empty(); - diagnostics.extend(parse_errs); + let (ast, parse_status) = pipeline::parse_ast::(file_id, content, tokens, diagnostics); - if let (Some(ast), true) = (&ast, lex_ok && parse_ok) { - unstable_features.check_program(ast, diagnostics); + if lex_ok && parse_status { + let () = pipeline::post_check(unstable_features, ast.as_ref(), diagnostics); } // TODO: We should return parsed result if we found errors, but because analyzing in `ast` module @@ -2034,7 +2221,11 @@ impl ChumskyParse for FunctionParam { identifier .then_ignore(just(Token::Colon)) .then(ty) - .map(|(identifier, ty)| Self { identifier, ty }) + .map_with(|(identifier, ty), e| Self { + identifier, + ty, + span: e.span(), + }) } } @@ -2613,20 +2804,20 @@ struct EnumArmHead { } impl EnumArmHead { - fn into_arm(self, expression: Arc) -> EnumMatchArm { + fn into_arm(self, expression: Arc, span: Span) -> EnumMatchArm { EnumMatchArm { enum_path: self.enum_path, variant: self.variant, bindings: self.bindings, expression, + span, } } } /// One parsed match arm. An enum variant head or a built-in pattern, -/// plus the arm body. The head classification decides below whether the -/// whole match becomes an [`EnumMatch`] or a binary [`Match`]. -type ParsedMatchArm = (Either, Arc); +/// plus the arm body and its source boundaries. +type ParsedMatchArm = (Either, Arc, Span); /// Parser for the head of an enum match arm: `EnumName::Variant` with /// optional payload bindings `(pattern: Type, ...)`. @@ -2709,7 +2900,7 @@ where .with_span(e.span()), ); } - (head, expression) + (head, expression, e.span()) }) } @@ -2719,6 +2910,7 @@ fn placeholder_match(scrutinee: Arc, span: Span) -> SingleExpression let fallback_arm = MatchArm { expression: Arc::new(Expression::empty(Span::DUMMY)), pattern: MatchPattern::False, + span: Span::DUMMY, }; SingleExpressionInner::Match(Match { scrutinee, @@ -2765,11 +2957,12 @@ fn assemble_match_arms( // reports missing variants. The binary arm-count error below would be misleading there. let (enum_arms, builtin_arms): (Vec, Vec) = arms .into_iter() - .partition_map(|(head, expression)| match head { - Either::Left(head) => Either::Left(head.into_arm(expression)), + .partition_map(|(head, expression, arm_span)| match head { + Either::Left(head) => Either::Left(head.into_arm(expression, arm_span)), Either::Right(pattern) => Either::Right(MatchArm { pattern, expression, + span: arm_span, }), }); @@ -2950,6 +3143,12 @@ impl AsRef for Function { } } +impl AsRef for FunctionParam { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for Assignment { fn as_ref(&self) -> &Span { &self.span @@ -2986,6 +3185,18 @@ impl AsRef for Match { } } +impl AsRef for MatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + +impl AsRef for EnumMatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for UseDecl { fn as_ref(&self) -> &Span { &self.span @@ -3306,10 +3517,12 @@ impl crate::ArbitraryRec for Match { left: MatchArm { pattern: pat_l, expression: expr_l, + span: Span::DUMMY, }, right: MatchArm { pattern: pat_r, expression: expr_r, + span: Span::DUMMY, }, span: Span::DUMMY, }) @@ -3334,172 +3547,531 @@ mod test { } } - #[test] - fn test_reject_redefined_builtin_type() { - let ty = TypeAlias::parse_from_str("type Ctx8 = u32") - .expect_err("Redefining built-in alias should be rejected"); + mod type_alias { + use super::*; - assert!(ty - .error() - .to_string() - .contains("Type alias `Ctx8` is already exists as built-in alias")); - } + #[test] + fn test_reject_redefined_builtin_type() { + let ty = TypeAlias::parse_from_str("type Ctx8 = u32") + .expect_err("Redefining built-in alias should be rejected"); + + assert!(ty + .error() + .to_string() + .contains("Type alias `Ctx8` is already exists as built-in alias")); + } - #[test] - fn test_fragment_rejects_version_directive() { - let err = TypeAlias::parse_from_str("simc \"*\"; type Ctx8 = u32") - .expect_err("a version directive must not be accepted in a fragment"); + #[test] + fn test_fragment_rejects_version_directive() { + let err = TypeAlias::parse_from_str("simc \"*\"; type Ctx8 = u32") + .expect_err("a version directive must not be accepted in a fragment"); - assert!(matches!(err.error(), Error::ReservedSimcKeyword)); + assert!(matches!(err.error(), Error::ReservedSimcKeyword)); + } } - #[test] - fn test_double_colon() { - let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; - let mut diagnostics = DiagnosticManager::new(); + mod regular_parsing { + use super::*; - let parsed_program = Program::parse_from_str_with_errors( - MAIN_MODULE, - input, - &UnstableFeatures::all(), - &mut diagnostics, - ); + #[test] + fn test_double_colon() { + let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; + let mut diagnostics = DiagnosticManager::new(); - assert!(parsed_program.is_none()); - assert!(diagnostics.to_string().contains("Expected '::', found ':'")); - } + let parsed_program = Program::parse_from_str_with_errors( + MAIN_MODULE, + input, + &UnstableFeatures::all(), + &mut diagnostics, + ); - #[test] - fn test_double_double_colon() { - let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; - let mut diagnostics = DiagnosticManager::new(); + assert!(parsed_program.is_none()); + assert!(diagnostics.to_string().contains("Expected '::', found ':'")); + } - let parsed_program = Program::parse_from_str_with_errors( - MAIN_MODULE, - input, - &UnstableFeatures::all(), - &mut diagnostics, - ); + #[test] + fn test_double_double_colon() { + let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; + let mut diagnostics = DiagnosticManager::new(); - assert!(parsed_program.is_none()); - assert!( - diagnostics - .to_string() - .contains("Expected identifier, found ::"), - "the second :: should be reported as the error site" - ); - } + let parsed_program = Program::parse_from_str_with_errors( + MAIN_MODULE, + input, + &UnstableFeatures::all(), + &mut diagnostics, + ); - /// Parse `input` and return whether it was rejected and the collected error text. - fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { - let mut diagnostics = DiagnosticManager::new(); - let program = - Program::parse_from_str_with_errors(MAIN_MODULE, input, features, &mut diagnostics); + assert!(parsed_program.is_none()); + assert!( + diagnostics + .to_string() + .contains("Expected identifier, found ::"), + "the second :: should be reported as the error site" + ); + } - let rejected = program.is_none(); - let text = diagnostics.to_string(); + /// Parse `input` and return whether it was rejected and the collected error text. + fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { + let mut diagnostics = DiagnosticManager::new(); + let program = + Program::parse_from_str_with_errors(MAIN_MODULE, input, features, &mut diagnostics); - (rejected, text) - } + let rejected = program.is_none(); + let text = diagnostics.to_string(); - #[test] - fn inverted_empty_span_from_token_gap_does_not_panic() { - // Fuzz-found (compile_text). - // - // The input is irreducible. The broken `fn` prefix forces the parser into delimiter recovery. - // The only path that requests an empty span at a lex-error gap and the - // NUL after `enum` creates that gap. - // Chumsky builds such spans as `next_token.start .. previous_token.end`, which is inverted - // across the gap and panicked the strict `Span` constructor. - // - // Simplifying any part (even NUL to a space) loses the crash. - let src = "fn`u({?\u{12}$0;;enum\0===lHf\u{15}"; - let mut diagnostics = DiagnosticManager::new(); - let program = Program::parse_from_str_with_errors( - MAIN_MODULE, - src, - &UnstableFeatures::all(), - &mut diagnostics, - ); - - assert!(program.is_none(), "garbage input must be rejected"); - assert!(diagnostics.has_errors()); - } - - #[test] - fn recovery_synchronizes_at_enum_declaration() { - // Discriminating setup: an invalid prefix followed by a malformed - // enum. Without `Token::Enum` in the synchronization set the whole - // enum is swallowed into the prefix's recovery span and only one - // error is reported; with it, the enum parses as its own item and - // its malformation is reported as a second error. - let mut diagnostics = DiagnosticManager::new(); - let program = Program::parse_from_str_with_errors( - MAIN_MODULE, - "let invalid = 1;\nenum Action { A B }\nfn main() {}", - &UnstableFeatures::all(), - &mut diagnostics, - ); - assert!(program.is_none(), "the invalid program must be rejected"); - assert_eq!( - 2, - diagnostics.error_count(), - "the invalid prefix and the malformed enum must each report an error" - ); - } - - #[test] - fn recovery_after_malformed_enum_reaches_next_item() { - let (rejected, _text) = parse_with( - "enum Action { A B }\nfn main() {}", - &UnstableFeatures::all(), - ); - assert!(rejected, "a malformed enum must fail compilation"); - } - - #[test] - fn malformed_construct_containing_enum_keyword_reports_errors() { - // `enum` inside a badly malformed nested construct may be mistaken - // for an item boundary; compilation must still fail cleanly. - let (rejected, _text) = parse_with( - "fn broken() { let x = (enum; }\nfn main() {}", - &UnstableFeatures::all(), - ); - assert!(rejected, "a malformed construct must fail compilation"); - } - - #[test] - fn test_gated_syntax_is_rejected_without_features() { - // Real `use`/`mod` syntax: rejected under none() (naming the feature + a - // -Z hint), accepted under all(). Delete when `imports` stabilizes. - for input in [ - "use crate::foo::bar;\nfn main() { }", - "mod inner { }\nfn main() { }", - ] { + (rejected, text) + } + + #[test] + fn inverted_empty_span_from_token_gap_does_not_panic() { + // Fuzz-found (compile_text). + // + // The input is irreducible. The broken `fn` prefix forces the parser into delimiter recovery. + // The only path that requests an empty span at a lex-error gap and the + // NUL after `enum` creates that gap. + // Chumsky builds such spans as `next_token.start .. previous_token.end`, which is inverted + // across the gap and panicked the strict `Span` constructor. + // + // Simplifying any part (even NUL to a space) loses the crash. + let src = "fn`u({?\u{12}$0;;enum\0===lHf\u{15}"; + let mut diagnostics = DiagnosticManager::new(); + let program = Program::parse_from_str_with_errors( + MAIN_MODULE, + src, + &UnstableFeatures::all(), + &mut diagnostics, + ); + + assert!(program.is_none(), "garbage input must be rejected"); + assert!(diagnostics.has_errors()); + } + + #[test] + fn recovery_synchronizes_at_enum_declaration() { + // Discriminating setup: an invalid prefix followed by a malformed + // enum. Without `Token::Enum` in the synchronization set the whole + // enum is swallowed into the prefix's recovery span and only one + // error is reported; with it, the enum parses as its own item and + // its malformation is reported as a second error. + let mut diagnostics = DiagnosticManager::new(); + let program = Program::parse_from_str_with_errors( + MAIN_MODULE, + "let invalid = 1;\nenum Action { A B }\nfn main() {}", + &UnstableFeatures::all(), + &mut diagnostics, + ); + assert!(program.is_none(), "the invalid program must be rejected"); + assert_eq!( + 2, + diagnostics.error_count(), + "the invalid prefix and the malformed enum must each report an error" + ); + } + + #[test] + fn recovery_after_malformed_enum_reaches_next_item() { + let (rejected, _text) = parse_with( + "enum Action { A B }\nfn main() {}", + &UnstableFeatures::all(), + ); + assert!(rejected, "a malformed enum must fail compilation"); + } + + #[test] + fn malformed_construct_containing_enum_keyword_reports_errors() { + // `enum` inside a badly malformed nested construct may be mistaken + // for an item boundary; compilation must still fail cleanly. + let (rejected, _text) = parse_with( + "fn broken() { let x = (enum; }\nfn main() {}", + &UnstableFeatures::all(), + ); + assert!(rejected, "a malformed construct must fail compilation"); + } + + #[test] + fn test_gated_syntax_is_rejected_without_features() { + // Real `use`/`mod` syntax: rejected under none() (naming the feature + a + // -Z hint), accepted under all(). Delete when `imports` stabilizes. + for input in [ + "use crate::foo::bar;\nfn main() { }", + "mod inner { }\nfn main() { }", + ] { + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + rejected, + "gated syntax must be rejected without features (if this feature was \ + just stabilized, delete this test):\n{input}" + ); + assert!( + error.contains("imports") && error.contains("-Z"), + "rejection should name the feature and suggest -Z, got:\n{error}" + ); + + let (rejected, error) = parse_with(input, &UnstableFeatures::all()); + assert!( + !rejected, + "the same syntax must parse with all features enabled:\n{error}" + ); + } + } + + #[test] + fn test_type_heavy_program_needs_no_features() { + // Types are traversed by `RequireFeature` but not gated, so a type-heavy, + // import-free program must parse with no features (no false-positive gates). + let input = r#"type Alias = u32; +fn pick(e: Either) -> u32 { + match e { + Left(a: u32) => a, + Right(b: u32) => b, + } +} +fn main() { + let casted: u32 = <(u16, u16)>::into((0xbeef, 0xbabe)); + let chosen: Alias = pick(Left(casted)); + assert!(jet::eq_32(chosen, chosen)); +} +"#; + // `parse_from_str_with_errors` returns `Some` only when no errors were + // collected, so a non-rejection already proves there were no gate errors. let (rejected, error) = parse_with(input, &UnstableFeatures::none()); assert!( - rejected, - "gated syntax must be rejected without features (if this feature was \ - just stabilized, delete this test):\n{input}" + !rejected, + "type-heavy but import-free program should parse with no features:\n{error}" ); + } + + #[test] + fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { + // The gate check skips error-recovered ASTs, so a program with + // both a syntax error and gated syntax reports only the syntax error. + let input = "use crate::foo;\nfn main( {"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "broken program must be rejected"); assert!( - error.contains("imports") && error.contains("-Z"), - "rejection should name the feature and suggest -Z, got:\n{error}" + !error.contains("-Z"), + "syntax error should not produce a spurious feature-gate hint, got:\n{error}" ); + } - let (rejected, error) = parse_with(input, &UnstableFeatures::all()); + #[test] + fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { + // Lexer-side companion to the case above: the stray `@` lexes with an + // error but recovers a cleanly-parsing `use` AST, and a program that + // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. + let input = "use crate@::foo;\nfn main() { }"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "program with a lexer error must be rejected"); assert!( - !rejected, - "the same syntax must parse with all features enabled:\n{error}" + !error.contains("-Z"), + "lexer error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } + + #[test] + fn test_use_decl_display_round_trips_with_aliases() { + for input in [ + "use lib::A::foo;", + "use lib::A::foo as bar;", + "use lib::A::{foo, bar as baz};", + ] { + let program = parse::Program::parse_from_str(input).expect("parsing works"); + assert_eq!(program.to_string(), format!("{input}\n")); + } + } + + #[test] + fn statement_spans_exclude_terminating_semicolons() { + let input = "fn main() { let value: u8 = 0; value; }"; + let program = Program::parse_from_str(input).expect("program parses"); + + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + let ExpressionInner::Block(statements, None) = function.body().inner() else { + panic!("expected a block containing only statements"); + }; + + assert_eq!( + statements[0].span().to_slice(input), + Some("let value: u8 = 0") + ); + assert_eq!(statements[1].span().to_slice(input), Some("value")); + } + + fn parse_item(input: &str) -> Item { + let program = parse::Program::parse_from_str(input).expect("parsing should succeed"); + program.items().first().expect("expected one item").clone() + } + + #[test] + fn test_enum_declaration_basic() { + let item = parse_item("enum Path { Inherit, ColdSpend, RefreshSpend, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration, got {item:?}"); + }; + assert_eq!(decl.name().as_inner(), "Path"); + assert_eq!(decl.variants().len(), 3); + assert_eq!(decl.variants()[0].name().as_inner(), "Inherit"); + assert_eq!(decl.variants()[2].name().as_inner(), "RefreshSpend"); + } + + #[test] + fn test_enum_declaration_pub() { + let item = parse_item("pub enum Color { Red, Green, Blue, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!(decl.visibility(), &Visibility::Public); + assert_eq!(decl.name().as_inner(), "Color"); + } + + #[test] + fn test_enum_declaration_display_round_trip() { + let input = "enum Path { Inherit, ColdSpend, RefreshSpend, }"; + let item = parse_item(input); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!( + decl.to_string(), + "enum Path { Inherit, ColdSpend, RefreshSpend, }" ); } + + #[test] + fn test_enum_declaration_reserved_name() { + for reserved in RESERVED_PATTERN_NAMES { + let result = Program::parse_from_str(&format!("enum {reserved} {{ A, B, }}")); + let error = result.expect_err(&format!("enum name {reserved} is reserved")); + assert!( + error.to_string().contains("reserved"), + "error should say the name is reserved: {error}" + ); + } + } } - #[test] - fn test_type_heavy_program_needs_no_features() { - // Types are traversed by `RequireFeature` but not gated, so a type-heavy, - // import-free program must parse with no features (no false-positive gates). - let input = r#"type Alias = u32; + #[cfg(feature = "fmt")] + mod fmt_parsing { + use super::*; + + /// Parse `input`, appending any formatter diagnostics to `diagnostics`. + fn parse_with_diagnostics<'src>( + input: &'src str, + features: &UnstableFeatures, + diagnostics: &mut DiagnosticManager, + ) -> Option> { + Program::parse_with_errors_for_fmt(MAIN_MODULE, input, features, diagnostics) + } + + /// Parse `input` and return whether it was rejected and the collected error text. + fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); + + let rejected = program.is_none(); + let text = diagnostics.to_string(); + + (rejected, text) + } + + /// Parse `input` and return whether it was rejected and the collected error in `DiagnosticManager`. + fn parse_with_diagnosis( + input: &str, + features: &UnstableFeatures, + ) -> (bool, DiagnosticManager) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); + + let rejected = program.is_none(); + + (rejected, diagnostics) + } + + /// Parse `input` and return whether it was rejected and the collected error text together with `ParsedSource`. + fn parse_with_extended_out<'src>( + input: &'src str, + features: &UnstableFeatures, + ) -> (bool, String, Option>) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); + + let rejected = program.is_none(); + let text = diagnostics.to_string(); + + (rejected, text, program) + } + + fn assert_formatter_matches_regular_parser(input: &str, features: &UnstableFeatures) { + let mut regular_diagnostics = DiagnosticManager::new(); + let regular = Program::parse_from_str_with_errors( + MAIN_MODULE, + input, + features, + &mut regular_diagnostics, + ); + + let mut formatting_diagnostics = DiagnosticManager::new(); + let formatting = parse_with_diagnostics(input, features, &mut formatting_diagnostics); + + assert_eq!( + formatting.as_ref().map(ParsedSource::program), + regular.as_ref(), + "formatter and regular parsers must produce the same AST for {input:?}" + ); + assert_eq!( + formatting_diagnostics.error_count(), + regular_diagnostics.error_count(), + "formatter and regular parsers must report the same number of errors for {input:?}, \n\ + fmt errors: \n\t '{formatting_diagnostics}, \n regular errors: \n\t '{regular_diagnostics}" + ); + } + + fn assert_lossless_source(input: &str, parsed: &ParsedSource<'_>) { + let prefix = parsed.prefix(); + assert_eq!(prefix.file_id, MAIN_MODULE); + + let mut reconstructed = prefix + .to_slice(input) + .expect("the directive prefix must be a valid source slice") + .to_owned(); + let mut cursor = prefix.end; + + for (_, span) in parsed.tokens() { + assert_eq!(span.file_id, MAIN_MODULE); + assert!( + span.start >= prefix.end, + "formatter tokens must not overlap the preserved directive prefix" + ); + assert_eq!( + span.start, cursor, + "formatter token spans must be contiguous and in source order" + ); + + let text = span + .to_slice(input) + .expect("formatter token spans must be valid source slices"); + assert!( + !text.is_empty(), + "the formatter token stream must not contain empty source slices" + ); + reconstructed.push_str(text); + cursor = span.end; + } + + assert_eq!(cursor, input.len(), "formatter tokens must reach EOF"); + assert_eq!(reconstructed, input, "formatter input must be lossless"); + } + + #[test] + fn test_double_colon() { + let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; + + let (rejected, text) = parse_with(input, &UnstableFeatures::all()); + + assert!(rejected); + assert!(text.contains("Expected '::', found ':'")); + } + + #[test] + fn test_double_double_colon() { + let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; + + let (rejected, text) = parse_with(input, &UnstableFeatures::all()); + + assert!(rejected); + assert!( + text.contains("Expected identifier, found ::"), + "the second :: should be reported as the error site" + ); + } + + #[test] + fn inverted_empty_span_from_token_gap_does_not_panic() { + // Fuzz-found (compile_text). + // + // The input is irreducible. The broken `fn` prefix forces the parser into delimiter recovery. + // The only path that requests an empty span at a lex-error gap and the + // NUL after `enum` creates that gap. + // Chumsky builds such spans as `next_token.start .. previous_token.end`, which is inverted + // across the gap and panicked the strict `Span` constructor. + // + // Simplifying any part (even NUL to a space) loses the crash. + let src = "fn`u({?\u{12}$0;;enum\0===lHf\u{15}"; + let (rejected, text) = parse_with(src, &UnstableFeatures::all()); + + assert!(rejected, "garbage input must be rejected"); + assert!(!text.is_empty()); + } + + #[test] + fn recovery_synchronizes_at_enum_declaration() { + // Discriminating setup: an invalid prefix followed by a malformed + // enum. Without `Token::Enum` in the synchronization set the whole + // enum is swallowed into the prefix's recovery span and only one + // error is reported; with it, the enum parses as its own item and + // its malformation is reported as a second error. + + let src = "let invalid = 1;\nenum Action { A B }\nfn main() {}"; + let (rejected, diagnostics) = parse_with_diagnosis(src, &UnstableFeatures::all()); + + assert!(rejected, "the invalid program must be rejected"); + assert_eq!( + 2, + diagnostics.error_count(), + "the invalid prefix and the malformed enum must each report an error" + ); + } + + #[test] + fn recovery_after_malformed_enum_reaches_next_item() { + let src = "enum Action { A B }\nfn main() {}"; + let (rejected, _text) = parse_with(src, &UnstableFeatures::all()); + assert!(rejected, "a malformed enum must fail compilation"); + } + + #[test] + fn malformed_construct_containing_enum_keyword_reports_errors() { + // `enum` inside a badly malformed nested construct may be mistaken + // for an item boundary; compilation must still fail cleanly. + let src = "fn broken() { let x = (enum; }\nfn main() {}"; + let (rejected, _text) = parse_with(src, &UnstableFeatures::all()); + assert!(rejected, "a malformed construct must fail compilation"); + } + + #[test] + fn test_gated_syntax_is_rejected_without_features() { + // Real `use`/`mod` syntax: rejected under none() (naming the feature + a + // -Z hint), accepted under all(). Delete when `imports` stabilizes. + for input in [ + "use crate::foo::bar;\nfn main() { }", + "mod inner { }\nfn main() { }", + ] { + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + rejected, + "gated syntax must be rejected without features (if this feature was \ + just stabilized, delete this test):\n{input}" + ); + assert!( + error.contains("imports") && error.contains("-Z"), + "rejection should name the feature and suggest -Z, got:\n{error}" + ); + + let (rejected, error) = parse_with(input, &UnstableFeatures::all()); + assert!( + !rejected, + "the same syntax must parse with all features enabled:\n{error}" + ); + } + } + + #[test] + fn test_type_heavy_program_needs_no_features() { + // Types are traversed by `RequireFeature` but not gated, so a type-heavy, + // import-free program must parse with no features (no false-positive gates). + let input = r#"type Alias = u32; fn pick(e: Either) -> u32 { match e { Left(a: u32) => a, @@ -3512,102 +4084,241 @@ fn main() { assert!(jet::eq_32(chosen, chosen)); } "#; - // `parse_from_str_with_errors` returns `Some` only when no errors were - // collected, so a non-rejection already proves there were no gate errors. - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!( - !rejected, - "type-heavy but import-free program should parse with no features:\n{error}" - ); - } - - #[test] - fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { - // The gate check skips error-recovered ASTs, so a program with - // both a syntax error and gated syntax reports only the syntax error. - let input = "use crate::foo;\nfn main( {"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "broken program must be rejected"); - assert!( - !error.contains("-Z"), - "syntax error should not produce a spurious feature-gate hint, got:\n{error}" - ); - } - - #[test] - fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { - // Lexer-side companion to the case above: the stray `@` lexes with an - // error but recovers a cleanly-parsing `use` AST, and a program that - // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. - let input = "use crate@::foo;\nfn main() { }"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "program with a lexer error must be rejected"); - assert!( - !error.contains("-Z"), - "lexer error should not produce a spurious feature-gate hint, got:\n{error}" - ); - } - - #[test] - fn test_use_decl_display_round_trips_with_aliases() { - for input in [ - "use lib::A::foo;", - "use lib::A::foo as bar;", - "use lib::A::{foo, bar as baz};", - ] { - let program = parse::Program::parse_from_str(input).expect("parsing works"); - assert_eq!(program.to_string(), format!("{input}\n")); - } - } - - fn parse_item(input: &str) -> Item { - let program = parse::Program::parse_from_str(input).expect("parsing should succeed"); - program.items().first().expect("expected one item").clone() - } - - #[test] - fn test_enum_declaration_basic() { - let item = parse_item("enum Path { Inherit, ColdSpend, RefreshSpend, }"); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration, got {item:?}"); - }; - assert_eq!(decl.name().as_inner(), "Path"); - assert_eq!(decl.variants().len(), 3); - assert_eq!(decl.variants()[0].name().as_inner(), "Inherit"); - assert_eq!(decl.variants()[2].name().as_inner(), "RefreshSpend"); - } + // `parse_from_str_with_errors` returns `Some` only when no errors were + // collected, so a non-rejection already proves there were no gate errors. + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + !rejected, + "type-heavy but import-free program should parse with no features:\n{error}" + ); + } - #[test] - fn test_enum_declaration_pub() { - let item = parse_item("pub enum Color { Red, Green, Blue, }"); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration"); - }; - assert_eq!(decl.visibility(), &Visibility::Public); - assert_eq!(decl.name().as_inner(), "Color"); - } + #[test] + fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { + // The gate check skips error-recovered ASTs, so a program with + // both a syntax error and gated syntax reports only the syntax error. + let input = "use crate::foo;\nfn main( {"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "broken program must be rejected"); + assert!( + !error.contains("-Z"), + "syntax error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } - #[test] - fn test_enum_declaration_display_round_trip() { - let input = "enum Path { Inherit, ColdSpend, RefreshSpend, }"; - let item = parse_item(input); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration"); - }; - assert_eq!( - decl.to_string(), - "enum Path { Inherit, ColdSpend, RefreshSpend, }" - ); - } + #[test] + fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { + // Lexer-side companion to the case above: the stray `@` lexes with an + // error but recovers a cleanly-parsing `use` AST, and a program that + // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. + let input = "use crate@::foo;\nfn main() { }"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "program with a lexer error must be rejected"); + assert!( + !error.contains("-Z"), + "lexer error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } + + #[test] + fn test_use_decl_display_round_trips_with_aliases() { + for input in [ + "use lib::A::foo;", + "use lib::A::foo as bar;", + "use lib::A::{foo, bar as baz};", + ] { + let (rejected, _, program) = + parse_with_extended_out(input, &UnstableFeatures::all()); + assert!(!rejected, "parsing works"); + assert_eq!(program.unwrap().program.to_string(), format!("{input}\n")); + } + } + + #[test] + fn statement_spans_exclude_terminating_semicolons() { + let input = "fn main() { let value: u8 = 0; value; }"; + + let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::none()); + assert!(!rejected, "parsing works"); + let program = program.unwrap().program; + + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + let ExpressionInner::Block(statements, None) = function.body().inner() else { + panic!("expected a block containing only statements"); + }; + + assert_eq!( + statements[0].span().to_slice(input), + Some("let value: u8 = 0") + ); + assert_eq!(statements[1].span().to_slice(input), Some("value")); + } + + fn parse_item(input: &str) -> Item { + let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::all()); + assert!(!rejected, "parsing works"); + let program = program.unwrap().program; + + program.items().first().expect("expected one item").clone() + } + + #[test] + fn test_enum_declaration_basic() { + let item = parse_item("enum Path { Inherit, ColdSpend, RefreshSpend, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration, got {item:?}"); + }; + assert_eq!(decl.name().as_inner(), "Path"); + assert_eq!(decl.variants().len(), 3); + assert_eq!(decl.variants()[0].name().as_inner(), "Inherit"); + assert_eq!(decl.variants()[2].name().as_inner(), "RefreshSpend"); + } + + #[test] + fn test_enum_declaration_pub() { + let item = parse_item("pub enum Color { Red, Green, Blue, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!(decl.visibility(), &Visibility::Public); + assert_eq!(decl.name().as_inner(), "Color"); + } + + #[test] + fn test_enum_declaration_display_round_trip() { + let input = "enum Path { Inherit, ColdSpend, RefreshSpend, }"; + let item = parse_item(input); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!( + decl.to_string(), + "enum Path { Inherit, ColdSpend, RefreshSpend, }" + ); + } + + #[test] + fn test_enum_declaration_reserved_name() { + for reserved in RESERVED_PATTERN_NAMES { + let input = format!("enum {reserved} {{ A, B, }}"); + let (rejected, error) = parse_with(&input, &UnstableFeatures::none()); + assert!(rejected, "enum name {reserved} is reserved"); + assert!( + error.contains("reserved"), + "error should say the name is reserved: {error}" + ); + } + } + + #[test] + fn formatting_parse_is_lossless_after_directive_prescan() { + let input = "// header\r\nsimc \"*\";\r\n\t/* block */ fn main() { // body\r\n\t}\r"; + + let (_rejected, _error, parsed) = + parse_with_extended_out(input, &UnstableFeatures::none()); + let parsed = parsed.expect("formatting parse should succeed"); + + assert_eq!( + parsed.prefix().to_slice(input), + Some("// header\r\nsimc \"*\";") + ); + for trivia in [ + crate::lexer::TriviaKind::BlockComment, + crate::lexer::TriviaKind::LineComment, + crate::lexer::TriviaKind::Newline, + crate::lexer::TriviaKind::Whitespace, + ] { + assert!( + parsed.tokens().iter().any( + |(token, _)| matches!(token, FmtToken::Trivia(token_trivia) if token_trivia.kind() == trivia) + ), + "formatter token stream should retain {trivia:?}" + ); + } + assert_lossless_source(input, &parsed); + assert_eq!(parsed.program().items().len(), 1); + } + + #[test] + fn formatting_parse_matches_regular_parser_pipeline() { + let no_features = UnstableFeatures::none(); + let all_features = UnstableFeatures::all(); + + for (input, features) in [ + ( + "/* header */\r\nfn\tmain() { // comment\r\n}\r", + &no_features, + ), + ("// version\r\nsimc \"*\";\r\nfn main() {}", &no_features), + ("use crate::foo::bar;\nfn main() {}", &no_features), + ("use crate::foo::bar;\nfn main() {}", &all_features), + ("fn main( {", &no_features), + ("fn main() { @// comment }", &no_features), + ] { + assert_formatter_matches_regular_parser(input, features); + } + } + + #[test] + fn formatting_parse_ignores_preexisting_errors() { + let input = "fn main() {}"; + let mut diagnostics = DiagnosticManager::new(); + diagnostics.push(Diagnostic::global(Error::CannotParse { + msg: "an error from an earlier source file".to_owned(), + })); + let before = diagnostics.error_count(); + + let parsed = parse_with_diagnostics(input, &UnstableFeatures::none(), &mut diagnostics); + + assert!( + parsed.is_some(), + "valid source must still produce ParsedSource" + ); + assert_eq!(diagnostics.error_count(), before); + assert_eq!(diagnostics.diagnostics().len(), before); + } + + #[test] + fn formatting_parse_stops_after_invalid_directive() { + for (input, is_malformed) in [ + ("simc \"*\"\nfn broken( @", true), + ("simc \">99.0.0\"; @", false), + ] { + let (rejected, diagnostics) = + parse_with_diagnosis(input, &UnstableFeatures::none()); + + assert!(rejected, "invalid directive must abort formatting parse"); + assert_eq!( + diagnostics.error_count(), + 1, + "the invalid body must not add diagnostics after prescan aborts" + ); + if is_malformed { + assert!(matches!( + diagnostics.diagnostics()[0].error(), + Error::MalformedSimcDirective + )); + } else { + assert!(matches!( + diagnostics.diagnostics()[0].error(), + Error::SimcVersionMismatch { .. } + )); + } + } + } + + #[test] + fn formatting_parse_reports_missing_match_arm_comma() { + let input = "fn main() { match true { false => () true => (), } }"; + + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - #[test] - fn test_enum_declaration_reserved_name() { - for reserved in RESERVED_PATTERN_NAMES { - let result = Program::parse_from_str(&format!("enum {reserved} {{ A, B, }}")); - let error = result.expect_err(&format!("enum name {reserved} is reserved")); + assert!(rejected); assert!( - error.to_string().contains("reserved"), - "error should say the name is reserved: {error}" + error.contains("Missing ',' after a match arm that isn't block expression"), + "expected the missing-comma diagnostic, got {error:?}" ); } } diff --git a/tests/lex_lossless_coverage.rs b/tests/lex_lossless_coverage.rs new file mode 100644 index 00000000..954ae5e2 --- /dev/null +++ b/tests/lex_lossless_coverage.rs @@ -0,0 +1,103 @@ +#[cfg(feature = "fmt")] +mod lossless_test { + use simplicityhl::lexer::{lex_lossless, FmtTokens}; + + fn get_input(input_file: &str) -> String { + use std::fs; + use std::path::Path; + + let tests_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("examples"); + let input_path = tests_dir.join(input_file); + fs::read_to_string(&input_path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", input_path.display())) + } + + fn assert_lossless_fixture(input_file: &str) { + const DEFAULT_FILE_ID: usize = 0; + + let original_input = get_input(input_file); + + let (tokens, diagnostics) = lex_lossless(DEFAULT_FILE_ID, &original_input, 0); + assert!( + diagnostics.is_empty(), + "Diagnostic errors aren't empty: {diagnostics:?}" + ); + let tokens = tokens.expect("lossless lexing succeeds"); + let restored_input = restore_input_by_tokens(&tokens); + assert_eq!( + restored_input, original_input, + "lossless tokens must reproduce the input" + ); + } + + fn restore_input_by_tokens(tokens: &FmtTokens<'_>) -> String { + let mut buf = String::new(); + + let mut prev_span_end = None; + for (t, s) in tokens { + match prev_span_end { + None => { + let _ = prev_span_end.insert(s.end); + } + Some(x) => { + assert_eq!(x, s.start); + let _ = prev_span_end.insert(s.end); + } + } + buf.push_str(&t.to_string()); + } + buf + } + + macro_rules! test_lex_coverage { + ($($lossless_name:ident: $input:literal,)+) => { + $( + #[test] + fn $lossless_name() { + assert_lossless_fixture($input); + } + )+ + }; +} + + test_lex_coverage! { + array_fold_2n_lossless: "array_fold_2n.simf", + array_fold_lossless: "array_fold.simf", + cat_lossless: "cat.simf", + ctv_lossless: "ctv.simf", + escrow_with_delay_lossless: "escrow_with_delay.simf", + hash_loop_lossless: "hash_loop.simf", + hodl_vault_lossless: "hodl_vault.simf", + htlc_lossless: "htlc.simf", + last_will_lossless: "last_will.simf", + modules_lossless: "modules.simf", + non_interactive_fee_bump_lossless: "non_interactive_fee_bump.simf", + p2ms_lossless: "p2ms.simf", + p2pk_lossless: "p2pk.simf", + p2pkh_lossless: "p2pkh.simf", + pattern_matching_lossless: "pattern_matching.simf", + presigned_vault_lossless: "presigned_vault.simf", + reveal_collision_lossless: "reveal_collision.simf", + reveal_fix_point_lossless: "reveal_fix_point.simf", + sighash_all_anyonecanpay_lossless: "sighash_all_anyonecanpay.simf", + sighash_all_anyprevout_lossless: "sighash_all_anyprevout.simf", + sighash_all_anyprevoutanyscript_lossless: "sighash_all_anyprevoutanyscript.simf", + sighash_none_lossless: "sighash_none.simf", + sighash_single_lossless: "sighash_single.simf", + transfer_with_timeout_lossless: "transfer_with_timeout.simf", + local_crate_main_lossless: "local_crate/main.simf", + local_crate_math_lossless: "local_crate/math.simf", + simple_multidep_hashes_lossless: "simple_multidep/crypto/hashes.simf", + simple_multidep_arithmetic_lossless: "simple_multidep/math/arithmetic.simf", + simple_multidep_flattened_lossless: "simple_multidep/flattened.simf", + simple_multidep_main_lossless: "simple_multidep/main.simf", + single_dep_flattened_lossless: "single_dep/flattened.simf", + single_dep_main_lossless: "single_dep/main.simf", + single_dep_funcs_lossless: "single_dep/temp/funcs.simf", + single_dep_utils_lossless: "single_dep/temp/constants/utils.simf", + multiple_deps_main_lossless: "multiple_deps/main.simf", + multiple_deps_flattened_lossless: "multiple_deps/flattened.simf", + multiple_deps_simple_op_lossless: "multiple_deps/math/simple_op.simf", + multiple_deps_build_root_lossless: "multiple_deps/merkle/build_root.simf", + } +}