diff --git a/verible/verilog/preprocessor/verilog-preprocess.cc b/verible/verilog/preprocessor/verilog-preprocess.cc index 2f7711142..0d12e0e46 100644 --- a/verible/verilog/preprocessor/verilog-preprocess.cc +++ b/verible/verilog/preprocessor/verilog-preprocess.cc @@ -14,6 +14,7 @@ #include "verible/verilog/preprocessor/verilog-preprocess.h" +#include #include #include #include @@ -328,6 +329,91 @@ void VerilogPreprocess::RegisterMacroDefinition( // TODO(hzeller): multiline warning with 'previously defined here' location } +// Process token concatenation (`` operator) in a token sequence. +// This function scans for PP_TOKEN_CONCAT operators and concatenates +// adjacent tokens by combining their text and re-lexing the result. +// Whitespace around `` is removed per SystemVerilog standard. +// Chained concatenations (a``b``c) are processed left-to-right. +// Concatenated strings are stored in preprocess_data for lifetime management. +static void ProcessTokenConcatenation(verible::TokenSequence *tokens, + VerilogPreprocessData *preprocess_data) { + if (tokens->empty()) return; + + verible::TokenSequence result; + result.reserve(tokens->size()); + + for (size_t i = 0; i < tokens->size(); ++i) { + const auto &curr_token = (*tokens)[i]; + + // Check if next non-space token is `` operator. + size_t next_idx = i + 1; + while (next_idx < tokens->size() && + (*tokens)[next_idx].token_enum() == TK_SPACE) { + ++next_idx; + } + + if (next_idx < tokens->size() && + (*tokens)[next_idx].token_enum() == PP_TOKEN_CONCAT) { + // Handle chained concatenations (a``b``c) by accumulating text. + std::string accumulated(curr_token.text()); + + size_t scan_idx = next_idx; + while (scan_idx < tokens->size()) { + if ((*tokens)[scan_idx].token_enum() != PP_TOKEN_CONCAT) break; + + // Skip to token after ``, skipping whitespace. + size_t after_concat_idx = scan_idx + 1; + while (after_concat_idx < tokens->size() && + (*tokens)[after_concat_idx].token_enum() == TK_SPACE) { + ++after_concat_idx; + } + + if (after_concat_idx >= tokens->size()) break; + + accumulated += std::string((*tokens)[after_concat_idx].text()); + + // Check if there's another `` after this token. + scan_idx = after_concat_idx + 1; + while (scan_idx < tokens->size() && + (*tokens)[scan_idx].token_enum() == TK_SPACE) { + ++scan_idx; + } + + if (scan_idx >= tokens->size() || + (*tokens)[scan_idx].token_enum() != PP_TOKEN_CONCAT) { + i = after_concat_idx; + break; + } + } + + // Store the accumulated string in persistent storage. + preprocess_data->concatenated_strings.push_back(accumulated); + const std::string &concatenated = + preprocess_data->concatenated_strings.back(); + + // Re-lex the concatenated text to get the proper token type. + VerilogLexer concat_lexer(concatenated); + concat_lexer.DoNextToken(); + const verible::TokenInfo &lexed_token = concat_lexer.GetLastToken(); + + // Create a token with the correct type and pointing to persistent + // storage. + verible::TokenInfo new_token(lexed_token.token_enum(), concatenated); + + result.push_back(new_token); + continue; + } + + // Not part of concatenation - keep token unless it's PP_TOKEN_CONCAT + // itself. + if (curr_token.token_enum() != PP_TOKEN_CONCAT) { + result.push_back(curr_token); + } + } + + *tokens = std::move(result); +} + // This function expands a text. // The expanded tokens are saved as a TokenSequence, stored at // preprocess_data_.lexed_macros_backup Can be accessed directly after expansion @@ -371,6 +457,10 @@ absl::Status VerilogPreprocess::ExpandText( } expanded_lexed_sequence.push_back(last_token); } + + // Process token concatenation operators. + ProcessTokenConcatenation(&expanded_lexed_sequence, &preprocess_data_); + preprocess_data_.lexed_macros_backup.emplace_back(expanded_lexed_sequence); return absl::OkStatus(); } @@ -434,6 +524,10 @@ absl::Status VerilogPreprocess::ExpandMacro( } expanded_lexed_sequence.push_back(last_token); } + + // Process token concatenation operators. + ProcessTokenConcatenation(&expanded_lexed_sequence, &preprocess_data_); + preprocess_data_.lexed_macros_backup.emplace_back(expanded_lexed_sequence); return absl::OkStatus(); } diff --git a/verible/verilog/preprocessor/verilog-preprocess.h b/verible/verilog/preprocessor/verilog-preprocess.h index 568ed5e1b..d06c4ea54 100644 --- a/verible/verilog/preprocessor/verilog-preprocess.h +++ b/verible/verilog/preprocessor/verilog-preprocess.h @@ -27,14 +27,15 @@ // body text to lexer. This approach works if the definition text // does not depend on the start-condition state at the macro call site. // TODO(fangism): implement conditional evaluation policy (`ifdef, `else, ...) -// TODO(fangism): token concatenation, e.g. a``b -// This will produce tokens that are not in the original source text. +// Token concatenation (a``b) is now supported. +// This produces tokens that are not in the original source text. // TODO(fangism): token string-ification (turning symbol names into strings) // TODO(fangism): evaluate `defines inside `defines at expansion time. #ifndef VERIBLE_VERILOG_PREPROCESSOR_VERILOG_PREPROCESS_H_ #define VERIBLE_VERILOG_PREPROCESSOR_VERILOG_PREPROCESS_H_ +#include #include #include #include @@ -76,6 +77,11 @@ struct VerilogPreprocessData { verible::TokenStreamView preprocessed_token_stream; std::vector lexed_macros_backup; + // Storage for concatenated token text (from `` operator). + // These strings must persist for the lifetime of tokens that reference them. + // Using deque because it doesn't invalidate references when adding elements. + std::deque concatenated_strings; + // A backup memory that owns the content of the included files. std::vector> included_text_structure; diff --git a/verible/verilog/preprocessor/verilog-preprocess_test.cc b/verible/verilog/preprocessor/verilog-preprocess_test.cc index ae335679a..8f4b95d9c 100644 --- a/verible/verilog/preprocessor/verilog-preprocess_test.cc +++ b/verible/verilog/preprocessor/verilog-preprocess_test.cc @@ -1033,5 +1033,359 @@ TEST(VerilogPreprocessTest, << error.error_message; } +// Token concatenation (`` operator) tests. +TEST(VerilogPreprocessTest, TokenConcatenationBasic) { + // Basic token concatenation: a``b -> ab + const RawAndFiltered test_cases[] = { + {"[** Basic identifier concatenation **]", + R"( +`define CONCAT(a, b) a``b +module m; + wire `CONCAT(sig, nal); +endmodule)", + R"( +`define CONCAT(a, b) a``b +module m; + wire signal; +endmodule)"}, + + {"[** Concatenation to build wire name **]", + R"( +`define MAKE_WIRE(prefix, suffix) wire prefix``suffix +module m; + `MAKE_WIRE(data, _in); + `MAKE_WIRE(data, _out); +endmodule)", + R"( +`define MAKE_WIRE(prefix, suffix) wire prefix``suffix +module m; + wire data_in; + wire data_out; +endmodule)"}, + + {"[** Concatenation with underscore **]", + R"( +`define JOIN(a, b) a``_``b +module m; + wire `JOIN(foo, bar); +endmodule)", + R"( +`define JOIN(a, b) a``_``b +module m; + wire foo_bar; +endmodule)"}, + }; + + for (const RawAndFiltered &test : test_cases) { + PreprocessorTester expanded( + test.pp_input, VerilogPreprocess::Config({.expand_macros = true})); + EXPECT_TRUE(expanded.Status().ok()) + << expanded.Status() << " " << test.description; + PreprocessorTester equivalent( + test.equivalent, VerilogPreprocess::Config({.expand_macros = false})); + EXPECT_TRUE(equivalent.Status().ok()) + << equivalent.Status() << " " << test.description; + const auto &expanded_stream = expanded.Data().GetTokenStreamView(); + const auto &equivalent_stream = equivalent.Data().GetTokenStreamView(); + EXPECT_GT(expanded_stream.size(), 0) << test.description; + EXPECT_EQ(expanded_stream.size(), equivalent_stream.size()) + << test.description; + auto expanded_it = expanded_stream.begin(); + auto equivalent_it = equivalent_stream.begin(); + while (expanded_it != expanded_stream.end() && + equivalent_it != equivalent_stream.end()) { + EXPECT_EQ((*expanded_it)->text(), (*equivalent_it)->text()) + << test.description; + ++expanded_it; + ++equivalent_it; + } + } +} + +TEST(VerilogPreprocessTest, TokenConcatenationChained) { + // Chained concatenation: a``b``c -> abc + const RawAndFiltered test_cases[] = { + {"[** Triple chained concatenation **]", + R"( +`define TRIPLE(a, b, c) a``b``c +module m; + wire `TRIPLE(x, y, z); +endmodule)", + R"( +`define TRIPLE(a, b, c) a``b``c +module m; + wire xyz; +endmodule)"}, + + {"[** Four-way chained concatenation **]", + R"( +`define QUAD(a, b, c, d) a``b``c``d +module m; + wire `QUAD(a, b, c, d); +endmodule)", + R"( +`define QUAD(a, b, c, d) a``b``c``d +module m; + wire abcd; +endmodule)"}, + + {"[** Chained with underscores **]", + R"( +`define PATH(a, b, c) a``_``b``_``c +module m; + wire `PATH(mod, sub, sig); +endmodule)", + R"( +`define PATH(a, b, c) a``_``b``_``c +module m; + wire mod_sub_sig; +endmodule)"}, + }; + + for (const RawAndFiltered &test : test_cases) { + PreprocessorTester expanded( + test.pp_input, VerilogPreprocess::Config({.expand_macros = true})); + EXPECT_TRUE(expanded.Status().ok()) + << expanded.Status() << " " << test.description; + PreprocessorTester equivalent( + test.equivalent, VerilogPreprocess::Config({.expand_macros = false})); + EXPECT_TRUE(equivalent.Status().ok()) + << equivalent.Status() << " " << test.description; + const auto &expanded_stream = expanded.Data().GetTokenStreamView(); + const auto &equivalent_stream = equivalent.Data().GetTokenStreamView(); + EXPECT_GT(expanded_stream.size(), 0) << test.description; + EXPECT_EQ(expanded_stream.size(), equivalent_stream.size()) + << test.description; + } +} + +TEST(VerilogPreprocessTest, TokenConcatenationWithWhitespace) { + // Whitespace around `` should be removed per SystemVerilog standard. + const RawAndFiltered test_cases[] = { + {"[** Spaces around concatenation operator **]", + R"( +`define SPACED(a, b) a `` b +module m; + wire `SPACED(foo, bar); +endmodule)", + R"( +`define SPACED(a, b) a `` b +module m; + wire foobar; +endmodule)"}, + + {"[** Mixed spacing **]", + R"( +`define MIXED(a, b) a`` b +module m; + wire `MIXED(left, right); +endmodule)", + R"( +`define MIXED(a, b) a`` b +module m; + wire leftright; +endmodule)"}, + }; + + for (const RawAndFiltered &test : test_cases) { + PreprocessorTester expanded( + test.pp_input, VerilogPreprocess::Config({.expand_macros = true})); + EXPECT_TRUE(expanded.Status().ok()) + << expanded.Status() << " " << test.description; + PreprocessorTester equivalent( + test.equivalent, VerilogPreprocess::Config({.expand_macros = false})); + EXPECT_TRUE(equivalent.Status().ok()) + << equivalent.Status() << " " << test.description; + const auto &expanded_stream = expanded.Data().GetTokenStreamView(); + const auto &equivalent_stream = equivalent.Data().GetTokenStreamView(); + EXPECT_GT(expanded_stream.size(), 0) << test.description; + EXPECT_EQ(expanded_stream.size(), equivalent_stream.size()) + << test.description; + } +} + +TEST(VerilogPreprocessTest, TokenConcatenationModuleName) { + // Test concatenation to build module names. + const RawAndFiltered test_cases[] = { + {"[** Module name generation **]", + R"( +`define MOD(prefix) module prefix``_module; endmodule +`MOD(test))", + R"( +`define MOD(prefix) module prefix``_module; endmodule +module test_module; endmodule)"}, + + {"[** Instance name generation **]", + R"( +`define INST(type, suffix) type u_``suffix() +module m; + `INST(submod, inst); +endmodule)", + R"( +`define INST(type, suffix) type u_``suffix() +module m; + submod u_inst(); +endmodule)"}, + }; + + for (const RawAndFiltered &test : test_cases) { + PreprocessorTester expanded( + test.pp_input, VerilogPreprocess::Config({.expand_macros = true})); + EXPECT_TRUE(expanded.Status().ok()) + << expanded.Status() << " " << test.description; + PreprocessorTester equivalent( + test.equivalent, VerilogPreprocess::Config({.expand_macros = false})); + EXPECT_TRUE(equivalent.Status().ok()) + << equivalent.Status() << " " << test.description; + const auto &expanded_stream = expanded.Data().GetTokenStreamView(); + const auto &equivalent_stream = equivalent.Data().GetTokenStreamView(); + EXPECT_GT(expanded_stream.size(), 0) << test.description; + EXPECT_EQ(expanded_stream.size(), equivalent_stream.size()) + << test.description; + } +} + +TEST(VerilogPreprocessTest, TokenConcatenationNestedMacros) { + // Test concatenation with nested macro calls. + const RawAndFiltered test_cases[] = { + {"[** Nested macro with concatenation **]", + R"( +`define SUFFIX _out +`define MAKE(name) wire name```SUFFIX +module m; + `MAKE(data); +endmodule)", + R"( +`define SUFFIX _out +`define MAKE(name) wire name```SUFFIX +module m; + wire data_out; +endmodule)"}, + }; + + for (const RawAndFiltered &test : test_cases) { + PreprocessorTester expanded( + test.pp_input, VerilogPreprocess::Config({.expand_macros = true})); + EXPECT_TRUE(expanded.Status().ok()) + << expanded.Status() << " " << test.description; + PreprocessorTester equivalent( + test.equivalent, VerilogPreprocess::Config({.expand_macros = false})); + EXPECT_TRUE(equivalent.Status().ok()) + << equivalent.Status() << " " << test.description; + const auto &expanded_stream = expanded.Data().GetTokenStreamView(); + const auto &equivalent_stream = equivalent.Data().GetTokenStreamView(); + EXPECT_GT(expanded_stream.size(), 0) << test.description; + EXPECT_EQ(expanded_stream.size(), equivalent_stream.size()) + << test.description; + } +} + +TEST(VerilogPreprocessTest, TokenConcatenationWithNumbers) { + // Test concatenation that produces numeric literals. + PreprocessorTester tester( + R"( +`define WIDTH 8 +`define SIZED(w, val) w``'d``val +module m; + parameter P1 = `SIZED(32, 10); + parameter P2 = `SIZED(`WIDTH, 255); +endmodule)", + VerilogPreprocess::Config({.expand_macros = true})); + + EXPECT_TRUE(tester.Status().ok()) << tester.Status(); + + // Verify the preprocessed output contains the concatenated literals. + std::string combined; + for (const auto &tok : tester.PreprocessorData().preprocessed_token_stream) { + combined += std::string(tok->text()) + " "; + } + // Should contain "32'd10" as a single token (or components). + EXPECT_TRUE(absl::StrContains(combined, "32") && + absl::StrContains(combined, "10")) + << "Expected number concatenation, got: " << combined; +} + +TEST(VerilogPreprocessTest, TokenConcatenationPreservesTokenType) { + // Verify that concatenated tokens get properly re-lexed for correct type. + PreprocessorTester tester( + R"( +`define IDENT(a, b) a``b +module m; + wire `IDENT(my, wire); +endmodule)", + VerilogPreprocess::Config({.expand_macros = true})); + + EXPECT_TRUE(tester.Status().ok()) << tester.Status(); + + // Find the concatenated token and verify it's an identifier. + bool found_mywire = false; + for (const auto &tok : tester.PreprocessorData().preprocessed_token_stream) { + if (tok->text() == "mywire") { + found_mywire = true; + // Token should be SymbolIdentifier after re-lexing. + EXPECT_EQ(tok->token_enum(), SymbolIdentifier) + << "Expected SymbolIdentifier for 'mywire'"; + break; + } + } + EXPECT_TRUE(found_mywire) << "Expected to find concatenated token 'mywire'"; +} + +TEST(VerilogPreprocessTest, TokenConcatenationMultipleInOneMacro) { + // Test multiple concatenations in a single macro. + const RawAndFiltered test_cases[] = { + {"[** Two concatenations in one macro **]", + R"( +`define PAIR(a, b, c, d) a``b, c``d +module m; + wire `PAIR(foo, 1, bar, 2); +endmodule)", + R"( +`define PAIR(a, b, c, d) a``b, c``d +module m; + wire foo1, bar2; +endmodule)"}, + }; + + for (const RawAndFiltered &test : test_cases) { + PreprocessorTester expanded( + test.pp_input, VerilogPreprocess::Config({.expand_macros = true})); + EXPECT_TRUE(expanded.Status().ok()) + << expanded.Status() << " " << test.description; + PreprocessorTester equivalent( + test.equivalent, VerilogPreprocess::Config({.expand_macros = false})); + EXPECT_TRUE(equivalent.Status().ok()) + << equivalent.Status() << " " << test.description; + const auto &expanded_stream = expanded.Data().GetTokenStreamView(); + const auto &equivalent_stream = equivalent.Data().GetTokenStreamView(); + EXPECT_GT(expanded_stream.size(), 0) << test.description; + EXPECT_EQ(expanded_stream.size(), equivalent_stream.size()) + << test.description; + } +} + +TEST(VerilogPreprocessTest, TokenConcatenationEmptyParameter) { + // Test concatenation with empty parameter (edge case). + PreprocessorTester tester( + R"( +`define OPT(prefix, suffix=) prefix``suffix +module m; + wire `OPT(signal); + wire `OPT(data, _bus); +endmodule)", + VerilogPreprocess::Config({.expand_macros = true})); + + EXPECT_TRUE(tester.Status().ok()) << tester.Status(); + + std::string combined; + for (const auto &tok : tester.PreprocessorData().preprocessed_token_stream) { + combined += std::string(tok->text()) + " "; + } + EXPECT_TRUE(absl::StrContains(combined, "signal")) + << "Expected 'signal', got: " << combined; + EXPECT_TRUE(absl::StrContains(combined, "data_bus")) + << "Expected 'data_bus', got: " << combined; +} + } // namespace } // namespace verilog