From 81a9dac7e3bfe4ab9066160a9c439dd978441d50 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 25 Jun 2026 09:58:00 +0100 Subject: [PATCH 01/23] Start implementing treesitter declarations parsing --- .../frontend/fortran_treesitter_reader.py | 150 +++++++++++++++--- .../fortran_treesitter_reader/ftr_test.py | 144 ++++++++++++++--- 2 files changed, 246 insertions(+), 48 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index a3498b9244..304684bb85 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -39,7 +39,7 @@ import logging from typing import TYPE_CHECKING, Iterable, Union, Callable -from psyclone.psyir import nodes +from psyclone.psyir import nodes, symbols from psyclone.psyir.nodes.codeblock import TreeSitterCodeBlock, CodeBlock if TYPE_CHECKING: @@ -122,15 +122,6 @@ def __init__( self._ignore_comments = ignore_comments self._free_form = free_form self._conditional_openmp = conditional_openmp - # TODO #3038: Currently this reader uses a cursor pointer instead of - # passing around a parent argument all the time (like fparser's), but - # this can be re-evaluated if necessary. - self._psyir_cursor = None - # Map from treesitter node types to their handler routine - self.handlers = { - 'translation_unit': self._translation_unit, - 'module': self._module_handler, - } def generate_parse_tree_from_file(self, file_path) -> 'TSNode': ''' @@ -193,7 +184,11 @@ def generate_psyir(self, parse_tree: 'TSNode') -> nodes.Node: ''' return self.process_nodes(parse_tree)[0] - def process_nodes(self, tsnodes: Union["TSNode", Iterable["TSNode"]]): + def process_nodes( + self, + tsnodes: Union["TSNode", Iterable["TSNode"]], + symtab: Optional[symbols.SymbolTable] = None, + ): ''' Create the PSyIR that represents the supplied treesitter nodes. @@ -202,12 +197,14 @@ def process_nodes(self, tsnodes: Union["TSNode", Iterable["TSNode"]]): :returns: the equivalent PSyIR Node. ''' + if symtab is None: + symtab = symbols.SymbolTable() list_of_nodes = tsnodes if isinstance(tsnodes, Iterable) else [tsnodes] children = [] for tsnode in list_of_nodes: try: handler = self.get_handler(tsnode) - children.append(handler(tsnode)) + children.append(handler(tsnode, symtab)) except NotImplementedError as err: # TODO #3038: Add support for expression codeblocks and # aggregating contiguous codeblocks into a single one. @@ -229,13 +226,14 @@ def get_handler(self, tsnode: 'TSNode') -> Callable: :raises NotImplementedError: if the given node type does not have a handler for it. ''' - handler = self.handlers.get(tsnode.type) - if not handler: + try: + handler = getattr(self, f"_{tsnode.type}_handler") + except AttributeError: raise NotImplementedError( f"Unsupported '{tsnode.type}' tree-sitter node.") return handler - def _translation_unit(self, tsnode: 'TSNode') -> nodes.Node: + def _translation_unit_handler(self, tsnode: 'TSNode', _) -> nodes.Node: ''' Handle treesitter 'translation_unit' node. :param tsnode: the treesitter node the process. @@ -243,11 +241,12 @@ def _translation_unit(self, tsnode: 'TSNode') -> nodes.Node: :returns: the equivalent PSyIR Node. ''' file_container = nodes.FileContainer("") - self._psyir_cursor = file_container - file_container.children.extend(self.process_nodes(tsnode.children)) + file_container.children.extend( + self.process_nodes(tsnode.children, file_container.symbol_table) + ) return file_container - def _module_handler(self, tsnode: 'TSNode') -> nodes.Node: + def _module_handler(self, tsnode: 'TSNode', symtab: symbols.SymbolTable) -> nodes.Node: ''' Handle a treesitter 'module' node. :param tsnode: the treesitter node the process. @@ -260,9 +259,14 @@ def _module_handler(self, tsnode: 'TSNode') -> nodes.Node: module_name = None internal_proc = None implicit_statement = False - for child in tsnode.children: - if child.type == "module_statement": - _module_keyword, module_name = child.children + + # The first node is always the module statement + _module_keyword, module_name = tsnode.children[0].children + container = nodes.Container(to_str(module_name) if module_name else "") + + for child in tsnode.children[1:]: + if child.type == "variable_declaration": + self.process_nodes(child, container.symbol_table) elif child.type == "end_module_statement": pass elif child.type == "internal_procedures": @@ -276,8 +280,106 @@ def _module_handler(self, tsnode: 'TSNode') -> nodes.Node: if not implicit_statement: raise NotImplementedError( "Modules that allow implicit variables are not supported") - container = nodes.Container(to_str(module_name) if module_name else "") - self._psyir_cursor = container if internal_proc: - container.children.extend(self.process_nodes(internal_proc)) + container.children.extend(self.process_nodes(internal_proc, container.symbol_table)) return container + + def _subroutine_handler(self, tsnode: 'TSNode', symtab: symbols.SymbolTable) -> nodes.Node: + ''' Handle a treesitter 'subroutine' node. + + :param tsnode: the treesitter node the process. + + :returns: the equivalent PSyIR Node. + ''' + for child in tsnode.children: + if child.type == "subroutine_statement": + sub_name = child.children[1] + elif child.type == "end_subroutine_statement": + pass + rsymbol = symbols.RoutineSymbol(to_str(sub_name)) + subroutine = nodes.Routine(symbol=rsymbol) + return subroutine + + def _number_literal_handler(self, tsnode: 'TSNode', symtab: symbols.SymbolTable) -> None: + ''' Handle a treesitter 'variable_declaration' node. + + :param tsnode: the treesitter node the process. + + ''' + # For now only integers + return nodes.Literal(to_str(tsnode), symbols.ScalarType.integer_type()) + + def _variable_declaration_handler(self, tsnode: 'TSNode', symtab: symbols.SymbolTable) -> None: + ''' Handle a treesitter 'variable_declaration' node. + + :param tsnode: the treesitter node the process. + + ''' + map_intrinsic_type = { + 'integer': symbols.ScalarType.Intrinsic.INTEGER, + 'real': symbols.ScalarType.Intrinsic.REAL, + 'logical': symbols.ScalarType.Intrinsic.BOOLEAN, + 'character': symbols.ScalarType.Intrinsic.CHARACTER, + } + + # Initialise none mandatory attibutes to their defaults + type_qualifier = None + unknown = None + + for child in tsnode.children: + if child.type == "intrinsic_type": + intrinsic_type, kind = unpack2(child) + intrinsic_type = map_intrinsic_type[intrinsic_type.type] + if kind: + _left_parens, kind_expr, _right_parens = kind.children + km = self.optional_name_equals(kind_expr, symtab, ("kind", )) + precision = km['kind'] + # If it is 4 or 8 it has special values, but a precison expression + # is also supported + if isinstance(precision, nodes.Literal): + if precision.value == "4": + precision = symbols.ScalarType.Precision.SINGLE + elif precision.value == "8": + precision = symbols.ScalarType.Precision.DOUBLE + else: + precision = symbols.ScalarType.Precision.UNDEFINED + + elif child.type == "identifier": + identifier = to_str(child) + elif child.type == "::": + pass + elif child.type == "type_qualifier": + import pdb; pdb.set_trace() + else: + unknown = child + + if unknown: + # Add as a declaration comment + print(f"Unrecognised: {unknown.type}: {to_str(unknown)}") + import pdb; pdb.set_trace() + datatype = symbols.UnsupportedFortranType(to_str(tsnode)) + else: + datatype = symbols.ScalarType(intrinsic_type, precision) + symbol = symbols.DataSymbol(identifier, datatype) + symtab.add(symbol) + + def optional_name_equals(self, tsnode, symtab, names): + result = {} + if tsnode.type == "keyword_argument": + identifier, _equals, expr = tsnode.children + string_id = to_str(identifier) + if string_id not in names: + raise NotImplementedError("Unexpected") + result[string_id] = self.process_nodes(expr, symtab)[0] + else: + string_id = names[0] + result[string_id] = self.process_nodes(tsnode, symtab)[0] + return result + + +def unpack2(child): + if len(child.children) == 1: + return child.children[0], None + if len(child.children) == 2: + return child.children[0], child.children[1] + raise NotImplementedError("Unexpected") diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index a92d6330d0..830dfa95cb 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -35,7 +35,6 @@ # ----------------------------------------------------------------------------- ''' Performs tests on the treesitter PSyIR front-end ''' - import logging import pytest @@ -43,7 +42,7 @@ from psyclone.psyir.frontend.fortran_treesitter_reader import \ FortranTreeSitterReader -from psyclone.psyir.nodes import FileContainer, CodeBlock, Container +from psyclone.psyir import nodes as psyir_nodes, symbols as psyir_symbols from psyclone.tests.utilities import min_version_3_10 @@ -138,11 +137,11 @@ def test_generate_psyir(): end module test """ ptree = processor.generate_parse_tree_from_source(valid_code) - psyir = processor.generate_psyir(ptree) + root = processor.generate_psyir(ptree) - assert isinstance(psyir, FileContainer) - assert isinstance(psyir.children[0], Container) - assert isinstance(psyir.children[0].children[0], CodeBlock) + assert isinstance(root, psyir_nodes.FileContainer) + assert isinstance(root.children[0], psyir_nodes.Container) + assert isinstance(root.children[0].children[0], psyir_nodes.CodeBlock) # TODO #3416: Skip treesitter tests below 3.10 as they're unsupported by @@ -163,32 +162,129 @@ def test_codeblock_generation_and_messages(): end module test """ ptree = processor.generate_parse_tree_from_source(unsupported_code) - psyir = processor.generate_psyir(ptree) + root = processor.generate_psyir(ptree) - assert isinstance(psyir, FileContainer) - assert isinstance(psyir.children[0], CodeBlock) + assert isinstance(root, psyir_nodes.FileContainer) + assert isinstance(root.children[0], psyir_nodes.CodeBlock) expected = ( "PSyclone CodeBlock (unsupported code) reason:\n" "- Modules that allow implicit variables are not supported" ) - assert psyir.children[0].preceding_comment == expected + assert root.children[0].preceding_comment == expected - unsupported_code = """ - module test - implicit none - integer :: a - contains + +@min_version_3_10 +def test_subroutine(): + ''' + Test subroutine nodes. + ''' + processor = FortranTreeSitterReader() + + valid_code = """ subroutine mysub() end subroutine - end module test + subroutine mysub2() + end subroutine mysub2 """ - ptree = processor.generate_parse_tree_from_source(unsupported_code) - psyir = processor.generate_psyir(ptree) + ptree = processor.generate_parse_tree_from_source(valid_code) + root = processor.generate_psyir(ptree) - assert isinstance(psyir, FileContainer) - assert isinstance(psyir.children[0], CodeBlock) - expected = ( - "PSyclone CodeBlock (unsupported code) reason:\n" - "- Module has an unsupported 'variable_declaration' node" + # Check the tree is as expected + assert len(root.children) == 2 + assert isinstance(root.children[0], psyir_nodes.Routine) + assert root.children[0].name == "mysub" + assert isinstance(root.children[1], psyir_nodes.Routine) + assert root.children[1].name == "mysub2" + + # Check that the symbols have been added to the symbol table + assert len(root.symbol_table.symbols) == 2 + rsymbol1 = root.symbol_table.lookup("mysub") + rsymbol2 = root.symbol_table.lookup("mysub2") + assert root.children[0].symbol is rsymbol1 + assert root.children[1].symbol is rsymbol2 + assert isinstance(rsymbol1, psyir_symbols.RoutineSymbol) + assert isinstance(rsymbol2, psyir_symbols.RoutineSymbol) + + +@min_version_3_10 +def test_declarations(): + ''' + Test subroutine nodes. + ''' + processor = FortranTreeSitterReader() + + valid_code = """ + module test + implicit none + integer :: a + real :: b + end module + """ + ptree = processor.generate_parse_tree_from_source(valid_code) + root = processor.generate_psyir(ptree) + module = root.children[0] + + # Declarations do not add children nodes + assert len(module.children) == 0 + + # Check that the symbols have been added to the symbol table + assert len(module.symbol_table.symbols) == 2 + assert "a" in module.symbol_table + assert "b" in module.symbol_table + + +@pytest.mark.parametrize("fortran_type,psyir_type", [ + ("integer", psyir_symbols.ScalarType.integer_type()), + ("integer(kind=4)", psyir_symbols.ScalarType.integer_single_type()), + ("integer(8)", psyir_symbols.ScalarType.integer_double_type()), + ("real", psyir_symbols.ScalarType.real_type()), + ("real(4)", psyir_symbols.ScalarType.real_single_type()), + ("real(kind=8)", psyir_symbols.ScalarType.real_double_type()), + ("logical", psyir_symbols.ScalarType.boolean_type()), + ("character", psyir_symbols.ScalarType.character_type()), + ("integer, dimension(:)", psyir_symbols.ScalarType.integer_type()), +]) +def test_declarations_datatypes(fortran_type, psyir_type): + ''' + Test subroutine nodes. + ''' + processor = FortranTreeSitterReader() + + valid_code = f""" + module test + implicit none + {fortran_type} :: a + end module + """ + ptree = processor.generate_parse_tree_from_source(valid_code) + root = processor.generate_psyir(ptree) + module = root.children[0] + assert module.symbol_table.lookup("a").datatype == psyir_type, ( + f"{module.symbol_table.lookup("a").datatype} != {psyir_type}" + ) +@pytest.mark.parametrize("shape_string, psyir_shape", [ + ("(:)", psyir_nodes.Literal("10", psyir_symbols.ScalarType.integer_type())), +]) +def test_declarations_arrays_datatypes(shape_string, psyir_shape): + ''' + Test subroutine nodes. + ''' + processor = FortranTreeSitterReader() + + valid_code = f""" + module test + implicit none + integer(4), dimension{shape_string} :: a + end module + """ + ptree = processor.generate_parse_tree_from_source(valid_code) + root = processor.generate_psyir(ptree) + module = root.children[0] + + array_symbol = module.symbol_table.lookup("a") + assert isinstance(array_symbol.datatype, psyir_symbols.ArrayType) + assert isinstance(array_symbol.elemental_type, psyir_symbols.ScalarType.integer4_type()) + + assert module.symbol_table.lookup("a").datatype == psyir_type, ( + f"{module.symbol_table.lookup("a").datatype} != {psyir_type}" ) - assert psyir.children[0].preceding_comment == expected From 5290f8cd5dffba771bb3011a3bd5d4544e672c23 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 29 Jul 2026 16:43:05 +0100 Subject: [PATCH 02/23] #3083 Add more treesitter handlers --- .../frontend/fortran_treesitter_reader.py | 2142 ++++++++++++++++- .../fortran_treesitter_reader/ftr_test.py | 907 ++++++- 2 files changed, 2912 insertions(+), 137 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 304684bb85..895542fc65 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -36,8 +36,10 @@ ''' PSyIR TreeSitter Fortran reader ''' import codecs +from contextlib import contextmanager +from dataclasses import dataclass import logging -from typing import TYPE_CHECKING, Iterable, Union, Callable +from typing import Callable, Iterable, Iterator, Optional, TYPE_CHECKING, Union from psyclone.psyir import nodes, symbols from psyclone.psyir.nodes.codeblock import TreeSitterCodeBlock, CodeBlock @@ -75,11 +77,32 @@ def to_str(node: 'TSNode') -> str: return node.text.decode('utf8') if node.text else "" -class FortranTreeSitterReader(): - ''' Processes the TreeSitter parse_tree and converts it to PSyIR. +@dataclass(frozen=True) +class _CommonDeclAttributes: + ''' Properties shared by all entities of a fortran declaration (the lhs + of ::) + + :param base_type: common PSyIR datatype, or ``None`` if unsupported. + :param dimension: common DIMENSION argument list, if present. + :param intent: common INTENT qualifier, if present. + :param qualifiers: names of all declaration qualifiers. + :param unsupported: qualifiers not represented directly in PSyIR. + :param prefix: declaration text preceding ``::``. + ''' - Note: this class is in development, currently only generates - top-level Modules and CodeBlocks. + base_type: object + dimension: Optional['TSNode'] + intent: Optional['TSNode'] + qualifiers: frozenset[str] + unsupported: frozenset[str] + prefix: str + + +class FortranTreeSitterReader(): + ''' + Processes the TreeSitter parse_tree and converts it to PSyIR nodes. + Unsupported declarations retain their source in UnsupportedFortranType + while unsupported executable statements become TreeSitterCodeBlocks. The structure of the expected fortran parse tree can be found in the 'rules' section of: @@ -98,12 +121,62 @@ class FortranTreeSitterReader(): Defaults to False. Currently ignored. :param ignore_comments: whether to let the parser ignore comments. :param free_form: whether to parse using Fortran free_form syntax. - :param ignore_directives: whether to ignore directives while parsing. :param conditional_openmp: whether to parse conditional OpenMP statements. - - :raises TypeError: if any constructor argument is not of the expected type. ''' + # These nodes belong to a Fortran specification part and update a symbol + # table rather than producing executable PSyIR children. + _SPECIFICATION_TYPES = { + "use_statement", "variable_declaration", "derived_type_definition", + "interface" + } + + # Punctuation and grammar-only nodes are listed explicitly at each scope + # boundary. This makes it clear which tree-sitter children are consumed by + # the scope handler and prevents them from becoming accidental CodeBlocks. + _MODULE_NON_EXECUTABLE_TYPES = _SPECIFICATION_TYPES.union({ + "module_statement", "end_module_statement", "implicit_statement", + "internal_procedures", "public_statement", "private_statement" + }) + + # Centralising these maps documents the supported Fortran spellings and + # avoids recreating identical dictionaries for every parsed operation. + _UNARY_OPERATORS = { + "+": nodes.UnaryOperation.Operator.PLUS, + "-": nodes.UnaryOperation.Operator.MINUS, + ".not.": nodes.UnaryOperation.Operator.NOT, + } + _BINARY_OPERATORS = { + "+": nodes.BinaryOperation.Operator.ADD, + "-": nodes.BinaryOperation.Operator.SUB, + "*": nodes.BinaryOperation.Operator.MUL, + "/": nodes.BinaryOperation.Operator.DIV, + "**": nodes.BinaryOperation.Operator.POW, + "==": nodes.BinaryOperation.Operator.EQ, + ".eq.": nodes.BinaryOperation.Operator.EQ, + "/=": nodes.BinaryOperation.Operator.NE, + ".ne.": nodes.BinaryOperation.Operator.NE, + "<": nodes.BinaryOperation.Operator.LT, + ".lt.": nodes.BinaryOperation.Operator.LT, + "<=": nodes.BinaryOperation.Operator.LE, + ".le.": nodes.BinaryOperation.Operator.LE, + ">": nodes.BinaryOperation.Operator.GT, + ".gt.": nodes.BinaryOperation.Operator.GT, + ">=": nodes.BinaryOperation.Operator.GE, + ".ge.": nodes.BinaryOperation.Operator.GE, + ".and.": nodes.BinaryOperation.Operator.AND, + ".or.": nodes.BinaryOperation.Operator.OR, + ".eqv.": nodes.BinaryOperation.Operator.EQV, + ".neqv.": nodes.BinaryOperation.Operator.NEQV, + } + _INTENT_ACCESS = { + "in": symbols.ArgumentInterface.Access.READ, + "out": symbols.ArgumentInterface.Access.WRITE, + "inout": symbols.ArgumentInterface.Access.READWRITE, + } + + # These arguments intentionally mirror the other Fortran reader API. + # pylint: disable=too-many-arguments,too-many-positional-arguments def __init__( self, ignore_directives: bool = True, @@ -113,6 +186,17 @@ def __init__( free_form: bool = True, conditional_openmp: bool = True, ): + '''Create a Fortran tree-sitter reader. + + :param ignore_directives: whether directives are ignored. + :param last_comments_as_codeblocks: whether trailing comments in a + block are retained as CodeBlocks. + :param resolve_modules: whether imported modules are resolved. + :param ignore_comments: whether comments are ignored. + :param free_form: whether source is parsed as free-form Fortran. + :param conditional_openmp: whether conditional OpenMP statements are + parsed. + ''' # TODO #3038 Arguments are currently not used nor typechecked, but if # we decide this is the common reader interface, this can be done in a # super class instead of duplicate it here. @@ -122,6 +206,11 @@ def __init__( self._ignore_comments = ignore_comments self._free_form = free_form self._conditional_openmp = conditional_openmp + self._current_scope: Optional[symbols.SymbolTable] = None + + # --------------------------------------------------------------------- + # Source parsing and generic dispatch + # --------------------------------------------------------------------- def generate_parse_tree_from_file(self, file_path) -> 'TSNode': ''' @@ -187,36 +276,79 @@ def generate_psyir(self, parse_tree: 'TSNode') -> nodes.Node: def process_nodes( self, tsnodes: Union["TSNode", Iterable["TSNode"]], - symtab: Optional[symbols.SymbolTable] = None, ): - ''' - Create the PSyIR that represents the supplied treesitter nodes. + '''Create PSyIR for one or more tree-sitter nodes. - :param nodes: the list of nodes to process, for convenience it accepts - a single node or a list of them. + This is the statement-boundary dispatcher. Unsupported syntax is + deliberately caught here rather than in individual handlers so that + nested expression failures replace their whole Fortran statement with + one valid CodeBlock. - :returns: the equivalent PSyIR Node. + :param tsnodes: one tree-sitter node or an iterable of nodes. + + :returns: PSyIR nodes produced from the supplied tree-sitter nodes. + :rtype: list[:py:class:`psyclone.psyir.nodes.Node`] ''' - if symtab is None: - symtab = symbols.SymbolTable() list_of_nodes = tsnodes if isinstance(tsnodes, Iterable) else [tsnodes] children = [] for tsnode in list_of_nodes: try: handler = self.get_handler(tsnode) - children.append(handler(tsnode, symtab)) + result = handler(tsnode) + if result is not None: + children.append(result) except NotImplementedError as err: # TODO #3038: Add support for expression codeblocks and # aggregating contiguous codeblocks into a single one. - structure = CodeBlock.Structure.STATEMENT - code_block = TreeSitterCodeBlock(tsnode, structure) - code_block.append_preceding_comment( - f"PSyclone CodeBlock (unsupported code) reason:\n" - f"- {err}" - ) - children.append(code_block) + children.append(self._create_codeblock(tsnode, str(err))) return children + @staticmethod + def _create_codeblock( + tsnode: 'TSNode', reason: str + ) -> TreeSitterCodeBlock: + '''Create a statement CodeBlock for unsupported valid Fortran. + + Keeping this construction in one place guarantees that ordinary + dispatch and specification-part dispatch produce the same diagnostic. + + :param tsnode: tree-sitter node containing unsupported Fortran. + :param reason: human-readable explanation of the limitation. + + :returns: CodeBlock retaining the original tree-sitter node. + ''' + code_block = TreeSitterCodeBlock( + tsnode, CodeBlock.Structure.STATEMENT) + code_block.append_preceding_comment( + f"PSyclone CodeBlock (unsupported code) reason:\n" + f"- {reason}" + ) + return code_block + + def _process_specification_part( + self, tsnodes: Iterable['TSNode'] + ) -> list[TreeSitterCodeBlock]: + '''Populate a scope's symbol table from specification statements. + + Most specification handlers only update the current symbol table and + return ``None``. If one is unsupported, preserve just that statement + as a CodeBlock rather than replacing the enclosing module or routine. + + :param tsnodes: tree-sitter children of the Fortran scope. + + :returns: CodeBlocks for unsupported specification statements. + ''' + unsupported = [] + for tsnode in tsnodes: + if tsnode.type not in self._SPECIFICATION_TYPES: + continue + try: + self.get_handler(tsnode)(tsnode) + except NotImplementedError as err: + unsupported.append( + self._create_codeblock(tsnode, str(err))) + return unsupported + def get_handler(self, tsnode: 'TSNode') -> Callable: ''' :param tsnode: a given treesitter node. @@ -230,10 +362,142 @@ def get_handler(self, tsnode: 'TSNode') -> Callable: handler = getattr(self, f"_{tsnode.type}_handler") except AttributeError: raise NotImplementedError( - f"Unsupported '{tsnode.type}' tree-sitter node.") + f"Unsupported '{tsnode.type}' tree-sitter node.") from None return handler - def _translation_unit_handler(self, tsnode: 'TSNode', _) -> nodes.Node: + @contextmanager + def _using_scope( + self, symtab: symbols.SymbolTable + ) -> Iterator[None]: + ''' Make the given symtab the new parsing scope, but keep a reference + to the previous scope in order to restore it when leaving this new + scope (by graceful exit or an exception). + + :param symtab: symbol table for the scope being translated. + + :yields: while ``symtab`` is the reader's current scope. + ''' + previous_scope = self._current_scope + self._current_scope = symtab + try: + yield + finally: + self._current_scope = previous_scope + + @contextmanager + def _using_temporary_scope( + self, parent: nodes.ScopingNode + ) -> Iterator[nodes.ScopingNode]: + ''' + Like `_using_scope`, but it creates a dummy symbol_table and scope, + soft linked to the current parent. This is useful to create disposable + symbol tables when the resulting PSyIR does not need them, but they + still need to be linked to the parent scope. + + For example in the body of a derived type: + + .. code-block:: fortran + + module m + integer, parameter :: size = 10 + type myt + integer, dimension(size) :: array + end type + end module + + :param parent: real host scope used for lexical lookup. + + :yields: disposable ScopingNode. + ''' + symtab = symbols.SymbolTable() + temporary_scope = nodes.ScopingNode(symbol_table=symtab) + # This intentionally bypasses child validation. + # pylint: disable=protected-access + temporary_scope._parent = parent + self._current_scope = symtab + try: + yield temporary_scope + finally: + # Calling Node.detach() would be incorrect because the temporary + # scope was never inserted into the parent's child list. + temporary_scope._parent = None + symtab.detach() + self._current_scope = parent.symbol_table + + @staticmethod + @contextmanager + def _temporary_parent( + node: nodes.Routine, parent: nodes.ScopingNode + ) -> Iterator[None]: + '''Temporarily attach a node while its contents are translated. + + A routine handler returns its completed node to the bottom-up + dispatcher, which attaches it later. During translation, temporarily + attaching the Routine to its real parent enables ordinary PSyIR + lexical lookup. Detaching in ``finally`` also restores the dispatcher's + expectation that returned nodes are orphans. + + :param node: PSyIR node requiring temporary host association. + :param parent: real PSyIR parent of ``node``. + + :yields: while ``node`` is attached to ``parent``. + + :raises RuntimeError: if translation unexpectedly reparents ``node``. + ''' + # Attaching a Routine moves its RoutineSymbol into the parent table; + # detaching normally moves it back. Record the exact initial state + # because a forward declaration (e.g. an interface member) may already + # place the same symbol in the parent table too. + symbol = node.symbol + name = symbol.name + symbol_was_in_parent = ( + name in parent.symbol_table and + parent.symbol_table.lookup(name, scope_limit=parent) is symbol) + symbol_was_in_routine = ( + name in node.symbol_table and + node.symbol_table.lookup(name, scope_limit=node) is symbol) + original_interface = symbol.interface + + parent.children.append(node) + try: + yield + finally: + if node.parent is not parent: + raise RuntimeError( + "Temporarily attached PSyIR node was unexpectedly " + "reparented") + node.detach() + + # Restore the symbol ownership seen by the bottom-up dispatcher. + # This is particularly important while processing a list of + # sibling routines: later siblings must still see any forward + # symbol that existed in the host before this temporary attach. + symbol_is_in_parent = ( + name in parent.symbol_table and + parent.symbol_table.lookup( + name, scope_limit=parent) is symbol) + if symbol_was_in_parent and not symbol_is_in_parent: + parent.symbol_table.add(symbol) + elif not symbol_was_in_parent and symbol_is_in_parent: + parent.symbol_table.remove(symbol) + + symbol_is_in_routine = ( + name in node.symbol_table and + node.symbol_table.lookup(name, scope_limit=node) is symbol) + if symbol_was_in_routine and not symbol_is_in_routine: + node.symbol_table.add(symbol) + elif not symbol_was_in_routine and symbol_is_in_routine: + node.symbol_table.remove(symbol) + symbol.interface = original_interface + + + # --------------------------------------------------------------------- + # Parse-tree navigation and Fortran scope handlers + # --------------------------------------------------------------------- + + def _translation_unit_handler( + self, tsnode: 'TSNode' + ) -> nodes.Node: ''' Handle treesitter 'translation_unit' node. :param tsnode: the treesitter node the process. @@ -241,12 +505,62 @@ def _translation_unit_handler(self, tsnode: 'TSNode', _) -> nodes.Node: :returns: the equivalent PSyIR Node. ''' file_container = nodes.FileContainer("") - file_container.children.extend( - self.process_nodes(tsnode.children, file_container.symbol_table) - ) + with self._using_scope(file_container.symbol_table): + file_container.children.extend( + self.process_nodes(tsnode.children) + ) return file_container - def _module_handler(self, tsnode: 'TSNode', symtab: symbols.SymbolTable) -> nodes.Node: + @staticmethod + def _child(tsnode: 'TSNode', node_type: str) -> Optional['TSNode']: + '''Return the first direct child having the supplied type. + + :param tsnode: tree-sitter node whose children are searched. + :param node_type: tree-sitter type to find. + + :returns: matching child, or ``None`` if no child matches. + ''' + return next((child for child in tsnode.children + if child.type == node_type), None) + + @staticmethod + def _children(tsnode: 'TSNode', *node_types: str) -> list['TSNode']: + '''Return direct children having one of the supplied types. + + :param tsnode: tree-sitter node whose children are searched. + :param node_types: tree-sitter types to find. + + :returns: children whose type is in ``node_types``. + ''' + return [child for child in tsnode.children + if child.type in node_types] + + @staticmethod + def _split_extent( + tsnode: 'TSNode' + ) -> tuple[list['TSNode'], list['TSNode'], bool]: + '''Split the children of a bound or range around its first colon. + + Several Fortran grammar nodes represent ``lower:upper`` using the same + child layout. Keeping the token handling here lets callers focus on + their different semantics (declaration shape, array section, CASE + range or allocation shape). + + :param tsnode: tree-sitter node containing an optional colon. + + :returns: children before the colon, children after it, and whether a + colon was present. + ''' + colon = next((idx for idx, child in enumerate(tsnode.children) + if child.type == ":"), None) + if colon is None: + return list(tsnode.children), [], False + return (list(tsnode.children[:colon]), + list(tsnode.children[colon + 1:]), True) + + def _module_handler( + self, tsnode: 'TSNode' + ) -> nodes.Node: ''' Handle a treesitter 'module' node. :param tsnode: the treesitter node the process. @@ -256,130 +570,1704 @@ def _module_handler(self, tsnode: 'TSNode', symtab: symbols.SymbolTable) -> node :raises NotImplementedError: if the module has an unsupported child. :raises NotImplementedError: if the module permits implicit variables. ''' - module_name = None - internal_proc = None - implicit_statement = False - - # The first node is always the module statement - _module_keyword, module_name = tsnode.children[0].children - container = nodes.Container(to_str(module_name) if module_name else "") - - for child in tsnode.children[1:]: - if child.type == "variable_declaration": - self.process_nodes(child, container.symbol_table) - elif child.type == "end_module_statement": - pass - elif child.type == "internal_procedures": - internal_proc = child - elif child.type == "implicit_statement": - implicit_statement = True - else: - raise NotImplementedError( - f"Module has an unsupported '{child.type}' node") - - if not implicit_statement: + statement = self._child(tsnode, "module_statement") + name = self._child(statement, "name") if statement else None + if not any(child.type == "implicit_statement" and + "none" in [item.type for item in child.children] + for child in tsnode.children): raise NotImplementedError( "Modules that allow implicit variables are not supported") - if internal_proc: - container.children.extend(self.process_nodes(internal_proc, container.symbol_table)) + container = nodes.Container(to_str(name) if name else "") + + with self._using_scope(container.symbol_table): + visibility_map = self._process_access_statements( + tsnode.children) + unsupported_specs = self._process_specification_part( + tsnode.children) + + internal = self._child(tsnode, "internal_procedures") + container.children.extend(unsupported_specs) + if internal: + container.children.extend( + self.process_nodes( + [child for child in internal.children + if child.type != "contains_statement"])) + + container.children.extend(self.process_nodes( + [child for child in tsnode.children + if child.type not in self._MODULE_NON_EXECUTABLE_TYPES])) + self._apply_visibility(visibility_map) return container - def _subroutine_handler(self, tsnode: 'TSNode', symtab: symbols.SymbolTable) -> nodes.Node: + def _internal_procedures_handler( + self, tsnode: 'TSNode' + ) -> nodes.Node: + '''Reject internal subprograms within routines. + + Module handlers process their internal-procedures children directly + because PSyIR Containers can contain Routines. PSyIR Routines cannot + currently contain nested Routines. + + :param tsnode: internal-procedures tree-sitter node. + + :raises NotImplementedError: because nested Routines are unsupported. + ''' + del tsnode + raise NotImplementedError( + "Internal subprograms within a routine are not supported") + + def _subroutine_handler( + self, tsnode: 'TSNode' + ) -> nodes.Node: ''' Handle a treesitter 'subroutine' node. :param tsnode: the treesitter node the process. :returns: the equivalent PSyIR Node. ''' - for child in tsnode.children: - if child.type == "subroutine_statement": - sub_name = child.children[1] - elif child.type == "end_subroutine_statement": - pass - rsymbol = symbols.RoutineSymbol(to_str(sub_name)) - subroutine = nodes.Routine(symbol=rsymbol) - return subroutine + return self._routine_handler(tsnode, "subroutine") - def _number_literal_handler(self, tsnode: 'TSNode', symtab: symbols.SymbolTable) -> None: - ''' Handle a treesitter 'variable_declaration' node. + def _function_handler( + self, tsnode: 'TSNode' + ) -> nodes.Node: + '''Create a PSyIR Routine for a Fortran function. - :param tsnode: the treesitter node the process. + :param tsnode: function tree-sitter node. + :returns: PSyIR Routine representing the function. ''' - # For now only integers - return nodes.Literal(to_str(tsnode), symbols.ScalarType.integer_type()) + return self._routine_handler(tsnode, "function") - def _variable_declaration_handler(self, tsnode: 'TSNode', symtab: symbols.SymbolTable) -> None: - ''' Handle a treesitter 'variable_declaration' node. + def _program_handler( + self, tsnode: 'TSNode' + ) -> nodes.Node: + '''Create a PSyIR Routine for a main program. - :param tsnode: the treesitter node the process. + :param tsnode: program tree-sitter node. + + :returns: PSyIR Routine representing the program. + ''' + return self._routine_handler(tsnode, "program") + + def _routine_handler( + self, tsnode: 'TSNode', routine_kind: str + ) -> nodes.Routine: + '''Create PSyIR shared by programs, subroutines and functions. + + :param tsnode: tree-sitter node for the complete program unit. + :param routine_kind: one of ``program``, ``subroutine`` or + ``function``. + + :returns: translated PSyIR Routine. + ''' + parent_symtab = self._current_scope + if parent_symtab is None: + raise RuntimeError( + "A Routine must be translated within a current scope") + statement = self._child(tsnode, f"{routine_kind}_statement") + name_node = self._child(statement, "name") if statement else None + name = to_str(name_node) if name_node else routine_kind + parameters = (self._child(statement, "parameters") + if statement else None) + argument_names = tuple( + to_str(child) for child in parameters.children + if child.type == "identifier") if parameters else () + return_name, return_type = self._function_return_info( + statement, name, routine_kind) + + # Insert arguments before declarations so specify_argument_list() can + # retain source order. Declarations later complete these placeholders. + routine_table = symbols.SymbolTable() + for arg_name in argument_names: + routine_table.add(symbols.DataSymbol( + arg_name, symbols.UnresolvedType(), + interface=symbols.ArgumentInterface())) + if return_name and return_name not in routine_table: + routine_table.add(symbols.DataSymbol( + return_name, return_type or symbols.UnresolvedType())) + + rsymbol = self._create_routine_symbol( + name, statement, return_type) + routine = nodes.Routine( + rsymbol, is_program=routine_kind == "program", + symbol_table=routine_table) + + parent = parent_symtab.node + if not isinstance(parent, nodes.ScopingNode): + raise RuntimeError( + "A Routine must be translated within a PSyIR scope") + with self._temporary_parent(routine, parent): + with self._using_scope(routine.symbol_table): + visibility_map = self._process_access_statements( + tsnode.children) + unsupported_specs = self._process_specification_part( + tsnode.children) + + args = [routine.symbol_table.lookup(name) + for name in argument_names] + routine.symbol_table.specify_argument_list(args) + if return_name: + routine.return_symbol = routine.symbol_table.lookup( + return_name) + + specification = { + f"{routine_kind}_statement", + f"end_{routine_kind}_statement", + "implicit_statement", "public_statement", + "private_statement" + } + specification.update(self._SPECIFICATION_TYPES) + routine.children.extend(unsupported_specs) + routine.children.extend(self.process_nodes( + [child for child in tsnode.children + if child.type not in specification])) + self._apply_visibility(visibility_map) + return routine + + def _function_return_info( + self, statement: 'TSNode', routine_name: str, routine_kind: str + ) -> tuple[Optional[str], Optional[symbols.DataType]]: + '''Extract result name and datatype from a function statement. + + :param statement: opening program-unit statement. + :param routine_name: name of the program unit. + :param routine_kind: one of ``program``, ``subroutine`` or + ``function``. + + :returns: result-symbol name and datatype, both ``None`` for a + non-function. + ''' + if routine_kind != "function": + return None, None + + result = self._child(statement, "function_result") + result_name = self._child(result, "identifier") if result else None + return_name = to_str(result_name) if result_name else routine_name + type_node = next( + (child for child in statement.children + if child.type in ("intrinsic_type", "derived_type")), None) + if not type_node: + return return_name, None + try: + return return_name, self._datatype_from_type(type_node) + except (NotImplementedError, KeyError, TypeError): + return return_name, symbols.UnsupportedFortranType( + to_str(statement).strip()) + + def _create_routine_symbol( + self, name: str, statement: 'TSNode', return_type + ) -> symbols.RoutineSymbol: + '''Create or complete the RoutineSymbol for a program unit. + + An interface block may create the symbol before its implementation is + visited. Reusing it ensures interface members and the Routine node + refer to the same object. + + :param name: routine name. + :param statement: opening program-unit statement. + :param return_type: translated function return type, if any. + + :returns: RoutineSymbol representing the program unit. + ''' + parent_symtab = self._current_scope + if parent_symtab is None: + raise RuntimeError( + "A RoutineSymbol must be created within a current scope") + qualifiers = { + to_str(child).lower() for child in statement.children + if child.type == "procedure_qualifier"} + visibility = parent_symtab.default_visibility + try: + routine_symbol = parent_symtab.lookup(name) + except KeyError: + routine_symbol = None + if isinstance(routine_symbol, symbols.RoutineSymbol): + routine_symbol.datatype = ( + return_type or routine_symbol.datatype) + routine_symbol.is_pure = "pure" in qualifiers + routine_symbol.is_elemental = "elemental" in qualifiers + routine_symbol.visibility = visibility + return routine_symbol + return symbols.RoutineSymbol( + name, datatype=return_type or symbols.UnresolvedType(), + is_pure="pure" in qualifiers, + is_elemental="elemental" in qualifiers, + visibility=visibility) + + # --------------------------------------------------------------------- + # Literals, declarations and datatypes + # --------------------------------------------------------------------- + + def _number_literal_handler( + self, tsnode: 'TSNode' + ) -> nodes.Literal: + '''Translate an integer or real literal. + + :param tsnode: number-literal tree-sitter node. + + :returns: PSyIR integer or real Literal. + ''' + text = to_str(tsnode).lower() + value, _, kind = text.partition("_") + datatype = (symbols.ScalarType.real_type() + if any(char in value for char in ".ed") + else symbols.ScalarType.integer_type()) + if kind: + if kind == "4": + precision = symbols.ScalarType.Precision.SINGLE + elif kind == "8": + precision = symbols.ScalarType.Precision.DOUBLE + else: + precision = (int(kind) if kind.isdigit() + else nodes.Reference( + self._kind_symbol(kind))) + datatype = symbols.ScalarType( + datatype.intrinsic, precision) + return nodes.Literal(value, datatype) + + def _string_literal_handler( + self, tsnode: 'TSNode' + ) -> nodes.Literal: + '''Translate a character literal. + + :param tsnode: string-literal tree-sitter node. + + :returns: PSyIR character Literal. + ''' + text = to_str(tsnode) + return nodes.Literal(text[1:-1].replace(text[0] * 2, text[0]), + symbols.ScalarType.character_type()) + + def _boolean_literal_handler( + self, tsnode: 'TSNode' + ) -> nodes.Literal: + '''Translate a logical literal. + + :param tsnode: boolean-literal tree-sitter node. + + :returns: PSyIR boolean Literal. + ''' + value = to_str(tsnode).lower().strip(".") + return nodes.Literal(value, symbols.ScalarType.boolean_type()) + + def _variable_declaration_handler( + self, tsnode: 'TSNode' + ) -> None: + '''Translate every entity in a variable declaration. + + :param tsnode: variable-declaration tree-sitter node. + ''' + # A declaration has properties shared by every entity (type and + # attributes) followed by one or more entity-specific declarators. + type_node = next((child for child in tsnode.children + if child.type in + ("intrinsic_type", "derived_type")), None) + if not type_node: + raise NotImplementedError( + "A variable declaration has no supported type specification") + + qualifiers = self._children(tsnode, "type_qualifier") + qualifier_names = frozenset( + child.children[0].type + for child in qualifiers if child.children) + unsupported = qualifier_names.intersection({ + "pointer", "target", "optional", "value", "volatile", + "asynchronous", "contiguous" + }) + try: + base_type = self._datatype_from_type(type_node) + except (NotImplementedError, KeyError, TypeError): + base_type = None + + dimension = next( + (self._child(item, "argument_list") + for item in qualifiers + if item.children and item.children[0].type == "dimension"), None) + intent = next( + (item for item in qualifiers + if item.children and item.children[0].type == "intent"), None) + common_attr = _CommonDeclAttributes( + base_type, dimension, intent, qualifier_names, + frozenset(unsupported), + to_str(tsnode).split("::", maxsplit=1)[0].strip()) + + + for declarator in tsnode.children: + if declarator.type in ( + "identifier", "sized_declarator", "init_declarator"): + self._declare_entity(declarator, common_attr) + + def _declare_entity( + self, declarator: 'TSNode', common_attr: _CommonDeclAttributes + ): + '''Translate one entity and add it to the current symbol table. + :param declarator: identifier or entity-declarator tree-sitter node. + :param common_attr: properties shared by the complete declaration. ''' - map_intrinsic_type = { - 'integer': symbols.ScalarType.Intrinsic.INTEGER, - 'real': symbols.ScalarType.Intrinsic.REAL, - 'logical': symbols.ScalarType.Intrinsic.BOOLEAN, - 'character': symbols.ScalarType.Intrinsic.CHARACTER, + id_node = (declarator if declarator.type == "identifier" + else self._child(declarator, "identifier")) + name = to_str(id_node) + datatype, initial_value = self._declarator_datatype( + declarator, common_attr) + + visibility = self._current_scope.default_visibility + if "public" in common_attr.qualifiers: + visibility = symbols.Symbol.Visibility.PUBLIC + elif "private" in common_attr.qualifiers: + visibility = symbols.Symbol.Visibility.PRIVATE + + kwargs = {"visibility": visibility} + interface = self._declaration_interface(name, common_attr) + if interface is not None: + kwargs["interface"] = interface + if initial_value is not None: + kwargs["initial_value"] = initial_value + if "parameter" in common_attr.qualifiers: + kwargs["is_constant"] = True + declared_symbol = symbols.DataSymbol(name, datatype, **kwargs) + self._add_or_update_datasymbol(declared_symbol) + + def _declarator_datatype( + self, declarator: 'TSNode', common_attr: _CommonDeclAttributes + ): + '''Translate one entity's datatype and initial value. + + Entity-specific syntax must be handled independently. In particular, + an unsupported initializer on one entity must not make its siblings + unsupported. + + :param declarator: entity-declarator tree-sitter node. + :param common_attr: properties shared by the complete declaration. + + :returns: entity datatype and optional initial-value expression. + ''' + is_unsupported = bool(common_attr.unsupported) + initial_value = None + if declarator.type == "init_declarator": + expressions = [ + child for child in declarator.children + if child.type not in ("identifier", "=")] + if expressions: + try: + initial_value = self._expression(expressions[-1]) + except NotImplementedError: + is_unsupported = True + + # Array shape and initialisation belong to an individual entity, so + # derive a fresh datatype from the common base type. + datatype = common_attr.base_type + shape_node = self._child(declarator, "size") or common_attr.dimension + is_allocatable = "allocatable" in common_attr.qualifiers + if datatype and shape_node: + try: + shape = self._shape_from_node( + shape_node, is_allocatable) + datatype = symbols.ArrayType(datatype, shape) + except (NotImplementedError, TypeError): + datatype = None + elif is_allocatable: + datatype = None + + # UnsupportedFortranType is preferable to losing a declaration. + # Keep only this entity in the saved text to avoid duplicating sibling + # names when the Fortran backend emits the symbols. + if datatype is None or is_unsupported: + entity = to_str(declarator).strip() + datatype = symbols.UnsupportedFortranType( + f"{common_attr.prefix} :: {entity}") + return datatype, initial_value + + def _declaration_interface( + self, name: str, common_attr: _CommonDeclAttributes + ): + '''Return the PSyIR interface for one declared entity. + + Interfaces describe where a symbol is defined and how a dummy argument + may be accessed. They are independent of its datatype. + + :param name: declared entity name. + :param common_attr: properties shared by the complete declaration. + + :returns: symbol interface, or ``None`` for an automatic local. + ''' + symtab = self._current_scope + if name in symtab and symtab.lookup(name).is_argument: + access = symbols.ArgumentInterface.Access.UNKNOWN + if common_attr.intent: + access = next( + (self._INTENT_ACCESS[child.type] + for child in common_attr.intent.children + if child.type in self._INTENT_ACCESS), + access) + return symbols.ArgumentInterface(access) + if {"save", "parameter"}.intersection(common_attr.qualifiers): + return symbols.StaticInterface() + if isinstance(symtab.node, nodes.Container): + return symbols.DefaultModuleInterface() + return None + + def _add_or_update_datasymbol( + self, declared_symbol: symbols.DataSymbol + ): + '''Add a declared symbol or complete a dummy-argument placeholder. + + Routine arguments are inserted before declarations are processed so + their source order is known. Their later declaration therefore updates + the placeholder, while ordinary local declarations create a new + DataSymbol. + + :param declared_symbol: fully translated symbol for the entity. + + :raises NotImplementedError: if an existing symbol is not a + DataSymbol. + ''' + symtab = self._current_scope + if declared_symbol.name in symtab: + symbol = symtab.lookup(declared_symbol.name) + if not isinstance(symbol, symbols.DataSymbol): + raise NotImplementedError( + f"'{declared_symbol.name}' is already declared as a " + f"non-data symbol") + symbol.datatype = declared_symbol.datatype + symbol.visibility = declared_symbol.visibility + if not isinstance( + declared_symbol.interface, symbols.AutomaticInterface): + symbol.interface = declared_symbol.interface + if declared_symbol.initial_value is not None: + symbol.initial_value = declared_symbol.initial_value + if declared_symbol.is_constant: + symbol.is_constant = True + return + symtab.add(declared_symbol) + + def _datatype_from_type( + self, tsnode: 'TSNode' + ): + '''Return a PSyIR datatype for a tree-sitter type specification. + + :param tsnode: intrinsic- or derived-type tree-sitter node. + + :returns: translated PSyIR datatype. + + :raises NotImplementedError: if PSyIR cannot represent the type. + ''' + symtab = self._current_scope + if tsnode.type == "derived_type": + keyword = tsnode.children[0].type + name_node = self._child(tsnode, "type_name") + name = to_str(name_node) + if keyword == "class": + raise NotImplementedError( + "Polymorphic CLASS declarations are not supported") + try: + return self._lookup(name) + except KeyError: + datatype = symbols.DataTypeSymbol( + name, symbols.UnresolvedType()) + symtab.add(datatype) + return datatype + + intrinsic = tsnode.children[0].type + mapping = { + "integer": symbols.ScalarType.Intrinsic.INTEGER, + "real": symbols.ScalarType.Intrinsic.REAL, + "logical": symbols.ScalarType.Intrinsic.BOOLEAN, + "character": symbols.ScalarType.Intrinsic.CHARACTER, } + if intrinsic not in mapping: + if to_str(tsnode).lower().replace(" ", "_") == "double_precision": + return symbols.ScalarType.real_double_type() + raise NotImplementedError( + f"Intrinsic type '{intrinsic}' has no PSyIR representation") + precision = symbols.ScalarType.Precision.UNDEFINED + length = None + kind_node = self._child(tsnode, "kind") + if kind_node: + values = [child for child in kind_node.children + if child.type not in ("(", ")")] + if values: + value = values[0] + if value.type == "keyword_argument": + key = to_str(value.children[0]).lower() + value = value.children[-1] + if key == "len": + length = self._expression(value) + else: + precision = self._precision(value) + elif intrinsic == "character": + length = self._expression(value) + else: + precision = self._precision(value) + return symbols.ScalarType(mapping[intrinsic], precision, length) + + def _precision(self, tsnode: 'TSNode'): + '''Translate a kind expression into a ScalarType precision value. + + :param tsnode: tree-sitter node containing the kind expression. + + :returns: precision enumeration, integer or PSyIR Reference. + + :raises NotImplementedError: if the kind expression is unsupported. + ''' + expr = self._expression(tsnode) + if isinstance(expr, nodes.Literal) and expr.value.isdigit(): + if expr.value == "4": + return symbols.ScalarType.Precision.SINGLE + if expr.value == "8": + return symbols.ScalarType.Precision.DOUBLE + return int(expr.value) + if isinstance(expr, nodes.Reference): + # A bare Symbol is a forward reference created before its role was + # known. Exact type checking is intentional: specialised Symbol + # subclasses must not be changed into a DataSymbol. + # pylint: disable=unidiomatic-typecheck + if type(expr.symbol) is symbols.Symbol: + expr.symbol.specialise( + symbols.DataSymbol, + datatype=symbols.ScalarType.integer_type()) + return expr + raise NotImplementedError("A kind must be a literal or named symbol") + + def _kind_symbol( + self, name: str + ) -> symbols.DataSymbol: + '''Look up or create a symbol used as a kind parameter. + + :param name: name of the kind parameter. + + :returns: DataSymbol representing the kind parameter. + + :raises NotImplementedError: if the name resolves to another symbol + type. + ''' + symtab = self._current_scope + try: + symbol = self._lookup(name) + except KeyError: + symbol = symbols.DataSymbol( + name, symbols.ScalarType.integer_type(), + interface=symbols.UnresolvedInterface()) + symtab.add(symbol) + if not isinstance(symbol, symbols.DataSymbol): + raise NotImplementedError( + f"Kind parameter '{name}' is not a data symbol") + return symbol + + def _shape_from_node( + self, tsnode: 'TSNode', is_allocatable: bool = False + ) -> list: + '''Translate a declaration size or argument list into an array + shape. - # Initialise none mandatory attibutes to their defaults - type_qualifier = None - unknown = None + :param tsnode: size or argument-list tree-sitter node. + :param is_allocatable: whether open bounds are deferred rather than + assumed shape. + :returns: PSyIR ArrayType shape entries. + ''' + result = [] for child in tsnode.children: - if child.type == "intrinsic_type": - intrinsic_type, kind = unpack2(child) - intrinsic_type = map_intrinsic_type[intrinsic_type.type] - if kind: - _left_parens, kind_expr, _right_parens = kind.children - km = self.optional_name_equals(kind_expr, symtab, ("kind", )) - precision = km['kind'] - # If it is 4 or 8 it has special values, but a precison expression - # is also supported - if isinstance(precision, nodes.Literal): - if precision.value == "4": - precision = symbols.ScalarType.Precision.SINGLE - elif precision.value == "8": - precision = symbols.ScalarType.Precision.DOUBLE + if child.type in ("(", ")", ","): + continue + if child.type == "extent_specifier": + before, after, has_colon = self._split_extent(child) + if not has_colon: + result.append(self._expression(before[0])) + continue + if not before and not after: + result.append(symbols.ArrayType.Extent.DEFERRED + if is_allocatable else + symbols.ArrayType.Extent.ATTRIBUTE) + elif before and not after: + result.append( + (self._expression(before[0]), + symbols.ArrayType.Extent.ATTRIBUTE)) + elif after and not before: + result.append(self._expression(after[0])) else: - precision = symbols.ScalarType.Precision.UNDEFINED - - elif child.type == "identifier": - identifier = to_str(child) - elif child.type == "::": - pass - elif child.type == "type_qualifier": - import pdb; pdb.set_trace() + result.append((self._expression(before[0]), + self._expression(after[0]))) else: - unknown = child + result.append(self._expression(child)) + return result + + def _process_access_statements( + self, tsnodes: Iterable['TSNode'] + ) -> dict[str, symbols.Symbol.Visibility]: + '''Record default and name-specific visibility for a scope. + + :param tsnodes: tree-sitter children of the current scope. + + :returns: explicit visibility indexed by normalised symbol name. + ''' + symtab = self._current_scope + visibility_map = {} + for tsnode in tsnodes: + if tsnode.type not in ("public_statement", "private_statement"): + continue + visibility = (symbols.Symbol.Visibility.PUBLIC + if tsnode.type == "public_statement" else + symbols.Symbol.Visibility.PRIVATE) + names = [to_str(child) for child in tsnode.children + if child.type in ("identifier", "name", "type_name", + "method_name")] + if names: + visibility_map.update({name.lower(): visibility + for name in names}) + else: + symtab.default_visibility = visibility + return visibility_map + + def _apply_visibility( + self, visibility_map: dict[str, symbols.Symbol.Visibility] + ): + '''Apply name-specific access rules after a scope is populated. + + Fortran access statements apply irrespective of source order. Delaying + this step until all declarations and contained routines have been + processed means the map remains local to its scope handler. + + :param visibility_map: explicit visibility indexed by symbol name. + ''' + symtab = self._current_scope + for name, visibility in visibility_map.items(): + # An unsupported specification may survive only as a CodeBlock and + # therefore have no symbol to update. + if name in symtab: + symtab.lookup( + name, scope_limit=symtab.node).visibility = visibility - if unknown: - # Add as a declaration comment - print(f"Unrecognised: {unknown.type}: {to_str(unknown)}") - import pdb; pdb.set_trace() - datatype = symbols.UnsupportedFortranType(to_str(tsnode)) + def _lookup(self, name: str): + '''Look up a symbol using the current PSyIR scope hierarchy. + + :param name: symbol name. + + :returns: symbol found in this or an enclosing scope. + + :raises KeyError: if no matching symbol exists. + ''' + return self._current_scope.lookup(name) + + # --------------------------------------------------------------------- + # Other specification-part symbols + # --------------------------------------------------------------------- + + def _use_statement_handler( + self, tsnode: 'TSNode' + ) -> None: + '''Translate a USE statement into container and imported symbols. + + :param tsnode: use-statement tree-sitter node. + + :raises NotImplementedError: if the module name conflicts with an + existing non-container symbol. + ''' + symtab = self._current_scope + module_node = self._child(tsnode, "module_name") + module_name = to_str(module_node) + intrinsic = any(child.type == "intrinsic" for child in tsnode.children) + included = self._child(tsnode, "included_items") + wildcard = included is None + try: + container = symtab.lookup(module_name) + except KeyError: + container = symbols.ContainerSymbol( + module_name, wildcard_import=wildcard, + is_intrinsic=intrinsic, + visibility=symtab.default_visibility) + symtab.add(container) + if not isinstance(container, symbols.ContainerSymbol): + raise NotImplementedError( + f"USE module '{module_name}' conflicts with another symbol") + container.wildcard_import = wildcard + + if included: + for child in included.children: + if child.type == "identifier": + local_name = to_str(child) + self._add_imported_symbol( + local_name, local_name, container) + elif child.type in ("rename", "use_rename", "use_alias"): + names = [item for item in child.children + if item.type in + ("identifier", "name", "local_name")] + if len(names) == 2: + self._add_imported_symbol( + to_str(names[0]), to_str(names[1]), + container) + + def _add_imported_symbol( + self, local_name: str, remote_name: str, + container: symbols.ContainerSymbol + ): + '''Add one symbol imported from a container. + + :param local_name: name used for the symbol in this scope. + :param remote_name: original name in the imported container. + :param container: symbol representing the imported module. + ''' + symtab = self._current_scope + interface = symbols.ImportInterface( + container, orig_name=(remote_name + if remote_name != local_name else None)) + try: + existing = symtab.lookup(local_name) + except KeyError: + symtab.add(symbols.Symbol( + local_name, visibility=symtab.default_visibility, + interface=interface)) + else: + if existing is not container: + existing.interface = interface + + def _derived_type_definition_handler( + self, tsnode: 'TSNode' + ) -> None: + '''Translate a simple Fortran derived-type definition. + + :param tsnode: derived-type-definition tree-sitter node. + + :raises NotImplementedError: if the type name conflicts with an + existing non-datatype symbol. + ''' + symtab = self._current_scope + statement = self._child(tsnode, "derived_type_statement") + name_node = self._child(statement, "type_name") if statement else None + name = to_str(name_node) + unsupported = any(child.type == "derived_type_procedures" + for child in tsnode.children) + datatype = None + if not unsupported: + component_table = symbols.SymbolTable() + parent = symtab.node + if not isinstance(parent, nodes.ScopingNode): + raise RuntimeError( + "A derived type must be translated within a PSyIR scope") + with self._using_temporary_scope(parent): + visibility_map = self._process_access_statements( + tsnode.children) + try: + for declaration in self._children( + tsnode, "variable_declaration"): + self._variable_declaration_handler(declaration) + self._apply_visibility(visibility_map) + datatype = symbols.StructureType() + for component in self._current_scope.datasymbols: + datatype.add( + component.name, component.datatype, + component.visibility, + component.initial_value) + except (NotImplementedError, TypeError, ValueError): + datatype = None + if datatype is None: + datatype = symbols.UnsupportedFortranType(to_str(tsnode).strip()) + + visibility = symtab.default_visibility + access = self._child(statement, "access_specifier") + if access: + visibility = (symbols.Symbol.Visibility.PRIVATE + if "private" in to_str(access).lower() else + symbols.Symbol.Visibility.PUBLIC) + try: + existing = symtab.lookup(name) + except KeyError: + symtab.add(symbols.DataTypeSymbol( + name, datatype, visibility=visibility)) else: - datatype = symbols.ScalarType(intrinsic_type, precision) - symbol = symbols.DataSymbol(identifier, datatype) - symtab.add(symbol) - - def optional_name_equals(self, tsnode, symtab, names): - result = {} - if tsnode.type == "keyword_argument": - identifier, _equals, expr = tsnode.children - string_id = to_str(identifier) - if string_id not in names: - raise NotImplementedError("Unexpected") - result[string_id] = self.process_nodes(expr, symtab)[0] + if not isinstance(existing, symbols.DataTypeSymbol): + raise NotImplementedError( + f"Derived type '{name}' conflicts with another symbol") + existing.datatype = datatype + existing.visibility = visibility + + def _interface_handler( + self, tsnode: 'TSNode' + ) -> None: + '''Translate a named interface containing procedure declarations. + + :param tsnode: interface tree-sitter node. + + :raises NotImplementedError: if the interface form or a member is + unsupported. + ''' + symtab = self._current_scope + statement = self._child(tsnode, "interface_statement") + name_node = (self._child(statement, "name") if statement else None) + if not name_node: + raise NotImplementedError( + "Abstract and operator interfaces are not supported") + name = to_str(name_node) + routines = [] + for procedure in self._children(tsnode, "procedure_statement"): + from_container = "module" in [ + child.type for child in procedure.children[0].children] + for method in self._children(procedure, "method_name"): + routine_name = to_str(method) + try: + routine = symtab.lookup(routine_name) + except KeyError: + routine = symbols.RoutineSymbol(routine_name) + symtab.add(routine) + if not isinstance(routine, symbols.RoutineSymbol): + raise NotImplementedError( + f"Interface member '{routine_name}' is not a routine") + routines.append((routine, from_container)) + if not routines: + raise NotImplementedError( + "Interfaces containing routine bodies are not supported") + symtab.add(symbols.GenericInterfaceSymbol( + name, routines, visibility=symtab.default_visibility)) + + # --------------------------------------------------------------------- + # Expressions and references + # --------------------------------------------------------------------- + + def _identifier_handler( + self, tsnode: 'TSNode' + ) -> nodes.Reference: + '''Translate an identifier into a symbol reference. + + :param tsnode: identifier tree-sitter node. + + :returns: PSyIR Reference to the resolved symbol. + ''' + symtab = self._current_scope + name = to_str(tsnode).lower() + try: + symbol = self._lookup(name) + except KeyError: + symbol = symbols.DataSymbol( + name, symbols.UnresolvedType(), + interface=symbols.UnresolvedInterface()) + symtab.add(symbol) + return nodes.Reference(symbol) + + def _parenthesized_expression_handler( + self, tsnode: 'TSNode' + ): + '''Discard parentheses while preserving the enclosed expression. + + :param tsnode: parenthesized-expression tree-sitter node. + + :returns: translated expression inside the parentheses. + + :raises NotImplementedError: if the parse-tree shape is unexpected. + ''' + content = [child for child in tsnode.children + if child.type not in ("(", ")")] + if len(content) != 1: + raise NotImplementedError( + "Unexpected parenthesized expression structure") + return self._expression(content[0]) + + def _unary_expression_handler( + self, tsnode: 'TSNode' + ): + '''Translate a unary arithmetic expression. + + :param tsnode: unary-expression tree-sitter node. + + :returns: PSyIR UnaryOperation. + ''' + return self._operation(tsnode) + + def _logical_expression_handler( + self, tsnode: 'TSNode' + ): + '''Translate a logical expression. + + :param tsnode: logical-expression tree-sitter node. + + :returns: PSyIR operation representing the expression. + ''' + return self._operation(tsnode) + + def _relational_expression_handler( + self, tsnode: 'TSNode' + ): + '''Translate a relational expression. + + :param tsnode: relational-expression tree-sitter node. + + :returns: PSyIR BinaryOperation. + ''' + return self._operation(tsnode) + + def _math_expression_handler( + self, tsnode: 'TSNode' + ): + '''Translate an arithmetic expression. + + :param tsnode: math-expression tree-sitter node. + + :returns: PSyIR BinaryOperation. + ''' + return self._operation(tsnode) + + def _operation( + self, tsnode: 'TSNode' + ): + '''Translate a unary or binary operation node. + + :param tsnode: operation tree-sitter node. + + :returns: PSyIR UnaryOperation or BinaryOperation. + + :raises NotImplementedError: if the operator or tree shape is + unsupported. + ''' + if len(tsnode.children) == 2: + operator = to_str(tsnode.children[0]).lower() + if operator not in self._UNARY_OPERATORS: + raise NotImplementedError( + f"Unsupported unary operator '{operator}'") + return nodes.UnaryOperation.create( + self._UNARY_OPERATORS[operator], + self._expression(tsnode.children[1])) + if len(tsnode.children) == 3: + operator = to_str(tsnode.children[1]).lower() + if operator not in self._BINARY_OPERATORS: + raise NotImplementedError( + f"Unsupported binary operator '{operator}'") + return nodes.BinaryOperation.create( + self._BINARY_OPERATORS[operator], + self._expression(tsnode.children[0]), + self._expression(tsnode.children[2])) + raise NotImplementedError("Unexpected operation structure") + + def _call_expression_handler( + self, tsnode: 'TSNode' + ): + '''Translate an array reference, intrinsic, or function call. + + Tree-sitter uses the same syntactic node for all three forms. The + symbol datatype resolves the ambiguity: a declared array produces an + ArrayReference, a recognised intrinsic produces an IntrinsicCall, and + other names produce a Call. + + :param tsnode: call-expression tree-sitter node. + + :returns: PSyIR ArrayReference, IntrinsicCall, StructureReference or + Call. + + :raises NotImplementedError: if the expression cannot be classified + or its argument form is unsupported. + ''' + symtab = self._current_scope + name_node = tsnode.children[0] + if name_node.type == "derived_type_member_expression": + return self._structure_reference( + name_node, + trailing_arguments=self._child(tsnode, "argument_list")) + name = to_str(name_node).lower() + argument_list = self._child(tsnode, "argument_list") + try: + symbol = self._lookup(name) + except KeyError: + symbol = None + + if isinstance(symbol, symbols.DataSymbol) and isinstance( + symbol.datatype, symbols.ArrayType): + indices = self._arguments( + argument_list, array_symbol=symbol) + if any(isinstance(arg, tuple) for arg in indices): + raise NotImplementedError( + "Named subscripts are not supported") + return nodes.ArrayReference.create(symbol, indices) + + arguments = self._arguments(argument_list) + intrinsic = next( + (item for item in nodes.IntrinsicCall.Intrinsic + if item.name.lower() == name), None) + if intrinsic: + try: + return nodes.IntrinsicCall.create(intrinsic, arguments) + except (TypeError, ValueError): + # Preserve parsed intrinsics even if the current PSyIR + # signature validation is stricter than the grammar. + raise NotImplementedError( + f"Unsupported argument form for intrinsic '{name}'" + ) from None + + if not isinstance(symbol, symbols.RoutineSymbol): + if symbol is not None and not isinstance( + symbol, symbols.DataTypeSymbol): + raise NotImplementedError( + f"'{name}(...)' cannot be classified as an array or call") + symbol = symbols.RoutineSymbol(name) + if name not in symtab: + symtab.add(symbol) + return nodes.Call.create(symbol, arguments) + + def _arguments( + self, tsnode: Optional['TSNode'], + array_symbol: Optional[symbols.DataSymbol] = None + ) -> list: + '''Translate an argument or array-subscript list. + + :param tsnode: argument-list tree-sitter node, or ``None``. + :param array_symbol: array being indexed when ranges are permitted. + + :returns: positional expressions and ``(name, expression)`` tuples. + + :raises NotImplementedError: if a range occurs outside an array + reference. + ''' + if tsnode is None: + return [] + result = [] + dimension = 0 + for child in tsnode.children: + if child.type in ("(", ")", ","): + continue + dimension += 1 + if child.type == "keyword_argument": + key = to_str(child.children[0]) + result.append((key, self._expression(child.children[-1]))) + elif child.type == "extent_specifier": + if array_symbol is None: + raise NotImplementedError( + "Ranges are only supported in array references") + result.append(self._range( + child, array_symbol, dimension)) + else: + result.append(self._expression(child)) + return result + + def _range( + self, tsnode: 'TSNode', symbol: symbols.DataSymbol, dimension: int + ) -> nodes.Range: + '''Translate an array-section triplet. + + Omitted bounds are made explicit with LBOUND and UBOUND calls because + a PSyIR Range always has start, stop and step children. + + :param tsnode: extent-specifier tree-sitter node. + :param symbol: DataSymbol for the indexed array. + :param dimension: one-based array dimension being indexed. + + :returns: PSyIR Range with explicit bounds. + + :raises NotImplementedError: if the range is malformed. + ''' + before, after, has_colon = self._split_extent(tsnode) + if not has_colon: + raise NotImplementedError("Malformed array range") + # A section can have a second colon before its step. + after = [child for child in after if child.type != ":"] + dim = nodes.Literal(str(dimension), + symbols.ScalarType.integer_type()) + start = (self._expression(before[0]) if before else + nodes.IntrinsicCall.create( + nodes.IntrinsicCall.Intrinsic.LBOUND, + [nodes.Reference(symbol), ("dim", dim.copy())])) + stop = (self._expression(after[0]) if after else + nodes.IntrinsicCall.create( + nodes.IntrinsicCall.Intrinsic.UBOUND, + [nodes.Reference(symbol), ("dim", dim.copy())])) + step = (self._expression(after[1]) + if len(after) > 1 else None) + return nodes.Range.create(start, stop, step) + + def _derived_type_member_expression_handler( + self, tsnode: 'TSNode' + ): + '''Translate a structure-component reference. + + :param tsnode: derived-type-member-expression tree-sitter node. + + :returns: PSyIR structure reference. + ''' + return self._structure_reference(tsnode) + + def _structure_reference( + self, tsnode: 'TSNode', + trailing_arguments: Optional['TSNode'] = None + ): + '''Translate nested and indexed structure-component references. + + :param tsnode: tree-sitter node describing the structure access. + :param trailing_arguments: optional indices attached outside + ``tsnode`` by a wrapping call-expression node. + + :returns: StructureReference or ArrayOfStructuresReference. + + :raises NotImplementedError: if the access or base symbol is + unsupported. + ''' + symtab = self._current_scope + name, indices, members = self._decompose_structure(tsnode) + if trailing_arguments: + arguments = self._arguments(trailing_arguments) + if not members or any(isinstance(arg, tuple) + for arg in arguments): + raise NotImplementedError( + "Unsupported structure member array access") + members[-1] = (members[-1], arguments) + try: + symbol = self._lookup(name) + except KeyError: + symbol = symbols.DataSymbol( + name, symbols.UnresolvedType(), + interface=symbols.UnresolvedInterface()) + symtab.add(symbol) + if not isinstance(symbol, symbols.DataSymbol): + raise NotImplementedError( + "A structure base must be a data symbol") + if indices: + return nodes.ArrayOfStructuresReference.create( + symbol, indices, members) + return nodes.StructureReference.create(symbol, members) + + def _decompose_structure( + self, tsnode: 'TSNode' + ) -> tuple[str, list, list]: + '''Return base name, base indices and member descriptors. + + Tree-sitter nests call-expression and member-expression nodes for + accesses such as ``items(i)%vector(2)%x``. Flattening that syntax into + the descriptor format expected by PSyIR keeps node construction out of + this recursive routine. + + :param tsnode: identifier, call or member-expression tree-sitter node. + + :returns: base name, base indices and PSyIR member descriptors. + + :raises NotImplementedError: if the access shape is unsupported. + ''' + if tsnode.type == "identifier": + return to_str(tsnode).lower(), [], [] + if tsnode.type == "call_expression": + base = tsnode.children[0] + name, indices, members = self._decompose_structure(base) + arguments = self._arguments( + self._child(tsnode, "argument_list")) + if any(isinstance(arg, tuple) for arg in arguments): + raise NotImplementedError( + "Named arguments in structure accesses are not supported") + if members: + members[-1] = (members[-1], arguments) + else: + indices = arguments + return name, indices, members + if tsnode.type == "derived_type_member_expression": + name, indices, members = self._decompose_structure( + tsnode.children[0]) + member = self._child(tsnode, "type_member") + if member is None: + raise NotImplementedError( + "Malformed structure component access") + members.append(to_str(member).lower()) + return name, indices, members + raise NotImplementedError( + f"Unsupported structure access base '{tsnode.type}'") + + def _array_literal_handler( + self, tsnode: 'TSNode' + ): + '''Translate a simple array constructor. + + :param tsnode: array-literal tree-sitter node. + + :returns: PSyIR ArrayConstructor. + + :raises NotImplementedError: for an implied-DO constructor. + ''' + if self._child(tsnode, "implied_do_loop_expression"): + raise NotImplementedError( + "Array constructors with implied-DO loops are not supported") + elems = [self._expression(child) for child in tsnode.children + if child.type not in ("[", "]", "(/", "/)", ",")] + return nodes.ArrayConstructor.create(elems) + + def _expression(self, tsnode: 'TSNode'): + '''Translate one expression, allowing failures to reach its + statement. + + :param tsnode: expression tree-sitter node. + + :returns: translated PSyIR DataNode. + ''' + return self.get_handler(tsnode)(tsnode) + + def _comment_handler( + self, tsnode: 'TSNode' + ) -> None: + '''Ignore comments when requested. + + Comment attachment will be added when the reader options cease to be + compatibility-only. Until then, comments must not turn otherwise + supported source into CodeBlocks. + + :param tsnode: comment tree-sitter node. + + :raises NotImplementedError: if comment preservation was requested. + ''' + del tsnode + if self._ignore_comments: + return None + raise NotImplementedError("Comment preservation is not yet supported") + + # --------------------------------------------------------------------- + # Executable statements and control flow + # --------------------------------------------------------------------- + + def _assignment_statement_handler( + self, tsnode: 'TSNode' + ) -> nodes.Assignment: + '''Translate an intrinsic assignment. + + :param tsnode: assignment-statement tree-sitter node. + + :returns: PSyIR Assignment. + + :raises NotImplementedError: if the tree shape is unexpected. + ''' + if len(tsnode.children) != 3: + raise NotImplementedError("Unexpected assignment structure") + return nodes.Assignment.create( + self._expression(tsnode.children[0]), + self._expression(tsnode.children[2])) + + def _pointer_association_statement_handler( + self, tsnode: 'TSNode' + ) -> nodes.Assignment: + '''Translate a simple pointer assignment. + + :param tsnode: pointer-association-statement tree-sitter node. + + :returns: pointer-annotated PSyIR Assignment. + + :raises NotImplementedError: for bounds remapping. + ''' + if len(tsnode.children) != 3: + raise NotImplementedError( + "Pointer assignment with bounds remapping is not supported") + assignment = nodes.Assignment(is_pointer=True) + assignment.children = [ + self._expression(tsnode.children[0]), + self._expression(tsnode.children[2])] + return assignment + + def _subroutine_call_handler( + self, tsnode: 'TSNode' + ): + '''Translate a CALL statement. + + :param tsnode: subroutine-call tree-sitter node. + + :returns: PSyIR Call. + + :raises NotImplementedError: if the called object is unsupported. + ''' + name_node = next((child for child in tsnode.children + if child.type == "identifier"), None) + if not name_node: + raise NotImplementedError( + "Calls through type-bound procedures are not supported") + symtab = self._current_scope + name = to_str(name_node).lower() + try: + symbol = self._lookup(name) + except KeyError: + symbol = symbols.RoutineSymbol(name, datatype=symbols.NoType()) + symtab.add(symbol) + # As above, only a bare forward-reference Symbol may be specialised. + # pylint: disable=unidiomatic-typecheck + if type(symbol) is symbols.Symbol: + symbol.specialise(symbols.RoutineSymbol, + datatype=symbols.NoType()) + if not isinstance(symbol, symbols.RoutineSymbol): + raise NotImplementedError( + f"Called object '{name}' is not a routine") + args = self._arguments(self._child(tsnode, "argument_list")) + return nodes.Call.create(symbol, args) + + def _keyword_statement_handler( + self, tsnode: 'TSNode' + ): + '''Translate a no-argument keyword statement. + + :param tsnode: keyword-statement tree-sitter node. + + :returns: PSyIR Return for a RETURN statement. + + :raises NotImplementedError: if the keyword has no PSyIR node. + ''' + keyword = tsnode.children[0].type + if keyword == "return": + return nodes.Return() + raise NotImplementedError( + f"Fortran '{keyword.upper()}' has no PSyIR node") + + def _if_statement_handler( + self, tsnode: 'TSNode' + ) -> nodes.IfBlock: + '''Translate block and single-line IF statements. + + :param tsnode: if-statement tree-sitter node. + + :returns: root PSyIR IfBlock. + + :raises NotImplementedError: if the statement has no condition. + ''' + condition_node = self._child(tsnode, "parenthesized_expression") + if not condition_node: + raise NotImplementedError("IF statement has no condition") + structural = { + "if", "parenthesized_expression", "then", + "end_if_statement", "else_clause", "elseif_clause" + } + body_nodes = [child for child in tsnode.children + if child.type not in structural] + else_clause = self._child(tsnode, "else_clause") + else_ifs = self._children(tsnode, "elseif_clause") + annotations = [] + if not self._child(tsnode, "end_if_statement"): + annotations.append("was_single_stmt") + if_body = self.process_nodes(body_nodes) + else_body = None + if else_clause: + else_body = self.process_nodes( + [child for child in else_clause.children + if child.type != "else"]) + for else_if in reversed(else_ifs): + else_body = [self._if_clause(else_if, else_body)] + result = nodes.IfBlock.create( + self._expression(condition_node), if_body, else_body) + result.annotations.extend(annotations) + return result + + def _if_clause( + self, tsnode: 'TSNode', + final_else: Optional[list[nodes.Node]] = None + ) -> nodes.IfBlock: + '''Translate an ELSE IF clause recursively. + + :param tsnode: elseif-clause tree-sitter node. + :param final_else: PSyIR statements for a following ELSE clause. + + :returns: annotated PSyIR IfBlock. + ''' + condition = self._child(tsnode, "parenthesized_expression") + structural = {"else", "if", "parenthesized_expression", "then", + "else_clause", "elseif_clause"} + body = self.process_nodes( + [child for child in tsnode.children + if child.type not in structural]) + trailing = self._child(tsnode, "elseif_clause") + otherwise = ( + [self._if_clause(trailing, final_else)] if trailing else + self.process_nodes( + [child for child in + (self._child(tsnode, "else_clause").children + if self._child(tsnode, "else_clause") else []) + if child.type != "else"]) or final_else) + result = nodes.IfBlock.create( + self._expression(condition), body, otherwise) + result.annotations.append("was_elseif") + return result + + def _do_loop_handler( + self, tsnode: 'TSNode' + ): + '''Translate counted, conditional and unconditional DO loops. + + :param tsnode: do-loop tree-sitter node. + + :returns: PSyIR Loop for counted DO, otherwise WhileLoop. + + :raises NotImplementedError: if counted-loop control is unsupported. + ''' + statement = self._child(tsnode, "do_statement") + control = self._child(statement, "loop_control_expression") + while_node = self._child(statement, "while_statement") + body = self.process_nodes( + [child for child in tsnode.children + if child.type not in ("do_statement", + "end_do_loop_statement")]) + if control: + parts = [child for child in control.children + if child.type not in ("=", ",")] + if len(parts) not in (3, 4): + raise NotImplementedError( + "Unsupported counted DO loop control") + variable_ref = self._identifier_handler(parts[0]) + variable = variable_ref.symbol + if not isinstance(variable, symbols.DataSymbol): + raise NotImplementedError( + "A DO variable must be a data symbol") + if not isinstance(variable.datatype, symbols.ScalarType): + if isinstance(variable.datatype, symbols.UnresolvedType): + variable.datatype = symbols.ScalarType.integer_type() + else: + raise NotImplementedError( + "A DO variable must be a scalar integer") + step = (self._expression(parts[3]) if len(parts) == 4 + else nodes.Literal( + "1", symbols.ScalarType.integer_type())) + return nodes.Loop.create( + variable, self._expression(parts[1]), + self._expression(parts[2]), step, body) + if while_node: + condition = self._child( + while_node, "parenthesized_expression") + return nodes.WhileLoop.create( + self._expression(condition), body) + result = nodes.WhileLoop.create( + nodes.Literal("true", symbols.ScalarType.boolean_type()), body) + result.annotations.append("was_unconditional") + return result + + def _where_statement_handler( + self, tsnode: 'TSNode' + ) -> nodes.IfBlock: + '''Translate block and single-statement WHERE constructs. + + :param tsnode: where-statement tree-sitter node. + + :returns: annotated PSyIR IfBlock. + ''' + condition = self._child(tsnode, "parenthesized_expression") + structural = {"where", "parenthesized_expression", + "elsewhere_clause", "end_where_statement"} + body = self.process_nodes( + [child for child in tsnode.children + if child.type not in structural]) + elsewhere = self._child(tsnode, "elsewhere_clause") + other = ( + self.process_nodes( + [child for child in elsewhere.children + if child.type != "elsewhere"]) + if elsewhere else None) + result = nodes.IfBlock.create( + self._expression(condition), body, other) + result.annotations.extend( + ["was_where"] if self._child( + tsnode, "end_where_statement") else + ["was_where", "was_single_stmt"]) + return result + + def _select_case_statement_handler( + self, tsnode: 'TSNode' + ): + '''Translate SELECT CASE into a nested annotated IF tree. + + Each non-default CASE becomes an IfBlock in the preceding block's + else-body. CASE DEFAULT becomes the final else-body. + + :param tsnode: select-case-statement tree-sitter node. + + :returns: root annotated PSyIR IfBlock. + + :raises NotImplementedError: if no conditional CASE can be produced. + ''' + selector_node = self._child( + self._child(tsnode, "selector"), "identifier") + if selector_node is None: + selector = self._expression( + [child for child in + self._child(tsnode, "selector").children + if child.type not in ("(", ")")][0]) else: - string_id = names[0] - result[string_id] = self.process_nodes(tsnode, symtab)[0] + selector = self._expression(selector_node) + cases = self._children(tsnode, "case_statement") + default_body = None + normal = [] + for case in cases: + if self._child(case, "default"): + default_body = self.process_nodes( + [child for child in case.children + if child.type not in ("case", "default")]) + else: + values = self._child(case, "case_value_range_list") + if values is None: + raise NotImplementedError( + "Malformed CASE value list") + structural = {"case", "(", ")", "case_value_range_list"} + body = self.process_nodes( + [child for child in case.children + if child.type not in structural]) + normal.append((values, body)) + current = default_body + for values, body in reversed(normal): + condition = self._case_condition(selector, values) + block = nodes.IfBlock.create( + condition, body, current) + block.annotations.append("was_case") + current = [block] + if current and len(current) == 1: + return current[0] + raise NotImplementedError( + "SELECT CASE with only a default clause has no PSyIR equivalent") + + def _case_condition( + self, selector, values: 'TSNode' + ): + '''Build a condition for one CASE value/range list. + + :param selector: translated SELECT CASE selector expression. + :param values: case-value-range-list tree-sitter node. + + :returns: PSyIR condition combining values with OR and range bounds + with AND. + ''' + conditions = [] + for child in values.children: + if child.type == ",": + continue + if child.type == "extent_specifier": + before, after, _ = self._split_extent(child) + parts = [] + if before: + parts.append(nodes.BinaryOperation.create( + nodes.BinaryOperation.Operator.GE, selector.copy(), + self._expression(before[0]))) + if after: + parts.append(nodes.BinaryOperation.create( + nodes.BinaryOperation.Operator.LE, selector.copy(), + self._expression(after[0]))) + condition = parts[0] if len(parts) == 1 else ( + nodes.BinaryOperation.create( + nodes.BinaryOperation.Operator.AND, + parts[0], parts[1])) + else: + condition = nodes.BinaryOperation.create( + nodes.BinaryOperation.Operator.EQ, selector.copy(), + self._expression(child)) + conditions.append(condition) + result = conditions[0] + for condition in conditions[1:]: + result = nodes.BinaryOperation.create( + nodes.BinaryOperation.Operator.OR, result, condition) return result + def _allocate_statement_handler( + self, tsnode: 'TSNode' + ): + '''Translate ALLOCATE using PSyIR's special intrinsic. + + :param tsnode: allocate-statement tree-sitter node. + + :returns: PSyIR ALLOCATE IntrinsicCall. + ''' + return self._memory_statement( + tsnode, nodes.IntrinsicCall.Intrinsic.ALLOCATE) + + def _deallocate_statement_handler( + self, tsnode: 'TSNode' + ): + '''Translate DEALLOCATE using PSyIR's special intrinsic. + + :param tsnode: deallocate-statement tree-sitter node. + + :returns: PSyIR DEALLOCATE IntrinsicCall. + ''' + return self._memory_statement( + tsnode, nodes.IntrinsicCall.Intrinsic.DEALLOCATE) -def unpack2(child): - if len(child.children) == 1: - return child.children[0], None - if len(child.children) == 2: - return child.children[0], child.children[1] - raise NotImplementedError("Unexpected") + def _nullify_statement_handler( + self, tsnode: 'TSNode' + ): + '''Translate NULLIFY using PSyIR's special intrinsic. + + :param tsnode: nullify-statement tree-sitter node. + + :returns: PSyIR NULLIFY IntrinsicCall. + ''' + return self._memory_statement( + tsnode, nodes.IntrinsicCall.Intrinsic.NULLIFY) + + def _memory_statement( + self, tsnode: 'TSNode', intrinsic + ): + '''Translate operands of allocation-related statements. + + ALLOCATE shape specifications become ArrayReference indices whose + Range children retain the requested lower and upper bounds. + + :param tsnode: allocation-related statement tree-sitter node. + :param intrinsic: ALLOCATE, DEALLOCATE or NULLIFY intrinsic. + + :returns: PSyIR IntrinsicCall. + + :raises NotImplementedError: if an object, bound or option is + unsupported. + ''' + args = [] + for child in tsnode.children: + if child.type in ( + "allocate", "deallocate", "nullify", "(", ")", ","): + continue + if child.type == "keyword_argument": + args.append((to_str(child.children[0]), + self._expression(child.children[-1]))) + elif child.type == "sized_allocation": + args.append(self._allocation_reference(child)) + else: + args.append(self._expression(child)) + try: + return nodes.IntrinsicCall.create(intrinsic, args) + except (TypeError, ValueError): + raise NotImplementedError( + f"Unsupported operands for {intrinsic.name}") from None + + def _allocation_reference( + self, tsnode: 'TSNode' + ) -> nodes.ArrayReference: + '''Translate an ALLOCATE object with explicit shape bounds. + + PSyIR represents the requested allocation shape as ArrayReference + Range indices. This is an allocation request rather than an ordinary + array access, but the representation permits the backend to reproduce + the original bounds. + + :param tsnode: sized-allocation tree-sitter node. + + :returns: ArrayReference containing one Range per allocated dimension. + + :raises NotImplementedError: if the object is not a data symbol. + ''' + ident = self._child(tsnode, "identifier") + reference = self._identifier_handler(ident) + if not isinstance(reference.symbol, symbols.DataSymbol): + raise NotImplementedError( + "An ALLOCATE object must be a data symbol") + size = self._child(tsnode, "size") + indices = [ + self._allocation_extent(extent) + for extent in size.children + if extent.type not in ("(", ")", ",")] + return nodes.ArrayReference.create(reference.symbol, indices) + + def _allocation_extent( + self, tsnode: 'TSNode' + ) -> nodes.Range: + '''Translate one requested allocation extent. + + Fortran's ``allocate(a(n))`` is equivalent to ``allocate(a(1:n))``; + the implicit lower bound is therefore made explicit in PSyIR. + + :param tsnode: extent or extent-specifier tree-sitter node. + + :returns: Range containing explicit lower and upper bounds. + + :raises NotImplementedError: if the allocation bound is malformed. + ''' + lower = nodes.Literal( + "1", symbols.ScalarType.integer_type()) + if tsnode.type != "extent_specifier": + return nodes.Range.create( + lower, self._expression(tsnode)) + + before, after, has_colon = self._split_extent(tsnode) + if not has_colon: + raise NotImplementedError("Malformed allocation bound") + if before: + lower = self._expression(before[0]) + if not after: + raise NotImplementedError( + "Allocation upper bound is required") + return nodes.Range.create( + lower, self._expression(after[0])) diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index 830dfa95cb..d41f8b408a 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -141,7 +141,49 @@ def test_generate_psyir(): assert isinstance(root, psyir_nodes.FileContainer) assert isinstance(root.children[0], psyir_nodes.Container) - assert isinstance(root.children[0].children[0], psyir_nodes.CodeBlock) + assert isinstance(root.children[0].children[0], psyir_nodes.Routine) + assert root.children[0].children[0].name == "mysub" + + +@min_version_3_10 +def test_current_scope_restored(): + '''Test that nested parsing does not leave mutable scope state behind.''' + processor = FortranTreeSitterReader() + valid_code = """ + module outer + implicit none + contains + subroutine inner() + end subroutine inner + end module outer + """ + processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + assert processor._current_scope is None + + +@min_version_3_10 +def test_routine_host_association(): + '''Test that a contained routine resolves a symbol from its host.''' + processor = FortranTreeSitterReader() + valid_code = """ + module host + implicit none + integer :: count + contains + subroutine work() + count = count + 1 + end subroutine work + end module host + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + container = root.children[0] + routine = container.children[0] + count = container.symbol_table.lookup("count") + assert "count" not in routine.symbol_table + assert [ref.symbol for ref in routine.walk(psyir_nodes.Reference)] == [ + count, count] # TODO #3416: Skip treesitter tests below 3.10 as they're unsupported by @@ -242,7 +284,6 @@ def test_declarations(): ("real(kind=8)", psyir_symbols.ScalarType.real_double_type()), ("logical", psyir_symbols.ScalarType.boolean_type()), ("character", psyir_symbols.ScalarType.character_type()), - ("integer, dimension(:)", psyir_symbols.ScalarType.integer_type()), ]) def test_declarations_datatypes(fortran_type, psyir_type): ''' @@ -260,12 +301,15 @@ def test_declarations_datatypes(fortran_type, psyir_type): root = processor.generate_psyir(ptree) module = root.children[0] assert module.symbol_table.lookup("a").datatype == psyir_type, ( - f"{module.symbol_table.lookup("a").datatype} != {psyir_type}" + f"{module.symbol_table.lookup('a').datatype} != {psyir_type}" ) -@pytest.mark.parametrize("shape_string, psyir_shape", [ - ("(:)", psyir_nodes.Literal("10", psyir_symbols.ScalarType.integer_type())), + + +@pytest.mark.parametrize("shape_string, extent", [ + ("(:)", psyir_symbols.ArrayType.Extent.ATTRIBUTE), + ("(10)", "10"), ]) -def test_declarations_arrays_datatypes(shape_string, psyir_shape): +def test_declarations_arrays_datatypes(shape_string, extent): ''' Test subroutine nodes. ''' @@ -283,8 +327,851 @@ def test_declarations_arrays_datatypes(shape_string, psyir_shape): array_symbol = module.symbol_table.lookup("a") assert isinstance(array_symbol.datatype, psyir_symbols.ArrayType) - assert isinstance(array_symbol.elemental_type, psyir_symbols.ScalarType.integer4_type()) + assert (array_symbol.datatype.elemental_type == + psyir_symbols.ScalarType.integer_single_type()) + shape = array_symbol.datatype.shape[0] + if isinstance(extent, str): + assert shape.upper.value == extent + assert shape.lower.value == "1" + else: + assert shape == extent - assert module.symbol_table.lookup("a").datatype == psyir_type, ( - f"{module.symbol_table.lookup("a").datatype} != {psyir_type}" - ) + +@min_version_3_10 +def test_program(): + '''Test a main-program unit.''' + processor = FortranTreeSitterReader() + valid_code = """ + program main + implicit none + end program main + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + assert isinstance(routine, psyir_nodes.Routine) + assert routine.is_program + assert routine.name == "main" + + +@min_version_3_10 +def test_use_rename(): + '''Test a renamed symbol in a USE ONLY statement.''' + processor = FortranTreeSitterReader() + valid_code = """ + program main + use kinds, only: local_kind => remote_kind + implicit none + end program main + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + container = table.lookup("kinds") + imported = table.lookup("local_kind") + assert isinstance(container, psyir_symbols.ContainerSymbol) + assert imported.interface.container_symbol is container + assert imported.interface.orig_name == "remote_kind" + + +@min_version_3_10 +def test_symbolic_kind(): + '''Test a symbolic kind expression.''' + processor = FortranTreeSitterReader() + valid_code = """ + program main + use kinds, only: local_kind + implicit none + integer(local_kind) :: value + end program main + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + kind = table.lookup("local_kind") + assert isinstance(kind, psyir_symbols.DataSymbol) + assert table.lookup("value").datatype.precision.symbol is kind + + +@min_version_3_10 +def test_parameter_declaration(): + '''Test a named constant declaration.''' + processor = FortranTreeSitterReader() + valid_code = """ + program main + implicit none + integer, parameter :: count = 4 + end program main + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + count = root.children[0].symbol_table.lookup("count") + assert count.is_constant + assert count.initial_value.value == "4" + + +@min_version_3_10 +def test_character_length(): + '''Test a character-length specification.''' + processor = FortranTreeSitterReader() + valid_code = """ + program main + implicit none + character(len=12) :: label + end program main + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + label = root.children[0].symbol_table.lookup("label") + assert label.datatype.length.value == "12" + + +@min_version_3_10 +def test_allocatable_declaration(): + '''Test an allocatable-array declaration.''' + processor = FortranTreeSitterReader() + valid_code = """ + program main + implicit none + real, allocatable :: values(:) + end program main + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + values = root.children[0].symbol_table.lookup("values") + assert values.datatype.shape == [ + psyir_symbols.ArrayType.Extent.DEFERRED] + + +@min_version_3_10 +def test_function_result(): + '''Test a named function-result symbol.''' + processor = FortranTreeSitterReader() + valid_code = """ + real function square(value) result(answer) + real :: value + answer = value * value + end function square + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + assert routine.return_symbol.name == "answer" + assert routine.return_symbol.datatype == \ + psyir_symbols.ScalarType.real_type() + + +@min_version_3_10 +def test_argument_order(): + '''Test the order of routine arguments.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine update(first, second) + real :: first, second + end subroutine update + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + assert [symbol.name for symbol in table.argument_list] == [ + "first", "second"] + + +@min_version_3_10 +@pytest.mark.parametrize("intent,access", [ + ("in", psyir_symbols.ArgumentInterface.Access.READ), + ("out", psyir_symbols.ArgumentInterface.Access.WRITE), + ("inout", psyir_symbols.ArgumentInterface.Access.READWRITE), +]) +def test_argument_intent(intent, access): + '''Test an INTENT attribute on a routine argument.''' + processor = FortranTreeSitterReader() + valid_code = f""" + subroutine update(value) + real, intent({intent}) :: value + end subroutine update + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + value = root.children[0].symbol_table.lookup("value") + assert value.interface.access == access + + +@min_version_3_10 +def test_pure_function(): + '''Test the PURE function qualifier.''' + processor = FortranTreeSitterReader() + valid_code = """ + pure real function identity(value) + real :: value + identity = value + end function identity + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + assert root.children[0].symbol.is_pure is True + + +@min_version_3_10 +def test_elemental_function(): + '''Test the ELEMENTAL function qualifier.''' + processor = FortranTreeSitterReader() + valid_code = """ + elemental real function identity(value) + real :: value + identity = value + end function identity + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + assert root.children[0].symbol.is_elemental is True + + +@min_version_3_10 +def test_unsupported_complex_datatype(): + '''Test the unsupported complex datatype.''' + processor = FortranTreeSitterReader() + valid_code = """ + module declarations + implicit none + complex :: coefficient + end module declarations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + coefficient = root.children[0].symbol_table.lookup("coefficient") + assert isinstance(coefficient.datatype, + psyir_symbols.UnsupportedFortranType) + assert coefficient.datatype.declaration == "complex :: coefficient" + + +@min_version_3_10 +def test_unsupported_pointer_datatype(): + '''Test entity-specific unsupported pointer datatypes.''' + processor = FortranTreeSitterReader() + valid_code = """ + module declarations + implicit none + integer, pointer :: first, second + end module declarations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + for name in ("first", "second"): + datatype = table.lookup(name).datatype + assert isinstance(datatype, psyir_symbols.UnsupportedFortranType) + assert datatype.declaration == f"integer, pointer :: {name}" + + +@min_version_3_10 +def test_unsupported_initialisation_is_entity_specific(): + '''Test that one unsupported initializer does not affect its sibling.''' + processor = FortranTreeSitterReader() + valid_code = """ + module declarations + implicit none + integer :: first = [(i, i=1,2)], second = 2 + end module declarations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + first = table.lookup("first") + second = table.lookup("second") + assert isinstance( + first.datatype, psyir_symbols.UnsupportedFortranType) + assert first.datatype.declaration == \ + "integer :: first = [(i, i=1,2)]" + assert isinstance(second.datatype, psyir_symbols.ScalarType) + assert second.initial_value.value == "2" + + +@min_version_3_10 +def test_save_attribute(): + '''Test the SAVE declaration attribute.''' + processor = FortranTreeSitterReader() + valid_code = """ + module declarations + implicit none + double precision, save :: accumulator + end module declarations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + accumulator = root.children[0].symbol_table.lookup("accumulator") + assert accumulator.datatype == \ + psyir_symbols.ScalarType.real_double_type() + assert isinstance(accumulator.interface, psyir_symbols.StaticInterface) + + +@min_version_3_10 +def test_logical_literal(): + '''Test a logical literal used as an initial value.''' + processor = FortranTreeSitterReader() + valid_code = """ + module declarations + implicit none + logical :: enabled = .true. + end module declarations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + enabled = root.children[0].symbol_table.lookup("enabled") + assert enabled.initial_value.value == "true" + + +@min_version_3_10 +def test_derived_type_definition(): + '''Test a simple derived-type definition.''' + processor = FortranTreeSitterReader() + valid_code = """ + module geometry + implicit none + type :: point + real :: x + end type point + end module geometry + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + point = root.children[0].symbol_table.lookup("point") + assert isinstance(point, psyir_symbols.DataTypeSymbol) + assert isinstance(point.datatype, psyir_symbols.StructureType) + assert list(point.datatype.components) == ["x"] + assert point.datatype.components["x"].datatype == \ + psyir_symbols.ScalarType.real_type() + + +@min_version_3_10 +def test_derived_type_component_host_association(): + '''Test that a component datatype resolves a kind from its host.''' + processor = FortranTreeSitterReader() + valid_code = """ + module geometry + implicit none + integer, parameter :: wp = 8 + type :: point + real(kind=wp) :: x + end type point + end module geometry + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + wp = table.lookup("wp") + point = table.lookup("point") + x_type = point.datatype.components["x"].datatype + assert x_type.precision.symbol is wp + + +@min_version_3_10 +def test_structure_reference(): + '''Test a scalar structure-component reference.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine get_x(item, value) + type(point) :: item + real :: value + value = item%x + end subroutine get_x + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + reference = root.children[0].children[0].rhs + assert isinstance(reference, psyir_nodes.StructureReference) + assert reference.symbol.name == "item" + assert reference.member.name == "x" + + +@min_version_3_10 +def test_array_of_structures_reference(): + '''Test an indexed array-of-structures component reference.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine get_value(items, value) + type(point) :: items(2) + real :: value + value = items(1)%vector(2) + end subroutine get_value + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + reference = root.children[0].children[0].rhs + assert isinstance(reference, + psyir_nodes.ArrayOfStructuresReference) + assert reference.indices[0].value == "1" + assert reference.member.name == "vector" + assert reference.member.indices[0].value == "2" + + +@min_version_3_10 +def test_multidimensional_explicit_array_bounds(): + '''Test explicit lower bounds in a multidimensional shape.''' + processor = FortranTreeSitterReader() + valid_code = """ + module array_shapes + implicit none + real :: field(-2:10, 20) + end module array_shapes + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + datatype = root.children[0].symbol_table.lookup("field").datatype + assert isinstance(datatype, psyir_symbols.ArrayType) + assert len(datatype.shape) == 2 + assert datatype.shape[0].lower.operator == \ + psyir_nodes.UnaryOperation.Operator.MINUS + assert datatype.shape[0].lower.children[0].value == "2" + assert datatype.shape[0].upper.value == "10" + assert datatype.shape[1].lower.value == "1" + assert datatype.shape[1].upper.value == "20" + + +@min_version_3_10 +def test_unary_operation(): + '''Test a unary arithmetic operation.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine operations(value) + real :: value + value = -value + end subroutine operations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + expression = root.children[0].children[0].rhs + assert expression.operator == psyir_nodes.UnaryOperation.Operator.MINUS + + +@min_version_3_10 +def test_binary_operation(): + '''Test a binary arithmetic operation.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine operations(value) + real :: value + value = value + 1.0 + end subroutine operations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + expression = root.children[0].children[0].rhs + assert expression.operator == psyir_nodes.BinaryOperation.Operator.ADD + + +@min_version_3_10 +def test_explicit_array_section(): + '''Test an array section with explicit start, stop and step.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine section(array) + real :: array(10) + array(2:8:2) = 0.0 + end subroutine section + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + section = root.children[0].children[0].lhs.indices[0] + assert isinstance(section, psyir_nodes.Range) + assert section.start.value == "2" + assert section.stop.value == "8" + assert section.step.value == "2" + + +@min_version_3_10 +def test_implicit_array_section_bounds(): + '''Test synthesized bounds for a whole-array section.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine section(array) + real :: array(10) + array(:) = array(:) + end subroutine section + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + assignment = root.children[0].children[0] + for reference in (assignment.lhs, assignment.rhs): + section = reference.indices[0] + assert section.start.intrinsic == \ + psyir_nodes.IntrinsicCall.Intrinsic.LBOUND + assert section.stop.intrinsic == \ + psyir_nodes.IntrinsicCall.Intrinsic.UBOUND + + +@min_version_3_10 +def test_intrinsic_call(): + '''Test an intrinsic function call.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine intrinsic(value) + real :: value + value = sin(value) + end subroutine intrinsic + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + call = root.children[0].children[0].rhs + assert isinstance(call, psyir_nodes.IntrinsicCall) + assert call.intrinsic == psyir_nodes.IntrinsicCall.Intrinsic.SIN + + +@min_version_3_10 +def test_named_call_argument(): + '''Test a named subroutine-call argument.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine caller(value) + integer :: value + call update(value, result=value) + end subroutine caller + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + call = root.children[0].children[0] + assert isinstance(call, psyir_nodes.Call) + assert call.argument_names == [None, "result"] + + +@min_version_3_10 +def test_array_constructor(): + '''Test a simple array constructor.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine constructor(values) + real :: values(3) + values = [1.0, 2.0, 3.0] + end subroutine constructor + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + constructor = root.children[0].children[0].rhs + assert isinstance(constructor, psyir_nodes.ArrayConstructor) + assert [element.value for element in constructor.children] == [ + "1.0", "2.0", "3.0"] + + +@min_version_3_10 +def test_pointer_assignment(): + '''Test a pointer assignment.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine associate(target, pointer) + integer, target :: target + integer, pointer :: pointer + pointer => target + end subroutine associate + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + assignment = root.children[0].children[0] + assert isinstance(assignment, psyir_nodes.Assignment) + assert assignment.is_pointer + + +@min_version_3_10 +def test_nullify_statement(): + '''Test a NULLIFY statement.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine clear(pointer) + integer, pointer :: pointer + nullify(pointer) + end subroutine clear + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + nullify = root.children[0].children[0] + assert nullify.intrinsic == psyir_nodes.IntrinsicCall.Intrinsic.NULLIFY + + +@min_version_3_10 +def test_if_construct(): + '''Test IF, ELSE IF and ELSE clauses.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine conditional(value) + integer :: value + if (value < 0) then + value = -value + else if (value == 0) then + value = 1 + else + value = value + 1 + end if + end subroutine conditional + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + ifblock = root.children[0].children[0] + assert isinstance(ifblock, psyir_nodes.IfBlock) + nested = ifblock.else_body.children[0] + assert "was_elseif" in nested.annotations + assert nested.else_body.children[0].rhs.operator == \ + psyir_nodes.BinaryOperation.Operator.ADD + + +@min_version_3_10 +def test_counted_do_loop(): + '''Test a counted DO loop.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine counted(limit) + integer :: limit, index + do index = 1, limit, 2 + limit = limit - 1 + end do + end subroutine counted + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + loop = root.children[0].children[0] + assert isinstance(loop, psyir_nodes.Loop) + assert loop.variable.name == "index" + assert loop.start_expr.value == "1" + assert loop.stop_expr.symbol.name == "limit" + assert loop.step_expr.value == "2" + + +@min_version_3_10 +def test_do_while_loop(): + '''Test a DO WHILE loop.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine conditional(value) + integer :: value + do while (value > 0) + value = value - 1 + end do + end subroutine conditional + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + loop = root.children[0].children[0] + assert isinstance(loop, psyir_nodes.WhileLoop) + assert loop.condition.operator == \ + psyir_nodes.BinaryOperation.Operator.GT + + +@min_version_3_10 +def test_unconditional_do_loop(): + '''Test an unconditional DO loop.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine unconditional() + do + return + end do + end subroutine unconditional + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + loop = root.children[0].children[0] + assert isinstance(loop, psyir_nodes.WhileLoop) + assert "was_unconditional" in loop.annotations + + +@min_version_3_10 +def test_where_construct(): + '''Test a WHERE construct with ELSEWHERE.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine mask(array) + real :: array(10) + where (array > 0.0) + array = sqrt(array) + elsewhere + array = 0.0 + end where + end subroutine mask + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + where = root.children[0].children[0] + assert isinstance(where, psyir_nodes.IfBlock) + assert where.annotations == ["was_where"] + assert where.else_body.children[0].rhs.value == "0.0" + + +@min_version_3_10 +def test_select_case_construct(): + '''Test SELECT CASE lowering.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine select_value(selector) + integer :: selector + select case(selector) + case(1) + selector = 2 + case(3:5, 8) + selector = 3 + case default + selector = 0 + end select + end subroutine select_value + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + case = root.children[0].children[0] + assert case.annotations == ["was_case"] + assert case.condition.operator == \ + psyir_nodes.BinaryOperation.Operator.EQ + second = case.else_body.children[0] + assert second.condition.operator == psyir_nodes.BinaryOperation.Operator.OR + assert second.else_body.children[0].rhs.value == "0" + + +@min_version_3_10 +def test_allocate_statement(): + '''Test an ALLOCATE statement.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine allocate_array(array, extent, status) + integer :: extent, status + real, allocatable :: array(:) + allocate(array(extent), stat=status) + end subroutine allocate_array + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + allocate = root.children[0].children[0] + assert allocate.intrinsic == \ + psyir_nodes.IntrinsicCall.Intrinsic.ALLOCATE + assert isinstance(allocate.arguments[0], psyir_nodes.ArrayReference) + assert allocate.argument_names == [None, "stat"] + + +@min_version_3_10 +def test_deallocate_statement(): + '''Test a DEALLOCATE statement.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine deallocate_array(array, status) + integer :: status + real, allocatable :: array(:) + deallocate(array, stat=status) + end subroutine deallocate_array + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + deallocate = root.children[0].children[0] + assert deallocate.intrinsic == \ + psyir_nodes.IntrinsicCall.Intrinsic.DEALLOCATE + assert deallocate.argument_names == [None, "stat"] + + +@min_version_3_10 +def test_stop_codeblock(): + '''Test the localized fallback for a STOP statement.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine stop_execution() + stop 1 + end subroutine stop_execution + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + codeblock = root.children[0].children[0] + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert "Unsupported 'stop_statement'" in codeblock.preceding_comment + + +@min_version_3_10 +def test_default_visibility(): + '''Test a module's default visibility.''' + processor = FortranTreeSitterReader() + valid_code = """ + module visibility + implicit none + private + contains + subroutine hidden() + end subroutine hidden + end module visibility + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + container = root.children[0] + assert container.symbol_table.default_visibility == \ + psyir_symbols.Symbol.Visibility.PRIVATE + assert container.symbol_table.lookup("hidden").visibility == \ + psyir_symbols.Symbol.Visibility.PRIVATE + + +@min_version_3_10 +def test_named_visibility(): + '''Test name-specific visibility for a contained routine.''' + processor = FortranTreeSitterReader() + valid_code = """ + module visibility + implicit none + private + public :: exposed + contains + subroutine exposed() + end subroutine exposed + end module visibility + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + exposed = root.children[0].symbol_table.lookup("exposed") + assert exposed.visibility == psyir_symbols.Symbol.Visibility.PUBLIC + + +@min_version_3_10 +def test_generic_interface(): + '''Test a named generic interface.''' + processor = FortranTreeSitterReader() + valid_code = """ + module dispatch + implicit none + interface apply + module procedure apply_integer, apply_real + end interface apply + end module dispatch + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + generic = root.children[0].symbol_table.lookup("apply") + assert isinstance(generic, psyir_symbols.GenericInterfaceSymbol) + assert [info.symbol.name for info in generic.routines] == [ + "apply_integer", "apply_real"] + assert all(info.from_container for info in generic.routines) + + +@min_version_3_10 +def test_ignored_comment(): + '''Test that an ignored comment does not create a CodeBlock.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine commented(value) + integer :: value + ! This comment must not create a CodeBlock. + value = 1 + end subroutine commented + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + assert len(routine.children) == 1 + assert isinstance(routine.children[0], psyir_nodes.Assignment) + + +@min_version_3_10 +def test_implied_do_codeblock(): + '''Test the localized fallback for an implied-DO array constructor.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine constructor(values) + real :: values(3) + integer :: index + values = [(real(index), index=1,3)] + end subroutine constructor + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + codeblock = root.children[0].children[0] + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert ("Array constructors with implied-DO loops are not supported" in + codeblock.preceding_comment) From cc94d6cd4dac2f438e8a0006d73c6dc801dfe7d1 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 30 Jul 2026 13:03:01 +0100 Subject: [PATCH 03/23] #3083 Improve treesitter frontend implementation --- .../frontend/fortran_treesitter_reader.py | 326 +++++++++--------- 1 file changed, 164 insertions(+), 162 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 895542fc65..ce78ecb993 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -40,6 +40,7 @@ from dataclasses import dataclass import logging from typing import Callable, Iterable, Iterator, Optional, TYPE_CHECKING, Union +from collections.abc import Generator from psyclone.psyir import nodes, symbols from psyclone.psyir.nodes.codeblock import TreeSitterCodeBlock, CodeBlock @@ -77,6 +78,35 @@ def to_str(node: 'TSNode') -> str: return node.text.decode('utf8') if node.text else "" +def direct_child_of_type( + tsnode: Optional['TSNode'], node_type: str +) -> Generator['TSNode']: + '''Return the first direct child having the supplied type. + + :param tsnode: tree-sitter node whose children are searched. + :param node_type: tree-sitter type to find. + + :returns: matching child, or ``None`` if no child matches. + ''' + if tsnode: + for child in tsnode.children: + if child.type == node_type: + yield child + + +def next_of_type( + tsnode: Optional['TSNode'], node_type: str +) -> Optional['TSNode']: + '''Return the first direct child having the supplied type. + + :param tsnode: tree-sitter node whose children are searched. + :param node_type: tree-sitter type to find. + + :returns: matching child, or ``None`` if no child matches. + ''' + return next(direct_child_of_type(tsnode, node_type), None) + + @dataclass(frozen=True) class _CommonDeclAttributes: ''' Properties shared by all entities of a fortran declaration (the lhs @@ -209,7 +239,7 @@ def __init__( self._current_scope: Optional[symbols.SymbolTable] = None # --------------------------------------------------------------------- - # Source parsing and generic dispatch + # Public methods # --------------------------------------------------------------------- def generate_parse_tree_from_file(self, file_path) -> 'TSNode': @@ -271,18 +301,76 @@ def generate_psyir(self, parse_tree: 'TSNode') -> nodes.Node: :returns: the equivalent PSyIR Node. ''' - return self.process_nodes(parse_tree)[0] + return self._process_nodes(parse_tree)[0] + + # Utility methods + + @contextmanager + def _using_scope( + self, symtab: symbols.SymbolTable + ) -> Iterator[None]: + ''' Make the given symtab the new parsing scope, but keep a reference + to the previous scope in order to restore it when leaving this new + scope (by graceful exit or an exception). + + :param symtab: symbol table for the scope being translated. + + :yields: while ``symtab`` is the reader's current scope. + ''' + previous_scope = self._current_scope + self._current_scope = symtab + try: + yield + finally: + self._current_scope = previous_scope + + @contextmanager + def _using_temporary_scope( + self, parent: nodes.ScopingNode + ) -> Iterator[nodes.ScopingNode]: + ''' + Like `_using_scope`, but it creates a dummy symbol_table and scope, + soft linked to the current parent. This is useful to create disposable + symbol tables when the resulting PSyIR does not need them, but they + still need to be linked to the parent scope. + + For example in the body of a derived type: + + .. code-block:: fortran - def process_nodes( + module m + integer, parameter :: size = 10 + type myt + integer, dimension(size) :: array + end type + end module + + :param parent: real host scope used for lexical lookup. + + :yields: disposable ScopingNode. + ''' + symtab = symbols.SymbolTable() + temporary_scope = nodes.ScopingNode(symbol_table=symtab) + # This intentionally bypasses child validation. + # pylint: disable=protected-access + temporary_scope._parent = parent + self._current_scope = symtab + try: + yield temporary_scope + finally: + # Remove soft link + temporary_scope._parent = None + symtab.detach() + self._current_scope = parent.symbol_table + + def _process_nodes( self, tsnodes: Union["TSNode", Iterable["TSNode"]], ): - '''Create PSyIR for one or more tree-sitter nodes. - - This is the statement-boundary dispatcher. Unsupported syntax is + ''' + This is the tsnodes handler dispatcher. Unsupported syntax is deliberately caught here rather than in individual handlers so that - nested expression failures replace their whole Fortran statement with - one valid CodeBlock. + contiguours unsupprted nodes are places in a single CodeBlock. :param tsnodes: one tree-sitter node or an iterable of nodes. @@ -365,65 +453,6 @@ def get_handler(self, tsnode: 'TSNode') -> Callable: f"Unsupported '{tsnode.type}' tree-sitter node.") from None return handler - @contextmanager - def _using_scope( - self, symtab: symbols.SymbolTable - ) -> Iterator[None]: - ''' Make the given symtab the new parsing scope, but keep a reference - to the previous scope in order to restore it when leaving this new - scope (by graceful exit or an exception). - - :param symtab: symbol table for the scope being translated. - - :yields: while ``symtab`` is the reader's current scope. - ''' - previous_scope = self._current_scope - self._current_scope = symtab - try: - yield - finally: - self._current_scope = previous_scope - - @contextmanager - def _using_temporary_scope( - self, parent: nodes.ScopingNode - ) -> Iterator[nodes.ScopingNode]: - ''' - Like `_using_scope`, but it creates a dummy symbol_table and scope, - soft linked to the current parent. This is useful to create disposable - symbol tables when the resulting PSyIR does not need them, but they - still need to be linked to the parent scope. - - For example in the body of a derived type: - - .. code-block:: fortran - - module m - integer, parameter :: size = 10 - type myt - integer, dimension(size) :: array - end type - end module - - :param parent: real host scope used for lexical lookup. - - :yields: disposable ScopingNode. - ''' - symtab = symbols.SymbolTable() - temporary_scope = nodes.ScopingNode(symbol_table=symtab) - # This intentionally bypasses child validation. - # pylint: disable=protected-access - temporary_scope._parent = parent - self._current_scope = symtab - try: - yield temporary_scope - finally: - # Calling Node.detach() would be incorrect because the temporary - # scope was never inserted into the parent's child list. - temporary_scope._parent = None - symtab.detach() - self._current_scope = parent.symbol_table - @staticmethod @contextmanager def _temporary_parent( @@ -490,7 +519,6 @@ def _temporary_parent( node.symbol_table.remove(symbol) symbol.interface = original_interface - # --------------------------------------------------------------------- # Parse-tree navigation and Fortran scope handlers # --------------------------------------------------------------------- @@ -507,34 +535,10 @@ def _translation_unit_handler( file_container = nodes.FileContainer("") with self._using_scope(file_container.symbol_table): file_container.children.extend( - self.process_nodes(tsnode.children) + self._process_nodes(tsnode.children) ) return file_container - @staticmethod - def _child(tsnode: 'TSNode', node_type: str) -> Optional['TSNode']: - '''Return the first direct child having the supplied type. - - :param tsnode: tree-sitter node whose children are searched. - :param node_type: tree-sitter type to find. - - :returns: matching child, or ``None`` if no child matches. - ''' - return next((child for child in tsnode.children - if child.type == node_type), None) - - @staticmethod - def _children(tsnode: 'TSNode', *node_types: str) -> list['TSNode']: - '''Return direct children having one of the supplied types. - - :param tsnode: tree-sitter node whose children are searched. - :param node_types: tree-sitter types to find. - - :returns: children whose type is in ``node_types``. - ''' - return [child for child in tsnode.children - if child.type in node_types] - @staticmethod def _split_extent( tsnode: 'TSNode' @@ -570,8 +574,8 @@ def _module_handler( :raises NotImplementedError: if the module has an unsupported child. :raises NotImplementedError: if the module permits implicit variables. ''' - statement = self._child(tsnode, "module_statement") - name = self._child(statement, "name") if statement else None + statement = next_of_type(tsnode, "module_statement") + name = next_of_type(statement, "name") if not any(child.type == "implicit_statement" and "none" in [item.type for item in child.children] for child in tsnode.children): @@ -585,15 +589,15 @@ def _module_handler( unsupported_specs = self._process_specification_part( tsnode.children) - internal = self._child(tsnode, "internal_procedures") + internal = next_of_type(tsnode, "internal_procedures") container.children.extend(unsupported_specs) if internal: container.children.extend( - self.process_nodes( + self._process_nodes( [child for child in internal.children if child.type != "contains_statement"])) - container.children.extend(self.process_nodes( + container.children.extend(self._process_nodes( [child for child in tsnode.children if child.type not in self._MODULE_NON_EXECUTABLE_TYPES])) self._apply_visibility(visibility_map) @@ -664,11 +668,10 @@ def _routine_handler( if parent_symtab is None: raise RuntimeError( "A Routine must be translated within a current scope") - statement = self._child(tsnode, f"{routine_kind}_statement") - name_node = self._child(statement, "name") if statement else None + statement = next_of_type(tsnode, f"{routine_kind}_statement") + name_node = next_of_type(statement, "name") name = to_str(name_node) if name_node else routine_kind - parameters = (self._child(statement, "parameters") - if statement else None) + parameters = next_of_type(statement, "parameters") argument_names = tuple( to_str(child) for child in parameters.children if child.type == "identifier") if parameters else () @@ -718,7 +721,7 @@ def _routine_handler( } specification.update(self._SPECIFICATION_TYPES) routine.children.extend(unsupported_specs) - routine.children.extend(self.process_nodes( + routine.children.extend(self._process_nodes( [child for child in tsnode.children if child.type not in specification])) self._apply_visibility(visibility_map) @@ -740,8 +743,8 @@ def _function_return_info( if routine_kind != "function": return None, None - result = self._child(statement, "function_result") - result_name = self._child(result, "identifier") if result else None + result = next_of_type(statement, "function_result") + result_name = next_of_type(result, "identifier") return_name = to_str(result_name) if result_name else routine_name type_node = next( (child for child in statement.children @@ -866,7 +869,9 @@ def _variable_declaration_handler( raise NotImplementedError( "A variable declaration has no supported type specification") - qualifiers = self._children(tsnode, "type_qualifier") + # qualifiers = direct_child_of_type(tsnode, "type_qualifier") + qualifiers = [child for child in tsnode.children + if child.type == "type_qualifier"] qualifier_names = frozenset( child.children[0].type for child in qualifiers if child.children) @@ -880,8 +885,7 @@ def _variable_declaration_handler( base_type = None dimension = next( - (self._child(item, "argument_list") - for item in qualifiers + (next_of_type(item, "argument_list") for item in qualifiers if item.children and item.children[0].type == "dimension"), None) intent = next( (item for item in qualifiers @@ -891,7 +895,6 @@ def _variable_declaration_handler( frozenset(unsupported), to_str(tsnode).split("::", maxsplit=1)[0].strip()) - for declarator in tsnode.children: if declarator.type in ( "identifier", "sized_declarator", "init_declarator"): @@ -906,7 +909,7 @@ def _declare_entity( :param common_attr: properties shared by the complete declaration. ''' id_node = (declarator if declarator.type == "identifier" - else self._child(declarator, "identifier")) + else next_of_type(declarator, "identifier")) name = to_str(id_node) datatype, initial_value = self._declarator_datatype( declarator, common_attr) @@ -957,7 +960,7 @@ def _declarator_datatype( # Array shape and initialisation belong to an individual entity, so # derive a fresh datatype from the common base type. datatype = common_attr.base_type - shape_node = self._child(declarator, "size") or common_attr.dimension + shape_node = next_of_type(declarator, "size") or common_attr.dimension is_allocatable = "allocatable" in common_attr.qualifiers if datatype and shape_node: try: @@ -1055,7 +1058,7 @@ def _datatype_from_type( symtab = self._current_scope if tsnode.type == "derived_type": keyword = tsnode.children[0].type - name_node = self._child(tsnode, "type_name") + name_node = next_of_type(tsnode, "type_name") name = to_str(name_node) if keyword == "class": raise NotImplementedError( @@ -1082,7 +1085,7 @@ def _datatype_from_type( f"Intrinsic type '{intrinsic}' has no PSyIR representation") precision = symbols.ScalarType.Precision.UNDEFINED length = None - kind_node = self._child(tsnode, "kind") + kind_node = next_of_type(tsnode, "kind") if kind_node: values = [child for child in kind_node.children if child.type not in ("(", ")")] @@ -1264,10 +1267,10 @@ def _use_statement_handler( existing non-container symbol. ''' symtab = self._current_scope - module_node = self._child(tsnode, "module_name") + module_node = next_of_type(tsnode, "module_name") module_name = to_str(module_node) intrinsic = any(child.type == "intrinsic" for child in tsnode.children) - included = self._child(tsnode, "included_items") + included = next_of_type(tsnode, "included_items") wildcard = included is None try: container = symtab.lookup(module_name) @@ -1332,14 +1335,13 @@ def _derived_type_definition_handler( existing non-datatype symbol. ''' symtab = self._current_scope - statement = self._child(tsnode, "derived_type_statement") - name_node = self._child(statement, "type_name") if statement else None + statement = next_of_type(tsnode, "derived_type_statement") + name_node = next_of_type(statement, "type_name") name = to_str(name_node) unsupported = any(child.type == "derived_type_procedures" for child in tsnode.children) datatype = None if not unsupported: - component_table = symbols.SymbolTable() parent = symtab.node if not isinstance(parent, nodes.ScopingNode): raise RuntimeError( @@ -1348,7 +1350,7 @@ def _derived_type_definition_handler( visibility_map = self._process_access_statements( tsnode.children) try: - for declaration in self._children( + for declaration in direct_child_of_type( tsnode, "variable_declaration"): self._variable_declaration_handler(declaration) self._apply_visibility(visibility_map) @@ -1364,7 +1366,7 @@ def _derived_type_definition_handler( datatype = symbols.UnsupportedFortranType(to_str(tsnode).strip()) visibility = symtab.default_visibility - access = self._child(statement, "access_specifier") + access = next_of_type(statement, "access_specifier") if access: visibility = (symbols.Symbol.Visibility.PRIVATE if "private" in to_str(access).lower() else @@ -1392,17 +1394,17 @@ def _interface_handler( unsupported. ''' symtab = self._current_scope - statement = self._child(tsnode, "interface_statement") - name_node = (self._child(statement, "name") if statement else None) + statement = next_of_type(tsnode, "interface_statement") + name_node = next_of_type(statement, "name") if not name_node: raise NotImplementedError( "Abstract and operator interfaces are not supported") name = to_str(name_node) routines = [] - for procedure in self._children(tsnode, "procedure_statement"): + for procedure in direct_child_of_type(tsnode, "procedure_statement"): from_container = "module" in [ child.type for child in procedure.children[0].children] - for method in self._children(procedure, "method_name"): + for method in direct_child_of_type(procedure, "method_name"): routine_name = to_str(method) try: routine = symtab.lookup(routine_name) @@ -1559,9 +1561,9 @@ def _call_expression_handler( if name_node.type == "derived_type_member_expression": return self._structure_reference( name_node, - trailing_arguments=self._child(tsnode, "argument_list")) + trailing_arguments=next_of_type(tsnode, "argument_list")) name = to_str(name_node).lower() - argument_list = self._child(tsnode, "argument_list") + argument_list = next_of_type(tsnode, "argument_list") try: symbol = self._lookup(name) except KeyError: @@ -1742,7 +1744,7 @@ def _decompose_structure( base = tsnode.children[0] name, indices, members = self._decompose_structure(base) arguments = self._arguments( - self._child(tsnode, "argument_list")) + next_of_type(tsnode, "argument_list")) if any(isinstance(arg, tuple) for arg in arguments): raise NotImplementedError( "Named arguments in structure accesses are not supported") @@ -1754,7 +1756,7 @@ def _decompose_structure( if tsnode.type == "derived_type_member_expression": name, indices, members = self._decompose_structure( tsnode.children[0]) - member = self._child(tsnode, "type_member") + member = next_of_type(tsnode, "type_member") if member is None: raise NotImplementedError( "Malformed structure component access") @@ -1774,7 +1776,7 @@ def _array_literal_handler( :raises NotImplementedError: for an implied-DO constructor. ''' - if self._child(tsnode, "implied_do_loop_expression"): + if next_of_type(tsnode, "implied_do_loop_expression"): raise NotImplementedError( "Array constructors with implied-DO loops are not supported") elems = [self._expression(child) for child in tsnode.children @@ -1881,7 +1883,7 @@ def _subroutine_call_handler( if not isinstance(symbol, symbols.RoutineSymbol): raise NotImplementedError( f"Called object '{name}' is not a routine") - args = self._arguments(self._child(tsnode, "argument_list")) + args = self._arguments(next_of_type(tsnode, "argument_list")) return nodes.Call.create(symbol, args) def _keyword_statement_handler( @@ -1912,7 +1914,7 @@ def _if_statement_handler( :raises NotImplementedError: if the statement has no condition. ''' - condition_node = self._child(tsnode, "parenthesized_expression") + condition_node = next_of_type(tsnode, "parenthesized_expression") if not condition_node: raise NotImplementedError("IF statement has no condition") structural = { @@ -1921,15 +1923,15 @@ def _if_statement_handler( } body_nodes = [child for child in tsnode.children if child.type not in structural] - else_clause = self._child(tsnode, "else_clause") - else_ifs = self._children(tsnode, "elseif_clause") + else_clause = next_of_type(tsnode, "else_clause") + else_ifs = list(direct_child_of_type(tsnode, "elseif_clause")) annotations = [] - if not self._child(tsnode, "end_if_statement"): + if not next_of_type(tsnode, "end_if_statement"): annotations.append("was_single_stmt") - if_body = self.process_nodes(body_nodes) + if_body = self._process_nodes(body_nodes) else_body = None if else_clause: - else_body = self.process_nodes( + else_body = self._process_nodes( [child for child in else_clause.children if child.type != "else"]) for else_if in reversed(else_ifs): @@ -1950,19 +1952,19 @@ def _if_clause( :returns: annotated PSyIR IfBlock. ''' - condition = self._child(tsnode, "parenthesized_expression") + condition = next_of_type(tsnode, "parenthesized_expression") structural = {"else", "if", "parenthesized_expression", "then", "else_clause", "elseif_clause"} - body = self.process_nodes( + body = self._process_nodes( [child for child in tsnode.children if child.type not in structural]) - trailing = self._child(tsnode, "elseif_clause") + trailing = next_of_type(tsnode, "elseif_clause") otherwise = ( [self._if_clause(trailing, final_else)] if trailing else - self.process_nodes( + self._process_nodes( [child for child in - (self._child(tsnode, "else_clause").children - if self._child(tsnode, "else_clause") else []) + (next_of_type(tsnode, "else_clause").children + if next_of_type(tsnode, "else_clause") else []) if child.type != "else"]) or final_else) result = nodes.IfBlock.create( self._expression(condition), body, otherwise) @@ -1980,10 +1982,10 @@ def _do_loop_handler( :raises NotImplementedError: if counted-loop control is unsupported. ''' - statement = self._child(tsnode, "do_statement") - control = self._child(statement, "loop_control_expression") - while_node = self._child(statement, "while_statement") - body = self.process_nodes( + statement = next_of_type(tsnode, "do_statement") + control = next_of_type(statement, "loop_control_expression") + while_node = next_of_type(statement, "while_statement") + body = self._process_nodes( [child for child in tsnode.children if child.type not in ("do_statement", "end_do_loop_statement")]) @@ -2011,7 +2013,7 @@ def _do_loop_handler( variable, self._expression(parts[1]), self._expression(parts[2]), step, body) if while_node: - condition = self._child( + condition = next_of_type( while_node, "parenthesized_expression") return nodes.WhileLoop.create( self._expression(condition), body) @@ -2029,22 +2031,22 @@ def _where_statement_handler( :returns: annotated PSyIR IfBlock. ''' - condition = self._child(tsnode, "parenthesized_expression") + condition = next_of_type(tsnode, "parenthesized_expression") structural = {"where", "parenthesized_expression", "elsewhere_clause", "end_where_statement"} - body = self.process_nodes( + body = self._process_nodes( [child for child in tsnode.children if child.type not in structural]) - elsewhere = self._child(tsnode, "elsewhere_clause") + elsewhere = next_of_type(tsnode, "elsewhere_clause") other = ( - self.process_nodes( + self._process_nodes( [child for child in elsewhere.children if child.type != "elsewhere"]) if elsewhere else None) result = nodes.IfBlock.create( self._expression(condition), body, other) result.annotations.extend( - ["was_where"] if self._child( + ["was_where"] if next_of_type( tsnode, "end_where_statement") else ["was_where", "was_single_stmt"]) return result @@ -2063,30 +2065,30 @@ def _select_case_statement_handler( :raises NotImplementedError: if no conditional CASE can be produced. ''' - selector_node = self._child( - self._child(tsnode, "selector"), "identifier") + selector_node = next_of_type( + next_of_type(tsnode, "selector"), "identifier") if selector_node is None: selector = self._expression( [child for child in - self._child(tsnode, "selector").children + next_of_type(tsnode, "selector").children if child.type not in ("(", ")")][0]) else: selector = self._expression(selector_node) - cases = self._children(tsnode, "case_statement") + cases = list(direct_child_of_type(tsnode, "case_statement")) default_body = None normal = [] for case in cases: - if self._child(case, "default"): - default_body = self.process_nodes( + if next_of_type(case, "default"): + default_body = self._process_nodes( [child for child in case.children if child.type not in ("case", "default")]) else: - values = self._child(case, "case_value_range_list") + values = next_of_type(case, "case_value_range_list") if values is None: raise NotImplementedError( "Malformed CASE value list") structural = {"case", "(", ")", "case_value_range_list"} - body = self.process_nodes( + body = self._process_nodes( [child for child in case.children if child.type not in structural]) normal.append((values, body)) @@ -2229,12 +2231,12 @@ def _allocation_reference( :raises NotImplementedError: if the object is not a data symbol. ''' - ident = self._child(tsnode, "identifier") + ident = next_of_type(tsnode, "identifier") reference = self._identifier_handler(ident) if not isinstance(reference.symbol, symbols.DataSymbol): raise NotImplementedError( "An ALLOCATE object must be a data symbol") - size = self._child(tsnode, "size") + size = next_of_type(tsnode, "size") indices = [ self._allocation_extent(extent) for extent in size.children From 14b308019e5a21e5231e5a19e11262650e2d4f5f Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 30 Jul 2026 14:46:17 +0100 Subject: [PATCH 04/23] #3083 Improve treesitter frontend implementation --- .../frontend/fortran_treesitter_reader.py | 318 ++++++------------ .../fortran_treesitter_reader/ftr_test.py | 103 +----- 2 files changed, 101 insertions(+), 320 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index ce78ecb993..446430708a 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -39,8 +39,8 @@ from contextlib import contextmanager from dataclasses import dataclass import logging -from typing import Callable, Iterable, Iterator, Optional, TYPE_CHECKING, Union -from collections.abc import Generator +from typing import Callable, Iterable, Optional, TYPE_CHECKING, Union +from collections.abc import Generator, Container from psyclone.psyir import nodes, symbols from psyclone.psyir.nodes.codeblock import TreeSitterCodeBlock, CodeBlock @@ -79,7 +79,7 @@ def to_str(node: 'TSNode') -> str: def direct_child_of_type( - tsnode: Optional['TSNode'], node_type: str + tsnode: Optional['TSNode'], types: str | Container[str] ) -> Generator['TSNode']: '''Return the first direct child having the supplied type. @@ -88,9 +88,10 @@ def direct_child_of_type( :returns: matching child, or ``None`` if no child matches. ''' + check_types = (types,) if isinstance(types, str) else types if tsnode: for child in tsnode.children: - if child.type == node_type: + if child.type in check_types: yield child @@ -236,7 +237,10 @@ def __init__( self._ignore_comments = ignore_comments self._free_form = free_form self._conditional_openmp = conditional_openmp - self._current_scope: Optional[symbols.SymbolTable] = None + # Keep a reference to the symbol table currently in scope, instead of + # having it as argument everywhere. The initial one here is a + # disposable instance (but prevents having to deal with the None type) + self._current_scope: symbols.SymbolTable = symbols.SymbolTable() # --------------------------------------------------------------------- # Public methods @@ -303,15 +307,17 @@ def generate_psyir(self, parse_tree: 'TSNode') -> nodes.Node: ''' return self._process_nodes(parse_tree)[0] + # --------------------------------------------------------------------- # Utility methods + # --------------------------------------------------------------------- @contextmanager def _using_scope( self, symtab: symbols.SymbolTable - ) -> Iterator[None]: + ) -> Generator[None]: ''' Make the given symtab the new parsing scope, but keep a reference to the previous scope in order to restore it when leaving this new - scope (by graceful exit or an exception). + scope (by a graceful exit or an exception). :param symtab: symbol table for the scope being translated. @@ -326,15 +332,14 @@ def _using_scope( @contextmanager def _using_temporary_scope( - self, parent: nodes.ScopingNode - ) -> Iterator[nodes.ScopingNode]: + self, parent: nodes.ScopingNode, + scope: Optional[nodes.ScopingNode] = None + ) -> Generator[None]: ''' - Like `_using_scope`, but it creates a dummy symbol_table and scope, - soft linked to the current parent. This is useful to create disposable - symbol tables when the resulting PSyIR does not need them, but they - still need to be linked to the parent scope. - - For example in the body of a derived type: + Like `_using_scope`, but soft-link the scope of the supplied parent + or a disposable ScopingNode if none is provided. This is useful when + the resulting PSyIR does not need the scope but lookup must still reach + the parent. For example in the body of a derived type: .. code-block:: fortran @@ -345,32 +350,41 @@ def _using_temporary_scope( end type end module - :param parent: real host scope used for lexical lookup. + :param parent: real scope used for lexical lookup. + :param scope: existing orphan scope to use, or ``None`` to create a + disposable one. + + :yields: while the provided or a disposable scope is soft-linked + to provide a temporary current scope. - :yields: disposable ScopingNode. + :raises ValueError: if the supplied scope already has a parent. ''' - symtab = symbols.SymbolTable() - temporary_scope = nodes.ScopingNode(symbol_table=symtab) + if scope: + if scope.parent is not None: + raise ValueError("The supplied scope must be an orphan") + else: + scope = nodes.ScopingNode(symbol_table=symbols.SymbolTable()) + + previous_scope = self._current_scope # This intentionally bypasses child validation. # pylint: disable=protected-access - temporary_scope._parent = parent - self._current_scope = symtab + scope._parent = parent + self._current_scope = scope.symbol_table try: - yield temporary_scope + yield finally: # Remove soft link - temporary_scope._parent = None - symtab.detach() - self._current_scope = parent.symbol_table + scope._parent = None + self._current_scope = previous_scope def _process_nodes( self, tsnodes: Union["TSNode", Iterable["TSNode"]], - ): + ) -> list[nodes.Node]: ''' This is the tsnodes handler dispatcher. Unsupported syntax is deliberately caught here rather than in individual handlers so that - contiguours unsupprted nodes are places in a single CodeBlock. + continuous unsupported nodes are placed in a single CodeBlock. :param tsnodes: one tree-sitter node or an iterable of nodes. @@ -381,7 +395,7 @@ def _process_nodes( children = [] for tsnode in list_of_nodes: try: - handler = self.get_handler(tsnode) + handler = self._get_handler(tsnode) result = handler(tsnode) if result is not None: children.append(result) @@ -397,9 +411,6 @@ def _create_codeblock( ) -> TreeSitterCodeBlock: '''Create a statement CodeBlock for unsupported valid Fortran. - Keeping this construction in one place guarantees that ordinary - dispatch and specification-part dispatch produce the same diagnostic. - :param tsnode: tree-sitter node containing unsupported Fortran. :param reason: human-readable explanation of the limitation. @@ -413,31 +424,7 @@ def _create_codeblock( ) return code_block - def _process_specification_part( - self, tsnodes: Iterable['TSNode'] - ) -> list[TreeSitterCodeBlock]: - '''Populate a scope's symbol table from specification statements. - - Most specification handlers only update the current symbol table and - return ``None``. If one is unsupported, preserve just that statement - as a CodeBlock rather than replacing the enclosing module or routine. - - :param tsnodes: tree-sitter children of the Fortran scope. - - :returns: CodeBlocks for unsupported specification statements. - ''' - unsupported = [] - for tsnode in tsnodes: - if tsnode.type not in self._SPECIFICATION_TYPES: - continue - try: - self.get_handler(tsnode)(tsnode) - except NotImplementedError as err: - unsupported.append( - self._create_codeblock(tsnode, str(err))) - return unsupported - - def get_handler(self, tsnode: 'TSNode') -> Callable: + def _get_handler(self, tsnode: 'TSNode') -> Callable: ''' :param tsnode: a given treesitter node. @@ -453,74 +440,8 @@ def get_handler(self, tsnode: 'TSNode') -> Callable: f"Unsupported '{tsnode.type}' tree-sitter node.") from None return handler - @staticmethod - @contextmanager - def _temporary_parent( - node: nodes.Routine, parent: nodes.ScopingNode - ) -> Iterator[None]: - '''Temporarily attach a node while its contents are translated. - - A routine handler returns its completed node to the bottom-up - dispatcher, which attaches it later. During translation, temporarily - attaching the Routine to its real parent enables ordinary PSyIR - lexical lookup. Detaching in ``finally`` also restores the dispatcher's - expectation that returned nodes are orphans. - - :param node: PSyIR node requiring temporary host association. - :param parent: real PSyIR parent of ``node``. - - :yields: while ``node`` is attached to ``parent``. - - :raises RuntimeError: if translation unexpectedly reparents ``node``. - ''' - # Attaching a Routine moves its RoutineSymbol into the parent table; - # detaching normally moves it back. Record the exact initial state - # because a forward declaration (e.g. an interface member) may already - # place the same symbol in the parent table too. - symbol = node.symbol - name = symbol.name - symbol_was_in_parent = ( - name in parent.symbol_table and - parent.symbol_table.lookup(name, scope_limit=parent) is symbol) - symbol_was_in_routine = ( - name in node.symbol_table and - node.symbol_table.lookup(name, scope_limit=node) is symbol) - original_interface = symbol.interface - - parent.children.append(node) - try: - yield - finally: - if node.parent is not parent: - raise RuntimeError( - "Temporarily attached PSyIR node was unexpectedly " - "reparented") - node.detach() - - # Restore the symbol ownership seen by the bottom-up dispatcher. - # This is particularly important while processing a list of - # sibling routines: later siblings must still see any forward - # symbol that existed in the host before this temporary attach. - symbol_is_in_parent = ( - name in parent.symbol_table and - parent.symbol_table.lookup( - name, scope_limit=parent) is symbol) - if symbol_was_in_parent and not symbol_is_in_parent: - parent.symbol_table.add(symbol) - elif not symbol_was_in_parent and symbol_is_in_parent: - parent.symbol_table.remove(symbol) - - symbol_is_in_routine = ( - name in node.symbol_table and - node.symbol_table.lookup(name, scope_limit=node) is symbol) - if symbol_was_in_routine and not symbol_is_in_routine: - node.symbol_table.add(symbol) - elif not symbol_was_in_routine and symbol_is_in_routine: - node.symbol_table.remove(symbol) - symbol.interface = original_interface - # --------------------------------------------------------------------- - # Parse-tree navigation and Fortran scope handlers + # Node handlers (and helper functions for those handlers) # --------------------------------------------------------------------- def _translation_unit_handler( @@ -539,29 +460,6 @@ def _translation_unit_handler( ) return file_container - @staticmethod - def _split_extent( - tsnode: 'TSNode' - ) -> tuple[list['TSNode'], list['TSNode'], bool]: - '''Split the children of a bound or range around its first colon. - - Several Fortran grammar nodes represent ``lower:upper`` using the same - child layout. Keeping the token handling here lets callers focus on - their different semantics (declaration shape, array section, CASE - range or allocation shape). - - :param tsnode: tree-sitter node containing an optional colon. - - :returns: children before the colon, children after it, and whether a - colon was present. - ''' - colon = next((idx for idx, child in enumerate(tsnode.children) - if child.type == ":"), None) - if colon is None: - return list(tsnode.children), [], False - return (list(tsnode.children[:colon]), - list(tsnode.children[colon + 1:]), True) - def _module_handler( self, tsnode: 'TSNode' ) -> nodes.Node: @@ -576,21 +474,14 @@ def _module_handler( ''' statement = next_of_type(tsnode, "module_statement") name = next_of_type(statement, "name") - if not any(child.type == "implicit_statement" and - "none" in [item.type for item in child.children] - for child in tsnode.children): - raise NotImplementedError( - "Modules that allow implicit variables are not supported") container = nodes.Container(to_str(name) if name else "") with self._using_scope(container.symbol_table): - visibility_map = self._process_access_statements( - tsnode.children) - unsupported_specs = self._process_specification_part( - tsnode.children) + visibility_map = self._process_access_statements(tsnode.children) + self._process_nodes( + direct_child_of_type(tsnode, self._SPECIFICATION_TYPES)) internal = next_of_type(tsnode, "internal_procedures") - container.children.extend(unsupported_specs) if internal: container.children.extend( self._process_nodes( @@ -603,23 +494,6 @@ def _module_handler( self._apply_visibility(visibility_map) return container - def _internal_procedures_handler( - self, tsnode: 'TSNode' - ) -> nodes.Node: - '''Reject internal subprograms within routines. - - Module handlers process their internal-procedures children directly - because PSyIR Containers can contain Routines. PSyIR Routines cannot - currently contain nested Routines. - - :param tsnode: internal-procedures tree-sitter node. - - :raises NotImplementedError: because nested Routines are unsupported. - ''' - del tsnode - raise NotImplementedError( - "Internal subprograms within a routine are not supported") - def _subroutine_handler( self, tsnode: 'TSNode' ) -> nodes.Node: @@ -699,32 +573,31 @@ def _routine_handler( if not isinstance(parent, nodes.ScopingNode): raise RuntimeError( "A Routine must be translated within a PSyIR scope") - with self._temporary_parent(routine, parent): - with self._using_scope(routine.symbol_table): - visibility_map = self._process_access_statements( - tsnode.children) - unsupported_specs = self._process_specification_part( - tsnode.children) - - args = [routine.symbol_table.lookup(name) - for name in argument_names] - routine.symbol_table.specify_argument_list(args) - if return_name: - routine.return_symbol = routine.symbol_table.lookup( - return_name) - - specification = { - f"{routine_kind}_statement", - f"end_{routine_kind}_statement", - "implicit_statement", "public_statement", - "private_statement" - } - specification.update(self._SPECIFICATION_TYPES) - routine.children.extend(unsupported_specs) - routine.children.extend(self._process_nodes( - [child for child in tsnode.children - if child.type not in specification])) - self._apply_visibility(visibility_map) + with self._using_temporary_scope(parent, routine): + visibility_map = self._process_access_statements( + tsnode.children) + self._process_nodes( + child for child in tsnode.children + if child.type in self._SPECIFICATION_TYPES) + + args = [routine.symbol_table.lookup(name) + for name in argument_names] + routine.symbol_table.specify_argument_list(args) + if return_name: + routine.return_symbol = routine.symbol_table.lookup( + return_name) + + specification = { + f"{routine_kind}_statement", + f"end_{routine_kind}_statement", + "implicit_statement", "public_statement", + "private_statement" + } + specification.update(self._SPECIFICATION_TYPES) + routine.children.extend(self._process_nodes( + [child for child in tsnode.children + if child.type not in specification])) + self._apply_visibility(visibility_map) return routine def _function_return_info( @@ -797,10 +670,6 @@ def _create_routine_symbol( is_elemental="elemental" in qualifiers, visibility=visibility) - # --------------------------------------------------------------------- - # Literals, declarations and datatypes - # --------------------------------------------------------------------- - def _number_literal_handler( self, tsnode: 'TSNode' ) -> nodes.Literal: @@ -1064,7 +933,7 @@ def _datatype_from_type( raise NotImplementedError( "Polymorphic CLASS declarations are not supported") try: - return self._lookup(name) + return self._current_scope.lookup(name) except KeyError: datatype = symbols.DataTypeSymbol( name, symbols.UnresolvedType()) @@ -1146,7 +1015,7 @@ def _kind_symbol( ''' symtab = self._current_scope try: - symbol = self._lookup(name) + symbol = self._current_scope.lookup(name) except KeyError: symbol = symbols.DataSymbol( name, symbols.ScalarType.integer_type(), @@ -1157,6 +1026,24 @@ def _kind_symbol( f"Kind parameter '{name}' is not a data symbol") return symbol + @staticmethod + def _split_extent( + tsnode: 'TSNode' + ) -> tuple[list['TSNode'], list['TSNode'], bool]: + ''' Split the children of a ``lower:upper``-style construct. + + :param tsnode: tree-sitter node containing an optional colon. + + :returns: children before the colon, children after it, and whether a + colon was present. + ''' + colon = next((idx for idx, child in enumerate(tsnode.children) + if child.type == ":"), None) + if colon is None: + return list(tsnode.children), [], False + return (list(tsnode.children[:colon]), + list(tsnode.children[colon + 1:]), True) + def _shape_from_node( self, tsnode: 'TSNode', is_allocatable: bool = False ) -> list: @@ -1241,17 +1128,6 @@ def _apply_visibility( symtab.lookup( name, scope_limit=symtab.node).visibility = visibility - def _lookup(self, name: str): - '''Look up a symbol using the current PSyIR scope hierarchy. - - :param name: symbol name. - - :returns: symbol found in this or an enclosing scope. - - :raises KeyError: if no matching symbol exists. - ''' - return self._current_scope.lookup(name) - # --------------------------------------------------------------------- # Other specification-part symbols # --------------------------------------------------------------------- @@ -1437,7 +1313,7 @@ def _identifier_handler( symtab = self._current_scope name = to_str(tsnode).lower() try: - symbol = self._lookup(name) + symbol = self._current_scope.lookup(name) except KeyError: symbol = symbols.DataSymbol( name, symbols.UnresolvedType(), @@ -1565,7 +1441,7 @@ def _call_expression_handler( name = to_str(name_node).lower() argument_list = next_of_type(tsnode, "argument_list") try: - symbol = self._lookup(name) + symbol = self._current_scope.lookup(name) except KeyError: symbol = None @@ -1708,7 +1584,7 @@ def _structure_reference( "Unsupported structure member array access") members[-1] = (members[-1], arguments) try: - symbol = self._lookup(name) + symbol = self._current_scope.lookup(name) except KeyError: symbol = symbols.DataSymbol( name, symbols.UnresolvedType(), @@ -1791,7 +1667,7 @@ def _expression(self, tsnode: 'TSNode'): :returns: translated PSyIR DataNode. ''' - return self.get_handler(tsnode)(tsnode) + return self._get_handler(tsnode)(tsnode) def _comment_handler( self, tsnode: 'TSNode' @@ -1871,7 +1747,7 @@ def _subroutine_call_handler( symtab = self._current_scope name = to_str(name_node).lower() try: - symbol = self._lookup(name) + symbol = self._current_scope.lookup(name) except KeyError: symbol = symbols.RoutineSymbol(name, datatype=symbols.NoType()) symtab.add(symbol) diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index d41f8b408a..2b078a10d3 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -45,6 +45,10 @@ from psyclone.psyir import nodes as psyir_nodes, symbols as psyir_symbols from psyclone.tests.utilities import min_version_3_10 +# TODO #3416: Skip treesitter tests below 3.10 as they're unsupported by +# treesitter. +pytestmark = min_version_3_10 + def test_constructor(): ''' Test the constructor and its arguments ''' @@ -69,9 +73,6 @@ def test_constructor(): # TODO #3038 Typecheck arguments -# TODO #3416: Skip treesitter tests below 3.10 as they're unsupported by -# treesitter. -@min_version_3_10 def test_generate_parse_tree(tmpdir_factory, caplog): ''' Test that generate_parse_tree returns treesitter trees or appropriate @@ -118,9 +119,6 @@ def test_generate_parse_tree(tmpdir_factory, caplog): assert isinstance(ptree, TSNode) -# TODO #3416: Skip treesitter tests below 3.10 as they're unsupported by -# treesitter. -@min_version_3_10 def test_generate_psyir(): ''' Test that generate_psyir transforms treesitter parse trees to @@ -145,24 +143,6 @@ def test_generate_psyir(): assert root.children[0].children[0].name == "mysub" -@min_version_3_10 -def test_current_scope_restored(): - '''Test that nested parsing does not leave mutable scope state behind.''' - processor = FortranTreeSitterReader() - valid_code = """ - module outer - implicit none - contains - subroutine inner() - end subroutine inner - end module outer - """ - processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) - assert processor._current_scope is None - - -@min_version_3_10 def test_routine_host_association(): '''Test that a contained routine resolves a symbol from its host.''' processor = FortranTreeSitterReader() @@ -186,36 +166,6 @@ def test_routine_host_association(): count, count] -# TODO #3416: Skip treesitter tests below 3.10 as they're unsupported by -# treesitter. -@min_version_3_10 -def test_codeblock_generation_and_messages(): - ''' - Test that NotImplementedErrors are caught and converted to CodeBlocks - with the appropriate associated comment - ''' - processor = FortranTreeSitterReader() - - unsupported_code = """ - module test - contains - subroutine mysub() - end subroutine - end module test - """ - ptree = processor.generate_parse_tree_from_source(unsupported_code) - root = processor.generate_psyir(ptree) - - assert isinstance(root, psyir_nodes.FileContainer) - assert isinstance(root.children[0], psyir_nodes.CodeBlock) - expected = ( - "PSyclone CodeBlock (unsupported code) reason:\n" - "- Modules that allow implicit variables are not supported" - ) - assert root.children[0].preceding_comment == expected - - -@min_version_3_10 def test_subroutine(): ''' Test subroutine nodes. @@ -248,7 +198,6 @@ def test_subroutine(): assert isinstance(rsymbol2, psyir_symbols.RoutineSymbol) -@min_version_3_10 def test_declarations(): ''' Test subroutine nodes. @@ -337,7 +286,6 @@ def test_declarations_arrays_datatypes(shape_string, extent): assert shape == extent -@min_version_3_10 def test_program(): '''Test a main-program unit.''' processor = FortranTreeSitterReader() @@ -354,7 +302,6 @@ def test_program(): assert routine.name == "main" -@min_version_3_10 def test_use_rename(): '''Test a renamed symbol in a USE ONLY statement.''' processor = FortranTreeSitterReader() @@ -374,7 +321,6 @@ def test_use_rename(): assert imported.interface.orig_name == "remote_kind" -@min_version_3_10 def test_symbolic_kind(): '''Test a symbolic kind expression.''' processor = FortranTreeSitterReader() @@ -393,7 +339,6 @@ def test_symbolic_kind(): assert table.lookup("value").datatype.precision.symbol is kind -@min_version_3_10 def test_parameter_declaration(): '''Test a named constant declaration.''' processor = FortranTreeSitterReader() @@ -410,7 +355,6 @@ def test_parameter_declaration(): assert count.initial_value.value == "4" -@min_version_3_10 def test_character_length(): '''Test a character-length specification.''' processor = FortranTreeSitterReader() @@ -426,7 +370,6 @@ def test_character_length(): assert label.datatype.length.value == "12" -@min_version_3_10 def test_allocatable_declaration(): '''Test an allocatable-array declaration.''' processor = FortranTreeSitterReader() @@ -443,7 +386,6 @@ def test_allocatable_declaration(): psyir_symbols.ArrayType.Extent.DEFERRED] -@min_version_3_10 def test_function_result(): '''Test a named function-result symbol.''' processor = FortranTreeSitterReader() @@ -461,7 +403,6 @@ def test_function_result(): psyir_symbols.ScalarType.real_type() -@min_version_3_10 def test_argument_order(): '''Test the order of routine arguments.''' processor = FortranTreeSitterReader() @@ -477,7 +418,6 @@ def test_argument_order(): "first", "second"] -@min_version_3_10 @pytest.mark.parametrize("intent,access", [ ("in", psyir_symbols.ArgumentInterface.Access.READ), ("out", psyir_symbols.ArgumentInterface.Access.WRITE), @@ -497,7 +437,6 @@ def test_argument_intent(intent, access): assert value.interface.access == access -@min_version_3_10 def test_pure_function(): '''Test the PURE function qualifier.''' processor = FortranTreeSitterReader() @@ -512,7 +451,6 @@ def test_pure_function(): assert root.children[0].symbol.is_pure is True -@min_version_3_10 def test_elemental_function(): '''Test the ELEMENTAL function qualifier.''' processor = FortranTreeSitterReader() @@ -527,7 +465,6 @@ def test_elemental_function(): assert root.children[0].symbol.is_elemental is True -@min_version_3_10 def test_unsupported_complex_datatype(): '''Test the unsupported complex datatype.''' processor = FortranTreeSitterReader() @@ -545,7 +482,6 @@ def test_unsupported_complex_datatype(): assert coefficient.datatype.declaration == "complex :: coefficient" -@min_version_3_10 def test_unsupported_pointer_datatype(): '''Test entity-specific unsupported pointer datatypes.''' processor = FortranTreeSitterReader() @@ -564,7 +500,6 @@ def test_unsupported_pointer_datatype(): assert datatype.declaration == f"integer, pointer :: {name}" -@min_version_3_10 def test_unsupported_initialisation_is_entity_specific(): '''Test that one unsupported initializer does not affect its sibling.''' processor = FortranTreeSitterReader() @@ -587,7 +522,6 @@ def test_unsupported_initialisation_is_entity_specific(): assert second.initial_value.value == "2" -@min_version_3_10 def test_save_attribute(): '''Test the SAVE declaration attribute.''' processor = FortranTreeSitterReader() @@ -605,7 +539,6 @@ def test_save_attribute(): assert isinstance(accumulator.interface, psyir_symbols.StaticInterface) -@min_version_3_10 def test_logical_literal(): '''Test a logical literal used as an initial value.''' processor = FortranTreeSitterReader() @@ -621,7 +554,6 @@ def test_logical_literal(): assert enabled.initial_value.value == "true" -@min_version_3_10 def test_derived_type_definition(): '''Test a simple derived-type definition.''' processor = FortranTreeSitterReader() @@ -643,7 +575,6 @@ def test_derived_type_definition(): psyir_symbols.ScalarType.real_type() -@min_version_3_10 def test_derived_type_component_host_association(): '''Test that a component datatype resolves a kind from its host.''' processor = FortranTreeSitterReader() @@ -665,7 +596,6 @@ def test_derived_type_component_host_association(): assert x_type.precision.symbol is wp -@min_version_3_10 def test_structure_reference(): '''Test a scalar structure-component reference.''' processor = FortranTreeSitterReader() @@ -684,7 +614,6 @@ def test_structure_reference(): assert reference.member.name == "x" -@min_version_3_10 def test_array_of_structures_reference(): '''Test an indexed array-of-structures component reference.''' processor = FortranTreeSitterReader() @@ -705,7 +634,6 @@ def test_array_of_structures_reference(): assert reference.member.indices[0].value == "2" -@min_version_3_10 def test_multidimensional_explicit_array_bounds(): '''Test explicit lower bounds in a multidimensional shape.''' processor = FortranTreeSitterReader() @@ -728,7 +656,6 @@ def test_multidimensional_explicit_array_bounds(): assert datatype.shape[1].upper.value == "20" -@min_version_3_10 def test_unary_operation(): '''Test a unary arithmetic operation.''' processor = FortranTreeSitterReader() @@ -744,7 +671,6 @@ def test_unary_operation(): assert expression.operator == psyir_nodes.UnaryOperation.Operator.MINUS -@min_version_3_10 def test_binary_operation(): '''Test a binary arithmetic operation.''' processor = FortranTreeSitterReader() @@ -760,7 +686,6 @@ def test_binary_operation(): assert expression.operator == psyir_nodes.BinaryOperation.Operator.ADD -@min_version_3_10 def test_explicit_array_section(): '''Test an array section with explicit start, stop and step.''' processor = FortranTreeSitterReader() @@ -779,7 +704,6 @@ def test_explicit_array_section(): assert section.step.value == "2" -@min_version_3_10 def test_implicit_array_section_bounds(): '''Test synthesized bounds for a whole-array section.''' processor = FortranTreeSitterReader() @@ -800,7 +724,6 @@ def test_implicit_array_section_bounds(): psyir_nodes.IntrinsicCall.Intrinsic.UBOUND -@min_version_3_10 def test_intrinsic_call(): '''Test an intrinsic function call.''' processor = FortranTreeSitterReader() @@ -817,7 +740,6 @@ def test_intrinsic_call(): assert call.intrinsic == psyir_nodes.IntrinsicCall.Intrinsic.SIN -@min_version_3_10 def test_named_call_argument(): '''Test a named subroutine-call argument.''' processor = FortranTreeSitterReader() @@ -834,7 +756,6 @@ def test_named_call_argument(): assert call.argument_names == [None, "result"] -@min_version_3_10 def test_array_constructor(): '''Test a simple array constructor.''' processor = FortranTreeSitterReader() @@ -852,7 +773,6 @@ def test_array_constructor(): "1.0", "2.0", "3.0"] -@min_version_3_10 def test_pointer_assignment(): '''Test a pointer assignment.''' processor = FortranTreeSitterReader() @@ -870,7 +790,6 @@ def test_pointer_assignment(): assert assignment.is_pointer -@min_version_3_10 def test_nullify_statement(): '''Test a NULLIFY statement.''' processor = FortranTreeSitterReader() @@ -886,7 +805,6 @@ def test_nullify_statement(): assert nullify.intrinsic == psyir_nodes.IntrinsicCall.Intrinsic.NULLIFY -@min_version_3_10 def test_if_construct(): '''Test IF, ELSE IF and ELSE clauses.''' processor = FortranTreeSitterReader() @@ -912,7 +830,6 @@ def test_if_construct(): psyir_nodes.BinaryOperation.Operator.ADD -@min_version_3_10 def test_counted_do_loop(): '''Test a counted DO loop.''' processor = FortranTreeSitterReader() @@ -934,7 +851,6 @@ def test_counted_do_loop(): assert loop.step_expr.value == "2" -@min_version_3_10 def test_do_while_loop(): '''Test a DO WHILE loop.''' processor = FortranTreeSitterReader() @@ -954,7 +870,6 @@ def test_do_while_loop(): psyir_nodes.BinaryOperation.Operator.GT -@min_version_3_10 def test_unconditional_do_loop(): '''Test an unconditional DO loop.''' processor = FortranTreeSitterReader() @@ -972,7 +887,6 @@ def test_unconditional_do_loop(): assert "was_unconditional" in loop.annotations -@min_version_3_10 def test_where_construct(): '''Test a WHERE construct with ELSEWHERE.''' processor = FortranTreeSitterReader() @@ -994,7 +908,6 @@ def test_where_construct(): assert where.else_body.children[0].rhs.value == "0.0" -@min_version_3_10 def test_select_case_construct(): '''Test SELECT CASE lowering.''' processor = FortranTreeSitterReader() @@ -1022,7 +935,6 @@ def test_select_case_construct(): assert second.else_body.children[0].rhs.value == "0" -@min_version_3_10 def test_allocate_statement(): '''Test an ALLOCATE statement.''' processor = FortranTreeSitterReader() @@ -1042,7 +954,6 @@ def test_allocate_statement(): assert allocate.argument_names == [None, "stat"] -@min_version_3_10 def test_deallocate_statement(): '''Test a DEALLOCATE statement.''' processor = FortranTreeSitterReader() @@ -1061,7 +972,6 @@ def test_deallocate_statement(): assert deallocate.argument_names == [None, "stat"] -@min_version_3_10 def test_stop_codeblock(): '''Test the localized fallback for a STOP statement.''' processor = FortranTreeSitterReader() @@ -1077,7 +987,6 @@ def test_stop_codeblock(): assert "Unsupported 'stop_statement'" in codeblock.preceding_comment -@min_version_3_10 def test_default_visibility(): '''Test a module's default visibility.''' processor = FortranTreeSitterReader() @@ -1099,7 +1008,6 @@ def test_default_visibility(): psyir_symbols.Symbol.Visibility.PRIVATE -@min_version_3_10 def test_named_visibility(): '''Test name-specific visibility for a contained routine.''' processor = FortranTreeSitterReader() @@ -1119,7 +1027,6 @@ def test_named_visibility(): assert exposed.visibility == psyir_symbols.Symbol.Visibility.PUBLIC -@min_version_3_10 def test_generic_interface(): '''Test a named generic interface.''' processor = FortranTreeSitterReader() @@ -1140,7 +1047,6 @@ def test_generic_interface(): assert all(info.from_container for info in generic.routines) -@min_version_3_10 def test_ignored_comment(): '''Test that an ignored comment does not create a CodeBlock.''' processor = FortranTreeSitterReader() @@ -1158,7 +1064,6 @@ def test_ignored_comment(): assert isinstance(routine.children[0], psyir_nodes.Assignment) -@min_version_3_10 def test_implied_do_codeblock(): '''Test the localized fallback for an implied-DO array constructor.''' processor = FortranTreeSitterReader() From a959e8073ca782d66f69a553554bb34c7463ed60 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 5 Aug 2026 14:38:35 +0100 Subject: [PATCH 05/23] Rename treesitter frontend support class name --- .../psyir/frontend/fortran_treesitter_reader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 446430708a..384df596e8 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -109,7 +109,7 @@ def next_of_type( @dataclass(frozen=True) -class _CommonDeclAttributes: +class _SharedDeclAttributes: ''' Properties shared by all entities of a fortran declaration (the lhs of ::) @@ -759,7 +759,7 @@ def _variable_declaration_handler( intent = next( (item for item in qualifiers if item.children and item.children[0].type == "intent"), None) - common_attr = _CommonDeclAttributes( + common_attr = _SharedDeclAttributes( base_type, dimension, intent, qualifier_names, frozenset(unsupported), to_str(tsnode).split("::", maxsplit=1)[0].strip()) @@ -770,7 +770,7 @@ def _variable_declaration_handler( self._declare_entity(declarator, common_attr) def _declare_entity( - self, declarator: 'TSNode', common_attr: _CommonDeclAttributes + self, declarator: 'TSNode', common_attr: _SharedDeclAttributes ): '''Translate one entity and add it to the current symbol table. @@ -801,7 +801,7 @@ def _declare_entity( self._add_or_update_datasymbol(declared_symbol) def _declarator_datatype( - self, declarator: 'TSNode', common_attr: _CommonDeclAttributes + self, declarator: 'TSNode', common_attr: _SharedDeclAttributes ): '''Translate one entity's datatype and initial value. @@ -851,7 +851,7 @@ def _declarator_datatype( return datatype, initial_value def _declaration_interface( - self, name: str, common_attr: _CommonDeclAttributes + self, name: str, common_attr: _SharedDeclAttributes ): '''Return the PSyIR interface for one declared entity. From 0ad138ca6625c6613d6ff68184ed34dd726f8a23 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 12 Aug 2026 11:48:03 +0100 Subject: [PATCH 06/23] Introduce treesitter redirection handlers to simplify code --- .../frontend/fortran_treesitter_reader.py | 183 ++++-------------- 1 file changed, 34 insertions(+), 149 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index de043cd2b7..76f095b971 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -178,6 +178,20 @@ class FortranTreeSitterReader(): "inout": symbols.ArgumentInterface.Access.READWRITE, } + # Some tree-sitter node types share a handler. + _HANDLER_REDIRECTIONS = { + "subroutine": "_routine_handler", + "function": "_routine_handler", + "program": "_routine_handler", + "unary_expression": "_operation", + "logical_expression": "_operation", + "relational_expression": "_operation", + "math_expression": "_operation", + "allocate_statement": "_memory_statement", + "deallocate_statement": "_memory_statement", + "nullify_statement": "_memory_statement", + } + # These arguments intentionally mirror the other Fortran reader API. # pylint: disable=too-many-arguments,too-many-positional-arguments def __init__( @@ -214,10 +228,6 @@ def __init__( # disposable instance (but prevents having to deal with the None type) self._current_scope: symbols.SymbolTable = symbols.SymbolTable() - # --------------------------------------------------------------------- - # Public methods - # --------------------------------------------------------------------- - def generate_parse_tree_from_file(self, file_path) -> 'TSNode': ''' Use the provided file to generate a treesitter parsetree. @@ -279,10 +289,6 @@ def generate_psyir(self, parse_tree: 'TSNode') -> nodes.Node: ''' return self._process_nodes(parse_tree)[0] - # --------------------------------------------------------------------- - # Utility methods - # --------------------------------------------------------------------- - @contextmanager def _using_scope( self, symtab: symbols.SymbolTable @@ -405,16 +411,19 @@ def _get_handler(self, tsnode: 'TSNode') -> Callable: :raises NotImplementedError: if the given node type does not have a handler for it. ''' - try: - handler = getattr(self, f"_{tsnode.type}_handler") - except AttributeError: - raise NotImplementedError( - f"Unsupported '{tsnode.type}' tree-sitter node.") from None - return handler + # Some nodes use a common handler + redirection = self._HANDLER_REDIRECTIONS.get(tsnode.type) + if redirection: + return getattr(self, redirection) + + # Otherwise use the handler that matches its name + handler = getattr(self, f"_{tsnode.type}_handler", None) + if handler is not None: + return handler - # --------------------------------------------------------------------- - # Node handlers (and helper functions for those handlers) - # --------------------------------------------------------------------- + # If at this point we still don't have a handler, it is unsupported + raise NotImplementedError( + f"Unsupported '{tsnode.type}' tree-sitter node.") from None def _translation_unit_handler( self, tsnode: 'TSNode' @@ -466,50 +475,15 @@ def _module_handler( self._apply_visibility(visibility_map) return container - def _subroutine_handler( - self, tsnode: 'TSNode' - ) -> nodes.Node: - ''' Handle a treesitter 'subroutine' node. - - :param tsnode: the treesitter node the process. - - :returns: the equivalent PSyIR Node. - ''' - return self._routine_handler(tsnode, "subroutine") - - def _function_handler( - self, tsnode: 'TSNode' - ) -> nodes.Node: - '''Create a PSyIR Routine for a Fortran function. - - :param tsnode: function tree-sitter node. - - :returns: PSyIR Routine representing the function. - ''' - return self._routine_handler(tsnode, "function") - - def _program_handler( - self, tsnode: 'TSNode' - ) -> nodes.Node: - '''Create a PSyIR Routine for a main program. - - :param tsnode: program tree-sitter node. - - :returns: PSyIR Routine representing the program. - ''' - return self._routine_handler(tsnode, "program") - def _routine_handler( - self, tsnode: 'TSNode', routine_kind: str + self, tsnode: 'TSNode' ) -> nodes.Routine: '''Create PSyIR shared by programs, subroutines and functions. :param tsnode: tree-sitter node for the complete program unit. - :param routine_kind: one of ``program``, ``subroutine`` or - ``function``. - :returns: translated PSyIR Routine. ''' + routine_kind = tsnode.type parent_symtab = self._current_scope if parent_symtab is None: raise RuntimeError( @@ -1100,10 +1074,6 @@ def _apply_visibility( symtab.lookup( name, scope_limit=symtab.node).visibility = visibility - # --------------------------------------------------------------------- - # Other specification-part symbols - # --------------------------------------------------------------------- - def _use_statement_handler( self, tsnode: 'TSNode' ) -> None: @@ -1269,10 +1239,6 @@ def _interface_handler( symtab.add(symbols.GenericInterfaceSymbol( name, routines, visibility=symtab.default_visibility)) - # --------------------------------------------------------------------- - # Expressions and references - # --------------------------------------------------------------------- - def _identifier_handler( self, tsnode: 'TSNode' ) -> nodes.Reference: @@ -1311,50 +1277,6 @@ def _parenthesized_expression_handler( "Unexpected parenthesized expression structure") return self._expression(content[0]) - def _unary_expression_handler( - self, tsnode: 'TSNode' - ): - '''Translate a unary arithmetic expression. - - :param tsnode: unary-expression tree-sitter node. - - :returns: PSyIR UnaryOperation. - ''' - return self._operation(tsnode) - - def _logical_expression_handler( - self, tsnode: 'TSNode' - ): - '''Translate a logical expression. - - :param tsnode: logical-expression tree-sitter node. - - :returns: PSyIR operation representing the expression. - ''' - return self._operation(tsnode) - - def _relational_expression_handler( - self, tsnode: 'TSNode' - ): - '''Translate a relational expression. - - :param tsnode: relational-expression tree-sitter node. - - :returns: PSyIR BinaryOperation. - ''' - return self._operation(tsnode) - - def _math_expression_handler( - self, tsnode: 'TSNode' - ): - '''Translate an arithmetic expression. - - :param tsnode: math-expression tree-sitter node. - - :returns: PSyIR BinaryOperation. - ''' - return self._operation(tsnode) - def _operation( self, tsnode: 'TSNode' ): @@ -1659,10 +1581,6 @@ def _comment_handler( return None raise NotImplementedError("Comment preservation is not yet supported") - # --------------------------------------------------------------------- - # Executable statements and control flow - # --------------------------------------------------------------------- - def _assignment_statement_handler( self, tsnode: 'TSNode' ) -> nodes.Assignment: @@ -1993,44 +1911,8 @@ def _case_condition( nodes.BinaryOperation.Operator.OR, result, condition) return result - def _allocate_statement_handler( - self, tsnode: 'TSNode' - ): - '''Translate ALLOCATE using PSyIR's special intrinsic. - - :param tsnode: allocate-statement tree-sitter node. - - :returns: PSyIR ALLOCATE IntrinsicCall. - ''' - return self._memory_statement( - tsnode, nodes.IntrinsicCall.Intrinsic.ALLOCATE) - - def _deallocate_statement_handler( - self, tsnode: 'TSNode' - ): - '''Translate DEALLOCATE using PSyIR's special intrinsic. - - :param tsnode: deallocate-statement tree-sitter node. - - :returns: PSyIR DEALLOCATE IntrinsicCall. - ''' - return self._memory_statement( - tsnode, nodes.IntrinsicCall.Intrinsic.DEALLOCATE) - - def _nullify_statement_handler( - self, tsnode: 'TSNode' - ): - '''Translate NULLIFY using PSyIR's special intrinsic. - - :param tsnode: nullify-statement tree-sitter node. - - :returns: PSyIR NULLIFY IntrinsicCall. - ''' - return self._memory_statement( - tsnode, nodes.IntrinsicCall.Intrinsic.NULLIFY) - def _memory_statement( - self, tsnode: 'TSNode', intrinsic + self, tsnode: 'TSNode' ): '''Translate operands of allocation-related statements. @@ -2038,13 +1920,16 @@ def _memory_statement( Range children retain the requested lower and upper bounds. :param tsnode: allocation-related statement tree-sitter node. - :param intrinsic: ALLOCATE, DEALLOCATE or NULLIFY intrinsic. - :returns: PSyIR IntrinsicCall. :raises NotImplementedError: if an object, bound or option is unsupported. ''' + intrinsic = { + "allocate_statement": nodes.IntrinsicCall.Intrinsic.ALLOCATE, + "deallocate_statement": nodes.IntrinsicCall.Intrinsic.DEALLOCATE, + "nullify_statement": nodes.IntrinsicCall.Intrinsic.NULLIFY, + }[tsnode.type] args = [] for child in tsnode.children: if child.type in ( From a2eb6f846650d8fa5efadb7a96888bdee06acd43 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 12 Aug 2026 12:54:55 +0100 Subject: [PATCH 07/23] Add treesitter support for creating CodeBlock.Structure.STATEMENTs --- .../frontend/fortran_treesitter_reader.py | 21 ++++++----- .../fortran_treesitter_reader/ftr_test.py | 35 ++++++++++++++++--- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 76f095b971..883beea003 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -378,24 +378,25 @@ def _process_nodes( if result is not None: children.append(result) except NotImplementedError as err: - # TODO #3038: Add support for expression codeblocks and - # aggregating contiguous codeblocks into a single one. + # TODO #3038: Aggregate contiguous CodeBlocks. children.append(self._create_codeblock(tsnode, str(err))) return children @staticmethod def _create_codeblock( - tsnode: 'TSNode', reason: str + tsnode: 'TSNode', reason: str, + structure: CodeBlock.Structure = CodeBlock.Structure.STATEMENT ) -> TreeSitterCodeBlock: - '''Create a statement CodeBlock for unsupported valid Fortran. + '''Create a CodeBlock for unsupported valid Fortran. :param tsnode: tree-sitter node containing unsupported Fortran. :param reason: human-readable explanation of the limitation. + :param structure: whether the unsupported code is a statement or an + expression. :returns: CodeBlock retaining the original tree-sitter node. ''' - code_block = TreeSitterCodeBlock( - tsnode, CodeBlock.Structure.STATEMENT) + code_block = TreeSitterCodeBlock(tsnode, structure) code_block.append_preceding_comment( f"PSyclone CodeBlock (unsupported code) reason:\n" f"- {reason}" @@ -1559,9 +1560,13 @@ def _expression(self, tsnode: 'TSNode'): :param tsnode: expression tree-sitter node. - :returns: translated PSyIR DataNode. + :returns: translated PSyIR DataNode or an expression CodeBlock. ''' - return self._get_handler(tsnode)(tsnode) + try: + return self._get_handler(tsnode)(tsnode) + except NotImplementedError as err: + return self._create_codeblock( + tsnode, str(err), CodeBlock.Structure.EXPRESSION) def _comment_handler( self, tsnode: 'TSNode' diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index 2c65a7d868..4565a4062c 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -485,10 +485,10 @@ def test_unsupported_initialisation_is_entity_specific(): table = root.children[0].symbol_table first = table.lookup("first") second = table.lookup("second") - assert isinstance( - first.datatype, psyir_symbols.UnsupportedFortranType) - assert first.datatype.declaration == \ - "integer :: first = [(i, i=1,2)]" + assert isinstance(first.datatype, psyir_symbols.ScalarType) + assert isinstance(first.initial_value, psyir_nodes.CodeBlock) + assert first.initial_value.structure == \ + psyir_nodes.CodeBlock.Structure.EXPRESSION assert isinstance(second.datatype, psyir_symbols.ScalarType) assert second.initial_value.value == "2" @@ -657,6 +657,28 @@ def test_binary_operation(): assert expression.operator == psyir_nodes.BinaryOperation.Operator.ADD +def test_unsupported_expression_codeblock(): + '''An unsupported expression becomes an expression CodeBlock without + replacing its enclosing statement. + ''' + processor = FortranTreeSitterReader() + valid_code = ''' + subroutine concatenate(value) + character(*) :: value + value = value // "suffix" + end subroutine concatenate + ''' + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + assignment = root.children[0].children[0] + assert isinstance(assignment, psyir_nodes.Assignment) + assert isinstance(assignment.rhs, psyir_nodes.CodeBlock) + assert assignment.rhs.structure == \ + psyir_nodes.CodeBlock.Structure.EXPRESSION + assert "Unsupported 'concatenation_expression'" in \ + assignment.rhs.preceding_comment + + def test_explicit_array_section(): '''Test an array section with explicit start, stop and step.''' processor = FortranTreeSitterReader() @@ -1047,7 +1069,10 @@ def test_implied_do_codeblock(): """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - codeblock = root.children[0].children[0] + assignment = root.children[0].children[0] + assert isinstance(assignment, psyir_nodes.Assignment) + codeblock = assignment.rhs assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert codeblock.structure == psyir_nodes.CodeBlock.Structure.EXPRESSION assert ("Array constructors with implied-DO loops are not supported" in codeblock.preceding_comment) From 1e12e356e33bbd2715ec18d1ed01d1d57b1d39ae Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 12 Aug 2026 13:14:09 +0100 Subject: [PATCH 08/23] In treesitter add a process_node expect argument to validate the resulting nodes --- .../frontend/fortran_treesitter_reader.py | 228 ++++++++++++------ 1 file changed, 148 insertions(+), 80 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 883beea003..56a1ac4be9 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -10,10 +10,12 @@ import codecs from contextlib import contextmanager from dataclasses import dataclass +from enum import Enum, auto import logging from typing import Callable, Iterable, Optional, TYPE_CHECKING, Union from collections.abc import Generator, Container +from psyclone.errors import InternalError from psyclone.psyir import nodes, symbols from psyclone.psyir.nodes.codeblock import TreeSitterCodeBlock, CodeBlock @@ -101,6 +103,15 @@ class _SharedDeclAttributes: prefix: str +class _NodeExpectation(Enum): + '''Expected result shape when processing tree-sitter nodes.''' + + LIST = auto() + NONE = auto() + ONE = auto() + EXPRESSION = auto() + + class FortranTreeSitterReader(): ''' Processes the TreeSitter parse_tree and converts it to PSyIR nodes. @@ -287,7 +298,7 @@ def generate_psyir(self, parse_tree: 'TSNode') -> nodes.Node: :returns: the equivalent PSyIR Node. ''' - return self._process_nodes(parse_tree)[0] + return self._process_nodes(parse_tree, _NodeExpectation.ONE) @contextmanager def _using_scope( @@ -358,16 +369,17 @@ def _using_temporary_scope( def _process_nodes( self, tsnodes: Union["TSNode", Iterable["TSNode"]], - ) -> list[nodes.Node]: + expect: _NodeExpectation, + ) -> list[nodes.Node] | nodes.Node | None: ''' This is the tsnodes handler dispatcher. Unsupported syntax is deliberately caught here rather than in individual handlers so that continuous unsupported nodes are placed in a single CodeBlock. :param tsnodes: one tree-sitter node or an iterable of nodes. + :param expect: expected number and kind of result nodes. :returns: PSyIR nodes produced from the supplied tree-sitter nodes. - :rtype: list[:py:class:`psyclone.psyir.nodes.Node`] ''' list_of_nodes = tsnodes if isinstance(tsnodes, Iterable) else [tsnodes] children = [] @@ -379,7 +391,29 @@ def _process_nodes( children.append(result) except NotImplementedError as err: # TODO #3038: Aggregate contiguous CodeBlocks. - children.append(self._create_codeblock(tsnode, str(err))) + structure = (CodeBlock.Structure.EXPRESSION + if expect is _NodeExpectation.EXPRESSION + else CodeBlock.Structure.STATEMENT) + children.append( + self._create_codeblock(tsnode, str(err), structure)) + + # Validate that the parsed nodes match the expectations of the caller + if expect in (_NodeExpectation.ONE, _NodeExpectation.EXPRESSION): + if len(children) != 1: + raise InternalError( + f"Only one node was expected in this location but got:\n" + f"{children}" + ) + return children[0] + if expect is _NodeExpectation.NONE: + if len(children) != 0: + raise InternalError( + f"No node was expected in this location but got:\n" + f"{children}" + ) + return None + if expect is not _NodeExpectation.LIST: + raise InternalError(f"Unsupported node expectation '{expect}'") return children @staticmethod @@ -438,7 +472,7 @@ def _translation_unit_handler( file_container = nodes.FileContainer("") with self._using_scope(file_container.symbol_table): file_container.children.extend( - self._process_nodes(tsnode.children) + self._process_nodes(tsnode.children, _NodeExpectation.LIST) ) return file_container @@ -461,18 +495,21 @@ def _module_handler( with self._using_scope(container.symbol_table): visibility_map = self._process_access_statements(tsnode.children) self._process_nodes( - direct_child_of_type(tsnode, self._SPECIFICATION_TYPES)) + direct_child_of_type(tsnode, self._SPECIFICATION_TYPES), + _NodeExpectation.NONE) internal = next_of_type(tsnode, "internal_procedures") if internal: container.children.extend( self._process_nodes( [child for child in internal.children - if child.type != "contains_statement"])) + if child.type != "contains_statement"], + _NodeExpectation.LIST)) container.children.extend(self._process_nodes( [child for child in tsnode.children - if child.type not in self._MODULE_NON_EXECUTABLE_TYPES])) + if child.type not in self._MODULE_NON_EXECUTABLE_TYPES], + _NodeExpectation.LIST)) self._apply_visibility(visibility_map) return container @@ -487,7 +524,7 @@ def _routine_handler( routine_kind = tsnode.type parent_symtab = self._current_scope if parent_symtab is None: - raise RuntimeError( + raise InternalError( "A Routine must be translated within a current scope") statement = next_of_type(tsnode, f"{routine_kind}_statement") name_node = next_of_type(statement, "name") @@ -518,14 +555,15 @@ def _routine_handler( parent = parent_symtab.node if not isinstance(parent, nodes.ScopingNode): - raise RuntimeError( + raise InternalError( "A Routine must be translated within a PSyIR scope") with self._using_temporary_scope(parent, routine): visibility_map = self._process_access_statements( tsnode.children) self._process_nodes( - child for child in tsnode.children - if child.type in self._SPECIFICATION_TYPES) + (child for child in tsnode.children + if child.type in self._SPECIFICATION_TYPES), + _NodeExpectation.NONE) args = [routine.symbol_table.lookup(name) for name in argument_names] @@ -543,7 +581,7 @@ def _routine_handler( specification.update(self._SPECIFICATION_TYPES) routine.children.extend(self._process_nodes( [child for child in tsnode.children - if child.type not in specification])) + if child.type not in specification], _NodeExpectation.LIST)) self._apply_visibility(visibility_map) return routine @@ -594,7 +632,7 @@ def _create_routine_symbol( ''' parent_symtab = self._current_scope if parent_symtab is None: - raise RuntimeError( + raise InternalError( "A RoutineSymbol must be created within a current scope") qualifiers = { to_str(child).lower() for child in statement.children @@ -769,7 +807,8 @@ def _declarator_datatype( if child.type not in ("identifier", "=")] if expressions: try: - initial_value = self._expression(expressions[-1]) + initial_value = self._process_nodes( + expressions[-1], _NodeExpectation.EXPRESSION) except NotImplementedError: is_unsupported = True @@ -911,11 +950,13 @@ def _datatype_from_type( key = to_str(value.children[0]).lower() value = value.children[-1] if key == "len": - length = self._expression(value) + length = self._process_nodes( + value, _NodeExpectation.EXPRESSION) else: precision = self._precision(value) elif intrinsic == "character": - length = self._expression(value) + length = self._process_nodes( + value, _NodeExpectation.EXPRESSION) else: precision = self._precision(value) return symbols.ScalarType(mapping[intrinsic], precision, length) @@ -929,7 +970,7 @@ def _precision(self, tsnode: 'TSNode'): :raises NotImplementedError: if the kind expression is unsupported. ''' - expr = self._expression(tsnode) + expr = self._process_nodes(tsnode, _NodeExpectation.EXPRESSION) if isinstance(expr, nodes.Literal) and expr.value.isdigit(): if expr.value == "4": return symbols.ScalarType.Precision.SINGLE @@ -1010,7 +1051,8 @@ def _shape_from_node( if child.type == "extent_specifier": before, after, has_colon = self._split_extent(child) if not has_colon: - result.append(self._expression(before[0])) + result.append(self._process_nodes( + before[0], _NodeExpectation.EXPRESSION)) continue if not before and not after: result.append(symbols.ArrayType.Extent.DEFERRED @@ -1018,15 +1060,20 @@ def _shape_from_node( symbols.ArrayType.Extent.ATTRIBUTE) elif before and not after: result.append( - (self._expression(before[0]), + (self._process_nodes( + before[0], _NodeExpectation.EXPRESSION), symbols.ArrayType.Extent.ATTRIBUTE)) elif after and not before: - result.append(self._expression(after[0])) + result.append(self._process_nodes( + after[0], _NodeExpectation.EXPRESSION)) else: - result.append((self._expression(before[0]), - self._expression(after[0]))) + result.append((self._process_nodes( + before[0], _NodeExpectation.EXPRESSION), + self._process_nodes( + after[0], _NodeExpectation.EXPRESSION))) else: - result.append(self._expression(child)) + result.append(self._process_nodes( + child, _NodeExpectation.EXPRESSION)) return result def _process_access_statements( @@ -1163,7 +1210,7 @@ def _derived_type_definition_handler( if not unsupported: parent = symtab.node if not isinstance(parent, nodes.ScopingNode): - raise RuntimeError( + raise InternalError( "A derived type must be translated within a PSyIR scope") with self._using_temporary_scope(parent): visibility_map = self._process_access_statements( @@ -1276,7 +1323,8 @@ def _parenthesized_expression_handler( if len(content) != 1: raise NotImplementedError( "Unexpected parenthesized expression structure") - return self._expression(content[0]) + return self._process_nodes( + content[0], _NodeExpectation.EXPRESSION) def _operation( self, tsnode: 'TSNode' @@ -1297,7 +1345,8 @@ def _operation( f"Unsupported unary operator '{operator}'") return nodes.UnaryOperation.create( self._UNARY_OPERATORS[operator], - self._expression(tsnode.children[1])) + self._process_nodes( + tsnode.children[1], _NodeExpectation.EXPRESSION)) if len(tsnode.children) == 3: operator = to_str(tsnode.children[1]).lower() if operator not in self._BINARY_OPERATORS: @@ -1305,8 +1354,10 @@ def _operation( f"Unsupported binary operator '{operator}'") return nodes.BinaryOperation.create( self._BINARY_OPERATORS[operator], - self._expression(tsnode.children[0]), - self._expression(tsnode.children[2])) + self._process_nodes( + tsnode.children[0], _NodeExpectation.EXPRESSION), + self._process_nodes( + tsnode.children[2], _NodeExpectation.EXPRESSION)) raise NotImplementedError("Unexpected operation structure") def _call_expression_handler( @@ -1397,7 +1448,8 @@ def _arguments( dimension += 1 if child.type == "keyword_argument": key = to_str(child.children[0]) - result.append((key, self._expression(child.children[-1]))) + result.append((key, self._process_nodes( + child.children[-1], _NodeExpectation.EXPRESSION))) elif child.type == "extent_specifier": if array_symbol is None: raise NotImplementedError( @@ -1405,7 +1457,8 @@ def _arguments( result.append(self._range( child, array_symbol, dimension)) else: - result.append(self._expression(child)) + result.append(self._process_nodes( + child, _NodeExpectation.EXPRESSION)) return result def _range( @@ -1431,15 +1484,18 @@ def _range( after = [child for child in after if child.type != ":"] dim = nodes.Literal(str(dimension), symbols.ScalarType.integer_type()) - start = (self._expression(before[0]) if before else + start = (self._process_nodes( + before[0], _NodeExpectation.EXPRESSION) if before else nodes.IntrinsicCall.create( nodes.IntrinsicCall.Intrinsic.LBOUND, [nodes.Reference(symbol), ("dim", dim.copy())])) - stop = (self._expression(after[0]) if after else + stop = (self._process_nodes( + after[0], _NodeExpectation.EXPRESSION) if after else nodes.IntrinsicCall.create( nodes.IntrinsicCall.Intrinsic.UBOUND, [nodes.Reference(symbol), ("dim", dim.copy())])) - step = (self._expression(after[1]) + step = (self._process_nodes( + after[1], _NodeExpectation.EXPRESSION) if len(after) > 1 else None) return nodes.Range.create(start, stop, step) @@ -1550,24 +1606,11 @@ def _array_literal_handler( if next_of_type(tsnode, "implied_do_loop_expression"): raise NotImplementedError( "Array constructors with implied-DO loops are not supported") - elems = [self._expression(child) for child in tsnode.children + elems = [self._process_nodes(child, _NodeExpectation.EXPRESSION) + for child in tsnode.children if child.type not in ("[", "]", "(/", "/)", ",")] return nodes.ArrayConstructor.create(elems) - def _expression(self, tsnode: 'TSNode'): - '''Translate one expression, allowing failures to reach its - statement. - - :param tsnode: expression tree-sitter node. - - :returns: translated PSyIR DataNode or an expression CodeBlock. - ''' - try: - return self._get_handler(tsnode)(tsnode) - except NotImplementedError as err: - return self._create_codeblock( - tsnode, str(err), CodeBlock.Structure.EXPRESSION) - def _comment_handler( self, tsnode: 'TSNode' ) -> None: @@ -1600,8 +1643,10 @@ def _assignment_statement_handler( if len(tsnode.children) != 3: raise NotImplementedError("Unexpected assignment structure") return nodes.Assignment.create( - self._expression(tsnode.children[0]), - self._expression(tsnode.children[2])) + self._process_nodes( + tsnode.children[0], _NodeExpectation.EXPRESSION), + self._process_nodes( + tsnode.children[2], _NodeExpectation.EXPRESSION)) def _pointer_association_statement_handler( self, tsnode: 'TSNode' @@ -1619,8 +1664,10 @@ def _pointer_association_statement_handler( "Pointer assignment with bounds remapping is not supported") assignment = nodes.Assignment(is_pointer=True) assignment.children = [ - self._expression(tsnode.children[0]), - self._expression(tsnode.children[2])] + self._process_nodes( + tsnode.children[0], _NodeExpectation.EXPRESSION), + self._process_nodes( + tsnode.children[2], _NodeExpectation.EXPRESSION)] return assignment def _subroutine_call_handler( @@ -1699,16 +1746,17 @@ def _if_statement_handler( annotations = [] if not next_of_type(tsnode, "end_if_statement"): annotations.append("was_single_stmt") - if_body = self._process_nodes(body_nodes) + if_body = self._process_nodes(body_nodes, _NodeExpectation.LIST) else_body = None if else_clause: else_body = self._process_nodes( [child for child in else_clause.children - if child.type != "else"]) + if child.type != "else"], _NodeExpectation.LIST) for else_if in reversed(else_ifs): else_body = [self._if_clause(else_if, else_body)] result = nodes.IfBlock.create( - self._expression(condition_node), if_body, else_body) + self._process_nodes(condition_node, _NodeExpectation.EXPRESSION), + if_body, else_body) result.annotations.extend(annotations) return result @@ -1728,7 +1776,7 @@ def _if_clause( "else_clause", "elseif_clause"} body = self._process_nodes( [child for child in tsnode.children - if child.type not in structural]) + if child.type not in structural], _NodeExpectation.LIST) trailing = next_of_type(tsnode, "elseif_clause") otherwise = ( [self._if_clause(trailing, final_else)] if trailing else @@ -1736,9 +1784,11 @@ def _if_clause( [child for child in (next_of_type(tsnode, "else_clause").children if next_of_type(tsnode, "else_clause") else []) - if child.type != "else"]) or final_else) + if child.type != "else"], _NodeExpectation.LIST) + or final_else) result = nodes.IfBlock.create( - self._expression(condition), body, otherwise) + self._process_nodes(condition, _NodeExpectation.EXPRESSION), + body, otherwise) result.annotations.append("was_elseif") return result @@ -1759,7 +1809,8 @@ def _do_loop_handler( body = self._process_nodes( [child for child in tsnode.children if child.type not in ("do_statement", - "end_do_loop_statement")]) + "end_do_loop_statement")], + _NodeExpectation.LIST) if control: parts = [child for child in control.children if child.type not in ("=", ",")] @@ -1777,17 +1828,21 @@ def _do_loop_handler( else: raise NotImplementedError( "A DO variable must be a scalar integer") - step = (self._expression(parts[3]) if len(parts) == 4 + step = (self._process_nodes( + parts[3], _NodeExpectation.EXPRESSION) if len(parts) == 4 else nodes.Literal( "1", symbols.ScalarType.integer_type())) return nodes.Loop.create( - variable, self._expression(parts[1]), - self._expression(parts[2]), step, body) + variable, + self._process_nodes(parts[1], _NodeExpectation.EXPRESSION), + self._process_nodes(parts[2], _NodeExpectation.EXPRESSION), + step, body) if while_node: condition = next_of_type( while_node, "parenthesized_expression") return nodes.WhileLoop.create( - self._expression(condition), body) + self._process_nodes(condition, _NodeExpectation.EXPRESSION), + body) result = nodes.WhileLoop.create( nodes.Literal("true", symbols.ScalarType.boolean_type()), body) result.annotations.append("was_unconditional") @@ -1807,15 +1862,16 @@ def _where_statement_handler( "elsewhere_clause", "end_where_statement"} body = self._process_nodes( [child for child in tsnode.children - if child.type not in structural]) + if child.type not in structural], _NodeExpectation.LIST) elsewhere = next_of_type(tsnode, "elsewhere_clause") other = ( self._process_nodes( [child for child in elsewhere.children - if child.type != "elsewhere"]) + if child.type != "elsewhere"], _NodeExpectation.LIST) if elsewhere else None) result = nodes.IfBlock.create( - self._expression(condition), body, other) + self._process_nodes(condition, _NodeExpectation.EXPRESSION), + body, other) result.annotations.extend( ["was_where"] if next_of_type( tsnode, "end_where_statement") else @@ -1839,12 +1895,14 @@ def _select_case_statement_handler( selector_node = next_of_type( next_of_type(tsnode, "selector"), "identifier") if selector_node is None: - selector = self._expression( + selector = self._process_nodes( [child for child in next_of_type(tsnode, "selector").children - if child.type not in ("(", ")")][0]) + if child.type not in ("(", ")")][0], + _NodeExpectation.EXPRESSION) else: - selector = self._expression(selector_node) + selector = self._process_nodes( + selector_node, _NodeExpectation.EXPRESSION) cases = list(direct_child_of_type(tsnode, "case_statement")) default_body = None normal = [] @@ -1852,7 +1910,8 @@ def _select_case_statement_handler( if next_of_type(case, "default"): default_body = self._process_nodes( [child for child in case.children - if child.type not in ("case", "default")]) + if child.type not in ("case", "default")], + _NodeExpectation.LIST) else: values = next_of_type(case, "case_value_range_list") if values is None: @@ -1861,7 +1920,7 @@ def _select_case_statement_handler( structural = {"case", "(", ")", "case_value_range_list"} body = self._process_nodes( [child for child in case.children - if child.type not in structural]) + if child.type not in structural], _NodeExpectation.LIST) normal.append((values, body)) current = default_body for values, body in reversed(normal): @@ -1896,11 +1955,13 @@ def _case_condition( if before: parts.append(nodes.BinaryOperation.create( nodes.BinaryOperation.Operator.GE, selector.copy(), - self._expression(before[0]))) + self._process_nodes( + before[0], _NodeExpectation.EXPRESSION))) if after: parts.append(nodes.BinaryOperation.create( nodes.BinaryOperation.Operator.LE, selector.copy(), - self._expression(after[0]))) + self._process_nodes( + after[0], _NodeExpectation.EXPRESSION))) condition = parts[0] if len(parts) == 1 else ( nodes.BinaryOperation.create( nodes.BinaryOperation.Operator.AND, @@ -1908,7 +1969,8 @@ def _case_condition( else: condition = nodes.BinaryOperation.create( nodes.BinaryOperation.Operator.EQ, selector.copy(), - self._expression(child)) + self._process_nodes( + child, _NodeExpectation.EXPRESSION)) conditions.append(condition) result = conditions[0] for condition in conditions[1:]: @@ -1942,11 +2004,14 @@ def _memory_statement( continue if child.type == "keyword_argument": args.append((to_str(child.children[0]), - self._expression(child.children[-1]))) + self._process_nodes( + child.children[-1], + _NodeExpectation.EXPRESSION))) elif child.type == "sized_allocation": args.append(self._allocation_reference(child)) else: - args.append(self._expression(child)) + args.append(self._process_nodes( + child, _NodeExpectation.EXPRESSION)) try: return nodes.IntrinsicCall.create(intrinsic, args) except (TypeError, ValueError): @@ -1999,15 +2064,18 @@ def _allocation_extent( "1", symbols.ScalarType.integer_type()) if tsnode.type != "extent_specifier": return nodes.Range.create( - lower, self._expression(tsnode)) + lower, self._process_nodes( + tsnode, _NodeExpectation.EXPRESSION)) before, after, has_colon = self._split_extent(tsnode) if not has_colon: raise NotImplementedError("Malformed allocation bound") if before: - lower = self._expression(before[0]) + lower = self._process_nodes( + before[0], _NodeExpectation.EXPRESSION) if not after: raise NotImplementedError( "Allocation upper bound is required") return nodes.Range.create( - lower, self._expression(after[0])) + lower, self._process_nodes( + after[0], _NodeExpectation.EXPRESSION)) From 7544ca26374bc895168cdc9f67b4337df9cb5885 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 12 Aug 2026 13:59:35 +0100 Subject: [PATCH 09/23] Remove treesitter unneeded class lists of node types --- .../frontend/fortran_treesitter_reader.py | 57 +++++++------------ 1 file changed, 20 insertions(+), 37 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 56a1ac4be9..2d5a192b25 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -138,21 +138,6 @@ class FortranTreeSitterReader(): :param conditional_openmp: whether to parse conditional OpenMP statements. ''' - # These nodes belong to a Fortran specification part and update a symbol - # table rather than producing executable PSyIR children. - _SPECIFICATION_TYPES = { - "use_statement", "variable_declaration", "derived_type_definition", - "interface" - } - - # Punctuation and grammar-only nodes are listed explicitly at each scope - # boundary. This makes it clear which tree-sitter children are consumed by - # the scope handler and prevents them from becoming accidental CodeBlocks. - _MODULE_NON_EXECUTABLE_TYPES = _SPECIFICATION_TYPES.union({ - "module_statement", "end_module_statement", "implicit_statement", - "internal_procedures", "public_statement", "private_statement" - }) - # Centralising these maps documents the supported Fortran spellings and # avoids recreating identical dictionaries for every parsed operation. _UNARY_OPERATORS = { @@ -494,9 +479,17 @@ def _module_handler( with self._using_scope(container.symbol_table): visibility_map = self._process_access_statements(tsnode.children) - self._process_nodes( - direct_child_of_type(tsnode, self._SPECIFICATION_TYPES), - _NodeExpectation.NONE) + + # Specification nodes precede executable nodes, so processing in + # source order ensures that symbols are declared before use. + structural = { + "module_statement", "end_module_statement", + "implicit_statement", "internal_procedures", + "public_statement", "private_statement" + } + container.children.extend(self._process_nodes( + [child for child in tsnode.children + if child.type not in structural], _NodeExpectation.LIST)) internal = next_of_type(tsnode, "internal_procedures") if internal: @@ -505,11 +498,6 @@ def _module_handler( [child for child in internal.children if child.type != "contains_statement"], _NodeExpectation.LIST)) - - container.children.extend(self._process_nodes( - [child for child in tsnode.children - if child.type not in self._MODULE_NON_EXECUTABLE_TYPES], - _NodeExpectation.LIST)) self._apply_visibility(visibility_map) return container @@ -560,28 +548,23 @@ def _routine_handler( with self._using_temporary_scope(parent, routine): visibility_map = self._process_access_statements( tsnode.children) - self._process_nodes( - (child for child in tsnode.children - if child.type in self._SPECIFICATION_TYPES), - _NodeExpectation.NONE) - - args = [routine.symbol_table.lookup(name) - for name in argument_names] - routine.symbol_table.specify_argument_list(args) - if return_name: - routine.return_symbol = routine.symbol_table.lookup( - return_name) - specification = { + structural = { f"{routine_kind}_statement", f"end_{routine_kind}_statement", "implicit_statement", "public_statement", "private_statement" } - specification.update(self._SPECIFICATION_TYPES) routine.children.extend(self._process_nodes( [child for child in tsnode.children - if child.type not in specification], _NodeExpectation.LIST)) + if child.type not in structural], _NodeExpectation.LIST)) + + args = [routine.symbol_table.lookup(name) + for name in argument_names] + routine.symbol_table.specify_argument_list(args) + if return_name: + routine.return_symbol = routine.symbol_table.lookup( + return_name) self._apply_visibility(visibility_map) return routine From 32cd25904a1b24affaa6b09a54a318e90133c64b Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 12 Aug 2026 15:08:30 +0100 Subject: [PATCH 10/23] Improve treesitter comments and store CommonDeclarationAttr as processed PSyir --- .../frontend/fortran_treesitter_reader.py | 159 +++++++++++------- .../fortran_treesitter_reader/ftr_test.py | 54 ++++++ 2 files changed, 151 insertions(+), 62 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 2d5a192b25..0623771a4e 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -5,7 +5,19 @@ # See the full LICENSE file in the project root for details. # ----------------------------------------------------------------------------- -''' PSyIR TreeSitter Fortran reader ''' +''' + +PSyIR fronted to ingest Fortran using the TreeSitter parse generator. + +The structure of the expected fortran parse tree can be found in the +'rules' section of: +https://github.com/stadelmanma/tree-sitter-fortran/blob/master/grammar.js + +To interpret the rules use: +https://tree-sitter.github.io/tree-sitter/creating-parsers/ +2-the-grammar-dsl.html + +''' import codecs from contextlib import contextmanager @@ -21,7 +33,8 @@ if TYPE_CHECKING: # Purposely inside typechecking because at runtime we want to lazily - # import the parser (only if it is actually used) + # import the parser only when it is actually used (import inside + # generate_parse_tree_from_source) from tree_sitter import Node as TSNode @@ -83,47 +96,57 @@ def next_of_type( @dataclass(frozen=True) -class _SharedDeclAttributes: +class _CommonDeclAttributes: ''' Properties shared by all entities of a fortran declaration (the lhs - of ::) + of ::). - :param base_type: common PSyIR datatype, or ``None`` if unsupported. - :param dimension: common DIMENSION argument list, if present. - :param intent: common INTENT qualifier, if present. + :param datatype: common PSyIR datatype or ``None`` if unsupported. + :param intent: common PSyIR argument access. :param qualifiers: names of all declaration qualifiers. :param unsupported: qualifiers not represented directly in PSyIR. - :param prefix: declaration text preceding ``::``. + :param prefix: string preceding ``::`` (this is needed in case the + entities end up as UnsupportedFortranType). ''' - base_type: object - dimension: Optional['TSNode'] - intent: Optional['TSNode'] + datatype: Union[symbols.DataType, symbols.DataTypeSymbol, None] + intent: symbols.ArgumentInterface.Access qualifiers: frozenset[str] unsupported: frozenset[str] prefix: str class _NodeExpectation(Enum): - '''Expected result shape when processing tree-sitter nodes.''' + '''Expected result of processing tree-sitter nodes.''' + #: Expect a list (of zero, one or multiple) PSyIR nodes LIST = auto() + #: Expect no result node (e.g. when processing a declaration) NONE = auto() + #: Expect exactly one node ONE = auto() + #: Expect exactly one node that is a DataNode EXPRESSION = auto() class FortranTreeSitterReader(): - ''' - Processes the TreeSitter parse_tree and converts it to PSyIR nodes. - Unsupported declarations retain their source in UnsupportedFortranType - while unsupported executable statements become TreeSitterCodeBlocks. + ''' Generate TreeSitter parse_trees and convert them to PSyIR nodes. + + The Reader works mostly by traversing the parse_tree with recursive + calls to the _process_nodes dispatching method. This method calls + the appropriate node handler for the given fparser node and creates + CodeBlocks if the handlers fail. - The structure of the expected fortran parse tree can be found in the - 'rules' section of: - https://github.com/stadelmanma/tree-sitter-fortran/blob/master/grammar.js - To interpret the rules use: - https://tree-sitter.github.io/tree-sitter/creating-parsers/ - 2-the-grammar-dsl.html + When Fortran scopes are found these are managed by the _using_scope + context manager, this utility keeps a stack of nested scopes with a + global reference to the top of the stack, this reference can be used + everywhere to access the current scope. + + Note that the implementation is incomplete, its main limitations are + that: + - the Reader parameters are ignored. + - fparser is still not isolated, so the performance penalty of importing + fparser is still paid when using treesitter. + - the coverage of Fortran supported is more limited than in fparser. :param ignore_directives: Whether directives should be ignored or not (default True). Currently ignored. @@ -138,8 +161,6 @@ class FortranTreeSitterReader(): :param conditional_openmp: whether to parse conditional OpenMP statements. ''' - # Centralising these maps documents the supported Fortran spellings and - # avoids recreating identical dictionaries for every parsed operation. _UNARY_OPERATORS = { "+": nodes.UnaryOperation.Operator.PLUS, "-": nodes.UnaryOperation.Operator.MINUS, @@ -174,7 +195,7 @@ class FortranTreeSitterReader(): "inout": symbols.ArgumentInterface.Access.READWRITE, } - # Some tree-sitter node types share a handler. + # Some tree-sitter node types share the same handler. _HANDLER_REDIRECTIONS = { "subroutine": "_routine_handler", "function": "_routine_handler", @@ -199,17 +220,7 @@ def __init__( free_form: bool = True, conditional_openmp: bool = True, ): - '''Create a Fortran tree-sitter reader. - - :param ignore_directives: whether directives are ignored. - :param last_comments_as_codeblocks: whether trailing comments in a - block are retained as CodeBlocks. - :param resolve_modules: whether imported modules are resolved. - :param ignore_comments: whether comments are ignored. - :param free_form: whether source is parsed as free-form Fortran. - :param conditional_openmp: whether conditional OpenMP statements are - parsed. - ''' + ''' Create a Fortran tree-sitter reader. ''' # TODO #3038 Arguments are currently not used nor typechecked, but if # we decide this is the common reader interface, this can be done in a # super class instead of duplicate it here. @@ -359,7 +370,7 @@ def _process_nodes( ''' This is the tsnodes handler dispatcher. Unsupported syntax is deliberately caught here rather than in individual handlers so that - continuous unsupported nodes are placed in a single CodeBlock. + continuous unsupported nodes can be placed in a single CodeBlock. :param tsnodes: one tree-sitter node or an iterable of nodes. :param expect: expected number and kind of result nodes. @@ -387,18 +398,25 @@ def _process_nodes( if len(children) != 1: raise InternalError( f"Only one node was expected in this location but got:\n" - f"{children}" + f"{[type(c).__name__ for c in children]}" ) + if expect is _NodeExpectation.EXPRESSION: + if not isinstance(children[0], nodes.DataNode): + raise InternalError( + f"A DataNode was expected in this location but got: " + f"{type(children[0]).__name__}" + ) return children[0] if expect is _NodeExpectation.NONE: if len(children) != 0: raise InternalError( f"No node was expected in this location but got:\n" - f"{children}" + f"{[type(c).__name__ for c in children]}" ) return None if expect is not _NodeExpectation.LIST: - raise InternalError(f"Unsupported node expectation '{expect}'") + raise InternalError( + f"Unsupported node expectation '{expect}'") return children @staticmethod @@ -717,18 +735,32 @@ def _variable_declaration_handler( "asynchronous", "contiguous" }) try: - base_type = self._datatype_from_type(type_node) + datatype = self._datatype_from_type(type_node) except (NotImplementedError, KeyError, TypeError): - base_type = None + datatype = None dimension = next( (next_of_type(item, "argument_list") for item in qualifiers if item.children and item.children[0].type == "dimension"), None) - intent = next( + is_allocatable = "allocatable" in qualifier_names + if datatype and dimension: + try: + shape = self._shape_from_node(dimension, is_allocatable) + datatype = symbols.ArrayType(datatype, shape) + except (NotImplementedError, TypeError): + datatype = None + + intent_node = next( (item for item in qualifiers if item.children and item.children[0].type == "intent"), None) - common_attr = _SharedDeclAttributes( - base_type, dimension, intent, qualifier_names, + intent = symbols.ArgumentInterface.Access.UNKNOWN + if intent_node: + intent = next( + (self._INTENT_ACCESS[child.type] + for child in intent_node.children + if child.type in self._INTENT_ACCESS), intent) + common_attr = _CommonDeclAttributes( + datatype, intent, qualifier_names, frozenset(unsupported), to_str(tsnode).split("::", maxsplit=1)[0].strip()) @@ -738,7 +770,7 @@ def _variable_declaration_handler( self._declare_entity(declarator, common_attr) def _declare_entity( - self, declarator: 'TSNode', common_attr: _SharedDeclAttributes + self, declarator: 'TSNode', common_attr: _CommonDeclAttributes ): '''Translate one entity and add it to the current symbol table. @@ -769,7 +801,7 @@ def _declare_entity( self._add_or_update_datasymbol(declared_symbol) def _declarator_datatype( - self, declarator: 'TSNode', common_attr: _SharedDeclAttributes + self, declarator: 'TSNode', common_attr: _CommonDeclAttributes ): '''Translate one entity's datatype and initial value. @@ -795,20 +827,30 @@ def _declarator_datatype( except NotImplementedError: is_unsupported = True - # Array shape and initialisation belong to an individual entity, so - # derive a fresh datatype from the common base type. - datatype = common_attr.base_type - shape_node = next_of_type(declarator, "size") or common_attr.dimension + # An entity-specific shape takes precedence over a shared DIMENSION + # attribute. The latter has already been translated into an ArrayType, + # from which the elemental type can be recovered. + datatype = common_attr.datatype + shape_node = next_of_type(declarator, "size") is_allocatable = "allocatable" in common_attr.qualifiers if datatype and shape_node: try: + elemental_type = ( + datatype.elemental_type + if isinstance(datatype, symbols.ArrayType) else datatype) + if isinstance(elemental_type, symbols.DataType): + elemental_type = elemental_type.copy() shape = self._shape_from_node( shape_node, is_allocatable) - datatype = symbols.ArrayType(datatype, shape) + datatype = symbols.ArrayType(elemental_type, shape) except (NotImplementedError, TypeError): datatype = None - elif is_allocatable: + elif is_allocatable and not isinstance(datatype, symbols.ArrayType): datatype = None + elif isinstance(datatype, symbols.DataType): + # A datatype can contain PSyIR expressions (e.g. array bounds or + # character length), which must not be shared between symbols. + datatype = datatype.copy() # UnsupportedFortranType is preferable to losing a declaration. # Keep only this entity in the saved text to avoid duplicating sibling @@ -820,7 +862,7 @@ def _declarator_datatype( return datatype, initial_value def _declaration_interface( - self, name: str, common_attr: _SharedDeclAttributes + self, name: str, common_attr: _CommonDeclAttributes ): '''Return the PSyIR interface for one declared entity. @@ -834,14 +876,7 @@ def _declaration_interface( ''' symtab = self._current_scope if name in symtab and symtab.lookup(name).is_argument: - access = symbols.ArgumentInterface.Access.UNKNOWN - if common_attr.intent: - access = next( - (self._INTENT_ACCESS[child.type] - for child in common_attr.intent.children - if child.type in self._INTENT_ACCESS), - access) - return symbols.ArgumentInterface(access) + return symbols.ArgumentInterface(common_attr.intent) if {"save", "parameter"}.intersection(common_attr.qualifiers): return symbols.StaticInterface() if isinstance(symtab.node, nodes.Container): diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index 4565a4062c..2c6260a301 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -257,6 +257,60 @@ def test_declarations_arrays_datatypes(shape_string, extent): assert shape == extent +def test_shared_dimension_is_translated_once(): + '''Test that a shared DIMENSION is translated once and its resulting + PSyIR expressions are copied for each declared entity. + ''' + class CountingReader(FortranTreeSitterReader): + '''Reader that counts declaration-shape translations.''' + + def __init__(self): + super().__init__() + self.shape_translations = 0 + + def _shape_from_node(self, *args, **kwargs): + '''Count calls while preserving the original implementation.''' + self.shape_translations += 1 + return super()._shape_from_node(*args, **kwargs) + + processor = CountingReader() + valid_code = """ + module test + implicit none + integer, parameter :: extent = 10 + real, dimension(extent) :: first, second + end module + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + first_type = table.lookup("first").datatype + second_type = table.lookup("second").datatype + + assert processor.shape_translations == 1 + assert first_type is not second_type + assert first_type.shape[0].upper is not second_type.shape[0].upper + assert first_type.shape[0].upper.symbol is table.lookup("extent") + assert second_type.shape[0].upper.symbol is table.lookup("extent") + + +def test_entity_dimension_overrides_shared_dimension(): + '''Test defensive handling of an entity shape together with DIMENSION.''' + processor = FortranTreeSitterReader() + valid_code = """ + module test + implicit none + real, dimension(10) :: field(20) + end module + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + datatype = root.children[0].symbol_table.lookup("field").datatype + + assert isinstance(datatype, psyir_symbols.ArrayType) + assert datatype.shape[0].upper.value == "20" + + def test_program(): '''Test a main-program unit.''' processor = FortranTreeSitterReader() From 149a0b01288d690baf92541d60fcd13cbf1c05a2 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 12 Aug 2026 16:30:40 +0100 Subject: [PATCH 11/23] Clean up tree sitter implementation --- .../frontend/fortran_treesitter_reader.py | 226 +++++++++--------- .../fortran_treesitter_reader/ftr_test.py | 13 +- 2 files changed, 115 insertions(+), 124 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 0623771a4e..f7a0c9f925 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -82,17 +82,22 @@ def direct_child_of_type( yield child -def next_of_type( +def child_of_type( tsnode: Optional['TSNode'], node_type: str ) -> Optional['TSNode']: - '''Return the first direct child having the supplied type. + ''' Return the direct child having the supplied type. :param tsnode: tree-sitter node whose children are searched. :param node_type: tree-sitter type to find. :returns: matching child, or ``None`` if no child matches. ''' - return next(direct_child_of_type(tsnode, node_type), None) + children = list(direct_child_of_type(tsnode, node_type)) + if len(children) == 0: + return None + elif len(children) > 1: + raise InternalError("Expected only 1") + return children[0] @dataclass(frozen=True) @@ -133,14 +138,21 @@ class FortranTreeSitterReader(): The Reader works mostly by traversing the parse_tree with recursive calls to the _process_nodes dispatching method. This method calls - the appropriate node handler for the given fparser node and creates - CodeBlocks if the handlers fail. + the appropriate node handler for the given treesitter node. When Fortran scopes are found these are managed by the _using_scope context manager, this utility keeps a stack of nested scopes with a global reference to the top of the stack, this reference can be used everywhere to access the current scope. + Generally, handlers can raise 3 types of errors: + - NotImplementedError, if they find valid Fortran that is currently + not supported. A parent node should catch it an convert it to a + CodeBlock or an UnsupportedType declaration. + - ValueError, if invalid Fortran is found. + - InternalError, if an unexpected state is found that likely points + to a bug in the Reader. + Note that the implementation is incomplete, its main limitations are that: - the Reader parameters are ignored. @@ -197,9 +209,9 @@ class FortranTreeSitterReader(): # Some tree-sitter node types share the same handler. _HANDLER_REDIRECTIONS = { - "subroutine": "_routine_handler", - "function": "_routine_handler", - "program": "_routine_handler", + "subroutine": "_procedure_handler", + "function": "_procedure_handler", + "program": "_procedure_handler", "unary_expression": "_operation", "logical_expression": "_operation", "relational_expression": "_operation", @@ -346,7 +358,7 @@ def _using_temporary_scope( ''' if scope: if scope.parent is not None: - raise ValueError("The supplied scope must be an orphan") + raise InternalError("The supplied scope must be an orphan") else: scope = nodes.ScopingNode(symbol_table=symbols.SymbolTable()) @@ -491,25 +503,24 @@ def _module_handler( :raises NotImplementedError: if the module has an unsupported child. :raises NotImplementedError: if the module permits implicit variables. ''' - statement = next_of_type(tsnode, "module_statement") - name = next_of_type(statement, "name") + statement = child_of_type(tsnode, "module_statement") + name = child_of_type(statement, "name") container = nodes.Container(to_str(name) if name else "") with self._using_scope(container.symbol_table): visibility_map = self._process_access_statements(tsnode.children) - # Specification nodes precede executable nodes, so processing in - # source order ensures that symbols are declared before use. - structural = { + # This nodes are already processed + skip = { "module_statement", "end_module_statement", "implicit_statement", "internal_procedures", "public_statement", "private_statement" } - container.children.extend(self._process_nodes( + self._process_nodes( [child for child in tsnode.children - if child.type not in structural], _NodeExpectation.LIST)) + if child.type not in skip], _NodeExpectation.NONE) - internal = next_of_type(tsnode, "internal_procedures") + internal = child_of_type(tsnode, "internal_procedures") if internal: container.children.extend( self._process_nodes( @@ -519,28 +530,24 @@ def _module_handler( self._apply_visibility(visibility_map) return container - def _routine_handler( + def _procedure_handler( self, tsnode: 'TSNode' ) -> nodes.Routine: - '''Create PSyIR shared by programs, subroutines and functions. + '''Handler shared by programs, subroutines and functions. - :param tsnode: tree-sitter node for the complete program unit. + :param tsnode: the procedure treesitter node. :returns: translated PSyIR Routine. ''' routine_kind = tsnode.type - parent_symtab = self._current_scope - if parent_symtab is None: - raise InternalError( - "A Routine must be translated within a current scope") - statement = next_of_type(tsnode, f"{routine_kind}_statement") - name_node = next_of_type(statement, "name") + signature = child_of_type(tsnode, f"{routine_kind}_statement") + name_node = child_of_type(signature, "name") name = to_str(name_node) if name_node else routine_kind - parameters = next_of_type(statement, "parameters") + parameters = child_of_type(signature, "parameters") argument_names = tuple( to_str(child) for child in parameters.children if child.type == "identifier") if parameters else () return_name, return_type = self._function_return_info( - statement, name, routine_kind) + signature, name, routine_kind) # Insert arguments before declarations so specify_argument_list() can # retain source order. Declarations later complete these placeholders. @@ -554,20 +561,22 @@ def _routine_handler( return_name, return_type or symbols.UnresolvedType())) rsymbol = self._create_routine_symbol( - name, statement, return_type) + name, signature, return_type) routine = nodes.Routine( rsymbol, is_program=routine_kind == "program", symbol_table=routine_table) - parent = parent_symtab.node + # A routine must be parsed inside a scope as it is also a declaration + parent_symtab = self._current_scope + parent = parent_symtab.node if parent_symtab else None if not isinstance(parent, nodes.ScopingNode): raise InternalError( "A Routine must be translated within a PSyIR scope") - with self._using_temporary_scope(parent, routine): - visibility_map = self._process_access_statements( - tsnode.children) - structural = { + with self._using_temporary_scope(parent, routine): + vis_map = self._process_access_statements(tsnode.children) + # This nodes are already processed + skip = { f"{routine_kind}_statement", f"end_{routine_kind}_statement", "implicit_statement", "public_statement", @@ -575,7 +584,7 @@ def _routine_handler( } routine.children.extend(self._process_nodes( [child for child in tsnode.children - if child.type not in structural], _NodeExpectation.LIST)) + if child.type not in skip], _NodeExpectation.LIST)) args = [routine.symbol_table.lookup(name) for name in argument_names] @@ -583,15 +592,15 @@ def _routine_handler( if return_name: routine.return_symbol = routine.symbol_table.lookup( return_name) - self._apply_visibility(visibility_map) + self._apply_visibility(vis_map) return routine def _function_return_info( - self, statement: 'TSNode', routine_name: str, routine_kind: str + self, signature: 'TSNode', routine_name: str, routine_kind: str ) -> tuple[Optional[str], Optional[symbols.DataType]]: '''Extract result name and datatype from a function statement. - :param statement: opening program-unit statement. + :param statement: node containing the function signature. :param routine_name: name of the program unit. :param routine_kind: one of ``program``, ``subroutine`` or ``function``. @@ -602,11 +611,11 @@ def _function_return_info( if routine_kind != "function": return None, None - result = next_of_type(statement, "function_result") - result_name = next_of_type(result, "identifier") + result = child_of_type(signature, "function_result") + result_name = child_of_type(result, "identifier") return_name = to_str(result_name) if result_name else routine_name type_node = next( - (child for child in statement.children + (child for child in signature.children if child.type in ("intrinsic_type", "derived_type")), None) if not type_node: return return_name, None @@ -614,10 +623,10 @@ def _function_return_info( return return_name, self._datatype_from_type(type_node) except (NotImplementedError, KeyError, TypeError): return return_name, symbols.UnsupportedFortranType( - to_str(statement).strip()) + to_str(signature).strip()) def _create_routine_symbol( - self, name: str, statement: 'TSNode', return_type + self, name: str, signature: 'TSNode', return_type ) -> symbols.RoutineSymbol: '''Create or complete the RoutineSymbol for a program unit. @@ -626,21 +635,17 @@ def _create_routine_symbol( refer to the same object. :param name: routine name. - :param statement: opening program-unit statement. + :param signature: the signature node of the routine. :param return_type: translated function return type, if any. :returns: RoutineSymbol representing the program unit. ''' - parent_symtab = self._current_scope - if parent_symtab is None: - raise InternalError( - "A RoutineSymbol must be created within a current scope") qualifiers = { - to_str(child).lower() for child in statement.children + to_str(child).lower() for child in signature.children if child.type == "procedure_qualifier"} - visibility = parent_symtab.default_visibility + visibility = self._current_scope.default_visibility try: - routine_symbol = parent_symtab.lookup(name) + routine_symbol = self._current_scope.lookup(name) except KeyError: routine_symbol = None if isinstance(routine_symbol, symbols.RoutineSymbol): @@ -693,8 +698,7 @@ def _string_literal_handler( :returns: PSyIR character Literal. ''' text = to_str(tsnode) - return nodes.Literal(text[1:-1].replace(text[0] * 2, text[0]), - symbols.ScalarType.character_type()) + return nodes.Literal(text[1:-1], symbols.ScalarType.character_type()) def _boolean_literal_handler( self, tsnode: 'TSNode' @@ -724,23 +728,24 @@ def _variable_declaration_handler( raise NotImplementedError( "A variable declaration has no supported type specification") - # qualifiers = direct_child_of_type(tsnode, "type_qualifier") qualifiers = [child for child in tsnode.children if child.type == "type_qualifier"] qualifier_names = frozenset( child.children[0].type for child in qualifiers if child.children) - unsupported = qualifier_names.intersection({ - "pointer", "target", "optional", "value", "volatile", - "asynchronous", "contiguous" - }) + supported_qualifiers = { + "allocatable", "dimension", "intent", "parameter", "private", + "public", "save" + } + # If we find anything else it will be UnsupportedType + unsupported = qualifier_names.difference(supported_qualifiers) try: datatype = self._datatype_from_type(type_node) except (NotImplementedError, KeyError, TypeError): datatype = None dimension = next( - (next_of_type(item, "argument_list") for item in qualifiers + (child_of_type(item, "argument_list") for item in qualifiers if item.children and item.children[0].type == "dimension"), None) is_allocatable = "allocatable" in qualifier_names if datatype and dimension: @@ -778,7 +783,7 @@ def _declare_entity( :param common_attr: properties shared by the complete declaration. ''' id_node = (declarator if declarator.type == "identifier" - else next_of_type(declarator, "identifier")) + else child_of_type(declarator, "identifier")) name = to_str(id_node) datatype, initial_value = self._declarator_datatype( declarator, common_attr) @@ -831,7 +836,7 @@ def _declarator_datatype( # attribute. The latter has already been translated into an ArrayType, # from which the elemental type can be recovered. datatype = common_attr.datatype - shape_node = next_of_type(declarator, "size") + shape_node = child_of_type(declarator, "size") is_allocatable = "allocatable" in common_attr.qualifiers if datatype and shape_node: try: @@ -931,7 +936,7 @@ def _datatype_from_type( symtab = self._current_scope if tsnode.type == "derived_type": keyword = tsnode.children[0].type - name_node = next_of_type(tsnode, "type_name") + name_node = child_of_type(tsnode, "type_name") name = to_str(name_node) if keyword == "class": raise NotImplementedError( @@ -958,7 +963,7 @@ def _datatype_from_type( f"Intrinsic type '{intrinsic}' has no PSyIR representation") precision = symbols.ScalarType.Precision.UNDEFINED length = None - kind_node = next_of_type(tsnode, "kind") + kind_node = child_of_type(tsnode, "kind") if kind_node: values = [child for child in kind_node.children if child.type not in ("(", ")")] @@ -996,16 +1001,14 @@ def _precision(self, tsnode: 'TSNode'): return symbols.ScalarType.Precision.DOUBLE return int(expr.value) if isinstance(expr, nodes.Reference): - # A bare Symbol is a forward reference created before its role was - # known. Exact type checking is intentional: specialised Symbol - # subclasses must not be changed into a DataSymbol. # pylint: disable=unidiomatic-typecheck if type(expr.symbol) is symbols.Symbol: + # All precisions must be integers expr.symbol.specialise( symbols.DataSymbol, datatype=symbols.ScalarType.integer_type()) return expr - raise NotImplementedError("A kind must be a literal or named symbol") + raise NotImplementedError("kind expressions are not supported") def _kind_symbol( self, name: str @@ -1027,9 +1030,6 @@ def _kind_symbol( name, symbols.ScalarType.integer_type(), interface=symbols.UnresolvedInterface()) symtab.add(symbol) - if not isinstance(symbol, symbols.DataSymbol): - raise NotImplementedError( - f"Kind parameter '{name}' is not a data symbol") return symbol @staticmethod @@ -1147,14 +1147,14 @@ def _use_statement_handler( :param tsnode: use-statement tree-sitter node. - :raises NotImplementedError: if the module name conflicts with an + :raises ValueError: if the module name conflicts with an existing non-container symbol. ''' symtab = self._current_scope - module_node = next_of_type(tsnode, "module_name") + module_node = child_of_type(tsnode, "module_name") module_name = to_str(module_node) intrinsic = any(child.type == "intrinsic" for child in tsnode.children) - included = next_of_type(tsnode, "included_items") + included = child_of_type(tsnode, "included_items") wildcard = included is None try: container = symtab.lookup(module_name) @@ -1165,7 +1165,7 @@ def _use_statement_handler( visibility=symtab.default_visibility) symtab.add(container) if not isinstance(container, symbols.ContainerSymbol): - raise NotImplementedError( + raise ValueError( f"USE module '{module_name}' conflicts with another symbol") container.wildcard_import = wildcard @@ -1219,8 +1219,8 @@ def _derived_type_definition_handler( existing non-datatype symbol. ''' symtab = self._current_scope - statement = next_of_type(tsnode, "derived_type_statement") - name_node = next_of_type(statement, "type_name") + statement = child_of_type(tsnode, "derived_type_statement") + name_node = child_of_type(statement, "type_name") name = to_str(name_node) unsupported = any(child.type == "derived_type_procedures" for child in tsnode.children) @@ -1250,7 +1250,7 @@ def _derived_type_definition_handler( datatype = symbols.UnsupportedFortranType(to_str(tsnode).strip()) visibility = symtab.default_visibility - access = next_of_type(statement, "access_specifier") + access = child_of_type(statement, "access_specifier") if access: visibility = (symbols.Symbol.Visibility.PRIVATE if "private" in to_str(access).lower() else @@ -1278,8 +1278,8 @@ def _interface_handler( unsupported. ''' symtab = self._current_scope - statement = next_of_type(tsnode, "interface_statement") - name_node = next_of_type(statement, "name") + statement = child_of_type(tsnode, "interface_statement") + name_node = child_of_type(statement, "name") if not name_node: raise NotImplementedError( "Abstract and operator interfaces are not supported") @@ -1334,13 +1334,9 @@ def _parenthesized_expression_handler( :returns: translated expression inside the parentheses. - :raises NotImplementedError: if the parse-tree shape is unexpected. ''' content = [child for child in tsnode.children if child.type not in ("(", ")")] - if len(content) != 1: - raise NotImplementedError( - "Unexpected parenthesized expression structure") return self._process_nodes( content[0], _NodeExpectation.EXPRESSION) @@ -1401,9 +1397,9 @@ def _call_expression_handler( if name_node.type == "derived_type_member_expression": return self._structure_reference( name_node, - trailing_arguments=next_of_type(tsnode, "argument_list")) + trailing_arguments=child_of_type(tsnode, "argument_list")) name = to_str(name_node).lower() - argument_list = next_of_type(tsnode, "argument_list") + argument_list = child_of_type(tsnode, "argument_list") try: symbol = self._current_scope.lookup(name) except KeyError: @@ -1589,7 +1585,7 @@ def _decompose_structure( base = tsnode.children[0] name, indices, members = self._decompose_structure(base) arguments = self._arguments( - next_of_type(tsnode, "argument_list")) + child_of_type(tsnode, "argument_list")) if any(isinstance(arg, tuple) for arg in arguments): raise NotImplementedError( "Named arguments in structure accesses are not supported") @@ -1601,7 +1597,7 @@ def _decompose_structure( if tsnode.type == "derived_type_member_expression": name, indices, members = self._decompose_structure( tsnode.children[0]) - member = next_of_type(tsnode, "type_member") + member = child_of_type(tsnode, "type_member") if member is None: raise NotImplementedError( "Malformed structure component access") @@ -1621,7 +1617,7 @@ def _array_literal_handler( :raises NotImplementedError: for an implied-DO constructor. ''' - if next_of_type(tsnode, "implied_do_loop_expression"): + if child_of_type(tsnode, "implied_do_loop_expression"): raise NotImplementedError( "Array constructors with implied-DO loops are not supported") elems = [self._process_nodes(child, _NodeExpectation.EXPRESSION) @@ -1632,20 +1628,12 @@ def _array_literal_handler( def _comment_handler( self, tsnode: 'TSNode' ) -> None: - '''Ignore comments when requested. - - Comment attachment will be added when the reader options cease to be - compatibility-only. Until then, comments must not turn otherwise - supported source into CodeBlocks. + '''Ignore comments. :param tsnode: comment tree-sitter node. - :raises NotImplementedError: if comment preservation was requested. ''' del tsnode - if self._ignore_comments: - return None - raise NotImplementedError("Comment preservation is not yet supported") def _assignment_statement_handler( self, tsnode: 'TSNode' @@ -1719,7 +1707,7 @@ def _subroutine_call_handler( if not isinstance(symbol, symbols.RoutineSymbol): raise NotImplementedError( f"Called object '{name}' is not a routine") - args = self._arguments(next_of_type(tsnode, "argument_list")) + args = self._arguments(child_of_type(tsnode, "argument_list")) return nodes.Call.create(symbol, args) def _keyword_statement_handler( @@ -1750,7 +1738,7 @@ def _if_statement_handler( :raises NotImplementedError: if the statement has no condition. ''' - condition_node = next_of_type(tsnode, "parenthesized_expression") + condition_node = child_of_type(tsnode, "parenthesized_expression") if not condition_node: raise NotImplementedError("IF statement has no condition") structural = { @@ -1759,10 +1747,10 @@ def _if_statement_handler( } body_nodes = [child for child in tsnode.children if child.type not in structural] - else_clause = next_of_type(tsnode, "else_clause") + else_clause = child_of_type(tsnode, "else_clause") else_ifs = list(direct_child_of_type(tsnode, "elseif_clause")) annotations = [] - if not next_of_type(tsnode, "end_if_statement"): + if not child_of_type(tsnode, "end_if_statement"): annotations.append("was_single_stmt") if_body = self._process_nodes(body_nodes, _NodeExpectation.LIST) else_body = None @@ -1789,19 +1777,19 @@ def _if_clause( :returns: annotated PSyIR IfBlock. ''' - condition = next_of_type(tsnode, "parenthesized_expression") + condition = child_of_type(tsnode, "parenthesized_expression") structural = {"else", "if", "parenthesized_expression", "then", "else_clause", "elseif_clause"} body = self._process_nodes( [child for child in tsnode.children if child.type not in structural], _NodeExpectation.LIST) - trailing = next_of_type(tsnode, "elseif_clause") + trailing = child_of_type(tsnode, "elseif_clause") otherwise = ( [self._if_clause(trailing, final_else)] if trailing else self._process_nodes( [child for child in - (next_of_type(tsnode, "else_clause").children - if next_of_type(tsnode, "else_clause") else []) + (child_of_type(tsnode, "else_clause").children + if child_of_type(tsnode, "else_clause") else []) if child.type != "else"], _NodeExpectation.LIST) or final_else) result = nodes.IfBlock.create( @@ -1821,9 +1809,9 @@ def _do_loop_handler( :raises NotImplementedError: if counted-loop control is unsupported. ''' - statement = next_of_type(tsnode, "do_statement") - control = next_of_type(statement, "loop_control_expression") - while_node = next_of_type(statement, "while_statement") + statement = child_of_type(tsnode, "do_statement") + control = child_of_type(statement, "loop_control_expression") + while_node = child_of_type(statement, "while_statement") body = self._process_nodes( [child for child in tsnode.children if child.type not in ("do_statement", @@ -1856,7 +1844,7 @@ def _do_loop_handler( self._process_nodes(parts[2], _NodeExpectation.EXPRESSION), step, body) if while_node: - condition = next_of_type( + condition = child_of_type( while_node, "parenthesized_expression") return nodes.WhileLoop.create( self._process_nodes(condition, _NodeExpectation.EXPRESSION), @@ -1875,13 +1863,13 @@ def _where_statement_handler( :returns: annotated PSyIR IfBlock. ''' - condition = next_of_type(tsnode, "parenthesized_expression") + condition = child_of_type(tsnode, "parenthesized_expression") structural = {"where", "parenthesized_expression", "elsewhere_clause", "end_where_statement"} body = self._process_nodes( [child for child in tsnode.children if child.type not in structural], _NodeExpectation.LIST) - elsewhere = next_of_type(tsnode, "elsewhere_clause") + elsewhere = child_of_type(tsnode, "elsewhere_clause") other = ( self._process_nodes( [child for child in elsewhere.children @@ -1891,7 +1879,7 @@ def _where_statement_handler( self._process_nodes(condition, _NodeExpectation.EXPRESSION), body, other) result.annotations.extend( - ["was_where"] if next_of_type( + ["was_where"] if child_of_type( tsnode, "end_where_statement") else ["was_where", "was_single_stmt"]) return result @@ -1910,12 +1898,12 @@ def _select_case_statement_handler( :raises NotImplementedError: if no conditional CASE can be produced. ''' - selector_node = next_of_type( - next_of_type(tsnode, "selector"), "identifier") + selector_node = child_of_type( + child_of_type(tsnode, "selector"), "identifier") if selector_node is None: selector = self._process_nodes( [child for child in - next_of_type(tsnode, "selector").children + child_of_type(tsnode, "selector").children if child.type not in ("(", ")")][0], _NodeExpectation.EXPRESSION) else: @@ -1925,13 +1913,13 @@ def _select_case_statement_handler( default_body = None normal = [] for case in cases: - if next_of_type(case, "default"): + if child_of_type(case, "default"): default_body = self._process_nodes( [child for child in case.children if child.type not in ("case", "default")], _NodeExpectation.LIST) else: - values = next_of_type(case, "case_value_range_list") + values = child_of_type(case, "case_value_range_list") if values is None: raise NotImplementedError( "Malformed CASE value list") @@ -2052,12 +2040,12 @@ def _allocation_reference( :raises NotImplementedError: if the object is not a data symbol. ''' - ident = next_of_type(tsnode, "identifier") + ident = child_of_type(tsnode, "identifier") reference = self._identifier_handler(ident) if not isinstance(reference.symbol, symbols.DataSymbol): raise NotImplementedError( "An ALLOCATE object must be a data symbol") - size = next_of_type(tsnode, "size") + size = child_of_type(tsnode, "size") indices = [ self._allocation_extent(extent) for extent in size.children diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index 2c6260a301..c0d1251447 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -507,13 +507,16 @@ def test_unsupported_complex_datatype(): assert coefficient.datatype.declaration == "complex :: coefficient" -def test_unsupported_pointer_datatype(): - '''Test entity-specific unsupported pointer datatypes.''' +@pytest.mark.parametrize("qualifier", ["pointer", "protected"]) +def test_unsupported_qualifier_datatype(qualifier): + '''Test entity-specific unsupported declaration qualifiers, including + those that are not in a predefined list. + ''' processor = FortranTreeSitterReader() - valid_code = """ + valid_code = f""" module declarations implicit none - integer, pointer :: first, second + integer, {qualifier} :: first, second end module declarations """ root = processor.generate_psyir( @@ -522,7 +525,7 @@ def test_unsupported_pointer_datatype(): for name in ("first", "second"): datatype = table.lookup(name).datatype assert isinstance(datatype, psyir_symbols.UnsupportedFortranType) - assert datatype.declaration == f"integer, pointer :: {name}" + assert datatype.declaration == f"integer, {qualifier} :: {name}" def test_unsupported_initialisation_is_entity_specific(): From ebfe78265edc3987158c137ce7da6dc287fb2691 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 13 Aug 2026 11:02:33 +0100 Subject: [PATCH 12/23] Improve treesitter testing --- .../frontend/fortran_treesitter_reader.py | 8 + .../fortran_treesitter_reader/ftr_test.py | 1755 ++++++++++++----- 2 files changed, 1297 insertions(+), 466 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index f7a0c9f925..4c57a88c0e 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -306,6 +306,8 @@ def generate_psyir(self, parse_tree: 'TSNode') -> nodes.Node: :returns: the equivalent PSyIR Node. ''' + # This is the public entry point, reset the scoping pointer + self._current_scope = symbols.SymbolTable() return self._process_nodes(parse_tree, _NodeExpectation.ONE) @contextmanager @@ -900,12 +902,18 @@ def _add_or_update_datasymbol( :param declared_symbol: fully translated symbol for the entity. + :raises ValueError: if the name is already used for an imported + module. :raises NotImplementedError: if an existing symbol is not a DataSymbol. ''' symtab = self._current_scope if declared_symbol.name in symtab: symbol = symtab.lookup(declared_symbol.name) + if isinstance(symbol, symbols.ContainerSymbol): + raise ValueError( + f"USE module '{symbol.name}' conflicts with another " + f"symbol") if not isinstance(symbol, symbols.DataSymbol): raise NotImplementedError( f"'{declared_symbol.name}' is already declared as a " diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index c0d1251447..b52b2be581 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -7,12 +7,15 @@ ''' Performs tests on the treesitter PSyIR front-end ''' import logging +from types import SimpleNamespace import pytest from tree_sitter import Node as TSNode -from psyclone.psyir.frontend.fortran_treesitter_reader import \ - FortranTreeSitterReader +from psyclone.errors import InternalError +from psyclone.psyir.frontend import fortran_treesitter_reader as ftr +from psyclone.psyir.frontend.fortran_treesitter_reader import ( + FortranTreeSitterReader, _CommonDeclAttributes, _NodeExpectation) from psyclone.psyir import nodes as psyir_nodes, symbols as psyir_symbols from psyclone.tests.utilities import min_version_3_10 @@ -21,6 +24,23 @@ pytestmark = min_version_3_10 +def _first_tsnode(tsnode, node_type): + ''' + :param tsnode: the tree-sitter tree to search. + :param node_type: the type to search. + + :returns: the first tree-sitter node of the requested type. + + ''' + if tsnode.type == node_type: + return tsnode + for child in tsnode.children: + result = _first_tsnode(child, node_type) + if result: + return result + return None + + def test_constructor(): ''' Test the constructor and its arguments ''' processor = FortranTreeSitterReader() @@ -41,8 +61,6 @@ def test_constructor(): assert processor._resolve_modules is True assert processor._last_comments_as_codeblocks is True - # TODO #3038 Typecheck arguments - def test_generate_parse_tree(tmpdir_factory, caplog): ''' @@ -114,33 +132,44 @@ def test_generate_psyir(): assert root.children[0].children[0].name == "mysub" -def test_routine_host_association(): - '''Test that a contained routine resolves a symbol from its host.''' +def test_process_node_expectation_errors(): + '''Test defensive validation of dispatcher result expectations.''' + valid_code = """ + subroutine assignment(first, second) + integer :: first, second + first = second + end subroutine assignment + """ + processor = FortranTreeSitterReader() + parse_tree = processor.generate_parse_tree_from_source(valid_code) + assignment = _first_tsnode(parse_tree, "assignment_statement") + + with pytest.raises(InternalError, match="Only one node was expected"): + processor._process_nodes([], _NodeExpectation.ONE) + with pytest.raises(InternalError, match="A DataNode was expected"): + processor._process_nodes(assignment, _NodeExpectation.EXPRESSION) + with pytest.raises(InternalError, match="Unsupported node expectation"): + processor._process_nodes([], None) + + +def test_program(): + '''Test a main-program unit.''' processor = FortranTreeSitterReader() valid_code = """ - module host + program main implicit none - integer :: count - contains - subroutine work() - count = count + 1 - end subroutine work - end module host + end program main """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - container = root.children[0] - routine = container.children[0] - count = container.symbol_table.lookup("count") - assert "count" not in routine.symbol_table - assert [ref.symbol for ref in routine.walk(psyir_nodes.Reference)] == [ - count, count] + routine = root.children[0] + assert isinstance(routine, psyir_nodes.Routine) + assert routine.is_program + assert routine.name == "main" -def test_subroutine(): - ''' - Test subroutine nodes. - ''' +def test_routines_nodes(): + ''' Test that routine nodes create a node and delcare a symbol ''' processor = FortranTreeSitterReader() valid_code = """ @@ -169,199 +198,157 @@ def test_subroutine(): assert isinstance(rsymbol2, psyir_symbols.RoutineSymbol) -def test_declarations(): - ''' - Test subroutine nodes. - ''' +def test_routine_symbol_association(): + '''Test that a contained routine resolves a symbol from its parent.''' processor = FortranTreeSitterReader() - valid_code = """ - module test - implicit none - integer :: a - real :: b - end module + module host + implicit none + integer :: count + contains + subroutine work() + count = count + 1 + end subroutine work + end module host """ - ptree = processor.generate_parse_tree_from_source(valid_code) - root = processor.generate_psyir(ptree) - module = root.children[0] - - # Declarations do not add children nodes - assert len(module.children) == 0 - - # Check that the symbols have been added to the symbol table - assert len(module.symbol_table.symbols) == 2 - assert "a" in module.symbol_table - assert "b" in module.symbol_table + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + container = root.children[0] + routine = container.children[0] + count = container.symbol_table.lookup("count") + assert "count" not in routine.symbol_table + assert [ref.symbol for ref in routine.walk(psyir_nodes.Reference)] == [ + count, count] -@pytest.mark.parametrize("fortran_type,psyir_type", [ - ("integer", psyir_symbols.ScalarType.integer_type()), - ("integer(kind=4)", psyir_symbols.ScalarType.integer_single_type()), - ("integer(8)", psyir_symbols.ScalarType.integer_double_type()), - ("real", psyir_symbols.ScalarType.real_type()), - ("real(4)", psyir_symbols.ScalarType.real_single_type()), - ("real(kind=8)", psyir_symbols.ScalarType.real_double_type()), - ("logical", psyir_symbols.ScalarType.boolean_type()), - ("character", psyir_symbols.ScalarType.character_type()), -]) -def test_declarations_datatypes(fortran_type, psyir_type): - ''' - Test subroutine nodes. - ''' +def test_function_result(): + '''Test a named function-result symbol.''' processor = FortranTreeSitterReader() - - valid_code = f""" - module test - implicit none - {fortran_type} :: a - end module + valid_code = """ + real function square(value) result(answer) + real :: value + answer = value * value + end function square """ - ptree = processor.generate_parse_tree_from_source(valid_code) - root = processor.generate_psyir(ptree) - module = root.children[0] - assert module.symbol_table.lookup("a").datatype == psyir_type, ( - f"{module.symbol_table.lookup('a').datatype} != {psyir_type}" - ) - + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + assert routine.return_symbol.name == "answer" + assert routine.return_symbol.datatype == \ + psyir_symbols.ScalarType.real_type() -@pytest.mark.parametrize("shape_string, extent", [ - ("(:)", psyir_symbols.ArrayType.Extent.ATTRIBUTE), - ("(10)", "10"), -]) -def test_declarations_arrays_datatypes(shape_string, extent): - ''' - Test subroutine nodes. - ''' - processor = FortranTreeSitterReader() - valid_code = f""" - module test - implicit none - integer(4), dimension{shape_string} :: a - end module +def test_function_return_type_variants(): + '''Test different types of expressing return types.''' + valid_code = """ + real function signature_type() + end function + function inner_type() + integer :: inner_type + end function inner_type + complex function unsupported() + end function unsupported """ - ptree = processor.generate_parse_tree_from_source(valid_code) - root = processor.generate_psyir(ptree) - module = root.children[0] - - array_symbol = module.symbol_table.lookup("a") - assert isinstance(array_symbol.datatype, psyir_symbols.ArrayType) - assert (array_symbol.datatype.elemental_type == - psyir_symbols.ScalarType.integer_single_type()) - shape = array_symbol.datatype.shape[0] - if isinstance(extent, str): - assert shape.upper.value == extent - assert shape.lower.value == "1" - else: - assert shape == extent + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + assert root.children[0].return_symbol.name == "signature_type" + assert (root.children[0].return_symbol.datatype == + psyir_symbols.ScalarType.real_type()) -def test_shared_dimension_is_translated_once(): - '''Test that a shared DIMENSION is translated once and its resulting - PSyIR expressions are copied for each declared entity. - ''' - class CountingReader(FortranTreeSitterReader): - '''Reader that counts declaration-shape translations.''' + assert root.children[1].return_symbol.name == "inner_type" + assert (root.children[1].return_symbol.datatype == + psyir_symbols.ScalarType.integer_type()) - def __init__(self): - super().__init__() - self.shape_translations = 0 + assert isinstance(root.children[2].return_symbol.datatype, + psyir_symbols.UnsupportedFortranType) - def _shape_from_node(self, *args, **kwargs): - '''Count calls while preserving the original implementation.''' - self.shape_translations += 1 - return super()._shape_from_node(*args, **kwargs) - processor = CountingReader() +def test_argument_order(): + '''Test the order of routine arguments.''' + processor = FortranTreeSitterReader() valid_code = """ - module test - implicit none - integer, parameter :: extent = 10 - real, dimension(extent) :: first, second - end module + subroutine update(first, second) + real :: first, second + end subroutine update """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) table = root.children[0].symbol_table - first_type = table.lookup("first").datatype - second_type = table.lookup("second").datatype - - assert processor.shape_translations == 1 - assert first_type is not second_type - assert first_type.shape[0].upper is not second_type.shape[0].upper - assert first_type.shape[0].upper.symbol is table.lookup("extent") - assert second_type.shape[0].upper.symbol is table.lookup("extent") + assert [symbol.name for symbol in table.argument_list] == [ + "first", "second"] -def test_entity_dimension_overrides_shared_dimension(): - '''Test defensive handling of an entity shape together with DIMENSION.''' +@pytest.mark.parametrize("intent,access", [ + ("in", psyir_symbols.ArgumentInterface.Access.READ), + ("out", psyir_symbols.ArgumentInterface.Access.WRITE), + ("inout", psyir_symbols.ArgumentInterface.Access.READWRITE), +]) +def test_argument_intent(intent, access): + '''Test an INTENT attribute on a routine argument.''' processor = FortranTreeSitterReader() - valid_code = """ - module test - implicit none - real, dimension(10) :: field(20) - end module + valid_code = f""" + subroutine update(value) + real, intent({intent}) :: value + end subroutine update """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - datatype = root.children[0].symbol_table.lookup("field").datatype - - assert isinstance(datatype, psyir_symbols.ArrayType) - assert datatype.shape[0].upper.value == "20" + value = root.children[0].symbol_table.lookup("value") + assert value.interface.access == access -def test_program(): - '''Test a main-program unit.''' +def test_pure_function(): + '''Test the PURE function qualifier.''' processor = FortranTreeSitterReader() valid_code = """ - program main - implicit none - end program main + pure real function identity(value) + real :: value + identity = value + end function identity """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - routine = root.children[0] - assert isinstance(routine, psyir_nodes.Routine) - assert routine.is_program - assert routine.name == "main" + assert root.children[0].symbol.is_pure is True -def test_use_rename(): - '''Test a renamed symbol in a USE ONLY statement.''' +def test_elemental_function(): + '''Test the ELEMENTAL function qualifier.''' processor = FortranTreeSitterReader() valid_code = """ - program main - use kinds, only: local_kind => remote_kind - implicit none - end program main + elemental real function identity(value) + real :: value + identity = value + end function identity """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - table = root.children[0].symbol_table - container = table.lookup("kinds") - imported = table.lookup("local_kind") - assert isinstance(container, psyir_symbols.ContainerSymbol) - assert imported.interface.container_symbol is container - assert imported.interface.orig_name == "remote_kind" + assert root.children[0].symbol.is_elemental is True -def test_symbolic_kind(): - '''Test a symbolic kind expression.''' +def test_declarations(): + ''' Test simple declarations ''' processor = FortranTreeSitterReader() + valid_code = """ - program main - use kinds, only: local_kind - implicit none - integer(local_kind) :: value - end program main + module test + implicit none + integer :: a + real :: b + end module """ - root = processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) - table = root.children[0].symbol_table - kind = table.lookup("local_kind") - assert isinstance(kind, psyir_symbols.DataSymbol) - assert table.lookup("value").datatype.precision.symbol is kind + ptree = processor.generate_parse_tree_from_source(valid_code) + root = processor.generate_psyir(ptree) + module = root.children[0] + + # Declarations do not add children nodes + assert len(module.children) == 0 + + # Check that the symbols have been added to the symbol table + assert len(module.symbol_table.symbols) == 2 + assert "a" in module.symbol_table + assert "b" in module.symbol_table def test_parameter_declaration(): @@ -380,13 +367,191 @@ def test_parameter_declaration(): assert count.initial_value.value == "4" -def test_character_length(): - '''Test a character-length specification.''' +def test_save_attribute(): + '''Test the SAVE declaration attribute.''' processor = FortranTreeSitterReader() valid_code = """ - program main + module declarations implicit none - character(len=12) :: label + double precision, save :: accumulator + end module declarations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + accumulator = root.children[0].symbol_table.lookup("accumulator") + assert accumulator.datatype == \ + psyir_symbols.ScalarType.real_double_type() + assert isinstance(accumulator.interface, psyir_symbols.StaticInterface) + + +@pytest.mark.parametrize("qualifier", ["pointer", "protected"]) +def test_unsupported_qualifier_datatype(qualifier): + '''Test entity-specific unsupported declaration qualifiers, including + those that are not in a predefined list. + ''' + processor = FortranTreeSitterReader() + valid_code = f""" + module declarations + implicit none + integer, {qualifier} :: first, second + end module declarations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + for name in ("first", "second"): + datatype = table.lookup(name).datatype + assert isinstance(datatype, psyir_symbols.UnsupportedFortranType) + assert datatype.declaration == f"integer, {qualifier} :: {name}" + + +def test_unsupported_complex_datatype(): + '''Test the unsupported complex datatype.''' + processor = FortranTreeSitterReader() + valid_code = """ + module declarations + implicit none + complex :: coefficient + end module declarations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + coefficient = root.children[0].symbol_table.lookup("coefficient") + assert isinstance(coefficient.datatype, + psyir_symbols.UnsupportedFortranType) + assert coefficient.datatype.declaration == "complex :: coefficient" + + +def test_class_declaration_is_unsupported(): + '''Test that a polymorphic declaration uses the unsupported fallback.''' + valid_code = """ + subroutine polymorphic(value) + class(item) :: value + end subroutine polymorphic + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + + assert isinstance(root.children[0].symbol_table.lookup("value").datatype, + psyir_symbols.UnsupportedFortranType) + + +def test_unsupported_initialisation_is_entity_specific(): + '''Test that one unsupported initializer does not affect its sibling.''' + processor = FortranTreeSitterReader() + valid_code = """ + module declarations + implicit none + integer :: first = [(i, i=1,2)], second = 2 + end module declarations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + first = table.lookup("first") + second = table.lookup("second") + assert isinstance(first.datatype, psyir_symbols.ScalarType) + assert isinstance(first.initial_value, psyir_nodes.CodeBlock) + assert first.initial_value.structure == \ + psyir_nodes.CodeBlock.Structure.EXPRESSION + assert isinstance(second.datatype, psyir_symbols.ScalarType) + assert second.initial_value.value == "2" + + +def test_unsupported_initializer_translation(monkeypatch): + '''Test the declaration fallback if expression processing itself fails.''' + valid_code = """ + module declarations + integer :: value = 1 + end module declarations + """ + processor = FortranTreeSitterReader() + parse_tree = processor.generate_parse_tree_from_source(valid_code) + original = processor._process_nodes + + def unsupported_number(tsnodes, expect): + '''Reject the initializer while processing everything else normally.''' + if (getattr(tsnodes, "type", None) == "number_literal" and + expect is _NodeExpectation.EXPRESSION): + raise NotImplementedError("unsupported test initializer") + return original(tsnodes, expect) + + monkeypatch.setattr(processor, "_process_nodes", unsupported_number) + root = processor.generate_psyir(parse_tree) + + assert isinstance(root.children[0].symbol_table.lookup("value").datatype, + psyir_symbols.UnsupportedFortranType) + + +@pytest.mark.parametrize("fortran_type,psyir_type", [ + ("integer", psyir_symbols.ScalarType.integer_type()), + ("integer(kind=4)", psyir_symbols.ScalarType.integer_single_type()), + ("integer(8)", psyir_symbols.ScalarType.integer_double_type()), + ("real", psyir_symbols.ScalarType.real_type()), + ("real(4)", psyir_symbols.ScalarType.real_single_type()), + ("real(kind=8)", psyir_symbols.ScalarType.real_double_type()), + ("logical", psyir_symbols.ScalarType.boolean_type()), + ("character", psyir_symbols.ScalarType.character_type()), +]) +def test_declarations_datatypes(fortran_type, psyir_type): + ''' Test base declaration datatypes ''' + processor = FortranTreeSitterReader() + + valid_code = f""" + module test + implicit none + {fortran_type} :: a + end module + """ + ptree = processor.generate_parse_tree_from_source(valid_code) + root = processor.generate_psyir(ptree) + module = root.children[0] + assert module.symbol_table.lookup("a").datatype == psyir_type + + +def test_datatype_kind_variants(): + '''Test symbolic, literal and unsupported kinds.''' + valid_code = """ + subroutine declarations() + integer(named_kind) :: symbolic + integer(16) :: wide + integer(1 + 1) :: unsupported_kind + end subroutine declarations + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + + assert isinstance(table.lookup("named_kind"), psyir_symbols.DataSymbol) + assert table.lookup("wide").datatype.precision == 16 + assert isinstance(table.lookup("unsupported_kind").datatype, + psyir_symbols.UnsupportedFortranType) + + # Also check symbolic links are connected + valid_code = """ + program main + use kinds, only: local_kind + implicit none + integer(local_kind) :: value + end program main + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + kind = table.lookup("local_kind") + assert isinstance(kind, psyir_symbols.DataSymbol) + assert table.lookup("value").datatype.precision.symbol is kind + + +def test_character_length(): + '''Test a character-length specification.''' + processor = FortranTreeSitterReader() + valid_code = """ + program main + implicit none + character(len=12) :: label end program main """ root = processor.generate_psyir( @@ -395,6 +560,95 @@ def test_character_length(): assert label.datatype.length.value == "12" +def test_logical_literal(): + '''Test a logical literal used as an initial value.''' + processor = FortranTreeSitterReader() + valid_code = """ + module declarations + implicit none + logical :: enabled = .true. + end module declarations + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + enabled = root.children[0].symbol_table.lookup("enabled") + assert enabled.initial_value.value == "true" + + +def test_literal_kind_variants_and_string(): + '''Test literal kind suffixes and a character literal.''' + valid_code = ''' + subroutine literals() + integer, parameter :: wp = 16 + integer :: single = 1_4, double = 1_8 + integer :: explicit = 1_16, symbolic = 1_wp + character(5) :: text = "hello" + end subroutine literals + ''' + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + + assert table.lookup("single").initial_value.datatype.precision == \ + psyir_symbols.ScalarType.Precision.SINGLE + assert table.lookup("double").initial_value.datatype.precision == \ + psyir_symbols.ScalarType.Precision.DOUBLE + assert table.lookup("explicit").initial_value.datatype.precision == 16 + precision = table.lookup("symbolic").initial_value.datatype.precision + assert precision.symbol is table.lookup("wp") + assert table.lookup("text").initial_value.value == "hello" + + +def test_new_literal_kind_symbol(): + '''Test that a named literal kind creates an unresolved kind symbol.''' + valid_code = """ + subroutine literal_kind() + integer :: value = 1_new_kind + end subroutine literal_kind + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + + assert isinstance(table.lookup("new_kind"), psyir_symbols.DataSymbol) + assert isinstance(table.lookup("new_kind").interface, + psyir_symbols.UnresolvedInterface) + assert table.lookup("value").initial_value.datatype.precision.symbol is \ + table.lookup("new_kind") + + +@pytest.mark.parametrize("shape_string, extent", [ + ("(:)", psyir_symbols.ArrayType.Extent.ATTRIBUTE), + ("(10)", "10"), +]) +def test_declarations_arrays_datatypes(shape_string, extent): + ''' Test array datatypes and its dimensions. ''' + processor = FortranTreeSitterReader() + + valid_code = f""" + module test + implicit none + integer(4), dimension{shape_string} :: a + end module + """ + ptree = processor.generate_parse_tree_from_source(valid_code) + root = processor.generate_psyir(ptree) + module = root.children[0] + + array_symbol = module.symbol_table.lookup("a") + assert isinstance(array_symbol.datatype, psyir_symbols.ArrayType) + assert (array_symbol.datatype.elemental_type == + psyir_symbols.ScalarType.integer_single_type()) + shape = array_symbol.datatype.shape[0] + if isinstance(extent, str): + assert shape.upper.value == extent + assert shape.lower.value == "1" + else: + assert shape == extent + + def test_allocatable_declaration(): '''Test an allocatable-array declaration.''' processor = FortranTreeSitterReader() @@ -411,175 +665,289 @@ def test_allocatable_declaration(): psyir_symbols.ArrayType.Extent.DEFERRED] -def test_function_result(): - '''Test a named function-result symbol.''' +def test_multidimensional_and_lower_bounded_arrays(): + '''Test explicit lower bounds in a multidimensional shape.''' processor = FortranTreeSitterReader() valid_code = """ - real function square(value) result(answer) - real :: value - answer = value * value - end function square + module array_shapes + implicit none + real :: field(-2:10, 20) + end module array_shapes """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - routine = root.children[0] - assert routine.return_symbol.name == "answer" - assert routine.return_symbol.datatype == \ - psyir_symbols.ScalarType.real_type() + datatype = root.children[0].symbol_table.lookup("field").datatype + assert isinstance(datatype, psyir_symbols.ArrayType) + assert len(datatype.shape) == 2 + assert datatype.shape[0].lower.operator == \ + psyir_nodes.UnaryOperation.Operator.MINUS + assert datatype.shape[0].lower.children[0].value == "2" + assert datatype.shape[0].upper.value == "10" + assert datatype.shape[1].lower.value == "1" + assert datatype.shape[1].upper.value == "20" -def test_argument_order(): - '''Test the order of routine arguments.''' +def test_shared_dimension_is_given_to_all_entities(): + '''Test that a shared DIMENSION are copied for each declared entity. + ''' processor = FortranTreeSitterReader() valid_code = """ - subroutine update(first, second) - real :: first, second - end subroutine update + module test + implicit none + integer, parameter :: extent = 10 + real, dimension(extent) :: first, second + end module """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) table = root.children[0].symbol_table - assert [symbol.name for symbol in table.argument_list] == [ - "first", "second"] + first_type = table.lookup("first").datatype + second_type = table.lookup("second").datatype + # They are not the same object, but copies + assert first_type is not second_type + assert first_type.shape[0].upper is not second_type.shape[0].upper + assert first_type.shape[0].upper.symbol is table.lookup("extent") + assert second_type.shape[0].upper.symbol is table.lookup("extent") -@pytest.mark.parametrize("intent,access", [ - ("in", psyir_symbols.ArgumentInterface.Access.READ), - ("out", psyir_symbols.ArgumentInterface.Access.WRITE), - ("inout", psyir_symbols.ArgumentInterface.Access.READWRITE), -]) -def test_argument_intent(intent, access): - '''Test an INTENT attribute on a routine argument.''' + +def test_entity_dimension_overrides_shared_dimension(): + '''Test handling of an entity shape together with DIMENSION.''' processor = FortranTreeSitterReader() - valid_code = f""" - subroutine update(value) - real, intent({intent}) :: value - end subroutine update + valid_code = """ + module test + implicit none + real, dimension(10) :: field(20) + end module """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - value = root.children[0].symbol_table.lookup("value") - assert value.interface.access == access + datatype = root.children[0].symbol_table.lookup("field").datatype + + assert isinstance(datatype, psyir_symbols.ArrayType) + assert datatype.shape[0].upper.value == "20" -def test_pure_function(): - '''Test the PURE function qualifier.''' +@pytest.mark.parametrize("valid_code", [ + """ + module declarations + real, dimension(10) :: values + end module declarations + """, + """ + module declarations + real :: values(10) + end module declarations + """, +]) +def test_unsupported_shape_translation(valid_code, monkeypatch): + '''Test failure translating shared and entity-specific array shapes.''' processor = FortranTreeSitterReader() + parse_tree = processor.generate_parse_tree_from_source(valid_code) + + def unsupported_shape(*_args, **_kwargs): + '''Stand in for an unsupported bound expression.''' + raise NotImplementedError("unsupported test shape") + + monkeypatch.setattr(processor, "_shape_from_node", unsupported_shape) + root = processor.generate_psyir(parse_tree) + + assert isinstance(root.children[0].symbol_table.lookup("values").datatype, + psyir_symbols.UnsupportedFortranType) + + +def test_direct_shape_and_argument_helpers(): + '''Test defensive extent splitting and absent argument handling.''' valid_code = """ - pure real function identity(value) - real :: value - identity = value - end function identity + subroutine shape(values) + integer :: values(10) + end subroutine shape """ - root = processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) - assert root.children[0].symbol.is_pure is True + processor = FortranTreeSitterReader() + parse_tree = processor.generate_parse_tree_from_source(valid_code) + number = _first_tsnode(parse_tree, "number_literal") + + before, after, has_colon = processor._split_extent(number) + assert not before + assert not after + assert not has_colon + assert not processor._arguments(None) + + malformed = SimpleNamespace( + type="extent_specifier", children=[number]) + assert processor._shape_from_node( + SimpleNamespace(children=[malformed]))[0].value == "10" + common = _CommonDeclAttributes( + psyir_symbols.ScalarType.integer_type(), + psyir_symbols.ArgumentInterface.Access.UNKNOWN, + frozenset(), frozenset(), "integer") + empty_initializer = SimpleNamespace( + type="init_declarator", children=[ + SimpleNamespace(type="identifier", text=b"value"), + SimpleNamespace(type="=", text=b"=")]) + datatype, initial = processor._declarator_datatype( + empty_initializer, common) + assert isinstance(datatype, psyir_symbols.ScalarType) + assert initial is None + + with pytest.raises(NotImplementedError, match="Malformed array range"): + processor._range(number, psyir_symbols.DataSymbol( + "values", psyir_symbols.ArrayType( + psyir_symbols.ScalarType.integer_type(), [10])), 1) + with pytest.raises(NotImplementedError, match="Malformed allocation"): + processor._allocation_extent(malformed) -def test_elemental_function(): - '''Test the ELEMENTAL function qualifier.''' +def test_default_visibility(): + '''Test a module's default visibility.''' processor = FortranTreeSitterReader() valid_code = """ - elemental real function identity(value) - real :: value - identity = value - end function identity + module visibility + implicit none + private + contains + subroutine hidden() + end subroutine hidden + end module visibility """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - assert root.children[0].symbol.is_elemental is True + container = root.children[0] + assert container.symbol_table.default_visibility == \ + psyir_symbols.Symbol.Visibility.PRIVATE + assert container.symbol_table.lookup("hidden").visibility == \ + psyir_symbols.Symbol.Visibility.PRIVATE -def test_unsupported_complex_datatype(): - '''Test the unsupported complex datatype.''' +def test_named_visibility(): + '''Test name-specific visibility for a contained routine.''' processor = FortranTreeSitterReader() valid_code = """ - module declarations + module visibility implicit none - complex :: coefficient - end module declarations + private + public :: exposed + contains + subroutine exposed() + end subroutine exposed + end module visibility """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - coefficient = root.children[0].symbol_table.lookup("coefficient") - assert isinstance(coefficient.datatype, - psyir_symbols.UnsupportedFortranType) - assert coefficient.datatype.declaration == "complex :: coefficient" + exposed = root.children[0].symbol_table.lookup("exposed") + assert exposed.visibility == psyir_symbols.Symbol.Visibility.PUBLIC -@pytest.mark.parametrize("qualifier", ["pointer", "protected"]) -def test_unsupported_qualifier_datatype(qualifier): - '''Test entity-specific unsupported declaration qualifiers, including - those that are not in a predefined list. - ''' - processor = FortranTreeSitterReader() - valid_code = f""" +def test_declaration_visibility_and_allocatable_scalar(): + '''Test declaration access attributes and unsupported scalar + ALLOCATABLE.''' + valid_code = """ module declarations - implicit none - integer, {qualifier} :: first, second + integer, public :: exposed + integer, private :: hidden + contains + subroutine local() + real, allocatable :: scalar + end subroutine local end module declarations """ + processor = FortranTreeSitterReader() root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - table = root.children[0].symbol_table - for name in ("first", "second"): - datatype = table.lookup(name).datatype - assert isinstance(datatype, psyir_symbols.UnsupportedFortranType) - assert datatype.declaration == f"integer, {qualifier} :: {name}" + container = root.children[0] + assert container.symbol_table.lookup("exposed").visibility == \ + psyir_symbols.Symbol.Visibility.PUBLIC + assert container.symbol_table.lookup("hidden").visibility == \ + psyir_symbols.Symbol.Visibility.PRIVATE + assert isinstance(container.children[0].symbol_table.lookup( + "scalar").datatype, psyir_symbols.UnsupportedFortranType) -def test_unsupported_initialisation_is_entity_specific(): - '''Test that one unsupported initializer does not affect its sibling.''' + +def test_use_rename(): + '''Test a renamed symbol in a USE ONLY statement.''' processor = FortranTreeSitterReader() valid_code = """ - module declarations + program main + use kinds, only: local_kind => remote_kind implicit none - integer :: first = [(i, i=1,2)], second = 2 - end module declarations + end program main """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) table = root.children[0].symbol_table - first = table.lookup("first") - second = table.lookup("second") - assert isinstance(first.datatype, psyir_symbols.ScalarType) - assert isinstance(first.initial_value, psyir_nodes.CodeBlock) - assert first.initial_value.structure == \ - psyir_nodes.CodeBlock.Structure.EXPRESSION - assert isinstance(second.datatype, psyir_symbols.ScalarType) - assert second.initial_value.value == "2" + container = table.lookup("kinds") + imported = table.lookup("local_kind") + assert isinstance(container, psyir_symbols.ContainerSymbol) + assert imported.interface.container_symbol is container + assert imported.interface.orig_name == "remote_kind" -def test_save_attribute(): - '''Test the SAVE declaration attribute.''' +def test_wildcard_and_defensive_import_branches(): + '''Test wildcard import and defensive malformed/identity import + handling.''' + valid_code = """ + module imports + use wildcard_source + end module imports + """ processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + assert root.children[0].symbol_table.lookup( + "wildcard_source").wildcard_import + + table = psyir_symbols.SymbolTable() + processor._current_scope = table + module_name = SimpleNamespace( + type="module_name", children=[], text=b"source") + malformed_rename = SimpleNamespace( + type="rename", children=[SimpleNamespace( + type="identifier", children=[], text=b"local")]) + included = SimpleNamespace( + type="included_items", children=[malformed_rename]) + use_statement = SimpleNamespace( + children=[module_name, included]) + processor._use_statement_handler(use_statement) + container = table.lookup("source") + + processor._add_imported_symbol("source", "source", container) + assert table.lookup("source") is container + + +def test_declaration_conflicts_with_import(): + '''Test that redeclaring an imported bare Symbol is localised.''' valid_code = """ - module declarations - implicit none - double precision, save :: accumulator - end module declarations + subroutine conflict() + use other, only: value + integer :: value + end subroutine conflict """ + processor = FortranTreeSitterReader() root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - accumulator = root.children[0].symbol_table.lookup("accumulator") - assert accumulator.datatype == \ - psyir_symbols.ScalarType.real_double_type() - assert isinstance(accumulator.interface, psyir_symbols.StaticInterface) + + codeblock = root.children[0].children[0] + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert "already declared as a non-data symbol" in \ + codeblock.preceding_comment -def test_logical_literal(): - '''Test a logical literal used as an initial value.''' - processor = FortranTreeSitterReader() +def test_name_conflict(): + '''Test that name conflicts in different declarations on the same scope + are invalid.''' valid_code = """ - module declarations - implicit none - logical :: enabled = .true. - end module declarations + module conflict + use other + integer :: other + end module conflict """ - root = processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) - enabled = root.children[0].symbol_table.lookup("enabled") - assert enabled.initial_value.value == "true" + processor = FortranTreeSitterReader() + + message = "USE module 'other' conflicts with another symbol" + with pytest.raises(ValueError, match=message): + processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) def test_derived_type_definition(): @@ -624,64 +992,145 @@ def test_derived_type_component_host_association(): assert x_type.precision.symbol is wp -def test_structure_reference(): - '''Test a scalar structure-component reference.''' - processor = FortranTreeSitterReader() +def test_unsupported_and_forward_declared_derived_types(): + '''Test unsupported type procedures and completion of a forward type.''' valid_code = """ - subroutine get_x(item, value) - type(point) :: item - real :: value - value = item%x - end subroutine get_x + module types + type :: with_procedure + contains + procedure :: method + end type with_procedure + type(forward) :: instance + type, private :: forward + integer :: value + end type forward + end module types """ + processor = FortranTreeSitterReader() root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - reference = root.children[0].children[0].rhs - assert isinstance(reference, psyir_nodes.StructureReference) - assert reference.symbol.name == "item" - assert reference.member.name == "x" + table = root.children[0].symbol_table + assert isinstance(table.lookup("with_procedure").datatype, + psyir_symbols.UnsupportedFortranType) + forward = table.lookup("forward") + assert isinstance(forward.datatype, psyir_symbols.StructureType) + assert forward.visibility == psyir_symbols.Symbol.Visibility.PRIVATE -def test_array_of_structures_reference(): - '''Test an indexed array-of-structures component reference.''' - processor = FortranTreeSitterReader() + +def test_invalid_derived_type_component_falls_back(): + '''Test a component-name conflict makes the whole type unsupported.''' valid_code = """ - subroutine get_value(items, value) - type(point) :: items(2) - real :: value - value = items(1)%vector(2) - end subroutine get_value + module types + type :: invalid + type(component_type) :: value + integer :: component_type + end type invalid + end module types """ + processor = FortranTreeSitterReader() root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - reference = root.children[0].children[0].rhs - assert isinstance(reference, - psyir_nodes.ArrayOfStructuresReference) - assert reference.indices[0].value == "1" - assert reference.member.name == "vector" - assert reference.member.indices[0].value == "2" + + assert isinstance(root.children[0].symbol_table.lookup("invalid").datatype, + psyir_symbols.UnsupportedFortranType) -def test_multidimensional_explicit_array_bounds(): - '''Test explicit lower bounds in a multidimensional shape.''' +def test_derived_type_name_conflict(): + '''Test a derived type whose name is already used by a data symbol.''' + valid_code = """ + module conflict + integer :: item + type :: item + integer :: value + end type item + end module conflict + """ + processor = FortranTreeSitterReader() + + with pytest.raises(InternalError, match="No node was expected"): + processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + + +def test_generic_interface(): + '''Test a named generic interface.''' processor = FortranTreeSitterReader() valid_code = """ - module array_shapes + module dispatch implicit none - real :: field(-2:10, 20) - end module array_shapes + interface apply + module procedure apply_integer, apply_real + end interface apply + end module dispatch """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - datatype = root.children[0].symbol_table.lookup("field").datatype - assert isinstance(datatype, psyir_symbols.ArrayType) - assert len(datatype.shape) == 2 - assert datatype.shape[0].lower.operator == \ - psyir_nodes.UnaryOperation.Operator.MINUS - assert datatype.shape[0].lower.children[0].value == "2" - assert datatype.shape[0].upper.value == "10" - assert datatype.shape[1].lower.value == "1" - assert datatype.shape[1].upper.value == "20" + generic = root.children[0].symbol_table.lookup("apply") + assert isinstance(generic, psyir_symbols.GenericInterfaceSymbol) + assert [info.symbol.name for info in generic.routines] == [ + "apply_integer", "apply_real"] + assert all(info.from_container for info in generic.routines) + + +def test_interface_routine_symbol_are_consistent(): + '''Test that a routine declared by an interface is reused by its body.''' + valid_code = """ + module routines + interface generic + module procedure implementation + end interface generic + contains + pure integer function implementation() + implementation = 1 + end function implementation + end module routines + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + container = root.children[0] + routine = container.children[0] + + assert routine.symbol is container.symbol_table.lookup("implementation") + assert routine.symbol.is_pure + assert routine.symbol.datatype == \ + psyir_symbols.ScalarType.integer_type() + + +@pytest.mark.parametrize("valid_code", [ + """ + module interfaces + abstract interface + subroutine method() + end subroutine method + end interface + end module interfaces + """, + """ + module interfaces + integer :: method + interface generic + module procedure method + end interface generic + end module interfaces + """, + """ + module interfaces + interface generic + subroutine method() + end subroutine method + end interface generic + end module interfaces + """, +]) +def test_unsupported_interface_forms(valid_code): + '''Test unsupported interface forms are detected in declaration context.''' + processor = FortranTreeSitterReader() + + with pytest.raises(InternalError, match="No node was expected"): + processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) def test_unary_operation(): @@ -774,6 +1223,83 @@ def test_implicit_array_section_bounds(): psyir_nodes.IntrinsicCall.Intrinsic.UBOUND +def test_array_constructor(): + '''Test a simple array constructor.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine constructor(values) + real :: values(3) + values = [1.0, 2.0, 3.0] + end subroutine constructor + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + constructor = root.children[0].children[0].rhs + assert isinstance(constructor, psyir_nodes.ArrayConstructor) + assert [element.value for element in constructor.children] == [ + "1.0", "2.0", "3.0"] + + +def test_implied_do_codeblock(): + '''Test the localized fallback for an implied-DO array constructor.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine constructor(values) + real :: values(3) + integer :: index + values = [(real(index), index=1,3)] + end subroutine constructor + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + assignment = root.children[0].children[0] + assert isinstance(assignment, psyir_nodes.Assignment) + codeblock = assignment.rhs + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert codeblock.structure == psyir_nodes.CodeBlock.Structure.EXPRESSION + assert ("Array constructors with implied-DO loops are not supported" in + codeblock.preceding_comment) + + +def test_unresolved_identifiers_and_call_forms(): + '''Test unresolved names, empty calls, ranges and invalid scalar calls.''' + valid_code = """ + subroutine expressions(result, scalar) + integer :: result, scalar + result = unknown + result = function_without_arguments() + result = function_with_range(1:2) + result = scalar(1) + end subroutine expressions + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + + assert routine.children[0].rhs.symbol.name == "unknown" + assert isinstance(routine.children[1].rhs, psyir_nodes.Call) + assert routine.children[1].rhs.arguments == () + for assignment in routine.children[2:]: + assert isinstance(assignment.rhs, psyir_nodes.CodeBlock) + + +def test_named_call_argument(): + '''Test a named subroutine-call argument.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine caller(value) + integer :: value + call update(value, result=value) + end subroutine caller + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + call = root.children[0].children[0] + assert isinstance(call, psyir_nodes.Call) + assert call.argument_names == [None, "result"] + + def test_intrinsic_call(): '''Test an intrinsic function call.''' processor = FortranTreeSitterReader() @@ -790,37 +1316,184 @@ def test_intrinsic_call(): assert call.intrinsic == psyir_nodes.IntrinsicCall.Intrinsic.SIN -def test_named_call_argument(): - '''Test a named subroutine-call argument.''' +def test_invalid_array_and_intrinsic_arguments(): + '''Test named array subscripts and an invalid intrinsic signature.''' + valid_code = """ + subroutine expressions(array, result) + integer :: array(10), result + result = array(dim=1) + result = sin() + end subroutine expressions + """ processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + + assert all(isinstance(assignment.rhs, psyir_nodes.CodeBlock) + for assignment in root.children[0].children) + assert "Named subscripts" in \ + root.children[0].children[0].rhs.preceding_comment + assert "Unsupported argument form" in \ + root.children[0].children[1].rhs.preceding_comment + + +def test_existing_routine_and_local_type_calls(): + '''Test calls resolved to an existing routine or local datatype name.''' valid_code = """ - subroutine caller(value) - integer :: value - call update(value, result=value) - end subroutine caller + module routines + interface generic + module procedure existing + end interface generic + contains + integer function existing() + existing = 1 + end function existing + subroutine caller(result) + integer :: result + type(local_type) :: value + result = existing() + value = local_type(1) + end subroutine caller + end module routines + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + caller = root.children[0].children[1] + + assert caller.children[0].rhs.routine.symbol.name == "existing" + assert isinstance(caller.symbol_table.lookup("local_type"), + psyir_symbols.DataTypeSymbol) + assert isinstance(caller.children[1].rhs, psyir_nodes.Call) + + +def test_call_statement_edge_cases(): + '''Test imported-symbol specialisation and invalid call targets.''' + valid_code = """ + subroutine calls(object) + use procedures, only: imported + type(item_type) :: object + integer :: data + call imported() + call data() + call object%method() + end subroutine calls + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + + assert isinstance(routine.children[0], psyir_nodes.Call) + assert isinstance(routine.symbol_table.lookup("imported"), + psyir_symbols.RoutineSymbol) + assert all(isinstance(node, psyir_nodes.CodeBlock) + for node in routine.children[1:]) + + +def test_structure_reference(): + '''Test a scalar structure-component reference.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine get_x(item, value) + type(point) :: item + real :: value + value = item%x + end subroutine get_x + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + reference = root.children[0].children[0].rhs + assert isinstance(reference, psyir_nodes.StructureReference) + assert reference.symbol.name == "item" + assert reference.member.name == "x" + + +def test_array_of_structures_reference(): + '''Test an indexed array-of-structures component reference.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine get_value(items, value) + type(point) :: items(2) + real :: value + value = items(1)%vector(2) + end subroutine get_value + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + reference = root.children[0].children[0].rhs + assert isinstance(reference, + psyir_nodes.ArrayOfStructuresReference) + assert reference.indices[0].value == "1" + assert reference.member.name == "vector" + assert reference.member.indices[0].value == "2" + + +def test_structure_reference_edge_cases(): + '''Test unknown, nested, named-index and non-data structure bases.''' + valid_code = """ + module structures + type :: item_type + integer :: field + end type item_type + contains + subroutine references(items, result) + type(item_type) :: items(2) + integer :: result + result = unknown%field + result = items(1)%vector(2)%field + result = items(dim=1)%field + result = item_type%field + end subroutine references + end module structures + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + statements = root.children[0].children[0].children + + assert isinstance(statements[0].rhs, psyir_nodes.StructureReference) + assert statements[0].rhs.symbol.name == "unknown" + assert isinstance(statements[1].rhs, + psyir_nodes.ArrayOfStructuresReference) + assert statements[1].rhs.member.member.name == "field" + assert isinstance(statements[2].rhs, psyir_nodes.CodeBlock) + assert isinstance(statements[3].rhs, psyir_nodes.CodeBlock) + + +def test_named_trailing_structure_index(): + '''Test a named index on the final component of a structure access.''' + valid_code = """ + subroutine structure(result) + integer :: result + result = item%values(dim=1) + end subroutine structure """ + processor = FortranTreeSitterReader() root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - call = root.children[0].children[0] - assert isinstance(call, psyir_nodes.Call) - assert call.argument_names == [None, "result"] + + codeblock = root.children[0].children[0].rhs + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert "Unsupported structure member array access" in \ + codeblock.preceding_comment -def test_array_constructor(): - '''Test a simple array constructor.''' +def test_ignored_comment(): + '''Test that an ignored comment does not create a CodeBlock.''' processor = FortranTreeSitterReader() valid_code = """ - subroutine constructor(values) - real :: values(3) - values = [1.0, 2.0, 3.0] - end subroutine constructor + subroutine commented(value) + integer :: value + ! This comment must not create a CodeBlock. + value = 1 + end subroutine commented """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - constructor = root.children[0].children[0].rhs - assert isinstance(constructor, psyir_nodes.ArrayConstructor) - assert [element.value for element in constructor.children] == [ - "1.0", "2.0", "3.0"] + routine = root.children[0] + assert len(routine.children) == 1 + assert isinstance(routine.children[0], psyir_nodes.Assignment) def test_pointer_assignment(): @@ -855,6 +1528,83 @@ def test_nullify_statement(): assert nullify.intrinsic == psyir_nodes.IntrinsicCall.Intrinsic.NULLIFY +def test_allocate_statement(): + '''Test an ALLOCATE statement.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine allocate_array(array, extent, status) + integer :: extent, status + real, allocatable :: array(:) + allocate(array(extent), stat=status) + end subroutine allocate_array + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + allocate = root.children[0].children[0] + assert allocate.intrinsic == \ + psyir_nodes.IntrinsicCall.Intrinsic.ALLOCATE + assert isinstance(allocate.arguments[0], psyir_nodes.ArrayReference) + assert allocate.argument_names == [None, "stat"] + + +def test_allocate_bounds_and_invalid_object(): + '''Test explicit allocation bounds, missing upper bound and an import.''' + valid_code = """ + subroutine bounds(first, second, third) + real, allocatable :: first(:), second(:), third(:) + allocate(first(2:10), second(:10)) + allocate(third(2:)) + end subroutine bounds + subroutine imported_object() + use source, only: array + allocate(array(2)) + end subroutine imported_object + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + allocation = root.children[0].children[0] + + assert allocation.arguments[0].indices[0].start.value == "2" + assert allocation.arguments[0].indices[0].stop.value == "10" + assert allocation.arguments[1].indices[0].start.value == "1" + assert isinstance(root.children[0].children[1], psyir_nodes.CodeBlock) + assert isinstance(root.children[1].children[0], psyir_nodes.CodeBlock) + + +def test_deallocate_statement(): + '''Test a DEALLOCATE statement.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine deallocate_array(array, status) + integer :: status + real, allocatable :: array(:) + deallocate(array, stat=status) + end subroutine deallocate_array + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + deallocate = root.children[0].children[0] + assert deallocate.intrinsic == \ + psyir_nodes.IntrinsicCall.Intrinsic.DEALLOCATE + assert deallocate.argument_names == [None, "stat"] + + +def test_stop_codeblock(): + '''Test the localized fallback for a STOP statement.''' + processor = FortranTreeSitterReader() + valid_code = """ + subroutine stop_execution() + stop 1 + end subroutine stop_execution + """ + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + codeblock = root.children[0].children[0] + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert "Unsupported 'stop_statement'" in codeblock.preceding_comment + + def test_if_construct(): '''Test IF, ELSE IF and ELSE clauses.''' processor = FortranTreeSitterReader() @@ -872,16 +1622,39 @@ def test_if_construct(): """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) + + # Top if construct ifblock = root.children[0].children[0] assert isinstance(ifblock, psyir_nodes.IfBlock) + + # Elseif nested = ifblock.else_body.children[0] + assert isinstance(nested, psyir_nodes.IfBlock) assert "was_elseif" in nested.annotations + + # Check final else body assert nested.else_body.children[0].rhs.operator == \ psyir_nodes.BinaryOperation.Operator.ADD -def test_counted_do_loop(): - '''Test a counted DO loop.''' +def test_single_statement_if(): + '''Test the single-statement IF annotation.''' + valid_code = """ + subroutine control(value) + integer :: value + if (value > 0) value = 1 + end subroutine control + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + + assert "was_single_stmt" in routine.children[0].annotations + + +def test_do_loop(): + '''Test a DO loop.''' processor = FortranTreeSitterReader() valid_code = """ subroutine counted(limit) @@ -937,6 +1710,54 @@ def test_unconditional_do_loop(): assert "was_unconditional" in loop.annotations +def test_do_variable_variants(): + '''Test implicit, imported and non-scalar DO variables.''' + valid_code = """ + subroutine implicit_variable() + do index = 1, 10 + end do + end subroutine implicit_variable + subroutine imported_variable() + use source, only: index + do index = 1, 10 + end do + end subroutine imported_variable + subroutine array_variable() + integer :: index(2) + do index = 1, 10 + end do + end subroutine array_variable + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + + implicit = root.children[0] + assert isinstance(implicit.children[0], psyir_nodes.Loop) + assert implicit.symbol_table.lookup("index").datatype == \ + psyir_symbols.ScalarType.integer_type() + assert isinstance(root.children[1].children[0], psyir_nodes.CodeBlock) + assert isinstance(root.children[2].children[0], psyir_nodes.CodeBlock) + + +def test_keyword_statement(): + '''Test that an unsupported CYCLE is represented by a CodeBlock.''' + valid_code = """ + subroutine control() + do + cycle + end do + end subroutine control + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + + assert isinstance(routine.children[0].loop_body.children[0], + psyir_nodes.CodeBlock) + + def test_where_construct(): '''Test a WHERE construct with ELSEWHERE.''' processor = FortranTreeSitterReader() @@ -985,151 +1806,153 @@ def test_select_case_construct(): assert second.else_body.children[0].rhs.value == "0" -def test_allocate_statement(): - '''Test an ALLOCATE statement.''' - processor = FortranTreeSitterReader() +def test_select_case_expression_and_open_ranges(): + '''Test an expression selector and lower- or upper-open CASE ranges.''' valid_code = """ - subroutine allocate_array(array, extent, status) - integer :: extent, status - real, allocatable :: array(:) - allocate(array(extent), stat=status) - end subroutine allocate_array + subroutine selection(value) + integer :: value + select case(value + 1) + case(:5) + value = 1 + case(8:) + value = 2 + end select + end subroutine selection """ - root = processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) - allocate = root.children[0].children[0] - assert allocate.intrinsic == \ - psyir_nodes.IntrinsicCall.Intrinsic.ALLOCATE - assert isinstance(allocate.arguments[0], psyir_nodes.ArrayReference) - assert allocate.argument_names == [None, "stat"] - - -def test_deallocate_statement(): - '''Test a DEALLOCATE statement.''' processor = FortranTreeSitterReader() - valid_code = """ - subroutine deallocate_array(array, status) - integer :: status - real, allocatable :: array(:) - deallocate(array, stat=status) - end subroutine deallocate_array - """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - deallocate = root.children[0].children[0] - assert deallocate.intrinsic == \ - psyir_nodes.IntrinsicCall.Intrinsic.DEALLOCATE - assert deallocate.argument_names == [None, "stat"] - + first = root.children[0].children[0] + second = first.else_body.children[0] -def test_stop_codeblock(): - '''Test the localized fallback for a STOP statement.''' - processor = FortranTreeSitterReader() - valid_code = """ - subroutine stop_execution() - stop 1 - end subroutine stop_execution - """ - root = processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) - codeblock = root.children[0].children[0] - assert isinstance(codeblock, psyir_nodes.CodeBlock) - assert "Unsupported 'stop_statement'" in codeblock.preceding_comment + assert first.condition.operator == psyir_nodes.BinaryOperation.Operator.LE + assert second.condition.operator == \ + psyir_nodes.BinaryOperation.Operator.GE -def test_default_visibility(): - '''Test a module's default visibility.''' - processor = FortranTreeSitterReader() +def test_select_case_with_only_default_is_unsupported(): + '''Test SELECT CASE with no conditional case becomes a CodeBlock.''' valid_code = """ - module visibility - implicit none - private - contains - subroutine hidden() - end subroutine hidden - end module visibility + subroutine selection(value) + integer :: value + select case(value) + case default + value = 1 + value = 2 + end select + end subroutine selection """ - root = processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) - container = root.children[0] - assert container.symbol_table.default_visibility == \ - psyir_symbols.Symbol.Visibility.PRIVATE - assert container.symbol_table.lookup("hidden").visibility == \ - psyir_symbols.Symbol.Visibility.PRIVATE - - -def test_named_visibility(): - '''Test name-specific visibility for a contained routine.''' processor = FortranTreeSitterReader() - valid_code = """ - module visibility - implicit none - private - public :: exposed - contains - subroutine exposed() - end subroutine exposed - end module visibility - """ root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - exposed = root.children[0].symbol_table.lookup("exposed") - assert exposed.visibility == psyir_symbols.Symbol.Visibility.PUBLIC - -def test_generic_interface(): - '''Test a named generic interface.''' - processor = FortranTreeSitterReader() - valid_code = """ - module dispatch - implicit none - interface apply - module procedure apply_integer, apply_real - end interface apply - end module dispatch - """ - root = processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) - generic = root.children[0].symbol_table.lookup("apply") - assert isinstance(generic, psyir_symbols.GenericInterfaceSymbol) - assert [info.symbol.name for info in generic.routines] == [ - "apply_integer", "apply_real"] - assert all(info.from_container for info in generic.routines) + assert isinstance(root.children[0].children[0], psyir_nodes.CodeBlock) + assert "only a default clause" in \ + root.children[0].children[0].preceding_comment -def test_ignored_comment(): - '''Test that an ignored comment does not create a CodeBlock.''' - processor = FortranTreeSitterReader() +# pylint: disable=too-many-locals +def test_malformed_operation_and_statement_guards(monkeypatch): + '''Test defensive guards for malformed expressions and statements.''' valid_code = """ - subroutine commented(value) - integer :: value - ! This comment must not create a CodeBlock. - value = 1 - end subroutine commented + subroutine guards(array) + real, allocatable :: array(:) + allocate(array(10)) + end subroutine guards """ - root = processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) - routine = root.children[0] - assert len(routine.children) == 1 - assert isinstance(routine.children[0], psyir_nodes.Assignment) - - -def test_implied_do_codeblock(): - '''Test the localized fallback for an implied-DO array constructor.''' processor = FortranTreeSitterReader() + parse_tree = processor.generate_parse_tree_from_source(valid_code) + number = _first_tsnode(parse_tree, "number_literal") + + bad_unary = SimpleNamespace(children=[ + SimpleNamespace(type="operator", text=b"?"), number]) + with pytest.raises(NotImplementedError, match="unary operator"): + processor._operation(bad_unary) + + bad_binary = SimpleNamespace(children=[ + number, SimpleNamespace(type="operator", text=b"?"), number]) + with pytest.raises(NotImplementedError, match="binary operator"): + processor._operation(bad_binary) + with pytest.raises(NotImplementedError, match="operation structure"): + processor._operation(SimpleNamespace(children=[])) + + with pytest.raises(NotImplementedError, match="assignment structure"): + processor._assignment_statement_handler( + SimpleNamespace(children=[])) + with pytest.raises(NotImplementedError, match="bounds remapping"): + processor._pointer_association_statement_handler( + SimpleNamespace(children=[])) + with pytest.raises(NotImplementedError, match="IF statement"): + processor._if_statement_handler(SimpleNamespace(children=[])) + + statement = SimpleNamespace( + type="do_statement", + children=[SimpleNamespace( + type="loop_control_expression", children=[])]) + loop = SimpleNamespace(type="do_loop", children=[statement]) + with pytest.raises(NotImplementedError, match="counted DO loop"): + processor._do_loop_handler(loop) + + identifier = _first_tsnode(parse_tree, "identifier") + selector = SimpleNamespace(type="selector", children=[identifier]) + malformed_case = SimpleNamespace(type="case_statement", children=[]) + select = SimpleNamespace(children=[selector, malformed_case]) + with pytest.raises(NotImplementedError, match="Malformed CASE"): + processor._select_case_statement_handler(select) + + malformed_member = SimpleNamespace( + type="derived_type_member_expression", + children=[SimpleNamespace( + type="identifier", children=[], text=b"item")]) + with pytest.raises(NotImplementedError, match="Malformed structure"): + processor._decompose_structure(malformed_member) + with pytest.raises(NotImplementedError, match="structure access base"): + processor._decompose_structure( + SimpleNamespace(type="number_literal", children=[])) + + allocate = _first_tsnode(parse_tree, "allocate_statement") + + def invalid_intrinsic(*_args, **_kwargs): + '''Simulate rejection by PSyIR intrinsic signature validation.''' + raise TypeError("invalid test operands") + + monkeypatch.setattr(psyir_nodes.IntrinsicCall, "create", + invalid_intrinsic) + with pytest.raises(NotImplementedError, match="Unsupported operands"): + processor._memory_statement(allocate) + + +def test_scope_and_handler_defensive_errors(): + '''Test scope ownership defensive checks.''' valid_code = """ - subroutine constructor(values) - real :: values(3) - integer :: index - values = [(real(index), index=1,3)] - end subroutine constructor + subroutine routine() + end subroutine routine + module types + type :: item + integer :: value + end type item + end module types """ - root = processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) - assignment = root.children[0].children[0] - assert isinstance(assignment, psyir_nodes.Assignment) - codeblock = assignment.rhs - assert isinstance(codeblock, psyir_nodes.CodeBlock) - assert codeblock.structure == psyir_nodes.CodeBlock.Structure.EXPRESSION - assert ("Array constructors with implied-DO loops are not supported" in - codeblock.preceding_comment) + processor = FortranTreeSitterReader() + parse_tree = processor.generate_parse_tree_from_source(valid_code) + routine = _first_tsnode(parse_tree, "subroutine") + derived = _first_tsnode(parse_tree, "derived_type_definition") + + duplicate = SimpleNamespace(children=[ + SimpleNamespace(type="name"), SimpleNamespace(type="name")]) + with pytest.raises(InternalError, match="Expected only 1"): + ftr.child_of_type(duplicate, "name") + + parent = psyir_nodes.ScopingNode(symbol_table=psyir_symbols.SymbolTable()) + attached = psyir_nodes.ScopingNode( + symbol_table=psyir_symbols.SymbolTable()) + attached._parent = parent + with pytest.raises(InternalError, match="must be an orphan"): + with processor._using_temporary_scope(parent, attached): + pass + attached._parent = None + + with pytest.raises(InternalError, match="Routine must be translated"): + processor._procedure_handler(routine) + with pytest.raises(InternalError, match="derived type must be translated"): + processor._derived_type_definition_handler(derived) From 5087395c506d9b7acd5a60d7646b696c10c05e0b Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 13 Aug 2026 11:14:41 +0100 Subject: [PATCH 13/23] Improve treesitter frontend utilities names and docstrings --- .../frontend/fortran_treesitter_reader.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 4c57a88c0e..f2c0ac1e04 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -65,15 +65,16 @@ def to_str(node: 'TSNode') -> str: return node.text.decode('utf8') if node.text else "" -def direct_child_of_type( +def iter_child_of_type( tsnode: Optional['TSNode'], types: str | Container[str] ) -> Generator['TSNode']: - '''Return the first direct child having the supplied type. + ''' Provides a generator to iterate over the provided tsnode + chidlren of the given type(s). :param tsnode: tree-sitter node whose children are searched. :param node_type: tree-sitter type to find. - :returns: matching child, or ``None`` if no child matches. + :yields: matching child, or ``None`` if no child matches. ''' check_types = (types,) if isinstance(types, str) else types if tsnode: @@ -83,16 +84,19 @@ def direct_child_of_type( def child_of_type( - tsnode: Optional['TSNode'], node_type: str + tsnode: Optional['TSNode'], node_type: str | Container[str] ) -> Optional['TSNode']: - ''' Return the direct child having the supplied type. + ''' Return the direct child having the supplied type(s). And validate + that is the only child of the supplied type. :param tsnode: tree-sitter node whose children are searched. - :param node_type: tree-sitter type to find. + :param node_type: tree-sitter type(s) to find. :returns: matching child, or ``None`` if no child matches. + + :raises InternalError: if more than one node of that type exists. ''' - children = list(direct_child_of_type(tsnode, node_type)) + children = list(iter_child_of_type(tsnode, node_type)) if len(children) == 0: return None elif len(children) > 1: @@ -1242,7 +1246,7 @@ def _derived_type_definition_handler( visibility_map = self._process_access_statements( tsnode.children) try: - for declaration in direct_child_of_type( + for declaration in iter_child_of_type( tsnode, "variable_declaration"): self._variable_declaration_handler(declaration) self._apply_visibility(visibility_map) @@ -1293,10 +1297,10 @@ def _interface_handler( "Abstract and operator interfaces are not supported") name = to_str(name_node) routines = [] - for procedure in direct_child_of_type(tsnode, "procedure_statement"): + for procedure in iter_child_of_type(tsnode, "procedure_statement"): from_container = "module" in [ child.type for child in procedure.children[0].children] - for method in direct_child_of_type(procedure, "method_name"): + for method in iter_child_of_type(procedure, "method_name"): routine_name = to_str(method) try: routine = symtab.lookup(routine_name) @@ -1756,7 +1760,7 @@ def _if_statement_handler( body_nodes = [child for child in tsnode.children if child.type not in structural] else_clause = child_of_type(tsnode, "else_clause") - else_ifs = list(direct_child_of_type(tsnode, "elseif_clause")) + else_ifs = list(iter_child_of_type(tsnode, "elseif_clause")) annotations = [] if not child_of_type(tsnode, "end_if_statement"): annotations.append("was_single_stmt") @@ -1917,7 +1921,7 @@ def _select_case_statement_handler( else: selector = self._process_nodes( selector_node, _NodeExpectation.EXPRESSION) - cases = list(direct_child_of_type(tsnode, "case_statement")) + cases = list(iter_child_of_type(tsnode, "case_statement")) default_body = None normal = [] for case in cases: From 15919c532ea0d7db1c4144d814cf3982dd8dc1aa Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 13 Aug 2026 11:48:49 +0100 Subject: [PATCH 14/23] Fix flake8 --- src/psyclone/psyir/frontend/fortran_treesitter_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index f2c0ac1e04..f5740c4b9e 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -94,7 +94,7 @@ def child_of_type( :returns: matching child, or ``None`` if no child matches. - :raises InternalError: if more than one node of that type exists. + :raises InternalError: if more than one node of that type exists. ''' children = list(iter_child_of_type(tsnode, node_type)) if len(children) == 0: From 7081ef2c37652b5252c4d94e85302bdfcc44110b Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 13 Aug 2026 14:39:07 +0100 Subject: [PATCH 15/23] Fix some issues in the Treesitter frontend and improve its test coverage --- .../frontend/fortran_treesitter_reader.py | 134 ++++-- .../fortran_treesitter_reader/ftr_test.py | 384 +++++++++++++++++- 2 files changed, 484 insertions(+), 34 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index f5740c4b9e..7627d4073a 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -522,9 +522,13 @@ def _module_handler( "implicit_statement", "internal_procedures", "public_statement", "private_statement" } - self._process_nodes( + # Specification statements normally only update the symbol table + # and therefore return no Node. Keep any unsupported statements + # as CodeBlocks so that valid Fortran is not lost (and, in + # particular, does not violate an expectation of no result). + container.children.extend(self._process_nodes( [child for child in tsnode.children - if child.type not in skip], _NodeExpectation.NONE) + if child.type not in skip], _NodeExpectation.LIST)) internal = child_of_type(tsnode, "internal_procedures") if internal: @@ -678,9 +682,17 @@ def _number_literal_handler( ''' text = to_str(tsnode).lower() value, _, kind = text.partition("_") - datatype = (symbols.ScalarType.real_type() - if any(char in value for char in ".ed") - else symbols.ScalarType.integer_type()) + is_real = any(char in value for char in ".ed") + # PSyIR stores all real exponents using ``e`` notation while the + # Fortran ``d`` exponent also specifies double precision. + has_double_exponent = "d" in value + if has_double_exponent: + value = value.replace("d", "e", 1) + datatype = ( + symbols.ScalarType.real_double_type() + if has_double_exponent else + symbols.ScalarType.real_type() + if is_real else symbols.ScalarType.integer_type()) if kind: if kind == "4": precision = symbols.ScalarType.Precision.SINGLE @@ -704,7 +716,30 @@ def _string_literal_handler( :returns: PSyIR character Literal. ''' text = to_str(tsnode) - return nodes.Literal(text[1:-1], symbols.ScalarType.character_type()) + quote_positions = [position for position in + (text.find("'"), text.find('"')) + if position >= 0] + if not quote_positions: + raise NotImplementedError( + "A character literal has no quote delimiter") + quote_position = min(quote_positions) + quote = text[quote_position] + if text[-1] != quote: + raise NotImplementedError( + "A character literal has mismatched quote delimiters") + + prefix = text[:quote_position] + datatype = symbols.ScalarType.character_type() + if prefix: + if not prefix.endswith("_") or len(prefix) == 1: + raise NotImplementedError( + "Unsupported character literal kind prefix") + kind = prefix[:-1].lower() + precision = (int(kind) if kind.isdigit() else + nodes.Reference(self._kind_symbol(kind))) + datatype = symbols.ScalarType( + symbols.ScalarType.Intrinsic.CHARACTER, precision) + return nodes.Literal(text[quote_position + 1:-1], datatype) def _boolean_literal_handler( self, tsnode: 'TSNode' @@ -928,7 +963,10 @@ def _add_or_update_datasymbol( declared_symbol.interface, symbols.AutomaticInterface): symbol.interface = declared_symbol.interface if declared_symbol.initial_value is not None: - symbol.initial_value = declared_symbol.initial_value + # The initial value belongs to the temporary symbol created + # for this declaration. Give the existing symbol its own + # expression tree when completing a forward reference. + symbol.initial_value = declared_symbol.initial_value.copy() if declared_symbol.is_constant: symbol.is_constant = True return @@ -1179,7 +1217,9 @@ def _use_statement_handler( if not isinstance(container, symbols.ContainerSymbol): raise ValueError( f"USE module '{module_name}' conflicts with another symbol") - container.wildcard_import = wildcard + # Multiple USE statements for the same module are cumulative. An + # ONLY list must therefore not undo a wildcard import seen earlier. + container.wildcard_import = container.wildcard_import or wildcard if included: for child in included.children: @@ -1427,6 +1467,19 @@ def _call_expression_handler( return nodes.ArrayReference.create(symbol, indices) arguments = self._arguments(argument_list) + + # Explicit imports initially create a bare Symbol. Once it is used as + # a function, specialise it in the same way as for a CALL statement. + # pylint: disable=unidiomatic-typecheck + if type(symbol) is symbols.Symbol: + symbol.specialise(symbols.RoutineSymbol) + + # Resolve declared routines (including generic interfaces) before + # considering intrinsic names since Fortran permits an intrinsic to + # be shadowed by a user procedure. + if isinstance(symbol, symbols.RoutineSymbol): + return nodes.Call.create(symbol, arguments) + intrinsic = next( (item for item in nodes.IntrinsicCall.Intrinsic if item.name.lower() == name), None) @@ -1440,14 +1493,13 @@ def _call_expression_handler( f"Unsupported argument form for intrinsic '{name}'" ) from None - if not isinstance(symbol, symbols.RoutineSymbol): - if symbol is not None and not isinstance( - symbol, symbols.DataTypeSymbol): - raise NotImplementedError( - f"'{name}(...)' cannot be classified as an array or call") - symbol = symbols.RoutineSymbol(name) - if name not in symtab: - symtab.add(symbol) + if symbol is not None and not isinstance( + symbol, symbols.DataTypeSymbol): + raise NotImplementedError( + f"'{name}(...)' cannot be classified as an array or call") + symbol = symbols.RoutineSymbol(name) + if name not in symtab: + symtab.add(symbol) return nodes.Call.create(symbol, arguments) def _arguments( @@ -1506,8 +1558,18 @@ def _range( before, after, has_colon = self._split_extent(tsnode) if not has_colon: raise NotImplementedError("Malformed array range") - # A section can have a second colon before its step. - after = [child for child in after if child.type != ":"] + # Preserve the field separated by a second colon. In particular, an + # absent upper bound in ``lower::step`` must not cause the step to be + # interpreted as the stop expression. + second_colon = next( + (idx for idx, child in enumerate(after) if child.type == ":"), + None) + if second_colon is None: + upper = after + step_nodes = [] + else: + upper = after[:second_colon] + step_nodes = after[second_colon + 1:] dim = nodes.Literal(str(dimension), symbols.ScalarType.integer_type()) start = (self._process_nodes( @@ -1516,13 +1578,13 @@ def _range( nodes.IntrinsicCall.Intrinsic.LBOUND, [nodes.Reference(symbol), ("dim", dim.copy())])) stop = (self._process_nodes( - after[0], _NodeExpectation.EXPRESSION) if after else + upper[0], _NodeExpectation.EXPRESSION) if upper else nodes.IntrinsicCall.create( nodes.IntrinsicCall.Intrinsic.UBOUND, [nodes.Reference(symbol), ("dim", dim.copy())])) step = (self._process_nodes( - after[1], _NodeExpectation.EXPRESSION) - if len(after) > 1 else None) + step_nodes[0], _NodeExpectation.EXPRESSION) + if step_nodes else None) return nodes.Range.create(start, stop, step) def _derived_type_member_expression_handler( @@ -1881,12 +1943,32 @@ def _where_statement_handler( body = self._process_nodes( [child for child in tsnode.children if child.type not in structural], _NodeExpectation.LIST) - elsewhere = child_of_type(tsnode, "elsewhere_clause") - other = ( - self._process_nodes( + elsewhere_clauses = list(iter_child_of_type( + tsnode, "elsewhere_clause")) + other = None + # Masked ELSEWHERE clauses have ELSE-IF semantics and are represented + # by nested IfBlocks. Constructing the chain backwards makes the + # following clause the else-body of the current masked clause. + for elsewhere in reversed(elsewhere_clauses): + mask = child_of_type( + elsewhere, "parenthesized_expression") + clause_body = self._process_nodes( [child for child in elsewhere.children - if child.type != "elsewhere"], _NodeExpectation.LIST) - if elsewhere else None) + if child.type not in + ("elsewhere", "parenthesized_expression")], + _NodeExpectation.LIST) + if mask: + nested = nodes.IfBlock.create( + self._process_nodes( + mask, _NodeExpectation.EXPRESSION), + clause_body, other) + nested.annotations.append("was_where") + other = [nested] + else: + if other is not None: + raise NotImplementedError( + "An unmasked ELSEWHERE must be the final clause") + other = clause_body result = nodes.IfBlock.create( self._process_nodes(condition, _NodeExpectation.EXPRESSION), body, other) diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index b52b2be581..26f82267ef 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -136,13 +136,17 @@ def test_process_node_expectation_errors(): '''Test defensive validation of dispatcher result expectations.''' valid_code = """ subroutine assignment(first, second) + ! This comment produces no PSyIR node. integer :: first, second first = second + second = 1 end subroutine assignment """ processor = FortranTreeSitterReader() parse_tree = processor.generate_parse_tree_from_source(valid_code) assignment = _first_tsnode(parse_tree, "assignment_statement") + comment = _first_tsnode(parse_tree, "comment") + number = _first_tsnode(parse_tree, "number_literal") with pytest.raises(InternalError, match="Only one node was expected"): processor._process_nodes([], _NodeExpectation.ONE) @@ -150,6 +154,10 @@ def test_process_node_expectation_errors(): processor._process_nodes(assignment, _NodeExpectation.EXPRESSION) with pytest.raises(InternalError, match="Unsupported node expectation"): processor._process_nodes([], None) + assert processor._process_nodes( + comment, _NodeExpectation.NONE) is None + with pytest.raises(InternalError, match="No node was expected"): + processor._process_nodes(number, _NodeExpectation.NONE) def test_program(): @@ -351,6 +359,52 @@ def test_declarations(): assert "b" in module.symbol_table +def test_forward_reference_completed_by_parameter_declaration(): + '''A named constant declaration completes a DataSymbol first created by + its use in an earlier array bound. + ''' + valid_code = """ + subroutine declarations() + integer :: values(extent) + integer, parameter :: extent = 10 + end subroutine declarations + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + extent = table.lookup("extent") + + assert extent.is_constant + assert extent.initial_value.value == "10" + assert isinstance(extent.interface, psyir_symbols.StaticInterface) + assert table.lookup("values").datatype.shape[0].upper.symbol is extent + + +def test_unsupported_module_specifications_are_preserved(): + '''Unsupported module specification statements become CodeBlocks rather + than causing translation to abort. + ''' + valid_code = """ + module specifications + integer :: value + save + common /block/ value + namelist /group/ value + end module specifications + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + children = root.children[0].children + + assert len(children) == 3 + assert all(isinstance(child, psyir_nodes.CodeBlock) + for child in children) + assert [child.parse_tree_nodes[0].type for child in children] == [ + "save_statement", "common_statement", "namelist_statement"] + + def test_parameter_declaration(): '''Test a named constant declaration.''' processor = FortranTreeSitterReader() @@ -560,6 +614,44 @@ def test_character_length(): assert label.datatype.length.value == "12" +def test_empty_kind_and_malformed_literal_declaration_nodes(): + '''Test defensive handling for malformed parser nodes after establishing + the expected declaration and literal node forms from Fortran input. + ''' + valid_code = """ + subroutine declarations() + character(3) :: text = 'abc' + end subroutine declarations + """ + processor = FortranTreeSitterReader() + parse_tree = processor.generate_parse_tree_from_source(valid_code) + string = _first_tsnode(parse_tree, "string_literal") + declaration = _first_tsnode(parse_tree, "variable_declaration") + intrinsic_type = _first_tsnode(parse_tree, "intrinsic_type") + + assert processor._string_literal_handler(string).value == "abc" + assert declaration.type == "variable_declaration" + for text, message in [ + (b"abc", "no quote delimiter"), + (b"'abc\"", "mismatched quote delimiters"), + (b"_'abc'", "kind prefix")]: + malformed = SimpleNamespace(text=text) + with pytest.raises(NotImplementedError, match=message): + processor._string_literal_handler(malformed) + + malformed_declaration = SimpleNamespace(children=[]) + with pytest.raises(NotImplementedError, match="no supported type"): + processor._variable_declaration_handler(malformed_declaration) + + empty_kind_type = SimpleNamespace( + type="intrinsic_type", + text=intrinsic_type.text, + children=[intrinsic_type.children[0], + SimpleNamespace(type="kind", children=[])]) + assert processor._datatype_from_type(empty_kind_type) == \ + psyir_symbols.ScalarType.character_type() + + def test_logical_literal(): '''Test a logical literal used as an initial value.''' processor = FortranTreeSitterReader() @@ -600,6 +692,36 @@ def test_literal_kind_variants_and_string(): assert table.lookup("text").initial_value.value == "hello" +def test_double_exponent_and_character_literal_kinds(): + '''Test normalisation of D exponents and kind-prefixed strings.''' + valid_code = """ + subroutine literals() + integer, parameter :: char_kind = 2 + real(kind=8) :: value + character(3) :: first, second + value = 1.0d0 + first = char_kind_'one' + second = 4_'two' + end subroutine literals + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + + double = routine.children[0].rhs + assert double.value == "1.0e0" + assert double.datatype.precision == \ + psyir_symbols.ScalarType.Precision.DOUBLE + symbolic = routine.children[1].rhs + assert symbolic.value == "one" + assert symbolic.datatype.precision.symbol is \ + routine.symbol_table.lookup("char_kind") + numeric = routine.children[2].rhs + assert numeric.value == "two" + assert numeric.datatype.precision == 4 + + def test_new_literal_kind_symbol(): '''Test that a named literal kind creates an unresolved kind symbol.''' valid_code = """ @@ -687,6 +809,28 @@ def test_multidimensional_and_lower_bounded_arrays(): assert datatype.shape[1].upper.value == "20" +def test_assumed_shape_explicit_lower_and_upper_bounds(): + '''Test the individual open-bound forms accepted in assumed-shape array + declarations. + ''' + valid_code = """ + subroutine shapes(lower_open, upper_open) + real :: lower_open(2:), upper_open(:10) + end subroutine shapes + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + lower_open = table.lookup("lower_open").datatype.shape[0] + upper_open = table.lookup("upper_open").datatype.shape[0] + + assert lower_open.lower.value == "2" + assert lower_open.upper == psyir_symbols.ArrayType.Extent.ATTRIBUTE + assert upper_open.lower.value == "1" + assert upper_open.upper.value == "10" + + def test_shared_dimension_is_given_to_all_entities(): '''Test that a shared DIMENSION are copied for each declared entity. ''' @@ -838,6 +982,26 @@ def test_named_visibility(): assert exposed.visibility == psyir_symbols.Symbol.Visibility.PUBLIC +def test_visibility_of_declared_and_unsupported_names(): + '''An access list updates declared symbols and safely ignores names whose + unsupported declarations did not create symbols. + ''' + valid_code = """ + module visibility + public :: exposed, unsupported_name + integer :: exposed + end module visibility + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + + assert table.lookup("exposed").visibility == \ + psyir_symbols.Symbol.Visibility.PUBLIC + assert "unsupported_name" not in table + + def test_declaration_visibility_and_allocatable_scalar(): '''Test declaration access attributes and unsupported scalar ALLOCATABLE.''' @@ -915,6 +1079,46 @@ def test_wildcard_and_defensive_import_branches(): assert table.lookup("source") is container +def test_repeated_use_preserves_wildcard_import(): + '''An ONLY import does not undo an earlier wildcard import from the same + module. + ''' + valid_code = """ + subroutine imports() + use source + use source, only: value + end subroutine imports + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + + assert table.lookup("source").wildcard_import + assert table.lookup("value").interface.container_symbol is \ + table.lookup("source") + + +def test_repeated_use_updates_existing_import_interface(): + '''A repeated explicit import updates the existing local symbol's remote + name while retaining its container. + ''' + valid_code = """ + subroutine imports() + use source, only: value + use source, only: value => remote_value + end subroutine imports + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + imported = table.lookup("value") + + assert imported.interface.container_symbol is table.lookup("source") + assert imported.interface.orig_name == "remote_value" + + def test_declaration_conflicts_with_import(): '''Test that redeclaring an imported bare Symbol is localised.''' valid_code = """ @@ -950,6 +1154,21 @@ def test_name_conflict(): processor.generate_parse_tree_from_source(valid_code)) +def test_use_conflicts_with_preceding_declaration(): + '''A USE statement cannot reuse the name of an existing data symbol.''' + valid_code = """ + subroutine conflict() + integer :: source + use source + end subroutine conflict + """ + processor = FortranTreeSitterReader() + + with pytest.raises(ValueError, match="USE module 'source' conflicts"): + processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + + def test_derived_type_definition(): '''Test a simple derived-type definition.''' processor = FortranTreeSitterReader() @@ -1037,7 +1256,9 @@ def test_invalid_derived_type_component_falls_back(): def test_derived_type_name_conflict(): - '''Test a derived type whose name is already used by a data symbol.''' + '''A derived type whose name is already used by a data symbol is + preserved as unsupported module specification code. + ''' valid_code = """ module conflict integer :: item @@ -1047,10 +1268,13 @@ def test_derived_type_name_conflict(): end module conflict """ processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) - with pytest.raises(InternalError, match="No node was expected"): - processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) + codeblock = root.children[0].children[0] + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert codeblock.parse_tree_nodes[0].type == \ + "derived_type_definition" def test_generic_interface(): @@ -1125,12 +1349,14 @@ def test_interface_routine_symbol_are_consistent(): """, ]) def test_unsupported_interface_forms(valid_code): - '''Test unsupported interface forms are detected in declaration context.''' + '''Unsupported interface forms are preserved in declaration context.''' processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) - with pytest.raises(InternalError, match="No node was expected"): - processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) + codeblock = root.children[0].children[0] + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert codeblock.parse_tree_nodes[0].type == "interface" def test_unary_operation(): @@ -1223,6 +1449,35 @@ def test_implicit_array_section_bounds(): psyir_nodes.IntrinsicCall.Intrinsic.UBOUND +def test_array_section_omitted_upper_bound_with_step(): + '''A step remains the third triplet field when the upper bound is + omitted. + ''' + valid_code = """ + subroutine section(array) + real :: array(10) + array(1::2) = 0.0 + array(::-1) = 1.0 + end subroutine section + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + first = root.children[0].children[0].lhs.indices[0] + second = root.children[0].children[1].lhs.indices[0] + + assert first.start.value == "1" + assert first.stop.intrinsic == \ + psyir_nodes.IntrinsicCall.Intrinsic.UBOUND + assert first.step.value == "2" + assert second.start.intrinsic == \ + psyir_nodes.IntrinsicCall.Intrinsic.LBOUND + assert second.stop.intrinsic == \ + psyir_nodes.IntrinsicCall.Intrinsic.UBOUND + assert second.step.operator == psyir_nodes.UnaryOperation.Operator.MINUS + assert second.step.children[0].value == "1" + + def test_array_constructor(): '''Test a simple array constructor.''' processor = FortranTreeSitterReader() @@ -1316,6 +1571,57 @@ def test_intrinsic_call(): assert call.intrinsic == psyir_nodes.IntrinsicCall.Intrinsic.SIN +def test_imported_function_call_specialises_symbol(): + '''A bare Symbol created by an explicit import becomes a RoutineSymbol + when used in a function call. + ''' + valid_code = """ + subroutine caller(result) + use procedures, only: imported + real :: result + result = imported() + end subroutine caller + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + + assert isinstance(routine.children[0].rhs, psyir_nodes.Call) + assert isinstance(routine.symbol_table.lookup("imported"), + psyir_symbols.RoutineSymbol) + + +def test_user_routine_resolved_before_intrinsic_name(): + '''A declared procedure whose name matches an intrinsic remains a user + call. + ''' + valid_code = """ + module routines + interface sin + module procedure user_sin + end interface sin + contains + real function user_sin() + user_sin = 0.0 + end function user_sin + subroutine caller(result) + real :: result + result = sin() + end subroutine caller + end module routines + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + call = root.children[0].children[1].children[0].rhs + + assert isinstance(call, psyir_nodes.Call) + assert not isinstance(call, psyir_nodes.IntrinsicCall) + assert isinstance(call.routine.symbol, + psyir_symbols.GenericInterfaceSymbol) + + def test_invalid_array_and_intrinsic_arguments(): '''Test named array subscripts and an invalid intrinsic signature.''' valid_code = """ @@ -1779,6 +2085,68 @@ def test_where_construct(): assert where.else_body.children[0].rhs.value == "0.0" +def test_masked_and_repeated_elsewhere_clauses(): + '''Masked ELSEWHERE clauses form a nested conditional chain and may be + followed by a final unmasked clause. + ''' + valid_code = """ + subroutine mask(array, first, second) + real :: array(10) + logical :: first(10), second(10) + where (first) + array = 1.0 + elsewhere (.not. first) + array = 2.0 + elsewhere (second) + array = 3.0 + elsewhere + array = 4.0 + end where + end subroutine mask + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + where = root.children[0].children[0] + first_masked = where.else_body.children[0] + second_masked = first_masked.else_body.children[0] + + assert isinstance(first_masked, psyir_nodes.IfBlock) + assert isinstance(second_masked, psyir_nodes.IfBlock) + assert first_masked.annotations == ["was_where"] + assert second_masked.annotations == ["was_where"] + assert first_masked.if_body.children[0].rhs.value == "2.0" + assert second_masked.if_body.children[0].rhs.value == "3.0" + assert second_masked.else_body.children[0].rhs.value == "4.0" + + +def test_unmasked_elsewhere_must_be_final(): + '''An unmasked ELSEWHERE followed by another clause is preserved as an + unsupported statement instead of constructing incorrect PSyIR. + ''' + valid_code = """ + subroutine mask(array, condition) + real :: array(10) + logical :: condition(10) + where (condition) + array = 1.0 + elsewhere + array = 2.0 + elsewhere (condition) + array = 3.0 + end where + end subroutine mask + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + codeblock = root.children[0].children[0] + + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert "unmasked ELSEWHERE must be the final clause" in \ + codeblock.preceding_comment + + def test_select_case_construct(): '''Test SELECT CASE lowering.''' processor = FortranTreeSitterReader() From a6eeecc9b65e3ac90b80a2c3f300c8481141ffe3 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 13 Aug 2026 14:58:14 +0100 Subject: [PATCH 16/23] Improve some aspects of the treesitter frontend generated PSyIR --- .../frontend/fortran_treesitter_reader.py | 93 +++++--- .../fortran_treesitter_reader/ftr_test.py | 216 ++++++++++++++++-- 2 files changed, 255 insertions(+), 54 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 7627d4073a..135e52ed84 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -603,6 +603,24 @@ def _procedure_handler( routine.return_symbol = routine.symbol_table.lookup( return_name) self._apply_visibility(vis_map) + + # PSyIR cannot represent implicitly declared data. Leave + # unresolved names alone only when a wildcard import or a + # CodeBlock could contain their declaration, matching the + # conditions under which the Fortran backend can preserve them. + fallback_context = ( + routine.symbol_table.wildcard_imports() or + routine.walk(nodes.CodeBlock)) + for symbol in routine.symbol_table.datasymbols: + if not isinstance(symbol.datatype, symbols.UnresolvedType): + continue + if (isinstance(symbol.interface, + symbols.UnresolvedInterface) and + fallback_context): + continue + raise NotImplementedError( + f"Implicit declaration of '{symbol.name}' is not " + "supported") return routine def _function_return_info( @@ -694,14 +712,10 @@ def _number_literal_handler( symbols.ScalarType.real_type() if is_real else symbols.ScalarType.integer_type()) if kind: - if kind == "4": - precision = symbols.ScalarType.Precision.SINGLE - elif kind == "8": - precision = symbols.ScalarType.Precision.DOUBLE - else: - precision = (int(kind) if kind.isdigit() - else nodes.Reference( - self._kind_symbol(kind))) + # A numeric Fortran KIND value is processor-specific and is not + # equivalent to either a byte size or PSyIR relative precision. + precision = (int(kind) if kind.isdigit() + else nodes.Reference(self._kind_symbol(kind))) datatype = symbols.ScalarType( datatype.intrinsic, precision) return nodes.Literal(value, datatype) @@ -739,7 +753,8 @@ def _string_literal_handler( nodes.Reference(self._kind_symbol(kind))) datatype = symbols.ScalarType( symbols.ScalarType.Intrinsic.CHARACTER, precision) - return nodes.Literal(text[quote_position + 1:-1], datatype) + value = text[quote_position + 1:-1].replace(quote * 2, quote) + return nodes.Literal(value, datatype) def _boolean_literal_handler( self, tsnode: 'TSNode' @@ -836,7 +851,8 @@ def _declare_entity( visibility = symbols.Symbol.Visibility.PRIVATE kwargs = {"visibility": visibility} - interface = self._declaration_interface(name, common_attr) + interface = self._declaration_interface( + name, common_attr, initial_value is not None) if interface is not None: kwargs["interface"] = interface if initial_value is not None: @@ -908,7 +924,8 @@ def _declarator_datatype( return datatype, initial_value def _declaration_interface( - self, name: str, common_attr: _CommonDeclAttributes + self, name: str, common_attr: _CommonDeclAttributes, + has_initial_value: bool = False ): '''Return the PSyIR interface for one declared entity. @@ -917,13 +934,15 @@ def _declaration_interface( :param name: declared entity name. :param common_attr: properties shared by the complete declaration. + :param has_initial_value: whether the entity has an initializer. :returns: symbol interface, or ``None`` for an automatic local. ''' symtab = self._current_scope if name in symtab and symtab.lookup(name).is_argument: return symbols.ArgumentInterface(common_attr.intent) - if {"save", "parameter"}.intersection(common_attr.qualifiers): + if (has_initial_value or + {"save", "parameter"}.intersection(common_attr.qualifiers)): return symbols.StaticInterface() if isinstance(symtab.node, nodes.Container): return symbols.DefaultModuleInterface() @@ -1045,11 +1064,9 @@ def _precision(self, tsnode: 'TSNode'): ''' expr = self._process_nodes(tsnode, _NodeExpectation.EXPRESSION) if isinstance(expr, nodes.Literal) and expr.value.isdigit(): - if expr.value == "4": - return symbols.ScalarType.Precision.SINGLE - if expr.value == "8": - return symbols.ScalarType.Precision.DOUBLE - return int(expr.value) + # Keep numeric KIND selectors as PSyIR expressions. An integer + # precision would instead mean a byte size to PSyIR backends. + return expr if isinstance(expr, nodes.Reference): # pylint: disable=unidiomatic-typecheck if type(expr.symbol) is symbols.Symbol: @@ -1127,6 +1144,13 @@ def _shape_from_node( if is_allocatable else symbols.ArrayType.Extent.ATTRIBUTE) elif before and not after: + if is_allocatable: + # PSyIR cannot currently distinguish a deferred upper + # bound with an explicit lower bound from an assumed- + # shape bound, so preserve the declaration verbatim. + raise NotImplementedError( + "An allocatable bound with an explicit lower " + "bound is not supported") result.append( (self._process_nodes( before[0], _NodeExpectation.EXPRESSION), @@ -1221,20 +1245,24 @@ def _use_statement_handler( # ONLY list must therefore not undo a wildcard import seen earlier. container.wildcard_import = container.wildcard_import or wildcard - if included: - for child in included.children: - if child.type == "identifier": - local_name = to_str(child) + import_items = list(included.children) if included else [] + # A rename list without ONLY is represented directly beneath the USE + # statement rather than inside ``included_items``. + import_items.extend( + child for child in tsnode.children + if child.type in ("rename", "use_rename", "use_alias")) + for child in import_items: + if child.type == "identifier": + local_name = to_str(child) + self._add_imported_symbol( + local_name, local_name, container) + elif child.type in ("rename", "use_rename", "use_alias"): + names = [item for item in child.children + if item.type in + ("identifier", "name", "local_name")] + if len(names) == 2: self._add_imported_symbol( - local_name, local_name, container) - elif child.type in ("rename", "use_rename", "use_alias"): - names = [item for item in child.children - if item.type in - ("identifier", "name", "local_name")] - if len(names) == 2: - self._add_imported_symbol( - to_str(names[0]), to_str(names[1]), - container) + to_str(names[0]), to_str(names[1]), container) def _add_imported_symbol( self, local_name: str, remote_name: str, @@ -2029,7 +2057,7 @@ def _select_case_statement_handler( condition, body, current) block.annotations.append("was_case") current = [block] - if current and len(current) == 1: + if normal and current and len(current) == 1: return current[0] raise NotImplementedError( "SELECT CASE with only a default clause has no PSyIR equivalent") @@ -2135,6 +2163,9 @@ def _allocation_reference( :raises NotImplementedError: if the object is not a data symbol. ''' ident = child_of_type(tsnode, "identifier") + if ident is None: + raise NotImplementedError( + "Allocations of structure components are not supported") reference = self._identifier_handler(ident) if not isinstance(reference.symbol, symbols.DataSymbol): raise NotImplementedError( diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index 26f82267ef..8e49b9c405 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -13,6 +13,7 @@ from tree_sitter import Node as TSNode from psyclone.errors import InternalError +from psyclone.psyir.backend.fortran import FortranWriter from psyclone.psyir.frontend import fortran_treesitter_reader as ftr from psyclone.psyir.frontend.fortran_treesitter_reader import ( FortranTreeSitterReader, _CommonDeclAttributes, _NodeExpectation) @@ -206,6 +207,27 @@ def test_routines_nodes(): assert isinstance(rsymbol2, psyir_symbols.RoutineSymbol) +def test_implicitly_declared_argument_falls_back(): + '''A valid implicitly typed argument is preserved in a procedure + CodeBlock because PSyIR cannot declare its type safely. + ''' + valid_code = """ + subroutine implicit_argument(value) + value = value + 1.0 + end subroutine implicit_argument + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + codeblock = root.children[0] + + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert "Implicit declaration of 'value'" in codeblock.preceding_comment + output = FortranWriter()(root) + assert "subroutine implicit_argument(value)" in output + assert "value = value + 1.0" in output + + def test_routine_symbol_association(): '''Test that a contained routine resolves a symbol from its parent.''' processor = FortranTreeSitterReader() @@ -438,6 +460,22 @@ def test_save_attribute(): assert isinstance(accumulator.interface, psyir_symbols.StaticInterface) +def test_initialized_local_has_static_interface(): + '''Fortran initialization implies SAVE for a local variable.''' + valid_code = """ + subroutine initialise() + integer :: value = 1 + end subroutine initialise + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + value = root.children[0].symbol_table.lookup("value") + + assert isinstance(value.interface, psyir_symbols.StaticInterface) + assert "integer, save :: value = 1" in FortranWriter()(root) + + @pytest.mark.parametrize("qualifier", ["pointer", "protected"]) def test_unsupported_qualifier_datatype(qualifier): '''Test entity-specific unsupported declaration qualifiers, including @@ -538,17 +576,17 @@ def unsupported_number(tsnodes, expect): psyir_symbols.UnsupportedFortranType) -@pytest.mark.parametrize("fortran_type,psyir_type", [ - ("integer", psyir_symbols.ScalarType.integer_type()), - ("integer(kind=4)", psyir_symbols.ScalarType.integer_single_type()), - ("integer(8)", psyir_symbols.ScalarType.integer_double_type()), - ("real", psyir_symbols.ScalarType.real_type()), - ("real(4)", psyir_symbols.ScalarType.real_single_type()), - ("real(kind=8)", psyir_symbols.ScalarType.real_double_type()), - ("logical", psyir_symbols.ScalarType.boolean_type()), - ("character", psyir_symbols.ScalarType.character_type()), +@pytest.mark.parametrize("fortran_type,intrinsic,kind", [ + ("integer", psyir_symbols.ScalarType.Intrinsic.INTEGER, None), + ("integer(kind=4)", psyir_symbols.ScalarType.Intrinsic.INTEGER, "4"), + ("integer(8)", psyir_symbols.ScalarType.Intrinsic.INTEGER, "8"), + ("real", psyir_symbols.ScalarType.Intrinsic.REAL, None), + ("real(4)", psyir_symbols.ScalarType.Intrinsic.REAL, "4"), + ("real(kind=8)", psyir_symbols.ScalarType.Intrinsic.REAL, "8"), + ("logical", psyir_symbols.ScalarType.Intrinsic.BOOLEAN, None), + ("character", psyir_symbols.ScalarType.Intrinsic.CHARACTER, None), ]) -def test_declarations_datatypes(fortran_type, psyir_type): +def test_declarations_datatypes(fortran_type, intrinsic, kind): ''' Test base declaration datatypes ''' processor = FortranTreeSitterReader() @@ -561,13 +599,21 @@ def test_declarations_datatypes(fortran_type, psyir_type): ptree = processor.generate_parse_tree_from_source(valid_code) root = processor.generate_psyir(ptree) module = root.children[0] - assert module.symbol_table.lookup("a").datatype == psyir_type + datatype = module.symbol_table.lookup("a").datatype + assert datatype.intrinsic == intrinsic + if kind: + assert isinstance(datatype.precision, psyir_nodes.Literal) + assert datatype.precision.value == kind + else: + assert datatype.precision == \ + psyir_symbols.ScalarType.Precision.UNDEFINED def test_datatype_kind_variants(): '''Test symbolic, literal and unsupported kinds.''' valid_code = """ subroutine declarations() + integer, parameter :: named_kind = 4 integer(named_kind) :: symbolic integer(16) :: wide integer(1 + 1) :: unsupported_kind @@ -579,7 +625,7 @@ def test_datatype_kind_variants(): table = root.children[0].symbol_table assert isinstance(table.lookup("named_kind"), psyir_symbols.DataSymbol) - assert table.lookup("wide").datatype.precision == 16 + assert table.lookup("wide").datatype.precision.value == "16" assert isinstance(table.lookup("unsupported_kind").datatype, psyir_symbols.UnsupportedFortranType) @@ -599,6 +645,30 @@ def test_datatype_kind_variants(): assert table.lookup("value").datatype.precision.symbol is kind +def test_numeric_kind_selectors_round_trip(): + '''Numeric KIND selectors remain KIND expressions for declarations and + integer values for literal suffixes. + ''' + valid_code = """ + subroutine kinds() + integer(kind=8) :: integer_value = 1_4 + real(kind=16) :: real_value = 1.0_8 + end subroutine kinds + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + + assert table.lookup("integer_value").datatype.precision.value == "8" + assert table.lookup("real_value").datatype.precision.value == "16" + assert table.lookup("integer_value").initial_value.datatype.precision == 4 + assert table.lookup("real_value").initial_value.datatype.precision == 8 + output = FortranWriter()(root) + assert "integer(kind=8), save :: integer_value = 1_4" in output + assert "real(kind=16), save :: real_value = 1.0_8" in output + + def test_character_length(): '''Test a character-length specification.''' processor = FortranTreeSitterReader() @@ -682,10 +752,8 @@ def test_literal_kind_variants_and_string(): processor.generate_parse_tree_from_source(valid_code)) table = root.children[0].symbol_table - assert table.lookup("single").initial_value.datatype.precision == \ - psyir_symbols.ScalarType.Precision.SINGLE - assert table.lookup("double").initial_value.datatype.precision == \ - psyir_symbols.ScalarType.Precision.DOUBLE + assert table.lookup("single").initial_value.datatype.precision == 4 + assert table.lookup("double").initial_value.datatype.precision == 8 assert table.lookup("explicit").initial_value.datatype.precision == 16 precision = table.lookup("symbolic").initial_value.datatype.precision assert precision.symbol is table.lookup("wp") @@ -722,6 +790,27 @@ def test_double_exponent_and_character_literal_kinds(): assert numeric.datatype.precision == 4 +def test_character_literal_doubled_delimiters(): + '''Doubled quote delimiters represent one quote in a character value.''' + valid_code = ''' + subroutine strings(first, second) + character(5) :: first, second + first = 'don''t' + second = "a ""word""" + end subroutine strings + ''' + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + + assert routine.children[0].rhs.value == "don't" + assert routine.children[1].rhs.value == 'a "word"' + output = FortranWriter()(root) + assert 'first = "don\'t"' in output + assert "second = 'a \"word\"'" in output + + def test_new_literal_kind_symbol(): '''Test that a named literal kind creates an unresolved kind symbol.''' valid_code = """ @@ -761,8 +850,9 @@ def test_declarations_arrays_datatypes(shape_string, extent): array_symbol = module.symbol_table.lookup("a") assert isinstance(array_symbol.datatype, psyir_symbols.ArrayType) - assert (array_symbol.datatype.elemental_type == - psyir_symbols.ScalarType.integer_single_type()) + assert array_symbol.datatype.elemental_type.intrinsic == \ + psyir_symbols.ScalarType.Intrinsic.INTEGER + assert array_symbol.datatype.elemental_type.precision.value == "4" shape = array_symbol.datatype.shape[0] if isinstance(extent, str): assert shape.upper.value == extent @@ -787,6 +877,27 @@ def test_allocatable_declaration(): psyir_symbols.ArrayType.Extent.DEFERRED] +def test_allocatable_with_explicit_lower_bound_is_preserved(): + '''PSyIR cannot distinguish a lower-bounded deferred extent from an + assumed-shape extent, so preserve this declaration verbatim. + ''' + valid_code = """ + subroutine declarations() + real, allocatable :: values(2:) + end subroutine declarations + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + values = root.children[0].symbol_table.lookup("values") + + assert isinstance(values.datatype, + psyir_symbols.UnsupportedFortranType) + assert values.datatype.declaration == \ + "real, allocatable :: values(2:)" + assert "real, allocatable :: values(2:)" in FortranWriter()(root) + + def test_multidimensional_and_lower_bounded_arrays(): '''Test explicit lower bounds in a multidimensional shape.''' processor = FortranTreeSitterReader() @@ -1047,6 +1158,32 @@ def test_use_rename(): assert imported.interface.orig_name == "remote_kind" +def test_use_rename_without_only(): + '''A rename-list alias directly beneath USE is retained as a wildcard + import with an explicit renamed symbol. + ''' + valid_code = """ + subroutine imports(value) + use source, local_value => remote_value + integer :: value + value = local_value + end subroutine imports + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + routine = root.children[0] + table = routine.symbol_table + container = table.lookup("source") + imported = table.lookup("local_value") + + assert container.wildcard_import + assert imported.interface.container_symbol is container + assert imported.interface.orig_name == "remote_value" + assert routine.children[0].rhs.symbol is imported + assert "use source, local_value=>remote_value" in FortranWriter()(root) + + def test_wildcard_and_defensive_import_branches(): '''Test wildcard import and defensive malformed/identity import handling.''' @@ -1853,6 +1990,34 @@ def test_allocate_statement(): assert allocate.argument_names == [None, "stat"] +def test_structure_component_allocation_falls_back(): + '''An allocation whose object is a structure component becomes a + statement CodeBlock instead of raising an unexpected exception. + ''' + valid_code = """ + module types + type :: item_type + real, allocatable :: values(:) + end type item_type + contains + subroutine allocate_component(object, extent) + type(item_type) :: object + integer :: extent + allocate(object%values(extent)) + end subroutine allocate_component + end module types + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + codeblock = root.children[0].children[0].children[0] + + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert "Allocations of structure components" in \ + codeblock.preceding_comment + assert "allocate(object%values(extent))" in FortranWriter()(root) + + def test_allocate_bounds_and_invalid_object(): '''Test explicit allocation bounds, missing upper bound and an import.''' valid_code = """ @@ -2199,14 +2364,15 @@ def test_select_case_expression_and_open_ranges(): def test_select_case_with_only_default_is_unsupported(): - '''Test SELECT CASE with no conditional case becomes a CodeBlock.''' + '''A default-only SELECT CASE remains a CodeBlock so evaluation of an + impure selector is not discarded. + ''' valid_code = """ subroutine selection(value) integer :: value - select case(value) + select case(next_value()) case default value = 1 - value = 2 end select end subroutine selection """ @@ -2214,9 +2380,13 @@ def test_select_case_with_only_default_is_unsupported(): root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - assert isinstance(root.children[0].children[0], psyir_nodes.CodeBlock) + codeblock = root.children[0].children[0] + assert isinstance(codeblock, psyir_nodes.CodeBlock) assert "only a default clause" in \ - root.children[0].children[0].preceding_comment + codeblock.preceding_comment + output = FortranWriter()(root) + assert "select case(next_value())" in output + assert "value = 1" in output # pylint: disable=too-many-locals From 20c80dfee2219499ebfc3c0dae0fa81bf152beaa Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 13 Aug 2026 15:01:40 +0100 Subject: [PATCH 17/23] Fix python 3.9 issues --- src/psyclone/psyir/frontend/fortran_treesitter_reader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 135e52ed84..6eb954ab7c 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -66,7 +66,7 @@ def to_str(node: 'TSNode') -> str: def iter_child_of_type( - tsnode: Optional['TSNode'], types: str | Container[str] + tsnode: Optional['TSNode'], types: Union[str, Container[str]] ) -> Generator['TSNode']: ''' Provides a generator to iterate over the provided tsnode chidlren of the given type(s). @@ -84,7 +84,7 @@ def iter_child_of_type( def child_of_type( - tsnode: Optional['TSNode'], node_type: str | Container[str] + tsnode: Optional['TSNode'], node_type: Union[str, Container[str]] ) -> Optional['TSNode']: ''' Return the direct child having the supplied type(s). And validate that is the only child of the supplied type. @@ -384,7 +384,7 @@ def _process_nodes( self, tsnodes: Union["TSNode", Iterable["TSNode"]], expect: _NodeExpectation, - ) -> list[nodes.Node] | nodes.Node | None: + ) -> Optional[Union[list[nodes.Node], nodes.Node]]: ''' This is the tsnodes handler dispatcher. Unsupported syntax is deliberately caught here rather than in individual handlers so that From 4337f610277309e16ff5bb9c204cd18ac1d4cd28 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Fri, 14 Aug 2026 09:44:57 +0100 Subject: [PATCH 18/23] Make sure treesitter codeblocks result in valid Fortran code --- src/psyclone/psyir/backend/fortran.py | 4 +- .../frontend/fortran_treesitter_reader.py | 133 ++++++++++++-- .../fortran_treesitter_reader/ftr_test.py | 173 ++++++++++++++++-- .../frontend/fparser2_kind_params_test.py | 24 ++- 4 files changed, 283 insertions(+), 51 deletions(-) diff --git a/src/psyclone/psyir/backend/fortran.py b/src/psyclone/psyir/backend/fortran.py index 13582ce2eb..1f17b51c3b 100644 --- a/src/psyclone/psyir/backend/fortran.py +++ b/src/psyclone/psyir/backend/fortran.py @@ -1650,8 +1650,8 @@ def codeblock_node(self, node): for line in node.get_fortran_lines(): result += f"{self._nindent}{line}\n" elif node.structure == CodeBlock.Structure.EXPRESSION: - for ast_node in node.parse_tree_nodes: - result += str(ast_node) + # No indent or newlines + result = "".join(node.get_fortran_lines()) else: raise VisitorError( f"Unsupported CodeBlock Structure '{node.structure}' found.") diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 6eb954ab7c..6a659c1ddd 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -532,6 +532,7 @@ def _module_handler( internal = child_of_type(tsnode, "internal_procedures") if internal: + self._predeclare_routines(internal.children) container.children.extend( self._process_nodes( [child for child in internal.children @@ -572,8 +573,15 @@ def _procedure_handler( rsymbol = self._create_routine_symbol( name, signature, return_type) + qualifiers = { + to_str(child).lower().replace("-", "_") + for child in signature.children + if child.type == "procedure_qualifier"} + is_recursive = (True if "recursive" in qualifiers else + False if "non_recursive" in qualifiers else None) routine = nodes.Routine( rsymbol, is_program=routine_kind == "program", + is_recursive=is_recursive, symbol_table=routine_table) # A routine must be parsed inside a scope as it is also a declaration @@ -589,8 +597,7 @@ def _procedure_handler( skip = { f"{routine_kind}_statement", f"end_{routine_kind}_statement", - "implicit_statement", "public_statement", - "private_statement" + "implicit_statement" } routine.children.extend(self._process_nodes( [child for child in tsnode.children @@ -650,8 +657,42 @@ def _function_return_info( try: return return_name, self._datatype_from_type(type_node) except (NotImplementedError, KeyError, TypeError): - return return_name, symbols.UnsupportedFortranType( - to_str(signature).strip()) + # UnsupportedFortranType must contain a valid entity declaration. + # A function prefix is not one, so retaining it on the result + # symbol makes the backend fail. Preserve the complete procedure + # as a CodeBlock instead. + raise NotImplementedError( + "Function return type is not supported") from None + + def _predeclare_routines(self, tsnodes: Iterable['TSNode']): + '''Create symbols for contained procedures before translating bodies. + + This makes calls in an earlier procedure resolve to the same symbol as + a procedure whose implementation occurs later in the source. + + :param tsnodes: children of an internal-procedures node. + ''' + symtab = self._current_scope + for procedure in tsnodes: + if procedure.type not in ("subroutine", "function"): + continue + signature = child_of_type( + procedure, f"{procedure.type}_statement") + name_node = child_of_type(signature, "name") + if name_node is None: + continue + name = to_str(name_node) + try: + _, return_type = self._function_return_info( + signature, name, procedure.type) + except NotImplementedError: + # Still establish identity for calls. Translation of the + # procedure itself will subsequently preserve its source. + return_type = None + routine_symbol = self._create_routine_symbol( + name, signature, return_type) + if name not in symtab: + symtab.add(routine_symbol) def _create_routine_symbol( self, name: str, signature: 'TSNode', return_type @@ -669,8 +710,12 @@ def _create_routine_symbol( :returns: RoutineSymbol representing the program unit. ''' qualifiers = { - to_str(child).lower() for child in signature.children + to_str(child).lower().replace("-", "_") + for child in signature.children if child.type == "procedure_qualifier"} + is_elemental = "elemental" in qualifiers + is_pure = ("pure" in qualifiers or + (is_elemental and "impure" not in qualifiers)) visibility = self._current_scope.default_visibility try: routine_symbol = self._current_scope.lookup(name) @@ -679,14 +724,14 @@ def _create_routine_symbol( if isinstance(routine_symbol, symbols.RoutineSymbol): routine_symbol.datatype = ( return_type or routine_symbol.datatype) - routine_symbol.is_pure = "pure" in qualifiers - routine_symbol.is_elemental = "elemental" in qualifiers + routine_symbol.is_pure = is_pure + routine_symbol.is_elemental = is_elemental routine_symbol.visibility = visibility return routine_symbol return symbols.RoutineSymbol( name, datatype=return_type or symbols.UnresolvedType(), - is_pure="pure" in qualifiers, - is_elemental="elemental" in qualifiers, + is_pure=is_pure, + is_elemental=is_elemental, visibility=visibility) def _number_literal_handler( @@ -1035,9 +1080,8 @@ def _datatype_from_type( kind_node = child_of_type(tsnode, "kind") if kind_node: values = [child for child in kind_node.children - if child.type not in ("(", ")")] - if values: - value = values[0] + if child.type not in ("(", ")", ",")] + for value in values: if value.type == "keyword_argument": key = to_str(value.children[0]).lower() value = value.children[-1] @@ -1182,6 +1226,13 @@ def _process_access_statements( for tsnode in tsnodes: if tsnode.type not in ("public_statement", "private_statement"): continue + # Access rules for defined operators and assignment cannot be + # represented in a SymbolTable. Reject the containing scope so + # that it is preserved verbatim as a CodeBlock. Module handlers + # otherwise skip access statements after this pre-processing + # pass, so relying on the statement handler alone would silently + # discard these forms. + self._public_statement_handler(tsnode) visibility = (symbols.Symbol.Visibility.PUBLIC if tsnode.type == "public_statement" else symbols.Symbol.Visibility.PRIVATE) @@ -1191,10 +1242,22 @@ def _process_access_statements( if names: visibility_map.update({name.lower(): visibility for name in names}) - else: + elif not any(child.type == "::" for child in tsnode.children): symtab.default_visibility = visibility return visibility_map + def _public_statement_handler(self, tsnode: 'TSNode') -> None: + '''Retain access-id forms that PSyIR cannot represent.''' + if any(child.type in ("operator", "assignment") + for child in tsnode.children): + raise NotImplementedError( + "Named operator and assignment access rules are not " + "supported") + + def _private_statement_handler(self, tsnode: 'TSNode') -> None: + '''Retain access-id forms that PSyIR cannot represent.''' + self._public_statement_handler(tsnode) + def _apply_visibility( self, visibility_map: dict[str, symbols.Symbol.Visibility] ): @@ -1302,8 +1365,13 @@ def _derived_type_definition_handler( statement = child_of_type(tsnode, "derived_type_statement") name_node = child_of_type(statement, "type_name") name = to_str(name_node) - unsupported = any(child.type == "derived_type_procedures" - for child in tsnode.children) + unsupported = ( + any(child.type == "derived_type_procedures" + for child in tsnode.children) or + any(child.type in ("base_type_specifier", "language_binding") + for child in statement.children) or + any(child.type == "sequence_statement" + for child in tsnode.children)) datatype = None if not unsupported: parent = symtab.node @@ -1327,7 +1395,8 @@ def _derived_type_definition_handler( except (NotImplementedError, TypeError, ValueError): datatype = None if datatype is None: - datatype = symbols.UnsupportedFortranType(to_str(tsnode).strip()) + datatype = self._unsupported_derived_type( + tsnode, statement, name) visibility = symtab.default_visibility access = child_of_type(statement, "access_specifier") @@ -1347,6 +1416,29 @@ def _derived_type_definition_handler( existing.datatype = datatype existing.visibility = visibility + @staticmethod + def _unsupported_derived_type( + tsnode: 'TSNode', statement: 'TSNode', name: str + ) -> symbols.UnsupportedFortranType: + '''Create a backend-compatible unsupported derived-type definition. + + :param tsnode: complete derived-type definition. + :param statement: opening derived-type statement. + :param name: derived-type name. + + :returns: unsupported datatype containing an equivalent definition. + ''' + declaration = to_str(tsnode).strip() + statement_text = to_str(statement).strip() + if "::" not in statement_text: + # The backend needs the separator in order to add visibility to + # an unsupported declaration. It is optional in this form of a + # Fortran derived-type statement, so add an equivalent one. + prefix = statement_text[:-len(name)].rstrip() + replacement = f"{prefix} :: {name}" + declaration = replacement + declaration[len(statement_text):] + return symbols.UnsupportedFortranType(declaration) + def _interface_handler( self, tsnode: 'TSNode' ) -> None: @@ -1722,6 +1814,10 @@ def _array_literal_handler( if child_of_type(tsnode, "implied_do_loop_expression"): raise NotImplementedError( "Array constructors with implied-DO loops are not supported") + if any(child.type in ("intrinsic_type", "derived_type") + for child in tsnode.children): + raise NotImplementedError( + "Typed array constructors are not supported") elems = [self._process_nodes(child, _NodeExpectation.EXPRESSION) for child in tsnode.children if child.type not in ("[", "]", "(/", "/)", ",")] @@ -1912,6 +2008,8 @@ def _do_loop_handler( :raises NotImplementedError: if counted-loop control is unsupported. ''' statement = child_of_type(tsnode, "do_statement") + if child_of_type(statement, "concurrent_statement"): + raise NotImplementedError("DO CONCURRENT is not supported") control = child_of_type(statement, "loop_control_expression") while_node = child_of_type(statement, "while_statement") body = self._process_nodes( @@ -2031,6 +2129,9 @@ def _select_case_statement_handler( else: selector = self._process_nodes( selector_node, _NodeExpectation.EXPRESSION) + if selector.walk(nodes.Call) or isinstance(selector, nodes.CodeBlock): + raise NotImplementedError( + "SELECT CASE selectors containing calls are not supported") cases = list(iter_child_of_type(tsnode, "case_statement")) default_body = None normal = [] diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index 8e49b9c405..3f372d9aaf 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -13,7 +13,6 @@ from tree_sitter import Node as TSNode from psyclone.errors import InternalError -from psyclone.psyir.backend.fortran import FortranWriter from psyclone.psyir.frontend import fortran_treesitter_reader as ftr from psyclone.psyir.frontend.fortran_treesitter_reader import ( FortranTreeSitterReader, _CommonDeclAttributes, _NodeExpectation) @@ -223,9 +222,6 @@ def test_implicitly_declared_argument_falls_back(): assert isinstance(codeblock, psyir_nodes.CodeBlock) assert "Implicit declaration of 'value'" in codeblock.preceding_comment - output = FortranWriter()(root) - assert "subroutine implicit_argument(value)" in output - assert "value = value + 1.0" in output def test_routine_symbol_association(): @@ -291,8 +287,7 @@ def test_function_return_type_variants(): assert (root.children[1].return_symbol.datatype == psyir_symbols.ScalarType.integer_type()) - assert isinstance(root.children[2].return_symbol.datatype, - psyir_symbols.UnsupportedFortranType) + assert isinstance(root.children[2], psyir_nodes.CodeBlock) def test_argument_order(): @@ -355,6 +350,20 @@ def test_elemental_function(): root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) assert root.children[0].symbol.is_elemental is True + assert root.children[0].symbol.is_pure is True + + +def test_recursive_qualifier(): + '''An explicit RECURSIVE prefix is retained on the Routine.''' + valid_code = """ + recursive subroutine recurse() + end subroutine recurse + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + + assert root.children[0].is_recursive is True def test_declarations(): @@ -473,7 +482,6 @@ def test_initialized_local_has_static_interface(): value = root.children[0].symbol_table.lookup("value") assert isinstance(value.interface, psyir_symbols.StaticInterface) - assert "integer, save :: value = 1" in FortranWriter()(root) @pytest.mark.parametrize("qualifier", ["pointer", "protected"]) @@ -664,9 +672,6 @@ def test_numeric_kind_selectors_round_trip(): assert table.lookup("real_value").datatype.precision.value == "16" assert table.lookup("integer_value").initial_value.datatype.precision == 4 assert table.lookup("real_value").initial_value.datatype.precision == 8 - output = FortranWriter()(root) - assert "integer(kind=8), save :: integer_value = 1_4" in output - assert "real(kind=16), save :: real_value = 1.0_8" in output def test_character_length(): @@ -684,6 +689,25 @@ def test_character_length(): assert label.datatype.length.value == "12" +def test_character_length_and_kind_selectors(): + '''LEN and KIND are both retained, irrespective of their order.''' + valid_code = """ + subroutine characters() + character(len=3, kind=2) :: first + character(kind=2, len=3) :: second + end subroutine characters + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + table = root.children[0].symbol_table + + for name in ("first", "second"): + datatype = table.lookup(name).datatype + assert datatype.length.value == "3" + assert datatype.precision.value == "2" + + def test_empty_kind_and_malformed_literal_declaration_nodes(): '''Test defensive handling for malformed parser nodes after establishing the expected declaration and literal node forms from Fortran input. @@ -806,9 +830,6 @@ def test_character_literal_doubled_delimiters(): assert routine.children[0].rhs.value == "don't" assert routine.children[1].rhs.value == 'a "word"' - output = FortranWriter()(root) - assert 'first = "don\'t"' in output - assert "second = 'a \"word\"'" in output def test_new_literal_kind_symbol(): @@ -895,7 +916,6 @@ def test_allocatable_with_explicit_lower_bound_is_preserved(): psyir_symbols.UnsupportedFortranType) assert values.datatype.declaration == \ "real, allocatable :: values(2:)" - assert "real, allocatable :: values(2:)" in FortranWriter()(root) def test_multidimensional_and_lower_bounded_arrays(): @@ -1093,6 +1113,22 @@ def test_named_visibility(): assert exposed.visibility == psyir_symbols.Symbol.Visibility.PUBLIC +@pytest.mark.parametrize("access_id", ["operator(+)", "assignment(=)"]) +def test_named_operator_access_is_preserved(access_id): + '''An unsupported access-id does not become default module visibility.''' + valid_code = f""" + module visibility + private :: {access_id} + integer :: value + end module visibility + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + + assert isinstance(root.children[0], psyir_nodes.CodeBlock) + + def test_visibility_of_declared_and_unsupported_names(): '''An access list updates declared symbols and safely ignores names whose unsupported declarations did not create symbols. @@ -1181,7 +1217,6 @@ def test_use_rename_without_only(): assert imported.interface.container_symbol is container assert imported.interface.orig_name == "remote_value" assert routine.children[0].rhs.symbol is imported - assert "use source, local_value=>remote_value" in FortranWriter()(root) def test_wildcard_and_defensive_import_branches(): @@ -1374,6 +1409,27 @@ def test_unsupported_and_forward_declared_derived_types(): assert forward.visibility == psyir_symbols.Symbol.Visibility.PRIVATE +@pytest.mark.parametrize("definition, marker", [ + ("type, extends(parent) :: child", "extends(parent)"), + ("type, bind(c) :: child", "bind(c)"), + ("type child\n sequence", "sequence")]) +def test_derived_type_attributes_are_preserved(definition, marker): + '''Attributes without StructureType representations retain their source.''' + valid_code = f""" + module types + {definition} + integer :: value + end type child + end module types + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + datatype = root.children[0].symbol_table.lookup("child").datatype + + assert isinstance(datatype, psyir_symbols.UnsupportedFortranType) + + def test_invalid_derived_type_component_falls_back(): '''Test a component-name conflict makes the whole type unsupported.''' valid_code = """ @@ -1632,6 +1688,23 @@ def test_array_constructor(): "1.0", "2.0", "3.0"] +def test_typed_array_constructor_codeblock(): + '''A typed constructor is preserved as one expression CodeBlock.''' + valid_code = """ + subroutine constructor(values) + integer :: values(2) + values = [integer :: 1, 2] + end subroutine constructor + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + constructor = root.children[0].children[0].rhs + + assert isinstance(constructor, psyir_nodes.CodeBlock) + assert constructor.parse_tree_nodes[0].type == "array_literal" + + def test_implied_do_codeblock(): '''Test the localized fallback for an implied-DO array constructor.''' processor = FortranTreeSitterReader() @@ -1810,6 +1883,31 @@ def test_existing_routine_and_local_type_calls(): assert isinstance(caller.children[1].rhs, psyir_nodes.Call) +def test_later_contained_routine_is_predeclared(): + '''A call to a later contained routine uses its container symbol.''' + valid_code = """ + module routines + contains + subroutine caller() + call later() + end subroutine caller + pure subroutine later() + end subroutine later + end module routines + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + container = root.children[0] + caller, later = container.children + call = caller.children[0] + + assert "later" not in caller.symbol_table + assert call.routine.symbol is later.symbol + assert call.routine.symbol is container.symbol_table.lookup("later") + assert call.routine.symbol.is_pure is True + + def test_call_statement_edge_cases(): '''Test imported-symbol specialisation and invalid call targets.''' valid_code = """ @@ -2015,7 +2113,6 @@ def test_structure_component_allocation_falls_back(): assert isinstance(codeblock, psyir_nodes.CodeBlock) assert "Allocations of structure components" in \ codeblock.preceding_comment - assert "allocate(object%values(extent))" in FortranWriter()(root) def test_allocate_bounds_and_invalid_object(): @@ -2181,6 +2278,24 @@ def test_unconditional_do_loop(): assert "was_unconditional" in loop.annotations +def test_do_concurrent_is_preserved(): + '''DO CONCURRENT is not mistaken for an unconditional DO.''' + valid_code = """ + subroutine concurrent_loop(limit) + integer :: limit, index + do concurrent(index=1:limit) + limit = limit - 1 + end do + end subroutine concurrent_loop + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + loop = root.children[0].children[0] + + assert isinstance(loop, psyir_nodes.CodeBlock) + + def test_do_variable_variants(): '''Test implicit, imported and non-scalar DO variables.''' valid_code = """ @@ -2363,6 +2478,27 @@ def test_select_case_expression_and_open_ranges(): psyir_nodes.BinaryOperation.Operator.GE +def test_select_case_call_selector_is_preserved(): + '''A selector call is not copied into multiple IF conditions.''' + valid_code = """ + subroutine selection(value) + integer :: value + select case(next_value()) + case(1, 2:3) + value = 1 + case(4) + value = 2 + end select + end subroutine selection + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + selection = root.children[0].children[0] + + assert isinstance(selection, psyir_nodes.CodeBlock) + + def test_select_case_with_only_default_is_unsupported(): '''A default-only SELECT CASE remains a CodeBlock so evaluation of an impure selector is not discarded. @@ -2382,11 +2518,8 @@ def test_select_case_with_only_default_is_unsupported(): codeblock = root.children[0].children[0] assert isinstance(codeblock, psyir_nodes.CodeBlock) - assert "only a default clause" in \ + assert "SELECT CASE selectors containing calls" in \ codeblock.preceding_comment - output = FortranWriter()(root) - assert "select case(next_value())" in output - assert "value = 1" in output # pylint: disable=too-many-locals diff --git a/src/psyclone/tests/psyir/frontend/fparser2_kind_params_test.py b/src/psyclone/tests/psyir/frontend/fparser2_kind_params_test.py index ef0ee030e8..0e3389970a 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_kind_params_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_kind_params_test.py @@ -15,7 +15,8 @@ from psyclone.psyir.frontend.fparser2 import (Fparser2Reader, _kind_find_or_create) -from psyclone.psyir.nodes import IntrinsicCall, KernelSchedule, Reference +from psyclone.psyir.nodes import ( + IntrinsicCall, KernelSchedule, Reference, BinaryOperation, Call) from psyclone.psyir.symbols import ( DataSymbol, ScalarType, UnsupportedFortranType, RoutineSymbol, SymbolTable, Symbol, UnresolvedType, ContainerSymbol, UnresolvedInterface) @@ -51,7 +52,10 @@ def test_process_declarations_kind_new_param(): ''' fake_parent, fp2spec = process_declarations("real(kind=wp) :: var1\n" - "real(kind=Wp) :: var2\n") + "real(kind=Wp) :: var2\n" + "real(kind=1+1) :: var3\n" + "real(kind=u()) :: var4\n") + var1_var = fake_parent.symbol_table.lookup("var1") assert isinstance(var1_var.datatype.precision, Reference) # Check that this has resulted in the creation of a new 'wp' symbol @@ -62,17 +66,11 @@ def test_process_declarations_kind_new_param(): # references the same 'wp' symbol. var2_var = fake_parent.symbol_table.lookup("var2") assert var2_var.datatype.precision == Reference(wp_var) - # Check that we get a symbol of unsupported type if the KIND expression has - # an unexpected structure - # Break the parse tree by changing Name('wp') into a str - fp2spec[0].items[0].items[1].items = ("(", "blah", ")") - # Change the variable name too to prevent a clash - fp2spec[0].children[2].children[0].items[0].string = "var3" - processor = Fparser2Reader() - processor.process_declarations(fake_parent, [fp2spec[0]], []) - sym = fake_parent.symbol_table.lookup("var3") - assert isinstance(sym, DataSymbol) - assert isinstance(sym.datatype, UnsupportedFortranType) + # Check that other kind types + var3_var = fake_parent.symbol_table.lookup("var3") + assert isinstance(var3_var.datatype.precision, BinaryOperation) + var4_var = fake_parent.symbol_table.lookup("var4") + assert isinstance(var4_var.datatype.precision, Call) @pytest.mark.usefixtures("f2008_parser") From 677e7358eed6dee7530aabbd818ca2e0db1da48d Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Mon, 17 Aug 2026 10:56:50 +0100 Subject: [PATCH 19/23] Improve treesitter capturing of UnsupportedFortranType symbols --- .../frontend/fortran_treesitter_reader.py | 58 +++++++++++-------- .../fortran_treesitter_reader/ftr_test.py | 41 +++++++------ 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 36070b2e6d..0ea90d2e8a 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -100,7 +100,7 @@ def child_of_type( if len(children) == 0: return None elif len(children) > 1: - raise InternalError("Expected only 1") + raise InternalError(f"Expected only 1 child of type {node_type}") return children[0] @@ -129,8 +129,10 @@ class _NodeExpectation(Enum): #: Expect a list (of zero, one or multiple) PSyIR nodes LIST = auto() - #: Expect no result node (e.g. when processing a declaration) + #: Expect no resulting node NONE = auto() + #: Expect no resulting node, NotImplemented should be UnsupportedTypes + SPECS = auto() #: Expect exactly one node ONE = auto() #: Expect exactly one node that is a DataNode @@ -369,7 +371,7 @@ def _using_temporary_scope( scope = nodes.ScopingNode(symbol_table=symbols.SymbolTable()) previous_scope = self._current_scope - # This intentionally bypasses child validation. + # Intentionally bypass bidirectional link and child validation. # pylint: disable=protected-access scope._parent = parent self._current_scope = scope.symbol_table @@ -386,9 +388,9 @@ def _process_nodes( expect: _NodeExpectation, ) -> Optional[Union[list[nodes.Node], nodes.Node]]: ''' - This is the tsnodes handler dispatcher. Unsupported syntax is - deliberately caught here rather than in individual handlers so that - continuous unsupported nodes can be placed in a single CodeBlock. + The tsnodes handler dispatcher. Unsupported syntax is deliberately + caught here rather than in individual handlers so that continuous + unsupported nodes can be placed in a single CodeBlock. :param tsnodes: one tree-sitter node or an iterable of nodes. :param expect: expected number and kind of result nodes. @@ -404,12 +406,26 @@ def _process_nodes( if result is not None: children.append(result) except NotImplementedError as err: - # TODO #3083: Aggregate contiguous CodeBlocks. - structure = (CodeBlock.Structure.EXPRESSION - if expect is _NodeExpectation.EXPRESSION - else CodeBlock.Structure.STATEMENT) - children.append( - self._create_codeblock(tsnode, str(err), structure)) + if expect is _NodeExpectation.SPECS: + # If it reaches this point it means we haven't even + # identified its name, but to not lose the statement + # we still add it in the symtab as an UnsupportedType + symbol = self._current_scope.new_symbol( + root_name="PSYCLONE_UNSUPPORTED", + symbol_type=symbols.DataSymbol, + datatype=symbols.UnsupportedFortranType(to_str(tsnode)) + ) + # TODO #3083: We need to hardcode the type_text until we + # remove the Fparser2 logic from that class + symbol.datatype._type_text = to_str(tsnode) + else: + # Everything else we store as Codeblocks + # TODO #3083: Aggregate contiguous CodeBlocks. + structure = (CodeBlock.Structure.EXPRESSION + if expect is _NodeExpectation.EXPRESSION + else CodeBlock.Structure.STATEMENT) + children.append( + self._create_codeblock(tsnode, str(err), structure)) # Validate that the parsed nodes match the expectations of the caller if expect in (_NodeExpectation.ONE, _NodeExpectation.EXPRESSION): @@ -425,7 +441,7 @@ def _process_nodes( f"{type(children[0]).__name__}" ) return children[0] - if expect is _NodeExpectation.NONE: + if expect in (_NodeExpectation.NONE, _NodeExpectation.SPECS): if len(children) != 0: raise InternalError( f"No node was expected in this location but got:\n" @@ -506,8 +522,6 @@ def _module_handler( :returns: the equivalent PSyIR Node. - :raises NotImplementedError: if the module has an unsupported child. - :raises NotImplementedError: if the module permits implicit variables. ''' statement = child_of_type(tsnode, "module_statement") name = child_of_type(statement, "name") @@ -522,14 +536,12 @@ def _module_handler( "implicit_statement", "internal_procedures", "public_statement", "private_statement" } - # Specification statements normally only update the symbol table - # and therefore return no Node. Keep any unsupported statements - # as CodeBlocks so that valid Fortran is not lost (and, in - # particular, does not violate an expectation of no result). - container.children.extend(self._process_nodes( + # Parse the specification part + self._process_nodes( [child for child in tsnode.children - if child.type not in skip], _NodeExpectation.LIST)) + if child.type not in skip], _NodeExpectation.SPECS) + # Parse the execution part internal = child_of_type(tsnode, "internal_procedures") if internal: self._predeclare_routines(internal.children) @@ -1358,7 +1370,7 @@ def _derived_type_definition_handler( :param tsnode: derived-type-definition tree-sitter node. - :raises NotImplementedError: if the type name conflicts with an + :raises ValueError: if the type name conflicts with an existing non-datatype symbol. ''' symtab = self._current_scope @@ -1411,7 +1423,7 @@ def _derived_type_definition_handler( name, datatype, visibility=visibility)) else: if not isinstance(existing, symbols.DataTypeSymbol): - raise NotImplementedError( + raise ValueError( f"Derived type '{name}' conflicts with another symbol") existing.datatype = datatype existing.visibility = visibility diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index 88a604497b..220ec09d24 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -415,8 +415,8 @@ def test_forward_reference_completed_by_parameter_declaration(): def test_unsupported_module_specifications_are_preserved(): - '''Unsupported module specification statements become CodeBlocks rather - than causing translation to abort. + '''Unsupported module specification statements become symbols with + UnsupportedFortranType ''' valid_code = """ module specifications @@ -429,13 +429,19 @@ def test_unsupported_module_specifications_are_preserved(): processor = FortranTreeSitterReader() root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - children = root.children[0].children - assert len(children) == 3 - assert all(isinstance(child, psyir_nodes.CodeBlock) - for child in children) - assert [child.parse_tree_nodes[0].type for child in children] == [ - "save_statement", "common_statement", "namelist_statement"] + num_of_unsupported = 0 + for symbol in root.children[0].symbol_table.symbols: + if isinstance(symbol.datatype, psyir_symbols.UnsupportedFortranType): + num_of_unsupported += 1 + # Each have a placeholder unsupported name and the correc type_text + if symbol.name == "PSYCLONE_UNSUPPORTED": + symbol.datatype.type_text == "save" + elif symbol.name == "PSYCLONE_UNSUPPORTED_1": + symbol.datatype.type_text == "common /block/ value" + elif symbol.name == "PSYCLONE_UNSUPPORTED_2": + symbol.datatype.type_text == "namelist /group/ value" + assert num_of_unsupported == 3 def test_parameter_declaration(): @@ -1452,7 +1458,7 @@ def test_invalid_derived_type_component_falls_back(): def test_derived_type_name_conflict(): '''A derived type whose name is already used by a data symbol is - preserved as unsupported module specification code. + rejected as an invalid name conflict. ''' valid_code = """ module conflict @@ -1463,13 +1469,11 @@ def test_derived_type_name_conflict(): end module conflict """ processor = FortranTreeSitterReader() - root = processor.generate_psyir( - processor.generate_parse_tree_from_source(valid_code)) + parse_tree = processor.generate_parse_tree_from_source(valid_code) - codeblock = root.children[0].children[0] - assert isinstance(codeblock, psyir_nodes.CodeBlock) - assert codeblock.parse_tree_nodes[0].type == \ - "derived_type_definition" + with pytest.raises(ValueError, + match="Derived type 'item' conflicts with another"): + processor.generate_psyir(parse_tree) def test_generic_interface(): @@ -1549,9 +1553,10 @@ def test_unsupported_interface_forms(valid_code): root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - codeblock = root.children[0].children[0] - assert isinstance(codeblock, psyir_nodes.CodeBlock) - assert codeblock.parse_tree_nodes[0].type == "interface" + symtab = root.children[0].symbol_table + assert len(root.children[0].children) == 0 + assert len(symtab.symbols) in (1, 2) + assert "end interface" in symtab.symbols[-1].datatype.type_text def test_unary_operation(): From 51def638dd10f8d6e1d615a2326576744ca24c0b Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Mon, 17 Aug 2026 11:39:20 +0100 Subject: [PATCH 20/23] #3083 Pin treesitter to a single version and remove defensive testing to states that the pinned version does not produce --- pyproject.toml | 4 +- .../frontend/fortran_treesitter_reader.py | 102 +++------ .../fortran_treesitter_reader/ftr_test.py | 202 +++++------------- 3 files changed, 86 insertions(+), 222 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 170bed599f..b3710c7b23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,11 +91,11 @@ test = [ "pytest-cov", "pytest-xdist", "tree-sitter", - "tree-sitter-fortran" + "tree-sitter-fortran==0.6.0" ] treesitter = [ "tree-sitter", - "tree-sitter-fortran" + "tree-sitter-fortran==0.6.0" ] [project.urls] diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 0ea90d2e8a..030ccfa20e 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -13,6 +13,9 @@ 'rules' section of: https://github.com/stadelmanma/tree-sitter-fortran/blob/master/grammar.js +Note that psyclone is pinned to a particular version of treesitter (in +pyproject.toml), use that branch instead of master to follow the grammar. + To interpret the rules use: https://tree-sitter.github.io/tree-sitter/creating-parsers/ 2-the-grammar-dsl.html @@ -57,11 +60,13 @@ def log_decode_error_handler(err) -> tuple[str, int]: codecs.register_error("treesitter-encoding", log_decode_error_handler) -def to_str(node: 'TSNode') -> str: +def to_str(node: Optional['TSNode']) -> str: ''' :param node: a given treesitter node. :returns: the string representing the node in utf8. ''' + if node is None: + return "" return node.text.decode('utf8') if node.text else "" @@ -94,7 +99,6 @@ def child_of_type( :returns: matching child, or ``None`` if no child matches. - :raises InternalError: if more than one node of that type exists. ''' children = list(iter_child_of_type(tsnode, node_type)) if len(children) == 0: @@ -691,8 +695,6 @@ def _predeclare_routines(self, tsnodes: Iterable['TSNode']): signature = child_of_type( procedure, f"{procedure.type}_statement") name_node = child_of_type(signature, "name") - if name_node is None: - continue name = to_str(name_node) try: _, return_type = self._function_return_info( @@ -787,24 +789,14 @@ def _string_literal_handler( :returns: PSyIR character Literal. ''' text = to_str(tsnode) - quote_positions = [position for position in - (text.find("'"), text.find('"')) - if position >= 0] - if not quote_positions: - raise NotImplementedError( - "A character literal has no quote delimiter") - quote_position = min(quote_positions) + quote_position = min(position for position in + (text.find("'"), text.find('"')) + if position >= 0) quote = text[quote_position] - if text[-1] != quote: - raise NotImplementedError( - "A character literal has mismatched quote delimiters") prefix = text[:quote_position] datatype = symbols.ScalarType.character_type() if prefix: - if not prefix.endswith("_") or len(prefix) == 1: - raise NotImplementedError( - "Unsupported character literal kind prefix") kind = prefix[:-1].lower() precision = (int(kind) if kind.isdigit() else nodes.Reference(self._kind_symbol(kind))) @@ -836,10 +828,7 @@ def _variable_declaration_handler( # attributes) followed by one or more entity-specific declarators. type_node = next((child for child in tsnode.children if child.type in - ("intrinsic_type", "derived_type")), None) - if not type_node: - raise NotImplementedError( - "A variable declaration has no supported type specification") + ("intrinsic_type", "derived_type"))) qualifiers = [child for child in tsnode.children if child.type == "type_qualifier"] @@ -1190,11 +1179,7 @@ def _shape_from_node( if child.type in ("(", ")", ","): continue if child.type == "extent_specifier": - before, after, has_colon = self._split_extent(child) - if not has_colon: - result.append(self._process_nodes( - before[0], _NodeExpectation.EXPRESSION)) - continue + before, after, _ = self._split_extent(child) if not before and not after: result.append(symbols.ArrayType.Extent.DEFERRED if is_allocatable else @@ -1533,30 +1518,20 @@ def _operation( :returns: PSyIR UnaryOperation or BinaryOperation. - :raises NotImplementedError: if the operator or tree shape is - unsupported. ''' if len(tsnode.children) == 2: operator = to_str(tsnode.children[0]).lower() - if operator not in self._UNARY_OPERATORS: - raise NotImplementedError( - f"Unsupported unary operator '{operator}'") return nodes.UnaryOperation.create( self._UNARY_OPERATORS[operator], self._process_nodes( tsnode.children[1], _NodeExpectation.EXPRESSION)) - if len(tsnode.children) == 3: - operator = to_str(tsnode.children[1]).lower() - if operator not in self._BINARY_OPERATORS: - raise NotImplementedError( - f"Unsupported binary operator '{operator}'") - return nodes.BinaryOperation.create( - self._BINARY_OPERATORS[operator], - self._process_nodes( - tsnode.children[0], _NodeExpectation.EXPRESSION), - self._process_nodes( - tsnode.children[2], _NodeExpectation.EXPRESSION)) - raise NotImplementedError("Unexpected operation structure") + operator = to_str(tsnode.children[1]).lower() + return nodes.BinaryOperation.create( + self._BINARY_OPERATORS[operator], + self._process_nodes( + tsnode.children[0], _NodeExpectation.EXPRESSION), + self._process_nodes( + tsnode.children[2], _NodeExpectation.EXPRESSION)) def _call_expression_handler( self, tsnode: 'TSNode' @@ -1685,11 +1660,8 @@ def _range( :returns: PSyIR Range with explicit bounds. - :raises NotImplementedError: if the range is malformed. ''' - before, after, has_colon = self._split_extent(tsnode) - if not has_colon: - raise NotImplementedError("Malformed array range") + before, after, _ = self._split_extent(tsnode) # Preserve the field separated by a second colon. In particular, an # absent upper bound in ``lower::step`` must not cause the step to be # interpreted as the stop expression. @@ -1800,17 +1772,11 @@ def _decompose_structure( else: indices = arguments return name, indices, members - if tsnode.type == "derived_type_member_expression": - name, indices, members = self._decompose_structure( - tsnode.children[0]) - member = child_of_type(tsnode, "type_member") - if member is None: - raise NotImplementedError( - "Malformed structure component access") - members.append(to_str(member).lower()) - return name, indices, members - raise NotImplementedError( - f"Unsupported structure access base '{tsnode.type}'") + name, indices, members = self._decompose_structure( + tsnode.children[0]) + member = child_of_type(tsnode, "type_member") + members.append(to_str(member).lower()) + return name, indices, members def _array_literal_handler( self, tsnode: 'TSNode' @@ -1854,10 +1820,7 @@ def _assignment_statement_handler( :returns: PSyIR Assignment. - :raises NotImplementedError: if the tree shape is unexpected. ''' - if len(tsnode.children) != 3: - raise NotImplementedError("Unexpected assignment structure") return nodes.Assignment.create( self._process_nodes( tsnode.children[0], _NodeExpectation.EXPRESSION), @@ -1873,11 +1836,7 @@ def _pointer_association_statement_handler( :returns: pointer-annotated PSyIR Assignment. - :raises NotImplementedError: for bounds remapping. ''' - if len(tsnode.children) != 3: - raise NotImplementedError( - "Pointer assignment with bounds remapping is not supported") assignment = nodes.Assignment(is_pointer=True) assignment.children = [ self._process_nodes( @@ -1946,11 +1905,8 @@ def _if_statement_handler( :returns: root PSyIR IfBlock. - :raises NotImplementedError: if the statement has no condition. ''' condition_node = child_of_type(tsnode, "parenthesized_expression") - if not condition_node: - raise NotImplementedError("IF statement has no condition") structural = { "if", "parenthesized_expression", "then", "end_if_statement", "else_clause", "elseif_clause" @@ -2032,9 +1988,6 @@ def _do_loop_handler( if control: parts = [child for child in control.children if child.type not in ("=", ",")] - if len(parts) not in (3, 4): - raise NotImplementedError( - "Unsupported counted DO loop control") variable_ref = self._identifier_handler(parts[0]) variable = variable_ref.symbol if not isinstance(variable, symbols.DataSymbol): @@ -2155,9 +2108,6 @@ def _select_case_statement_handler( _NodeExpectation.LIST) else: values = child_of_type(case, "case_value_range_list") - if values is None: - raise NotImplementedError( - "Malformed CASE value list") structural = {"case", "(", ")", "case_value_range_list"} body = self._process_nodes( [child for child in case.children @@ -2311,9 +2261,7 @@ def _allocation_extent( lower, self._process_nodes( tsnode, _NodeExpectation.EXPRESSION)) - before, after, has_colon = self._split_extent(tsnode) - if not has_colon: - raise NotImplementedError("Malformed allocation bound") + before, after, _ = self._split_extent(tsnode) if before: lower = self._process_nodes( before[0], _NodeExpectation.EXPRESSION) diff --git a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py index 220ec09d24..3d074e5cfd 100644 --- a/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py +++ b/src/psyclone/tests/psyir/frontend/fortran_treesitter_reader/ftr_test.py @@ -7,15 +7,13 @@ ''' Performs tests on the treesitter PSyIR front-end ''' import logging -from types import SimpleNamespace import pytest from tree_sitter import Node as TSNode from psyclone.errors import InternalError -from psyclone.psyir.frontend import fortran_treesitter_reader as ftr from psyclone.psyir.frontend.fortran_treesitter_reader import ( - FortranTreeSitterReader, _CommonDeclAttributes, _NodeExpectation) + FortranTreeSitterReader, _NodeExpectation) from psyclone.psyir import nodes as psyir_nodes, symbols as psyir_symbols from psyclone.tests.utilities import min_version_3_10 @@ -716,44 +714,6 @@ def test_character_length_and_kind_selectors(): assert datatype.precision.value == "2" -def test_empty_kind_and_malformed_literal_declaration_nodes(): - '''Test defensive handling for malformed parser nodes after establishing - the expected declaration and literal node forms from Fortran input. - ''' - valid_code = """ - subroutine declarations() - character(3) :: text = 'abc' - end subroutine declarations - """ - processor = FortranTreeSitterReader() - parse_tree = processor.generate_parse_tree_from_source(valid_code) - string = _first_tsnode(parse_tree, "string_literal") - declaration = _first_tsnode(parse_tree, "variable_declaration") - intrinsic_type = _first_tsnode(parse_tree, "intrinsic_type") - - assert processor._string_literal_handler(string).value == "abc" - assert declaration.type == "variable_declaration" - for text, message in [ - (b"abc", "no quote delimiter"), - (b"'abc\"", "mismatched quote delimiters"), - (b"_'abc'", "kind prefix")]: - malformed = SimpleNamespace(text=text) - with pytest.raises(NotImplementedError, match=message): - processor._string_literal_handler(malformed) - - malformed_declaration = SimpleNamespace(children=[]) - with pytest.raises(NotImplementedError, match="no supported type"): - processor._variable_declaration_handler(malformed_declaration) - - empty_kind_type = SimpleNamespace( - type="intrinsic_type", - text=intrinsic_type.text, - children=[intrinsic_type.children[0], - SimpleNamespace(type="kind", children=[])]) - assert processor._datatype_from_type(empty_kind_type) == \ - psyir_symbols.ScalarType.character_type() - - def test_logical_literal(): '''Test a logical literal used as an initial value.''' processor = FortranTreeSitterReader() @@ -1039,8 +999,8 @@ def unsupported_shape(*_args, **_kwargs): psyir_symbols.UnsupportedFortranType) -def test_direct_shape_and_argument_helpers(): - '''Test defensive extent splitting and absent argument handling.''' +def test_direct_extent_and_argument_helpers(): + '''Test extent splitting and absent argument handling.''' valid_code = """ subroutine shape(values) integer :: values(10) @@ -1056,30 +1016,6 @@ def test_direct_shape_and_argument_helpers(): assert not has_colon assert not processor._arguments(None) - malformed = SimpleNamespace( - type="extent_specifier", children=[number]) - assert processor._shape_from_node( - SimpleNamespace(children=[malformed]))[0].value == "10" - common = _CommonDeclAttributes( - psyir_symbols.ScalarType.integer_type(), - psyir_symbols.ArgumentInterface.Access.UNKNOWN, - frozenset(), frozenset(), "integer") - empty_initializer = SimpleNamespace( - type="init_declarator", children=[ - SimpleNamespace(type="identifier", text=b"value"), - SimpleNamespace(type="=", text=b"=")]) - datatype, initial = processor._declarator_datatype( - empty_initializer, common) - assert isinstance(datatype, psyir_symbols.ScalarType) - assert initial is None - - with pytest.raises(NotImplementedError, match="Malformed array range"): - processor._range(number, psyir_symbols.DataSymbol( - "values", psyir_symbols.ArrayType( - psyir_symbols.ScalarType.integer_type(), [10])), 1) - with pytest.raises(NotImplementedError, match="Malformed allocation"): - processor._allocation_extent(malformed) - def test_default_visibility(): '''Test a module's default visibility.''' @@ -1227,9 +1163,8 @@ def test_use_rename_without_only(): assert routine.children[0].rhs.symbol is imported -def test_wildcard_and_defensive_import_branches(): - '''Test wildcard import and defensive malformed/identity import - handling.''' +def test_wildcard_and_identity_import(): + '''Test a wildcard import and importing a container under its own name.''' valid_code = """ module imports use wildcard_source @@ -1238,25 +1173,14 @@ def test_wildcard_and_defensive_import_branches(): processor = FortranTreeSitterReader() root = processor.generate_psyir( processor.generate_parse_tree_from_source(valid_code)) - assert root.children[0].symbol_table.lookup( - "wildcard_source").wildcard_import + table = root.children[0].symbol_table + container = table.lookup("wildcard_source") + assert container.wildcard_import - table = psyir_symbols.SymbolTable() processor._current_scope = table - module_name = SimpleNamespace( - type="module_name", children=[], text=b"source") - malformed_rename = SimpleNamespace( - type="rename", children=[SimpleNamespace( - type="identifier", children=[], text=b"local")]) - included = SimpleNamespace( - type="included_items", children=[malformed_rename]) - use_statement = SimpleNamespace( - children=[module_name, included]) - processor._use_statement_handler(use_statement) - container = table.lookup("source") - - processor._add_imported_symbol("source", "source", container) - assert table.lookup("source") is container + processor._add_imported_symbol( + "wildcard_source", "wildcard_source", container) + assert table.lookup("wildcard_source") is container def test_repeated_use_preserves_wildcard_import(): @@ -1915,6 +1839,28 @@ def test_later_contained_routine_is_predeclared(): assert call.routine.symbol.is_pure is True +def test_predeclare_unsupported_routine(): + '''A routine with an unsupported return type is still predeclared with an + unresolved type. + ''' + valid_code = """ + module routines + contains + complex function unsupported() + end function unsupported + end module routines + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + container = root.children[0] + symbol = container.symbol_table.lookup("unsupported") + + assert isinstance(symbol, psyir_symbols.RoutineSymbol) + assert isinstance(symbol.datatype, psyir_symbols.UnresolvedType) + assert isinstance(container.children[0], psyir_nodes.CodeBlock) + + def test_call_statement_edge_cases(): '''Test imported-symbol specialisation and invalid call targets.''' valid_code = """ @@ -2529,9 +2475,28 @@ def test_select_case_with_only_default_is_unsupported(): codeblock.preceding_comment -# pylint: disable=too-many-locals -def test_malformed_operation_and_statement_guards(monkeypatch): - '''Test defensive guards for malformed expressions and statements.''' +def test_select_case_with_only_default_and_simple_selector(): + '''A default-only SELECT CASE with a simple selector is unsupported.''' + valid_code = """ + subroutine selection(value) + integer :: value + select case(value) + case default + value = 1 + end select + end subroutine selection + """ + processor = FortranTreeSitterReader() + root = processor.generate_psyir( + processor.generate_parse_tree_from_source(valid_code)) + + codeblock = root.children[0].children[0] + assert isinstance(codeblock, psyir_nodes.CodeBlock) + assert "only a default clause" in codeblock.preceding_comment + + +def test_invalid_memory_intrinsic_signature(monkeypatch): + '''Test handling of a memory intrinsic rejected by PSyIR validation.''' valid_code = """ subroutine guards(array) real, allocatable :: array(:) @@ -2540,54 +2505,6 @@ def test_malformed_operation_and_statement_guards(monkeypatch): """ processor = FortranTreeSitterReader() parse_tree = processor.generate_parse_tree_from_source(valid_code) - number = _first_tsnode(parse_tree, "number_literal") - - bad_unary = SimpleNamespace(children=[ - SimpleNamespace(type="operator", text=b"?"), number]) - with pytest.raises(NotImplementedError, match="unary operator"): - processor._operation(bad_unary) - - bad_binary = SimpleNamespace(children=[ - number, SimpleNamespace(type="operator", text=b"?"), number]) - with pytest.raises(NotImplementedError, match="binary operator"): - processor._operation(bad_binary) - with pytest.raises(NotImplementedError, match="operation structure"): - processor._operation(SimpleNamespace(children=[])) - - with pytest.raises(NotImplementedError, match="assignment structure"): - processor._assignment_statement_handler( - SimpleNamespace(children=[])) - with pytest.raises(NotImplementedError, match="bounds remapping"): - processor._pointer_association_statement_handler( - SimpleNamespace(children=[])) - with pytest.raises(NotImplementedError, match="IF statement"): - processor._if_statement_handler(SimpleNamespace(children=[])) - - statement = SimpleNamespace( - type="do_statement", - children=[SimpleNamespace( - type="loop_control_expression", children=[])]) - loop = SimpleNamespace(type="do_loop", children=[statement]) - with pytest.raises(NotImplementedError, match="counted DO loop"): - processor._do_loop_handler(loop) - - identifier = _first_tsnode(parse_tree, "identifier") - selector = SimpleNamespace(type="selector", children=[identifier]) - malformed_case = SimpleNamespace(type="case_statement", children=[]) - select = SimpleNamespace(children=[selector, malformed_case]) - with pytest.raises(NotImplementedError, match="Malformed CASE"): - processor._select_case_statement_handler(select) - - malformed_member = SimpleNamespace( - type="derived_type_member_expression", - children=[SimpleNamespace( - type="identifier", children=[], text=b"item")]) - with pytest.raises(NotImplementedError, match="Malformed structure"): - processor._decompose_structure(malformed_member) - with pytest.raises(NotImplementedError, match="structure access base"): - processor._decompose_structure( - SimpleNamespace(type="number_literal", children=[])) - allocate = _first_tsnode(parse_tree, "allocate_statement") def invalid_intrinsic(*_args, **_kwargs): @@ -2606,6 +2523,7 @@ def test_scope_and_handler_defensive_errors(): subroutine routine() end subroutine routine module types + private :: operator(+) type :: item integer :: value end type item @@ -2615,11 +2533,9 @@ def test_scope_and_handler_defensive_errors(): parse_tree = processor.generate_parse_tree_from_source(valid_code) routine = _first_tsnode(parse_tree, "subroutine") derived = _first_tsnode(parse_tree, "derived_type_definition") - - duplicate = SimpleNamespace(children=[ - SimpleNamespace(type="name"), SimpleNamespace(type="name")]) - with pytest.raises(InternalError, match="Expected only 1"): - ftr.child_of_type(duplicate, "name") + unsupported_access = _first_tsnode(parse_tree, "private_statement") + with pytest.raises(NotImplementedError, match="Named operator"): + processor._private_statement_handler(unsupported_access) parent = psyir_nodes.ScopingNode(symbol_table=psyir_symbols.SymbolTable()) attached = psyir_nodes.ScopingNode( From 705dcd4ac87750bf036bf0f68174b76d751ca780 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Mon, 17 Aug 2026 12:05:00 +0100 Subject: [PATCH 21/23] #3083 Simplify treesitter generator utility --- .../frontend/fortran_treesitter_reader.py | 175 +++++++++--------- 1 file changed, 91 insertions(+), 84 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 030ccfa20e..8163a30c4f 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -60,17 +60,15 @@ def log_decode_error_handler(err) -> tuple[str, int]: codecs.register_error("treesitter-encoding", log_decode_error_handler) -def to_str(node: Optional['TSNode']) -> str: +def to_str(node: 'TSNode') -> str: ''' :param node: a given treesitter node. :returns: the string representing the node in utf8. ''' - if node is None: - return "" return node.text.decode('utf8') if node.text else "" -def iter_child_of_type( +def children_of_type( tsnode: Optional['TSNode'], types: Union[str, Container[str]] ) -> Generator['TSNode']: ''' Provides a generator to iterate over the provided tsnode @@ -88,26 +86,6 @@ def iter_child_of_type( yield child -def child_of_type( - tsnode: Optional['TSNode'], node_type: Union[str, Container[str]] -) -> Optional['TSNode']: - ''' Return the direct child having the supplied type(s). And validate - that is the only child of the supplied type. - - :param tsnode: tree-sitter node whose children are searched. - :param node_type: tree-sitter type(s) to find. - - :returns: matching child, or ``None`` if no child matches. - - ''' - children = list(iter_child_of_type(tsnode, node_type)) - if len(children) == 0: - return None - elif len(children) > 1: - raise InternalError(f"Expected only 1 child of type {node_type}") - return children[0] - - @dataclass(frozen=True) class _CommonDeclAttributes: ''' Properties shared by all entities of a fortran declaration (the lhs @@ -527,8 +505,9 @@ def _module_handler( :returns: the equivalent PSyIR Node. ''' - statement = child_of_type(tsnode, "module_statement") - name = child_of_type(statement, "name") + statement = next( + children_of_type(tsnode, "module_statement"), None) + name = next(children_of_type(statement, "name"), None) container = nodes.Container(to_str(name) if name else "") with self._using_scope(container.symbol_table): @@ -546,7 +525,8 @@ def _module_handler( if child.type not in skip], _NodeExpectation.SPECS) # Parse the execution part - internal = child_of_type(tsnode, "internal_procedures") + internal = next( + children_of_type(tsnode, "internal_procedures"), None) if internal: self._predeclare_routines(internal.children) container.children.extend( @@ -566,10 +546,12 @@ def _procedure_handler( :returns: translated PSyIR Routine. ''' routine_kind = tsnode.type - signature = child_of_type(tsnode, f"{routine_kind}_statement") - name_node = child_of_type(signature, "name") + signature = next(children_of_type( + tsnode, f"{routine_kind}_statement"), None) + name_node = next(children_of_type(signature, "name"), None) name = to_str(name_node) if name_node else routine_kind - parameters = child_of_type(signature, "parameters") + parameters = next( + children_of_type(signature, "parameters"), None) argument_names = tuple( to_str(child) for child in parameters.children if child.type == "identifier") if parameters else () @@ -662,8 +644,9 @@ def _function_return_info( if routine_kind != "function": return None, None - result = child_of_type(signature, "function_result") - result_name = child_of_type(result, "identifier") + result = next( + children_of_type(signature, "function_result"), None) + result_name = next(children_of_type(result, "identifier"), None) return_name = to_str(result_name) if result_name else routine_name type_node = next( (child for child in signature.children @@ -692,9 +675,9 @@ def _predeclare_routines(self, tsnodes: Iterable['TSNode']): for procedure in tsnodes: if procedure.type not in ("subroutine", "function"): continue - signature = child_of_type( - procedure, f"{procedure.type}_statement") - name_node = child_of_type(signature, "name") + signature = next(children_of_type( + procedure, f"{procedure.type}_statement"), None) + name_node = next(children_of_type(signature, "name"), None) name = to_str(name_node) try: _, return_type = self._function_return_info( @@ -847,7 +830,8 @@ def _variable_declaration_handler( datatype = None dimension = next( - (child_of_type(item, "argument_list") for item in qualifiers + (next(children_of_type(item, "argument_list"), None) + for item in qualifiers if item.children and item.children[0].type == "dimension"), None) is_allocatable = "allocatable" in qualifier_names if datatype and dimension: @@ -885,7 +869,8 @@ def _declare_entity( :param common_attr: properties shared by the complete declaration. ''' id_node = (declarator if declarator.type == "identifier" - else child_of_type(declarator, "identifier")) + else next(children_of_type( + declarator, "identifier"), None)) name = to_str(id_node) datatype, initial_value = self._declarator_datatype( declarator, common_attr) @@ -939,7 +924,7 @@ def _declarator_datatype( # attribute. The latter has already been translated into an ArrayType, # from which the elemental type can be recovered. datatype = common_attr.datatype - shape_node = child_of_type(declarator, "size") + shape_node = next(children_of_type(declarator, "size"), None) is_allocatable = "allocatable" in common_attr.qualifiers if datatype and shape_node: try: @@ -1051,7 +1036,8 @@ def _datatype_from_type( symtab = self._current_scope if tsnode.type == "derived_type": keyword = tsnode.children[0].type - name_node = child_of_type(tsnode, "type_name") + name_node = next( + children_of_type(tsnode, "type_name"), None) name = to_str(name_node) if keyword == "class": raise NotImplementedError( @@ -1078,7 +1064,7 @@ def _datatype_from_type( f"Intrinsic type '{intrinsic}' has no PSyIR representation") precision = symbols.ScalarType.Precision.UNDEFINED length = None - kind_node = child_of_type(tsnode, "kind") + kind_node = next(children_of_type(tsnode, "kind"), None) if kind_node: values = [child for child in kind_node.children if child.type not in ("(", ")", ",")] @@ -1285,10 +1271,12 @@ def _use_statement_handler( existing non-container symbol. ''' symtab = self._current_scope - module_node = child_of_type(tsnode, "module_name") + module_node = next( + children_of_type(tsnode, "module_name"), None) module_name = to_str(module_node) intrinsic = any(child.type == "intrinsic" for child in tsnode.children) - included = child_of_type(tsnode, "included_items") + included = next( + children_of_type(tsnode, "included_items"), None) wildcard = included is None try: container = symtab.lookup(module_name) @@ -1359,8 +1347,10 @@ def _derived_type_definition_handler( existing non-datatype symbol. ''' symtab = self._current_scope - statement = child_of_type(tsnode, "derived_type_statement") - name_node = child_of_type(statement, "type_name") + statement = next( + children_of_type(tsnode, "derived_type_statement"), None) + name_node = next( + children_of_type(statement, "type_name"), None) name = to_str(name_node) unsupported = ( any(child.type == "derived_type_procedures" @@ -1379,7 +1369,7 @@ def _derived_type_definition_handler( visibility_map = self._process_access_statements( tsnode.children) try: - for declaration in iter_child_of_type( + for declaration in children_of_type( tsnode, "variable_declaration"): self._variable_declaration_handler(declaration) self._apply_visibility(visibility_map) @@ -1396,7 +1386,8 @@ def _derived_type_definition_handler( tsnode, statement, name) visibility = symtab.default_visibility - access = child_of_type(statement, "access_specifier") + access = next( + children_of_type(statement, "access_specifier"), None) if access: visibility = (symbols.Symbol.Visibility.PRIVATE if "private" in to_str(access).lower() else @@ -1447,17 +1438,18 @@ def _interface_handler( unsupported. ''' symtab = self._current_scope - statement = child_of_type(tsnode, "interface_statement") - name_node = child_of_type(statement, "name") + statement = next( + children_of_type(tsnode, "interface_statement"), None) + name_node = next(children_of_type(statement, "name"), None) if not name_node: raise NotImplementedError( "Abstract and operator interfaces are not supported") name = to_str(name_node) routines = [] - for procedure in iter_child_of_type(tsnode, "procedure_statement"): + for procedure in children_of_type(tsnode, "procedure_statement"): from_container = "module" in [ child.type for child in procedure.children[0].children] - for method in iter_child_of_type(procedure, "method_name"): + for method in children_of_type(procedure, "method_name"): routine_name = to_str(method) try: routine = symtab.lookup(routine_name) @@ -1556,9 +1548,11 @@ def _call_expression_handler( if name_node.type == "derived_type_member_expression": return self._structure_reference( name_node, - trailing_arguments=child_of_type(tsnode, "argument_list")) + trailing_arguments=next( + children_of_type(tsnode, "argument_list"), None)) name = to_str(name_node).lower() - argument_list = child_of_type(tsnode, "argument_list") + argument_list = next( + children_of_type(tsnode, "argument_list"), None) try: symbol = self._current_scope.lookup(name) except KeyError: @@ -1763,7 +1757,7 @@ def _decompose_structure( base = tsnode.children[0] name, indices, members = self._decompose_structure(base) arguments = self._arguments( - child_of_type(tsnode, "argument_list")) + next(children_of_type(tsnode, "argument_list"), None)) if any(isinstance(arg, tuple) for arg in arguments): raise NotImplementedError( "Named arguments in structure accesses are not supported") @@ -1774,7 +1768,7 @@ def _decompose_structure( return name, indices, members name, indices, members = self._decompose_structure( tsnode.children[0]) - member = child_of_type(tsnode, "type_member") + member = next(children_of_type(tsnode, "type_member"), None) members.append(to_str(member).lower()) return name, indices, members @@ -1789,7 +1783,8 @@ def _array_literal_handler( :raises NotImplementedError: for an implied-DO constructor. ''' - if child_of_type(tsnode, "implied_do_loop_expression"): + if next(children_of_type( + tsnode, "implied_do_loop_expression"), None): raise NotImplementedError( "Array constructors with implied-DO loops are not supported") if any(child.type in ("intrinsic_type", "derived_type") @@ -1876,7 +1871,8 @@ def _subroutine_call_handler( if not isinstance(symbol, symbols.RoutineSymbol): raise NotImplementedError( f"Called object '{name}' is not a routine") - args = self._arguments(child_of_type(tsnode, "argument_list")) + args = self._arguments(next( + children_of_type(tsnode, "argument_list"), None)) return nodes.Call.create(symbol, args) def _keyword_statement_handler( @@ -1906,17 +1902,19 @@ def _if_statement_handler( :returns: root PSyIR IfBlock. ''' - condition_node = child_of_type(tsnode, "parenthesized_expression") + condition_node = next(children_of_type( + tsnode, "parenthesized_expression"), None) structural = { "if", "parenthesized_expression", "then", "end_if_statement", "else_clause", "elseif_clause" } body_nodes = [child for child in tsnode.children if child.type not in structural] - else_clause = child_of_type(tsnode, "else_clause") - else_ifs = list(iter_child_of_type(tsnode, "elseif_clause")) + else_clause = next(children_of_type(tsnode, "else_clause"), None) + else_ifs = list(children_of_type(tsnode, "elseif_clause")) annotations = [] - if not child_of_type(tsnode, "end_if_statement"): + if not next(children_of_type( + tsnode, "end_if_statement"), None): annotations.append("was_single_stmt") if_body = self._process_nodes(body_nodes, _NodeExpectation.LIST) else_body = None @@ -1943,19 +1941,21 @@ def _if_clause( :returns: annotated PSyIR IfBlock. ''' - condition = child_of_type(tsnode, "parenthesized_expression") + condition = next(children_of_type( + tsnode, "parenthesized_expression"), None) structural = {"else", "if", "parenthesized_expression", "then", "else_clause", "elseif_clause"} body = self._process_nodes( [child for child in tsnode.children if child.type not in structural], _NodeExpectation.LIST) - trailing = child_of_type(tsnode, "elseif_clause") + trailing = next( + children_of_type(tsnode, "elseif_clause"), None) + else_clause = next(children_of_type(tsnode, "else_clause"), None) otherwise = ( [self._if_clause(trailing, final_else)] if trailing else self._process_nodes( [child for child in - (child_of_type(tsnode, "else_clause").children - if child_of_type(tsnode, "else_clause") else []) + (else_clause.children if else_clause else []) if child.type != "else"], _NodeExpectation.LIST) or final_else) result = nodes.IfBlock.create( @@ -1975,11 +1975,14 @@ def _do_loop_handler( :raises NotImplementedError: if counted-loop control is unsupported. ''' - statement = child_of_type(tsnode, "do_statement") - if child_of_type(statement, "concurrent_statement"): + statement = next(children_of_type(tsnode, "do_statement"), None) + if next(children_of_type( + statement, "concurrent_statement"), None): raise NotImplementedError("DO CONCURRENT is not supported") - control = child_of_type(statement, "loop_control_expression") - while_node = child_of_type(statement, "while_statement") + control = next(children_of_type( + statement, "loop_control_expression"), None) + while_node = next( + children_of_type(statement, "while_statement"), None) body = self._process_nodes( [child for child in tsnode.children if child.type not in ("do_statement", @@ -2009,8 +2012,8 @@ def _do_loop_handler( self._process_nodes(parts[2], _NodeExpectation.EXPRESSION), step, body) if while_node: - condition = child_of_type( - while_node, "parenthesized_expression") + condition = next(children_of_type( + while_node, "parenthesized_expression"), None) return nodes.WhileLoop.create( self._process_nodes(condition, _NodeExpectation.EXPRESSION), body) @@ -2028,21 +2031,22 @@ def _where_statement_handler( :returns: annotated PSyIR IfBlock. ''' - condition = child_of_type(tsnode, "parenthesized_expression") + condition = next(children_of_type( + tsnode, "parenthesized_expression"), None) structural = {"where", "parenthesized_expression", "elsewhere_clause", "end_where_statement"} body = self._process_nodes( [child for child in tsnode.children if child.type not in structural], _NodeExpectation.LIST) - elsewhere_clauses = list(iter_child_of_type( + elsewhere_clauses = list(children_of_type( tsnode, "elsewhere_clause")) other = None # Masked ELSEWHERE clauses have ELSE-IF semantics and are represented # by nested IfBlocks. Constructing the chain backwards makes the # following clause the else-body of the current masked clause. for elsewhere in reversed(elsewhere_clauses): - mask = child_of_type( - elsewhere, "parenthesized_expression") + mask = next(children_of_type( + elsewhere, "parenthesized_expression"), None) clause_body = self._process_nodes( [child for child in elsewhere.children if child.type not in @@ -2064,8 +2068,8 @@ def _where_statement_handler( self._process_nodes(condition, _NodeExpectation.EXPRESSION), body, other) result.annotations.extend( - ["was_where"] if child_of_type( - tsnode, "end_where_statement") else + ["was_where"] if next(children_of_type( + tsnode, "end_where_statement"), None) else ["was_where", "was_single_stmt"]) return result @@ -2083,12 +2087,14 @@ def _select_case_statement_handler( :raises NotImplementedError: if no conditional CASE can be produced. ''' - selector_node = child_of_type( - child_of_type(tsnode, "selector"), "identifier") + selector_syntax = next( + children_of_type(tsnode, "selector"), None) + selector_node = next( + children_of_type(selector_syntax, "identifier"), None) if selector_node is None: selector = self._process_nodes( [child for child in - child_of_type(tsnode, "selector").children + selector_syntax.children if child.type not in ("(", ")")][0], _NodeExpectation.EXPRESSION) else: @@ -2097,17 +2103,18 @@ def _select_case_statement_handler( if selector.walk(nodes.Call) or isinstance(selector, nodes.CodeBlock): raise NotImplementedError( "SELECT CASE selectors containing calls are not supported") - cases = list(iter_child_of_type(tsnode, "case_statement")) + cases = list(children_of_type(tsnode, "case_statement")) default_body = None normal = [] for case in cases: - if child_of_type(case, "default"): + if next(children_of_type(case, "default"), None): default_body = self._process_nodes( [child for child in case.children if child.type not in ("case", "default")], _NodeExpectation.LIST) else: - values = child_of_type(case, "case_value_range_list") + values = next(children_of_type( + case, "case_value_range_list"), None) structural = {"case", "(", ")", "case_value_range_list"} body = self._process_nodes( [child for child in case.children @@ -2225,7 +2232,7 @@ def _allocation_reference( :raises NotImplementedError: if the object is not a data symbol. ''' - ident = child_of_type(tsnode, "identifier") + ident = next(children_of_type(tsnode, "identifier"), None) if ident is None: raise NotImplementedError( "Allocations of structure components are not supported") @@ -2233,7 +2240,7 @@ def _allocation_reference( if not isinstance(reference.symbol, symbols.DataSymbol): raise NotImplementedError( "An ALLOCATE object must be a data symbol") - size = child_of_type(tsnode, "size") + size = next(children_of_type(tsnode, "size"), None) indices = [ self._allocation_extent(extent) for extent in size.children From 5e68154be56e852b4916fb9d88abc542adbfde31 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Mon, 17 Aug 2026 12:26:13 +0100 Subject: [PATCH 22/23] Fix get_fortran_lines for specific fparser codeblocks that already contain strings --- src/psyclone/psyir/nodes/codeblock.py | 4 +++- .../frontend/fparser2_kind_params_test.py | 24 ++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/psyclone/psyir/nodes/codeblock.py b/src/psyclone/psyir/nodes/codeblock.py index faa890b839..c7e7b60aa8 100644 --- a/src/psyclone/psyir/nodes/codeblock.py +++ b/src/psyclone/psyir/nodes/codeblock.py @@ -379,7 +379,9 @@ def get_fortran_lines(self) -> list[str]: ''' output = [] for node in self._parse_tree_nodes: - output.extend(node.tofortran().split("\n")) + output.extend( + node if isinstance(node, str) else + node.tofortran().split("\n")) return output diff --git a/src/psyclone/tests/psyir/frontend/fparser2_kind_params_test.py b/src/psyclone/tests/psyir/frontend/fparser2_kind_params_test.py index 0e3389970a..ef0ee030e8 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_kind_params_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_kind_params_test.py @@ -15,8 +15,7 @@ from psyclone.psyir.frontend.fparser2 import (Fparser2Reader, _kind_find_or_create) -from psyclone.psyir.nodes import ( - IntrinsicCall, KernelSchedule, Reference, BinaryOperation, Call) +from psyclone.psyir.nodes import IntrinsicCall, KernelSchedule, Reference from psyclone.psyir.symbols import ( DataSymbol, ScalarType, UnsupportedFortranType, RoutineSymbol, SymbolTable, Symbol, UnresolvedType, ContainerSymbol, UnresolvedInterface) @@ -52,10 +51,7 @@ def test_process_declarations_kind_new_param(): ''' fake_parent, fp2spec = process_declarations("real(kind=wp) :: var1\n" - "real(kind=Wp) :: var2\n" - "real(kind=1+1) :: var3\n" - "real(kind=u()) :: var4\n") - + "real(kind=Wp) :: var2\n") var1_var = fake_parent.symbol_table.lookup("var1") assert isinstance(var1_var.datatype.precision, Reference) # Check that this has resulted in the creation of a new 'wp' symbol @@ -66,11 +62,17 @@ def test_process_declarations_kind_new_param(): # references the same 'wp' symbol. var2_var = fake_parent.symbol_table.lookup("var2") assert var2_var.datatype.precision == Reference(wp_var) - # Check that other kind types - var3_var = fake_parent.symbol_table.lookup("var3") - assert isinstance(var3_var.datatype.precision, BinaryOperation) - var4_var = fake_parent.symbol_table.lookup("var4") - assert isinstance(var4_var.datatype.precision, Call) + # Check that we get a symbol of unsupported type if the KIND expression has + # an unexpected structure + # Break the parse tree by changing Name('wp') into a str + fp2spec[0].items[0].items[1].items = ("(", "blah", ")") + # Change the variable name too to prevent a clash + fp2spec[0].children[2].children[0].items[0].string = "var3" + processor = Fparser2Reader() + processor.process_declarations(fake_parent, [fp2spec[0]], []) + sym = fake_parent.symbol_table.lookup("var3") + assert isinstance(sym, DataSymbol) + assert isinstance(sym.datatype, UnsupportedFortranType) @pytest.mark.usefixtures("f2008_parser") From 5a717cbcaf79802de89828a2281af002e75aa23d Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Tue, 18 Aug 2026 15:46:41 +0100 Subject: [PATCH 23/23] #3083 Clean up treesitter frontend --- .../psyir/frontend/fortran_treesitter_reader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py index 8163a30c4f..f147419347 100644 --- a/src/psyclone/psyir/frontend/fortran_treesitter_reader.py +++ b/src/psyclone/psyir/frontend/fortran_treesitter_reader.py @@ -7,7 +7,7 @@ ''' -PSyIR fronted to ingest Fortran using the TreeSitter parse generator. +PSyIR frontend for the TreeSitter Fortran parser generator. The structure of the expected fortran parse tree can be found in the 'rules' section of: @@ -72,7 +72,7 @@ def children_of_type( tsnode: Optional['TSNode'], types: Union[str, Container[str]] ) -> Generator['TSNode']: ''' Provides a generator to iterate over the provided tsnode - chidlren of the given type(s). + children of the given type(s). :param tsnode: tree-sitter node whose children are searched. :param node_type: tree-sitter type to find. @@ -379,6 +379,9 @@ def _process_nodes( :returns: PSyIR nodes produced from the supplied tree-sitter nodes. ''' + if expect not in _NodeExpectation: + raise InternalError( + f"Unsupported node expectation '{expect}'") list_of_nodes = tsnodes if isinstance(tsnodes, Iterable) else [tsnodes] children = [] for tsnode in list_of_nodes: @@ -430,9 +433,6 @@ def _process_nodes( f"{[type(c).__name__ for c in children]}" ) return None - if expect is not _NodeExpectation.LIST: - raise InternalError( - f"Unsupported node expectation '{expect}'") return children @staticmethod