Add pluggable regex engine support

noxfile.py

@@ -44,9 +44,33 @@ def tests(session):
     build_and_check_dists(session)
 
     generated_files = os.listdir("dist/")
-    generated_sdist = os.path.join("dist/", generated_files[1])
+    sdists = [f for f in generated_files if f.endswith(".tar.gz")]
+    if not sdists:
+        session.error("No sdist (.tar.gz) found in dist/")
+    generated_sdist = max((os.path.join("dist/", f) for f in sdists), key=os.path.getmtime)
 
     session.install(generated_sdist)
 
     session.run("python", "tests/generate.py")
     session.run("py.test", "tests/", *session.posargs)
+
+
+# Exercises the optional pluggable regex_engine hook against Google's RE2
+# (tests/re2_engine_test.py). Kept as its own session so the main `tests`
+# session -- and the package itself -- stay free of a google-re2 dependency;
+# this one opts in explicitly.
+@nox.session
+def test_re2(session):
+    session.install("pytest")
+    session.install("google-re2")
+    build_and_check_dists(session)
+
+    generated_files = os.listdir("dist/")
+    sdists = [f for f in generated_files if f.endswith(".tar.gz")]
+    if not sdists:
+        session.error("No sdist (.tar.gz) found in dist/")
+    generated_sdist = max((os.path.join("dist/", f) for f in sdists), key=os.path.getmtime)
+
+    session.install(generated_sdist)
+
+    session.run("py.test", "tests/re2_engine_test.py", *session.posargs)

src/jsonata/functions.py

@@ -39,6 +39,7 @@
 from typing import Any, AnyStr, Mapping, NoReturn, Optional, Sequence, Callable, Type, Union
 
 from jsonata import datetimeutils, jexception, parser, utils
+from jsonata.regex_engine import CompiledPattern
 
 
 class Functions:
@@ -514,7 +515,7 @@ class RegexpMatch:
     # @returns {object} - structure that represents the match(es)
     #     
     @staticmethod
-    def evaluate_matcher(matcher: re.Pattern, string: Optional[str]) -> list[RegexpMatch]:
+    def evaluate_matcher(matcher: CompiledPattern, string: Optional[str]) -> list[RegexpMatch]:
         res = []
         matches = matcher.finditer(string)
         for m in matches:
@@ -537,7 +538,7 @@ def evaluate_matcher(matcher: re.Pattern, string: Optional[str]) -> list[RegexpM
     # @returns {Boolean} - true if str contains token
     #     
     @staticmethod
-    def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Optional[bool]:
+    def contains(string: Optional[str], token: Union[None, str, CompiledPattern]) -> Optional[bool]:
         # undefined inputs always return undefined
         if string is None:
             return None
@@ -549,7 +550,7 @@ def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Opti
 
         if isinstance(token, str):
             result = (string.find(str(token)) != - 1)
-        elif isinstance(token, re.Pattern):
+        elif Functions.is_regex(token):
             matches = Functions.evaluate_matcher(token, string)
             # if (dbg) System.out.println("match = "+matches)
             # result = (typeof matches !== 'undefined')
@@ -568,7 +569,7 @@ def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Opti
     # @returns {Array} The array of match objects
     #     
     @staticmethod
-    def match_(string: Optional[str], regex: Optional[re.Pattern], limit: Optional[int]) -> Optional[list[dict[str, Any]]]:
+    def match_(string: Optional[str], regex: Optional[CompiledPattern], limit: Optional[int]) -> Optional[list[dict[str, Any]]]:
         # undefined inputs always return undefined
         if string is None:
             return None
@@ -636,7 +637,7 @@ def safe_replacement(in_: str) -> str:
     # @return
     #     
     @staticmethod
-    def safe_replace_all(s: str, pattern: re.Pattern, replacement: Optional[Any]) -> Optional[str]:
+    def safe_replace_all(s: str, pattern: CompiledPattern, replacement: Optional[Any]) -> Optional[str]:
 
         if not (isinstance(replacement, str)):
             return Functions.safe_replace_all_fn(s, pattern, replacement)
@@ -647,7 +648,7 @@ def safe_replace_all(s: str, pattern: re.Pattern, replacement: Optional[Any]) ->
         r = None
         for i in range(0, 10):
             try:
-                r = re.sub(pattern, replacement, s)
+                r = pattern.sub(replacement, s)
                 break
             except Exception as e:
                 msg = str(e)
@@ -696,15 +697,15 @@ def to_jsonata_match(mr: re.Match[str]) -> dict[str, list[str]]:
     # @return
     #     
     @staticmethod
-    def safe_replace_all_fn(s: str, pattern: re.Pattern, fn: Optional[Any]) -> str:
+    def safe_replace_all_fn(s: str, pattern: CompiledPattern, fn: Optional[Any]) -> str:
         def replace_fn(t):
             res = Functions.func_apply(fn, [Functions.to_jsonata_match(t)])
             if isinstance(res, str):
                 return res
             else:
                 raise jexception.JException("D3012", -1)
 
-        r = re.sub(pattern, replace_fn, s)
+        r = pattern.sub(replace_fn, s)
         return r
 
     #
@@ -716,12 +717,12 @@ def replace_fn(t):
     # @return
     #     
     @staticmethod
-    def safe_replace_first(s: str, pattern: re.Pattern, replacement: str) -> Optional[str]:
+    def safe_replace_first(s: str, pattern: CompiledPattern, replacement: str) -> Optional[str]:
         replacement = Functions.safe_replacement(replacement)
         r = None
         for i in range(0, 10):
             try:
-                r = re.sub(pattern, replacement, s, count=1)
+                r = pattern.sub(replacement, s, 1)
                 break
             except Exception as e:
                 msg = str(e)
@@ -744,7 +745,7 @@ def safe_replace_first(s: str, pattern: re.Pattern, replacement: str) -> Optiona
         return r
 
     @staticmethod
-    def replace(string: Optional[str], pattern: Union[str, re.Pattern], replacement: Optional[Any], limit: Optional[int]) -> Optional[str]:
+    def replace(string: Optional[str], pattern: Union[str, CompiledPattern], replacement: Optional[Any], limit: Optional[int]) -> Optional[str]:
         if string is None:
             return None
 
@@ -938,7 +939,7 @@ def decode_url(string: Optional[str]) -> Optional[str]:
         return urllib.parse.unquote(string, errors="strict")
 
     @staticmethod
-    def split(string: Optional[str], pattern: Union[str, Optional[re.Pattern]], limit: Optional[float]) -> Optional[list[str]]:
+    def split(string: Optional[str], pattern: Union[str, Optional[CompiledPattern]], limit: Optional[float]) -> Optional[list[str]]:
         if string is None:
             return None
 
@@ -2019,6 +2020,23 @@ def append(arg1: Optional[Any], arg2: Optional[Any]) -> Optional[Any]:
     def is_lambda(result: Optional[Any]) -> bool:
         return isinstance(result, parser.Parser.Symbol) and result._jsonata_lambda
 
+    #
+    # Tests whether a value is a compiled regex, from the stdlib re module
+    # or from a pluggable regex_engine (e.g. re2) with a compatible interface.
+    #
+    @staticmethod
+    def is_regex(value: Optional[Any]) -> bool:
+        if isinstance(value, re.Pattern):
+            return True
+        if value is None or inspect.ismodule(value) or inspect.isclass(value):
+            return False
+        return (
+            callable(getattr(value, "search", None))
+            and callable(getattr(value, "finditer", None))
+            and callable(getattr(value, "sub", None))
+            and callable(getattr(value, "split", None))
+        )
+
     #
     # Return value from an object for a given key
     # @param {Object} input - Object/Array
@@ -2201,7 +2219,7 @@ def function_eval(expr: Optional[str], focus: Optional[Any]) -> Optional[Any]:
 
         ast = None
         try:
-            ast = jsonata.Jsonata(expr)
+            ast = jsonata.Jsonata(expr, jsonata.Jsonata.CURRENT.jsonata.regex_engine)
         except Exception as err:
             # error parsing the expression passed to $eval
             # populateMessage(err)

src/jsonata/jsonata.py

@@ -27,13 +27,13 @@
 import copy
 import inspect
 import math
-import re
 import sys
 import threading
 from dataclasses import dataclass
 from typing import Any, Callable, Mapping, MutableSequence, Optional, Sequence, Type, MutableMapping, Union
 
 from jsonata import functions, jexception, parser, signature as sig, timebox, utils
+from jsonata.regex_engine import RegexEngine, default_regex_engine
 
 
 #
@@ -1300,7 +1300,7 @@ def evaluate_apply_expression(self, expr: Optional[parser.Parser.Symbol], input:
         return result
 
     def is_function_like(self, o: Optional[Any]) -> bool:
-        return utils.Utils.is_function(o) or functions.Functions.is_lambda(o) or (isinstance(o, re.Pattern))
+        return utils.Utils.is_function(o) or functions.Functions.is_lambda(o) or functions.Functions.is_regex(o)
 
     CURRENT = threading.local()
     MUTEX = threading.Lock()
@@ -1477,7 +1477,7 @@ def apply_inner(self, proc: Optional[Any], args: Optional[Any], input: Optional[
                 #  }
             elif isinstance(proc, Jsonata.JLambda):
                 result = proc.call(input, validated_args)
-            elif isinstance(proc, re.Pattern):
+            elif functions.Functions.is_regex(proc):
                 _res = []
                 for s in validated_args:
                     if isinstance(s, str):
@@ -1886,12 +1886,17 @@ def _static_initializer() -> None:
     #
     # JSONata
     # @param {Object} expr - JSONata expression
+    # @param {Object} regex_engine - callable taking (pattern: str, flags:
+    #     regex_engine.RegexFlags) and returning a compiled pattern, used to
+    #     compile JSONata regex literals. Every engine, including the
+    #     default, must translate RegexFlags into its own native
+    #     representation -- see jsonata.regex_engine.default_regex_engine.
     # @returns Evaluated expression
     # @throws jexception.JException An exception if an error occured.
-    #      
+    #
     @staticmethod
-    def jsonata(expression: Optional[str]) -> 'Jsonata':
-        return Jsonata(expression)
+    def jsonata(expression: Optional[str], regex_engine: RegexEngine = default_regex_engine) -> 'Jsonata':
+        return Jsonata(expression, regex_engine)
 
     #
     # Internal constructor
@@ -1904,11 +1909,13 @@ def jsonata(expression: Optional[str]) -> 'Jsonata':
     ast: Optional[parser.Parser.Symbol]
     timestamp: int
     input: Optional[Any]
+    regex_engine: RegexEngine
 
-    def __init__(self, expr: Optional[str]) -> None:
+    def __init__(self, expr: Optional[str], regex_engine: RegexEngine = default_regex_engine) -> None:
+        self.regex_engine = regex_engine
         try:
             self.parser = Jsonata.get_parser()
-            self.ast = self.parser.parse(expr)  # , optionsRecover);
+            self.ast = self.parser.parse(expr, regex_engine)  # , optionsRecover);
             self.errors = self.ast.errors
             self.ast.errors = None  # delete ast.errors;
         except jexception.JException as err:
@@ -1931,13 +1938,6 @@ def __init__(self, expr: Optional[str]) -> None:
         #      return timestamp.getTime()
         #  }, "<:n>"))
 
-        # FIXED: options.RegexEngine not implemented in Java
-        #  if(options && options.RegexEngine) {
-        #      jsonata.RegexEngine = options.RegexEngine
-        #  } else {
-        #      jsonata.RegexEngine = RegExp
-        #  }
-
         # Set instance for this thread
         Jsonata.CURRENT.jsonata = self
 

src/jsonata/parser.py

@@ -28,6 +28,7 @@
 from typing import Any, MutableSequence, Optional, Sequence
 
 from jsonata import jexception, tokenizer, signature, utils
+from jsonata.regex_engine import RegexEngine, default_regex_engine
 
 
 # var parseSignature = require('./signature')
@@ -1413,11 +1414,11 @@ def object_parser(self, left: Optional[Symbol]) -> Symbol:
             res.type = "binary"
         return res
 
-    def parse(self, jsonata: Optional[str]) -> Symbol:
+    def parse(self, jsonata: Optional[str], regex_engine: RegexEngine = default_regex_engine) -> Symbol:
         self.source = jsonata
 
         # now invoke the tokenizer and the parser and return the syntax tree
-        self.lexer = tokenizer.Tokenizer(self.source)
+        self.lexer = tokenizer.Tokenizer(self.source, regex_engine)
         self.advance()
         # parse the tokens
         expr = self.expression(0)

src/jsonata/regex_engine.py

@@ -0,0 +1,59 @@
+#
+# Copyright Robert Yokota
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import re
+from dataclasses import dataclass
+from typing import Any, Callable, Optional, Protocol
+
+
+@dataclass
+class RegexFlags:
+    """
+    Flags parsed from a JSONata regex literal's /pattern/flags suffix.
+    Engines translate these into their own native flag representation.
+    """
+
+    case_insensitive: bool = False
+    multiline: bool = False
+
+
+class CompiledPattern(Protocol):
+    """
+    Structural type for a compiled regex: matches stdlib `re.Pattern`
+    as well as whatever a pluggable regex_engine returns (e.g. a
+    `google-re2` pattern object).
+    """
+
+    def search(self, string: str) -> Optional[Any]: ...
+    def finditer(self, string: str) -> Any: ...
+    def sub(self, repl: Any, string: str, count: int = 0) -> str: ...
+    def split(self, string: str, maxsplit: int = 0) -> list[str]: ...
+
+
+# Compiles a pattern into a CompiledPattern. Used for JSONata regex literals.
+RegexEngine = Callable[[str, RegexFlags], CompiledPattern]
+
+
+def default_regex_engine(pattern: str, flags: RegexFlags) -> re.Pattern:
+    """
+    The built-in stdlib `re`-backed engine; this is jsonata-python's default.
+    """
+    py_flags = 0
+    if flags.case_insensitive:
+        py_flags |= re.IGNORECASE
+    if flags.multiline:
+        py_flags |= re.MULTILINE
+    return re.compile(pattern, py_flags)

src/jsonata/signature.py

@@ -112,7 +112,7 @@ def get_symbol(self, value: Optional[Any]) -> str:
             symbol = "m"
         else:
             # first check to see if this is a function
-            if utils.Utils.is_function(value) or functions.Functions.is_lambda(value) or isinstance(value, re.Pattern):
+            if utils.Utils.is_function(value) or functions.Functions.is_lambda(value) or functions.Functions.is_regex(value):
                 symbol = "f"
             elif isinstance(value, str):
                 symbol = "s"