Skip to content

Improve default behavior of string search (NUL = new line) #6573

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Rot127 wants to merge 2 commits into
base: dev
Choose a base branch
from
Open

Improve default behavior of string search (NUL = new line) #6573

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b534320
Allow to set dotall and multiline flag also for normal regex patterns
Rot127 Jun 28, 2026
9ca8c12
String search sees NUL as newline by default.
Rot127 Jun 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions librz/core/cmd_descs/cmd_descs.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2901,6 +2901,7 @@ static const RzCmdDescDetailEntry slash_z_Examples_detail_entries[] = {
{ .text = "/z", .arg_str = " (ABC*)D li", .comment = "Search the exact string \"(ABC*)D\" but case insensitive." },
{ .text = "/z", .arg_str = " \\\\d\\\\sC*\\\\w ri", .comment = "Search the regular expression \"\\d\\sC*\\w\" but case insensitive." },
{ .text = "/z", .arg_str = " \"и.{3}м\" ei", .comment = "Search the extended regular expression \"и.{3}м\" but case insensitive." },
{ .text = "/z", .arg_str = " \"(*LF).+\" r ascii", .comment = "Search any ascii string with line feed is a newline." },
{ 0 },
};
static const RzCmdDescDetail slash_z_details[] = {
Expand Down
3 changes: 3 additions & 0 deletions librz/core/cmd_descs/cmd_search.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1132,6 +1132,9 @@ commands:
- text: "/z"
arg_str: " \"и.{3}м\" ei"
comment: "Search the extended regular expression \"и.{3}м\" but case insensitive."
- text: "/z"
arg_str: " \"(*LF).+\" r ascii"
comment: "Search any ascii string with line feed is a newline."
subcommands:
- name: "/z"
summary: String search.
Expand Down
3 changes: 2 additions & 1 deletion librz/include/rz_util/rz_regex.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ typedef struct {
RzRegex32 *re32;
};
void *jit_stack; ///< The JIT stack for this pattern. Must be used only by one thread at a time.
RzRegexCompContext *ccontext;
} RzRegexMulti;

typedef struct {
Expand All @@ -106,7 +107,7 @@ RZ_API RZ_OWN RzRegex16 *rz_regex_new_16(RZ_NONNULL const char *pattern, RzRegex
RZ_API RZ_OWN RzRegex32 *rz_regex_new_32(RZ_NONNULL const char *pattern, RzRegexFlags cflags, RzRegexFlags jflags,
RzRegexCompContext *ccontext);
RZ_API RZ_OWN RzRegexMulti *rz_regex_new_multi(RZ_NONNULL const char *pattern, RzRegexFlags cflags, RzRegexFlags jflags,
RzRegexCompContext *ccontext, RzRegexType type);
RzRegexType type);
RZ_API RZ_OWN RzRegex *rz_regex_new_bytes(RZ_NONNULL const ut8 *pattern, size_t pattern_len, RzRegexFlags cflags, RzRegexFlags jflags,
RzRegexCompContext *ccontext);
RZ_API void rz_regex_free(RZ_OWN RzRegex *regex);
Expand Down
8 changes: 4 additions & 4 deletions librz/search/string_search.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -422,19 +422,19 @@ static RzDetectedString *setup_str_regex(const char *re_pattern, RzRegexFlags cf
return NULL;
case RZ_STRING_ENC_UTF8:
case RZ_STRING_ENC_8BIT:
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, NULL, RZ_REGEX_UTF8);
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, RZ_REGEX_UTF8);
break;
case RZ_STRING_ENC_UTF16LE:
case RZ_STRING_ENC_UTF16BE:
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, NULL, RZ_REGEX_UTF16);
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, RZ_REGEX_UTF16);
break;
case RZ_STRING_ENC_UTF32LE:
case RZ_STRING_ENC_UTF32BE:
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, NULL, RZ_REGEX_UTF32);
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, RZ_REGEX_UTF32);
break;
}
} else {
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, NULL, RZ_REGEX_UTF8);
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, RZ_REGEX_UTF8);
}
if (!re) {
RZ_LOG_ERROR("Failed to compile regex pattern: '%s'\n", re_pattern);
Expand Down
42 changes: 28 additions & 14 deletions librz/util/regex.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,20 +227,19 @@ RZ_API RZ_OWN RzRegex32 *rz_regex_new_32(RZ_NONNULL const char *pattern, RzRegex
* \brief Compile an Regex pattern of \p type.
*
* NOTE: The pattern and matching will always be in the host's endianness.
* NOTE: The patterns are compiled with PCRE2_NEWLINE_NUL.
*
* \param pattern The regex pattern string. It must be an UTF-8 encoded string.
* \param cflags The compilation flags or zero for default.
* PCRE2_UCP | PCRE2_UTF | PCRE2_NO_UTF_CHECK | PCRE2_MATCH_INVALID_UTF are enforced currently.
* \param jflags The compilation flags for the JIT compiler.
* You can pass RZ_REGEX_JIT_PARTIAL_SOFT or RZ_REGEX_JIT_PARTIAL_HARD if you
* intend to use the pattern for partial matching. Otherwise set it to RZ_REGEX_DEFAULT.
* \param ccontext A compile context or NULL.
* \param type The string encoding type the pattern should match.
*
* \return The compiled regex or NULL in case of failure.
*/
RZ_API RZ_OWN RzRegexMulti *rz_regex_new_multi(RZ_NONNULL const char *pattern, RzRegexFlags cflags, RzRegexFlags jflags,
RzRegexCompContext *ccontext, RzRegexType type) {
RZ_API RZ_OWN RzRegexMulti *rz_regex_new_multi(RZ_NONNULL const char *pattern, RzRegexFlags cflags, RzRegexFlags jflags, RzRegexType type) {
RzRegexMulti *re = RZ_NEW0(RzRegexMulti);
if (!re) {
return NULL;
Expand All @@ -255,16 +254,28 @@ RZ_API RZ_OWN RzRegexMulti *rz_regex_new_multi(RZ_NONNULL const char *pattern, R
rz_warn_if_reached();
free(re);
return NULL;
case RZ_REGEX_UTF8:
case RZ_REGEX_UTF8: {
RzRegexCompContext *ccontext = pcre2_compile_context_create_8(NULL);
pcre2_set_newline_8(ccontext, PCRE2_NEWLINE_NUL);
re->re8 = rz_regex_new(pattern, cflags, jflags, ccontext);
re->ccontext = ccontext;
break;
case RZ_REGEX_UTF16:
}
case RZ_REGEX_UTF16: {
RzRegexCompContext *ccontext = pcre2_compile_context_create_16(NULL);
pcre2_set_newline_16(ccontext, PCRE2_NEWLINE_NUL);
re->re16 = rz_regex_new_16(pattern, cflags, jflags, ccontext);
re->ccontext = ccontext;
break;
case RZ_REGEX_UTF32:
}
case RZ_REGEX_UTF32: {
RzRegexCompContext *ccontext = pcre2_compile_context_create_32(NULL);
pcre2_set_newline_32(ccontext, PCRE2_NEWLINE_NUL);
re->re32 = rz_regex_new_32(pattern, cflags, jflags, ccontext);
re->ccontext = ccontext;
break;
}
}
if (!re->re8 && !re->re16 && !re->re32) {
free(re);
return NULL;
Expand Down Expand Up @@ -457,12 +468,15 @@ RZ_API void rz_regex_free_multi(RZ_NULLABLE RZ_OWN RzRegexMulti *regex_multi) {
switch (regex_multi->re_type) {
case RZ_REGEX_UTF8:
rz_regex_free(regex_multi->re8);
pcre2_compile_context_free_8(regex_multi->ccontext);
break;
case RZ_REGEX_UTF16:
rz_regex_free_16(regex_multi->re16);
pcre2_compile_context_free_16(regex_multi->ccontext);
break;
case RZ_REGEX_UTF32:
rz_regex_free_32(regex_multi->re32);
pcre2_compile_context_free_32(regex_multi->ccontext);
break;
}
#ifdef SUPPORTS_PCRE2_JIT
Expand Down Expand Up @@ -1458,6 +1472,14 @@ RZ_API RzRegexFlags rz_regex_parse_flag_desc(RZ_NULLABLE const char *re_flags_de
flags |= RZ_REGEX_LITERAL;
goto return_flags;
}
if (strchr(re_flags_desc, 'd')) {
fcount++;
flags |= RZ_REGEX_DOTALL;
}
if (strchr(re_flags_desc, 'm')) {
fcount++;
flags |= RZ_REGEX_MULTILINE;
}
if (strchr(re_flags_desc, 'r')) {
fcount++;
goto return_flags;
Expand All @@ -1470,14 +1492,6 @@ RZ_API RzRegexFlags rz_regex_parse_flag_desc(RZ_NULLABLE const char *re_flags_de
fcount++;
flags |= RZ_REGEX_EXTENDED_MORE;
}
if (strchr(re_flags_desc, 'm')) {
fcount++;
flags |= RZ_REGEX_MULTILINE;
}
if (strchr(re_flags_desc, 'd')) {
fcount++;
flags |= RZ_REGEX_DOTALL;
}
return_flags:
if (fcount != strlen(re_flags_desc)) {
RZ_LOG_ERROR("Flag combination '%s' is invalid.\n", re_flags_desc);
Expand Down
12 changes: 6 additions & 6 deletions test/db/cmd/cmd_help
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,11 +299,11 @@ e scr.columns=70
/z??~:-6..
EOF
EXPECT=<<EOF
| /z (ABC*)D li # Search the exact string "(ABC*)D" but case
insensitive.
| /z \\d\\sC*\\w ri # Search the regular expression "\d\sC*\w" but
case insensitive.
| /z "и.{3}м" ei # Search the extended regular expression
"и.{3}м" but case insensitive.
| /z \\d\\sC*\\w ri # Search the regular expression "\d\sC*\w" but
case insensitive.
| /z "и.{3}м" ei # Search the extended regular expression
"и.{3}м" but case insensitive.
| /z "(*LF).+" r ascii # Search any ascii string with line feed is a
newline.
EOF
RUN
27 changes: 27 additions & 0 deletions test/db/cmd/cmd_search_z
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1548,3 +1548,30 @@ EXPECT=<<EOF
EOF
EXPECT_ERR=
RUN

NAME=Strings with new lines and \0 terminators
FILE==
CMDS=<<EOF
# Searching in empty buffer should not find any strings
/z .+ ri ascii

# Now write strings with new lines and NUL byte mixed
wx 41414141414141004141414141410a0041414141414141004141414141410a00

# NUL is a new line by default
/z .+ r ascii

# Overwrite default newline (NUL), \0 can be matched now with dot.
/z "(*LF).+" r ascii
EOF
EXPECT=<<EOF
0x00000000 7 hit.string.ascii.0 7
0x00000008 7 hit.string.ascii.1 7
0x00000010 7 hit.string.ascii.2 7
0x00000018 7 hit.string.ascii.3 7
0x00000000 14 hit.string.ascii.0 14
0x0000000f 15 hit.string.ascii.1 15
0x0000001f 481 hit.string.ascii.2 1e1
EOF
EXPECT_ERR=
RUN
Expand Down
Loading