Skip to content

GH-50859: [C++][Parquet] Move JsonWriter to simdjson utilities #50990

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rok wants to merge 4 commits into
base: main
Choose a base branch
from
Open

GH-50859: [C++][Parquet] Move JsonWriter to simdjson utilities #50990

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5a1f424
GH-50859: [C++][Parquet] Move JsonWriter to simdjson utilities
rok Aug 27, 2026
86e85bd
simplify meson change
rok Aug 27, 2026
4d93dff
set ARROW_JSON for odbc build
rok Aug 27, 2026
ab2c41a
GH-50859: [C++][Parquet] Build ObjectParser with simdjson utilities
rok Aug 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,11 +411,11 @@ if(ARROW_AZURE)
set(ARROW_WITH_AZURE_SDK ON)
endif()

if(ARROW_JSON OR ARROW_FLIGHT_SQL_ODBC)
if(ARROW_JSON)
set(ARROW_WITH_RAPIDJSON ON)
endif()

if(ARROW_JSON)
if(ARROW_JSON OR ARROW_FLIGHT_SQL_ODBC)
set(ARROW_WITH_SIMDJSON ON)
endif()

Expand Down
1 change: 1 addition & 0 deletions cpp/meson.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ needs_testing = (get_option('testing').enabled()
or needs_integration
)
needs_json = get_option('json').enabled() or needs_testing
needs_simdjson = needs_json or needs_parquet
needs_brotli = get_option('brotli').enabled() or needs_fuzzing
needs_bz2 = get_option('bz2').enabled()
needs_lz4 = get_option('lz4').enabled()
Expand Down
10 changes: 9 additions & 1 deletion cpp/src/arrow/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -644,6 +644,9 @@ endif()
if(ARROW_WITH_OPENTELEMETRY)
list(APPEND ARROW_UTIL_SRCS util/tracing_internal.cc)
endif()
if(ARROW_WITH_SIMDJSON)
list(APPEND ARROW_UTIL_SRCS util/json_writer_internal.cc util/simdjson_internal.cc)
endif()
if(ARROW_WITH_SNAPPY)
list(APPEND ARROW_UTIL_SRCS util/compression_snappy.cc)
endif()
Expand All @@ -664,6 +667,12 @@ foreach(ARROW_UTIL_TARGET ${ARROW_UTIL_TARGETS})
target_link_libraries(${ARROW_UTIL_TARGET} PRIVATE ${ARROW_XSIMD})
endforeach()

if(ARROW_WITH_SIMDJSON)
foreach(ARROW_UTIL_TARGET ${ARROW_UTIL_TARGETS})
target_link_libraries(${ARROW_UTIL_TARGET} PRIVATE arrow::simdjson)
endforeach()
endif()

if(ARROW_USE_BOOST)
foreach(ARROW_UTIL_TARGET ${ARROW_UTIL_TARGETS})
target_link_libraries(${ARROW_UTIL_TARGET} PRIVATE Boost::headers)
Expand Down Expand Up @@ -1088,7 +1097,6 @@ if(ARROW_JSON)
json/chunker.cc
json/converter.cc
json/from_string.cc
json/json_writer_internal.cc
json/object_parser.cc
json/parser.cc
json/reader.cc)
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/dataset/file_json_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@
#include "arrow/dataset/plan.h"
#include "arrow/dataset/test_util_internal.h"
#include "arrow/filesystem/mockfs.h"
#include "arrow/json/json_writer_internal.h"
#include "arrow/json/parser.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/util.h"
#include "arrow/util/json_writer_internal.h"
#include "arrow/util/logging_internal.h"

namespace arrow {
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/extension/fixed_shape_tensor.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@

#include "arrow/array/array_nested.h"
#include "arrow/array/array_primitive.h"
#include "arrow/json/json_writer_internal.h"
#include "arrow/tensor.h"
#include "arrow/util/json_writer_internal.h"
#include "arrow/util/logging_internal.h"
#include "arrow/util/print_internal.h"
#include "arrow/util/simdjson_internal.h"
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/extension/opaque.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

#include <sstream>

#include "arrow/json/json_writer_internal.h"
#include "arrow/util/json_writer_internal.h"
#include "arrow/util/logging_internal.h"
#include "arrow/util/simdjson_internal.h"

Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/extension/variable_shape_tensor.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@
#include "arrow/extension/variable_shape_tensor.h"

#include "arrow/array/array_primitive.h"
#include "arrow/json/json_writer_internal.h"
#include "arrow/scalar.h"
#include "arrow/tensor.h"
#include "arrow/util/json_writer_internal.h"
#include "arrow/util/logging_internal.h"
#include "arrow/util/print_internal.h"
#include "arrow/util/simdjson_internal.h"
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/flight/sql/odbc/odbc_impl/json_converter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
#include <boost/beast/core/detail/base64.hpp>
#include "arrow/builder.h"
#include "arrow/flight/sql/odbc/odbc_impl/util.h"
#include "arrow/json/json_writer_internal.h"
#include "arrow/scalar.h"
#include "arrow/util/json_writer_internal.h"
#include "arrow/visitor.h"

using boost::beast::detail::base64::encode;
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/integration/json_integration.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@
#include "arrow/integration/json_internal.h"
#include "arrow/io/file.h"
#include "arrow/ipc/dictionary.h"
#include "arrow/json/json_writer_internal.h"
#include "arrow/record_batch.h"
#include "arrow/result.h"
#include "arrow/status.h"
#include "arrow/type.h"
#include "arrow/util/json_writer_internal.h"
#include "arrow/util/logging_internal.h"
#include "arrow/util/simdjson_internal.h"

Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/integration/json_integration_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
#include "arrow/ipc/reader.h"
#include "arrow/ipc/test_common.h"
#include "arrow/ipc/writer.h"
#include "arrow/json/json_writer_internal.h"
#include "arrow/pretty_print.h"
#include "arrow/status.h"
#include "arrow/testing/builder.h"
Expand All @@ -49,6 +48,7 @@
#include "arrow/type.h"
#include "arrow/type_fwd.h"
#include "arrow/util/io_util.h"
#include "arrow/util/json_writer_internal.h"
#include "arrow/util/simdjson_internal.h"

DEFINE_string(arrow, "", "Arrow file name");
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/integration/json_internal.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
#include "arrow/array/builder_time.h"
#include "arrow/extension_type.h"
#include "arrow/ipc/dictionary.h"
#include "arrow/json/json_writer_internal.h"
#include "arrow/record_batch.h"
#include "arrow/result.h"
#include "arrow/scalar.h"
Expand All @@ -48,6 +47,7 @@
#include "arrow/util/checked_cast.h"
#include "arrow/util/decimal.h"
#include "arrow/util/formatting.h"
#include "arrow/util/json_writer_internal.h"
#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging_internal.h"
#include "arrow/util/range.h"
Expand Down
1 change: 0 additions & 1 deletion cpp/src/arrow/json/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ add_arrow_test(test
chunker_test.cc
converter_test.cc
from_string_test.cc
json_writer_internal_test.cc
object_parser_test.cc
parser_test.cc
reader_test.cc
Expand Down
1 change: 0 additions & 1 deletion cpp/src/arrow/json/meson.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ exc = executable(
'chunker_test.cc',
'converter_test.cc',
'from_string_test.cc',
'json_writer_internal_test.cc',
'parser_test.cc',
'reader_test.cc',
],
Expand Down
41 changes: 24 additions & 17 deletions cpp/src/arrow/meson.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,22 @@
dl_dep = dependency('dl')
threads_dep = dependency('threads')

if needs_simdjson
simdjson_dep = dependency('simdjson', allow_fallback: false, required: false)

if not simdjson_dep.found()
cmake = import('cmake')
simdjson_opts = cmake.subproject_options()

simdjson_opts.add_cmake_defines({'SIMDJSON_EXCEPTIONS': 'OFF'})

simdjson_proj = cmake.subproject('simdjson', options: simdjson_opts)
simdjson_dep = simdjson_proj.dependency('simdjson')
endif
else
simdjson_dep = disabler()
endif

arrow_components = {
'arrow_array': {
'sources': [
Expand Down Expand Up @@ -224,6 +240,14 @@ arrow_util_srcs = [

arrow_util_deps = [threads_dep]

if needs_simdjson
arrow_util_srcs += [
'util/json_writer_internal.cc',

@wgtmac wgtmac Aug 28, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Meson can still compile Parquet encryption with JSON disabled, but json/object_parser.cc remains excluded because needs_json is false. Please build ObjectParser under the same simdjson gate, otherwise this configuration still fails to link.

@rok rok Aug 28, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added.

'util/simdjson_internal.cc',
]
arrow_util_deps += [simdjson_dep]
endif

if needs_brotli
arrow_util_srcs += ['util/compression_brotli.cc']
arrow_util_deps += [dependency('libbrotlidec'), dependency('libbrotlienc')]
Expand Down Expand Up @@ -318,22 +342,6 @@ else
rapidjson_dep = disabler()
endif

if needs_json or needs_integration
simdjson_dep = dependency('simdjson', allow_fallback: false, required: false)

if not simdjson_dep.found()
cmake = import('cmake')
simdjson_opts = cmake.subproject_options()

simdjson_opts.add_cmake_defines({'SIMDJSON_EXCEPTIONS': 'OFF'})

simdjson_proj = cmake.subproject('simdjson', options: simdjson_opts)
simdjson_dep = simdjson_proj.dependency('simdjson')
endif
else
simdjson_dep = disabler()
endif

azure_dep = disabler()
gcs_dep = disabler()
s3_dep = disabler()
Expand Down Expand Up @@ -530,7 +538,6 @@ if needs_json
'json/chunker.cc',
'json/converter.cc',
'json/from_string.cc',
'json/json_writer_internal.cc',
'json/object_parser.cc',
'json/options.cc',
'json/parser.cc',
Expand Down
8 changes: 8 additions & 0 deletions cpp/src/arrow/util/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,14 @@ add_arrow_test(utility-test
EXTRA_LINK_LIBS
${ARROW_UTILITY_TEST_LINK_LIBS})

if(ARROW_WITH_SIMDJSON)
add_arrow_test(json-writer-internal-test
SOURCES
json_writer_internal_test.cc
EXTRA_LINK_LIBS
simdjson::simdjson)

@rok rok Aug 27, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we use arrow::simdjson here instead? Both should work, but arrow alieas would hide the vendored/system simdjson.

@wgtmac wgtmac Aug 28, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

if(SIMDJSON_VENDORED)
  add_library(arrow::simdjson ALIAS simdjson)
else()
  add_library(arrow::simdjson ALIAS simdjson::simdjson)
endif()

It seems better to use arrow::simdjson based on the above statements? cc @kou

@rok rok Aug 28, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, changed to arrow::simdjson.

endif()

add_arrow_test(async-utility-test
SOURCES
async_generator_test.cc
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/json/json_writer_internal.cc → cpp/src/arrow/util/json_writer_internal.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

#include "arrow/json/json_writer_internal.h"
#include "arrow/util/json_writer_internal.h"
#include "arrow/util/simdjson_internal.h"

namespace arrow::json {
Expand Down
0 cpp/src/arrow/json/json_writer_internal.h → cpp/src/arrow/util/json_writer_internal.h
View file
Open in desktop
File renamed without changes.
2 changes: 1 addition & 1 deletion ...c/arrow/json/json_writer_internal_test.cc → ...c/arrow/util/json_writer_internal_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@

#include <gtest/gtest.h>

#include "arrow/json/json_writer_internal.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/util/json_writer_internal.h"

namespace sj = simdjson::ondemand;

Expand Down
8 changes: 7 additions & 1 deletion cpp/src/arrow/util/meson.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,12 @@ utility_test_srcs = [
'value_parsing_test.cc',
]

utility_test_deps = [arrow_test_dep_no_main]
if needs_simdjson
utility_test_srcs += ['json_writer_internal_test.cc']
utility_test_deps += [simdjson_dep]
endif

if host_machine.system() == 'windows'
# This manifest enables long file paths on Windows 10+
# See https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#enable-long-paths-in-windows-10-version-1607-and-later
Expand All @@ -235,7 +241,7 @@ endif
exc = executable(
'arrow-utility-test',
sources: utility_test_srcs,
dependencies: arrow_test_dep_no_main,
dependencies: utility_test_deps,
implicit_include_directories: false,
)
test('arrow-utility-test', exc)
Expand Down
Loading
Loading