Skip to content

Factorize benchmark result path setup #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mzl2233 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Factorize benchmark result path setup #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions benchmark_tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,17 @@ def rustfs_server():
subprocess.run(cmd, check=True) # noqa: S603


def _benchmark_result_path(test_results_dir: Path, benchmark_name: str) -> Path:
"""Return the JSON result path for a benchmark test."""
import os
import sys

py_vers = sys.version_info
test_dir = test_results_dir / Path(benchmark_name)
os.makedirs(test_dir, exist_ok=True)
return test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")


def _timeit(fn, iterations: int) -> float:
"""Measure execution time of a function."""
import time
Expand Down
74 changes: 21 additions & 53 deletions benchmark_tests/test_benchmark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,13 @@
import json
import os
import random
import sys
import tempfile
from pathlib import Path

import boto3
import pytest
from botocore.client import Config

from conftest import _timeit
from conftest import _benchmark_result_path, _timeit
from signurlarity import Client


Expand All @@ -21,10 +19,7 @@ def test_generate_presigned_post_perf_sync(rustfs_server, test_results_dir):
This is a non-failing, informational test: it prints timings and skips
if the signurlarity implementation is not available.
"""
py_vers = sys.version_info
test_dir = test_results_dir / Path("test_generate_presigned_post_perf")
os.makedirs(test_dir, exist_ok=True)
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
result_file = _benchmark_result_path(test_results_dir, "test_generate_presigned_post_perf")

rng = random.Random(42) # noqa: S311
bucket = "perf-bucket"
Expand Down Expand Up @@ -88,7 +83,7 @@ def run_light(n: int):

light_client.close()
results = {
"python_version": f"{py_vers.major}.{py_vers.minor}",
"python_version": result_file.stem.removeprefix("run_"),
"tested_method": "generate_presigned_post_sync",
"iterations": iterations,
"boto_total": t_boto,
Expand All @@ -108,10 +103,7 @@ def test_generate_presigned_url_perf_sync(rustfs_server, test_results_dir):
This benchmark compares boto3's generate_presigned_url with the custom
implementation that has zero boto3 dependencies.
"""
py_vers = sys.version_info
test_dir = test_results_dir / Path("test_generate_presigned_url_perf")
os.makedirs(test_dir, exist_ok=True)
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
result_file = _benchmark_result_path(test_results_dir, "test_generate_presigned_url_perf")
rng = random.Random(42) # noqa: S311
bucket = "perf-bucket"
key = "object.txt"
Expand Down Expand Up @@ -166,7 +158,7 @@ def run_custom(n: int):

light_client.close()
results = {
"python_version": f"{py_vers.major}.{py_vers.minor}",
"python_version": result_file.stem.removeprefix("run_"),
"tested_method": "generate_presigned_url_sync",
"iterations": iterations,
"boto_total": t_boto,
Expand All @@ -186,10 +178,7 @@ def test_head_bucket_perf_sync(rustfs_server, test_results_dir):
This benchmark compares boto3's head_bucket with the custom
implementation that has zero boto3 dependencies.
"""
py_vers = sys.version_info
test_dir = test_results_dir / Path("test_head_bucket_perf")
os.makedirs(test_dir, exist_ok=True)
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
result_file = _benchmark_result_path(test_results_dir, "test_head_bucket_perf")
bucket = "perf-bucket"

# Extract region from endpoint_url
Expand Down Expand Up @@ -223,7 +212,7 @@ def run_custom(n: int):

light_client.close()
results = {
"python_version": f"{py_vers.major}.{py_vers.minor}",
"python_version": result_file.stem.removeprefix("run_"),
"tested_method": "head_bucket_sync",
"iterations": iterations,
"boto_total": t_boto,
Expand All @@ -242,10 +231,7 @@ def test_head_object_perf_sync(rustfs_server, test_results_dir):
This benchmark compares boto3's head_object with the custom
implementation that has zero boto3 dependencies.
"""
py_vers = sys.version_info
test_dir = test_results_dir / Path("test_head_object_perf")
os.makedirs(test_dir, exist_ok=True)
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
result_file = _benchmark_result_path(test_results_dir, "test_head_object_perf")
bucket = "perf-object"
key = "perf-object.txt"

Expand Down Expand Up @@ -283,7 +269,7 @@ def run_custom(n: int):

light_client.close()
results = {
"python_version": f"{py_vers.major}.{py_vers.minor}",
"python_version": result_file.stem.removeprefix("run_"),
"tested_method": "head_object_sync",
"iterations": iterations,
"boto_total": t_boto,
Expand Down Expand Up @@ -322,10 +308,7 @@ def test_create_bucket_perf_sync(rustfs_server, test_results_dir):
This benchmark compares boto3's create_bucket with the signurlarity
implementation that uses httpx with AWS Signature V4.
"""
py_vers = sys.version_info
test_dir = test_results_dir / Path("test_create_bucket_perf")
os.makedirs(test_dir, exist_ok=True)
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
result_file = _benchmark_result_path(test_results_dir, "test_create_bucket_perf")

boto_client = boto3.client("s3", **rustfs_server)
light_client = Client(**rustfs_server)
Expand Down Expand Up @@ -361,7 +344,7 @@ def run_custom(n: int):

light_client.close()
results = {
"python_version": f"{py_vers.major}.{py_vers.minor}",
"python_version": result_file.stem.removeprefix("run_"),
"tested_method": "create_bucket_sync",
"iterations": iterations,
"boto_total": t_boto,
Expand Down Expand Up @@ -400,10 +383,7 @@ def test_delete_objects_perf_sync(rustfs_server, test_results_dir):
This benchmark compares boto3's delete_objects with the signurlarity
implementation that uses httpx with AWS Signature V4.
"""
py_vers = sys.version_info
test_dir = test_results_dir / Path("test_delete_objects_perf")
os.makedirs(test_dir, exist_ok=True)
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
result_file = _benchmark_result_path(test_results_dir, "test_delete_objects_perf")

bucket = "perf-delete-objects"
num_keys = 10
Expand Down Expand Up @@ -454,7 +434,7 @@ def run_custom(n: int):

light_client.close()
results = {
"python_version": f"{py_vers.major}.{py_vers.minor}",
"python_version": result_file.stem.removeprefix("run_"),
"tested_method": "delete_objects_sync",
"iterations": iterations,
"boto_total": t_boto,
Expand Down Expand Up @@ -492,10 +472,7 @@ def test_put_object_perf_sync(rustfs_server, test_results_dir):
Uploads a 1 KB object per iteration to a unique key, comparing boto3 and
signurlarity implementations.
"""
py_vers = sys.version_info
test_dir = test_results_dir / Path("test_put_object_perf")
os.makedirs(test_dir, exist_ok=True)
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
result_file = _benchmark_result_path(test_results_dir, "test_put_object_perf")

rng = random.Random(42) # noqa: S311
bucket = "perf-put-object"
Expand Down Expand Up @@ -535,7 +512,7 @@ def run_custom(n: int):

light_client.close()
results = {
"python_version": f"{py_vers.major}.{py_vers.minor}",
"python_version": result_file.stem.removeprefix("run_"),
"tested_method": "put_object_sync",
"iterations": iterations,
"boto_total": t_boto,
Expand Down Expand Up @@ -571,10 +548,7 @@ def test_list_objects_perf_sync(rustfs_server, test_results_dir):

Pre-populates 10 objects and benchmarks listing them with a prefix filter.
"""
py_vers = sys.version_info
test_dir = test_results_dir / Path("test_list_objects_perf")
os.makedirs(test_dir, exist_ok=True)
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
result_file = _benchmark_result_path(test_results_dir, "test_list_objects_perf")

bucket = "perf-list-objects"
prefix = "bench/"
Expand Down Expand Up @@ -607,7 +581,7 @@ def run_custom(n: int):

light_client.close()
results = {
"python_version": f"{py_vers.major}.{py_vers.minor}",
"python_version": result_file.stem.removeprefix("run_"),
"tested_method": "list_objects_sync",
"iterations": iterations,
"boto_total": t_boto,
Expand Down Expand Up @@ -643,10 +617,7 @@ def test_copy_object_perf_sync(rustfs_server, test_results_dir):

Pre-uploads a source object and benchmarks copying it to unique destination keys.
"""
py_vers = sys.version_info
test_dir = test_results_dir / Path("test_copy_object_perf")
os.makedirs(test_dir, exist_ok=True)
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
result_file = _benchmark_result_path(test_results_dir, "test_copy_object_perf")

rng = random.Random(42) # noqa: S311
bucket = "perf-copy-object"
Expand Down Expand Up @@ -695,7 +666,7 @@ def run_custom(n: int):

light_client.close()
results = {
"python_version": f"{py_vers.major}.{py_vers.minor}",
"python_version": result_file.stem.removeprefix("run_"),
"tested_method": "copy_object_sync",
"iterations": iterations,
"boto_total": t_boto,
Expand Down Expand Up @@ -731,10 +702,7 @@ def test_upload_file_perf_sync(rustfs_server, test_results_dir):

Uses a 1 KB temporary file and uploads to unique keys per iteration.
"""
py_vers = sys.version_info
test_dir = test_results_dir / Path("test_upload_file_perf")
os.makedirs(test_dir, exist_ok=True)
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
result_file = _benchmark_result_path(test_results_dir, "test_upload_file_perf")

rng = random.Random(42) # noqa: S311
bucket = "perf-upload-file"
Expand Down Expand Up @@ -784,7 +752,7 @@ def run_custom(n: int):
light_client.close()

results = {
"python_version": f"{py_vers.major}.{py_vers.minor}",
"python_version": result_file.stem.removeprefix("run_"),
"tested_method": "upload_file_sync",
"iterations": iterations,
"boto_total": t_boto,
Expand Down
Loading