Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 12 additions & 28 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,17 +83,8 @@ non-default path.

### 4. Verify Installation

```python
import asyncio
import openviking as ov

async def main():
client = ov.AsyncOpenViking(path="./test_data")
await client.initialize()
print("OpenViking initialized successfully!")
await client.close()

asyncio.run(main())
```bash
python -c "import openviking; print(openviking.__version__)"
```

### 5. Build Rust CLI (Optional)
Expand Down Expand Up @@ -121,7 +112,7 @@ openviking/
├── pyproject.toml # Python project and tooling configuration
├── Cargo.toml # Rust workspace configuration
├── openviking/ # Python SDK and server implementation
│ ├── client/ # Local and HTTP client implementations
│ ├── client/ # HTTP client compatibility exports
│ ├── connector/ # Data connectors
│ ├── core/ # Core data models and directory abstractions
│ ├── ingest/ # Ingestion pipeline
Expand Down Expand Up @@ -206,10 +197,10 @@ pytest tests/server/ -v
pytest tests/parse/ -v

# Run specific test file
pytest tests/client/test_lifecycle.py
pytest tests/client/test_http_client_config.py

# Run specific test
pytest tests/client/test_lifecycle.py::TestClientInitialization::test_initialize_success
pytest tests/client/test_http_client_config.py

# Run by keyword
pytest -k "search" -v
Expand All @@ -223,26 +214,19 @@ pytest --cov=openviking --cov-report=term-missing
Tests are organized in subdirectories under `tests/`. The project uses `asyncio_mode = "auto"`, so async tests do **not** need the `@pytest.mark.asyncio` decorator:

```python
# tests/client/test_example.py
from openviking import AsyncOpenViking


class TestAsyncOpenViking:
async def test_initialize(self, uninitialized_client: AsyncOpenViking):
await uninitialized_client.initialize()
assert uninitialized_client._service is not None
await uninitialized_client.close()

async def test_add_resource(self, client: AsyncOpenViking, sample_markdown_file):
result = await client.add_resource(
# tests/service/test_example.py
class TestResourceService:
async def test_add_resource(self, service, request_context, sample_markdown_file):
result = await service.resources.add_resource(
path=str(sample_markdown_file),
reason="test document"
ctx=request_context,
reason="test document",
)
assert "root_uri" in result
assert result["root_uri"].startswith("viking://")
```

Common fixtures are defined in `tests/conftest.py`, including `client` (initialized `AsyncOpenViking`), `uninitialized_client`, `temp_dir`, `sample_markdown_file`, and more.
Common fixtures are defined in `tests/conftest.py`, including the initialized `service`, `request_context`, `temp_dir`, and sample files.

---

Expand Down
44 changes: 13 additions & 31 deletions CONTRIBUTING_CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,17 +98,8 @@ export OPENVIKING_CONFIG_FILE=~/.openviking/ov.conf

### 4. 验证安装

```python
import asyncio
import openviking as ov

async def main():
client = ov.AsyncOpenViking(path="./test_data")
await client.initialize()
print("OpenViking initialized successfully!")
await client.close()

asyncio.run(main())
```bash
python -c "import openviking; print(openviking.__version__)"
```

### 5. 构建 Rust CLI(可选)
Expand Down Expand Up @@ -138,10 +129,8 @@ openviking/
├── third_party/ # 第三方依赖
│ └── agfs/ # AGFS 文件系统
├── openviking/ # Python SDK
│ ├── async_client.py # AsyncOpenViking 客户端
│ ├── sync_client.py # SyncOpenViking 客户端
│ ├── client/ # 本地与 HTTP 客户端实现
├── openviking/ # Python 服务端与核心实现
│ ├── client/ # HTTP 客户端兼容导出
│ ├── console/ # 独立 console UI 与代理服务
│ ├── core/ # 核心数据模型与目录抽象
│ ├── message/ # 会话消息与 part 模型
Expand Down Expand Up @@ -233,10 +222,10 @@ pytest tests/server/ -v
pytest tests/parse/ -v

# 运行特定测试文件
pytest tests/client/test_lifecycle.py
pytest tests/client/test_http_client_config.py

# 运行特定测试
pytest tests/client/test_lifecycle.py::TestClientInitialization::test_initialize_success
pytest tests/client/test_http_client_config.py

# 按关键字运行
pytest -k "search" -v
Expand All @@ -250,26 +239,19 @@ pytest --cov=openviking --cov-report=term-missing
测试按模块组织在 `tests/` 的子目录中。项目使用 `asyncio_mode = "auto"`,异步测试**不需要** `@pytest.mark.asyncio` 装饰器:

```python
# tests/client/test_example.py
from openviking import AsyncOpenViking


class TestAsyncOpenViking:
async def test_initialize(self, uninitialized_client: AsyncOpenViking):
await uninitialized_client.initialize()
assert uninitialized_client._service is not None
await uninitialized_client.close()

async def test_add_resource(self, client: AsyncOpenViking, sample_markdown_file):
result = await client.add_resource(
# tests/service/test_example.py
class TestResourceService:
async def test_add_resource(self, service, request_context, sample_markdown_file):
result = await service.resources.add_resource(
path=str(sample_markdown_file),
reason="test document"
ctx=request_context,
reason="test document",
)
assert "root_uri" in result
assert result["root_uri"].startswith("viking://")
```

常用 fixture 定义在 `tests/conftest.py` 中,包括 `client`(已初始化的 `AsyncOpenViking`)、`uninitialized_client`、`temp_dir`、`sample_markdown_file` 等
常用 fixture 定义在 `tests/conftest.py` 中,包括已初始化的 `service`、`request_context`、`temp_dir` 和示例文件等

---

Expand Down
44 changes: 13 additions & 31 deletions CONTRIBUTING_JA.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,17 +98,8 @@ export OPENVIKING_CONFIG_FILE=~/.openviking/ov.conf

### 4. インストールの確認

```python
import asyncio
import openviking as ov

async def main():
client = ov.AsyncOpenViking(path="./test_data")
await client.initialize()
print("OpenViking initialized successfully!")
await client.close()

asyncio.run(main())
```bash
python -c "import openviking; print(openviking.__version__)"
```

### 5. Rust CLIのビルド(オプション)
Expand Down Expand Up @@ -140,10 +131,8 @@ openviking/
├── third_party/ # サードパーティ依存関係
│ └── agfs/ # AGFSファイルシステム
├── openviking/ # Python SDK
│ ├── async_client.py # AsyncOpenVikingクライアント
│ ├── sync_client.py # SyncOpenVikingクライアント
│ ├── client/ # ローカル / HTTP クライアント実装
├── openviking/ # Pythonサーバーとコア実装
│ ├── client/ # HTTPクライアント互換エクスポート
│ ├── console/ # スタンドアロン console UI とプロキシサービス
│ ├── core/ # コアデータモデルとディレクトリ抽象
│ ├── message/ # セッションメッセージと part モデル
Expand Down Expand Up @@ -235,10 +224,10 @@ pytest tests/server/ -v
pytest tests/parse/ -v

# 特定のテストファイルの実行
pytest tests/client/test_lifecycle.py
pytest tests/client/test_http_client_config.py

# 特定のテストの実行
pytest tests/client/test_lifecycle.py::TestClientInitialization::test_initialize_success
pytest tests/client/test_http_client_config.py

# キーワードで実行
pytest -k "search" -v
Expand All @@ -252,26 +241,19 @@ pytest --cov=openviking --cov-report=term-missing
テストは`tests/`配下のサブディレクトリに整理されています。プロジェクトは`asyncio_mode = "auto"`を使用しているため、非同期テストに`@pytest.mark.asyncio`デコレーターは**不要**です:

```python
# tests/client/test_example.py
from openviking import AsyncOpenViking


class TestAsyncOpenViking:
async def test_initialize(self, uninitialized_client: AsyncOpenViking):
await uninitialized_client.initialize()
assert uninitialized_client._service is not None
await uninitialized_client.close()

async def test_add_resource(self, client: AsyncOpenViking, sample_markdown_file):
result = await client.add_resource(
# tests/service/test_example.py
class TestResourceService:
async def test_add_resource(self, service, request_context, sample_markdown_file):
result = await service.resources.add_resource(
path=str(sample_markdown_file),
reason="test document"
ctx=request_context,
reason="test document",
)
assert "root_uri" in result
assert result["root_uri"].startswith("viking://")
```

共通フィクスチャは`tests/conftest.py`に定義されており、`client`(初期化済み`AsyncOpenViking`)、`uninitialized_client`、`temp_dir`、`sample_markdown_file` などが含まれます
共通フィクスチャは`tests/conftest.py`に定義されており、初期化済みの`service`、`request_context`、`temp_dir`、サンプルファイルなどが含まれます

---

Expand Down
13 changes: 4 additions & 9 deletions benchmark/RAG/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,6 @@ RAG uses YAML configuration files to control the evaluation process. Each datase
4. **Path Configuration**:
- `dataset_dir`: Path to dataset file or directory
- `doc_output_dir`: Directory for processed documents
- `vector_store`: Directory for vector index storage
- `output_dir`: Directory for evaluation results
- `log_file`: Path to log file
5. **LLM Configuration**:
Expand Down Expand Up @@ -330,12 +329,8 @@ Output/
└── benchmark.log # Log file
```

**Vector Store Database Location:**
The vector index (document database) is stored in the path specified by `vector_store` in the configuration file. By default, this is:

```
datasets/{dataset_name}/viking_store_index_dir
```
**OpenViking Storage:**
The benchmark uses the OpenViking Server configured for the Python HTTP SDK. Storage and vector-index locations are owned by that Server rather than by the benchmark process.

#### File descriptions and examples

Expand Down Expand Up @@ -638,8 +633,8 @@ FinanceBench has 3 question types:

This project integrates with OpenViking through:

- Using `openviking` client for data ingestion and retrieval
- Configuring OpenViking connection via `ov.conf`
- Using the OpenViking Python HTTP SDK for data ingestion and retrieval
- Configuring the OpenViking connection via `ovcli.conf` or SDK environment variables
- Supporting dynamic loading of OpenViking's latest features

### Frequently Asked Questions (FAQ)
Expand Down
13 changes: 4 additions & 9 deletions benchmark/RAG/README_zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,6 @@ RAG 使用 YAML 配置文件来控制评估过程。每个数据集在 `config/`
4. **路径配置**:
- `dataset_dir`:数据集文件或目录的路径
- `doc_output_dir`:处理文档的目录
- `vector_store`:向量索引存储的目录
- `output_dir`:评估结果的目录
- `log_file`:日志文件的路径
5. **LLM 配置**:
Expand Down Expand Up @@ -330,12 +329,8 @@ Output/
└── benchmark.log # 日志文件
```

**向量存储数据库位置:**
向量索引(文档数据库)存储在配置文件中 `vector_store` 指定的路径中。默认情况下,这是:

```
datasets/{dataset_name}/viking_store_index_dir
```
**OpenViking 存储:**
Benchmark 使用 Python HTTP SDK 所配置的 OpenViking Server。内容和向量索引的存储位置由 Server 管理,而不是由 Benchmark 进程管理。

#### 文件描述和示例

Expand Down Expand Up @@ -638,8 +633,8 @@ FinanceBench 有 3 种问题类型:

本项目通过以下方式与 OpenViking 集成:

- 使用 `openviking` 客户端进行数据摄取和检索
- 通过 `ov.conf` 配置 OpenViking 连接
- 使用 OpenViking Python HTTP SDK 进行数据摄取和检索
- 通过 `ovcli.conf` 或 SDK 环境变量配置 OpenViking 连接
- 支持动态加载 OpenViking 的最新功能

### 常见问题(FAQ)
Expand Down
2 changes: 0 additions & 2 deletions benchmark/RAG/config/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ paths:
dataset_path: "datasets/{dataset_name}"
# Output directory for processed documents
doc_output_dir: "datasets/{dataset_name}/{dataset_name}_processed_docs"
# Vector index storage directory
vector_store: "datasets/{dataset_name}/viking_store_index_dir"
# Results output directory
output_dir: "Output/{dataset_name}/experiment_dev"
# Log file path
Expand Down
1 change: 0 additions & 1 deletion benchmark/RAG/config/financebench_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ execution:
paths:
dataset_path: "datasets/{dataset_name}/financebench_open_source.jsonl"
doc_output_dir: "ov_storage/{dataset_name}/{dataset_name}_processed_docs"
vector_store: "ov_storage/{dataset_name}/{dataset_name}_viking_store_index"
output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}"
log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}/benchmark.log"

Expand Down
1 change: 0 additions & 1 deletion benchmark/RAG/config/locomo_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ execution:
paths:
dataset_path: "datasets/{dataset_name}/locomo10.json"
doc_output_dir: "ov_storage/{dataset_name}/{dataset_name}_processed_docs"
vector_store: "ov_storage/{dataset_name}/{dataset_name}_viking_store_index"
output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}"
log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}/benchmark.log"

Expand Down
1 change: 0 additions & 1 deletion benchmark/RAG/config/qasper_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ execution:
paths:
dataset_path: "datasets/{dataset_name}"
doc_output_dir: "ov_storage/{dataset_name}/{dataset_name}_processed_docs"
vector_store: "ov_storage/{dataset_name}/{dataset_name}_viking_store_index"
output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}"
log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}/benchmark.log"

Expand Down
1 change: 0 additions & 1 deletion benchmark/RAG/config/syllabusqa_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ execution:
paths:
dataset_path: "datasets/{dataset_name}"
doc_output_dir: "ov_storage/{dataset_name}/{dataset_name}_processed_docs"
vector_store: "ov_storage/{dataset_name}/{dataset_name}_viking_store_index"
output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}"
log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}/benchmark.log"

Expand Down
4 changes: 2 additions & 2 deletions benchmark/RAG/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def main():
'retrieval_topk': retrieval_topk
}

path_keys = ['dataset_path', 'output_dir', 'vector_store', 'log_file', 'doc_output_dir']
path_keys = ['dataset_path', 'output_dir', 'log_file', 'doc_output_dir']
for key in path_keys:
if key in config.get('paths', {}):
original = config['paths'][key]
Expand Down Expand Up @@ -122,7 +122,7 @@ def main():
raise e

# 2. Vector Store
vector_store = VikingStoreWrapper(store_path=config['paths']['vector_store'])
vector_store = VikingStoreWrapper()

# 3. LLM Client
api_key = os.environ.get(
Expand Down
19 changes: 8 additions & 11 deletions benchmark/RAG/src/core/vector_store.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,21 @@
import os
import time
from typing import List
import sys
import time
from pathlib import Path
from typing import List

sys.path.append(str(Path(__file__).parent.parent))

from adapters.base import StandardDoc, StandardSample
import tiktoken
import openviking as ov
from adapters.base import StandardDoc
from openviking_sdk import SyncHTTPClient


class VikingStoreWrapper:
def __init__(self, store_path: str):
self.store_path = store_path
if not os.path.exists(store_path):
os.makedirs(store_path)

self.client = ov.SyncOpenViking(path=store_path)

def __init__(self):
self.client = SyncHTTPClient()
self.client.initialize()

try:
self.enc = tiktoken.get_encoding("cl100k_base")
except Exception as e:
Expand Down
Loading