enhance(backend): OpenTelemetryサポート再追加

.config/example.yml

@@ -329,6 +329,63 @@ id: 'aidx'
 #    #tracePropagationTargets:
 #    #  - 'internal-service.example'
 
+#   ┌─────────┐
+#───┘ Tracing └────────────────────────────────────────────────
+# OpenTelemetry distributed tracing (Traces only, opt-in).
+# Existing API and Queue spans are exported to an OTLP http/protobuf endpoint.
+#
+# Standard OTEL_* environment variables are honored for values omitted below:
+# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_EXPORTER_OTLP_ENDPOINT,
+# OTEL_EXPORTER_OTLP_HEADERS, OTEL_TRACES_SAMPLER, OTEL_TRACES_SAMPLER_ARG, etc.
+#
+# When this is enabled together with sentryForBackend, Misskey shares Sentry's
+# TracerProvider and adds an OTLP SpanProcessor to it. In that mode:
+# - sampleRate below is ignored; sampling follows sentryForBackend.options
+#   tracesSampleRate / tracesSampler.
+# - resourceAttributes below is ignored; set OTEL_SERVICE_NAME and
+#   OTEL_RESOURCE_ATTRIBUTES in the environment if you need to override them.
+# - spans produced by Sentry integrations are exported to both Sentry and OTLP.
+# - sentry-trace / baggage propagation to outbound remote requests is disabled
+#   by default while OTel is enabled. Set propagateTraceToRemote: true only if
+#   you intentionally want Sentry trace propagation for your deployment.
+
+#otelForBackend:
+#  endpoint: 'http://localhost:4318/v1/traces'
+#  #headers:
+#  #  authorization: 'Bearer xxxxx'
+#  #sampleRate: 1.0
+#  # Record PostgreSQL query spans. Disabled by default to avoid instrumentation
+#  # overhead when database-level diagnostics are not needed.
+#  #capturePgSpans: false
+#  # Include raw SQL text in PostgreSQL spans. Requires capturePgSpans and is
+#  # disabled by default because non-parameterized queries can expose values to
+#  # the OTLP Collector.
+#  #capturePgStatement: false
+#  # Include PostgreSQL connection-pool spans with an existing parent span.
+#  # Requires capturePgSpans and is disabled by default to avoid noisy spans
+#  # from connection churn.
+#  #capturePgConnectionSpans: false
+#  # Record Redis command spans. Disabled by default because subscribing to the
+#  # ioredis diagnostics channel adds work to every Redis command, including
+#  # commands without a parent span. Enable for development diagnostics or when
+#  # the added overhead is acceptable in production.
+#  #captureRedisCommandSpans: false
+#  # Include Redis startup/reconnect spans. Disabled by default to avoid noisy
+#  # connection churn; these spans are recorded even without an HTTP or job
+#  # parent span.
+#  #captureRedisConnectionSpans: false
+#  # Record Redis commands without an HTTP or job parent span. Disabled by
+#  # default because queue polling and background work can produce many root
+#  # traces; only applies when captureRedisCommandSpans is enabled.
+#  #captureRedisRootSpans: false
+#  #resourceAttributes:
+#  #  deployment.environment: 'production'
+#  #propagateTraceToRemote: false
+#  # Queue worker spans are independent traces linked to their enqueuer by default.
+#  # Set to 'parent' to make them child spans instead. This can create very large
+#  # traces for high-fan-out queues such as federation delivery.
+#  #jobTraceContextMode: 'link'
+
 #sentryForFrontend:
 #  vueIntegration:
 #    tracingOptions:

CHANGELOG.md

@@ -48,12 +48,21 @@
 - Fix: ノートの詳細表示で削除された引用元が表示されない問題を修正
 
 ### Server
+- Feat: OpenTelemetryサポート
+  - 詳細な設定はconfigファイルを参照してください。
+  - Sentryとの併用も可能です。Sentry併用時は、PostgreSQL Query と Redis command は Sentry で計装されます。
+  - 以下の自動計装をサポートしています。（計装対象にする項目は設定可能）
+    - PostgreSQL query
+    - Redis command
+    - 全ての受信HTTPリクエスト
+    - 全ての送信HTTPリクエスト
+    - ジョブキュー（エンキュー元のトレースを含む）
 - Feat: ログ基盤の刷新
   - API内部エラーのログに構造化属性と正規化したエラー情報を付与し、認証情報を自動的に秘匿するように（従来形式の表示は維持）
   - ログ全体の既定出力レベルとドメインごとの出力レベルを設定できるように
   - バックエンドのログを1行JSON形式で出力できるように
-  - SentryのTrace ContextをJSON形式のログへ関連付けられるように
-  - HTTPのAccess logをstatus class単位で出力できるように（開発時のリクエスト・レスポンス本文、SentryのTrace Contextにも対応）
+  - OpenTelemetryのTrace ContextをJSON形式のログへ関連付けられるように
+  - HTTPのAccess logをstatus class単位で出力できるように（開発時のリクエスト・レスポンス本文、OpenTelemetryのTrace Contextにも対応）
 - Enhance: Sentry バックエンドの自動計装を `sentryForBackend.disabledIntegrations` で個別に無効化できるように
 - Enhance: センシティブメディアの判定を外部サービス ([sensitive-detector](https://github.com/misskey-dev/sensitive-detector)) に分離し、`nsfwjs` / `@tensorflow/tfjs(-node)` の同梱と NSFW 判定モデルを廃止 (#16804)
 - Enhance: Node.js 22.22.2以降、24.17.0以降、26.4.0以降をサポートするように

packages/backend/package.json

@@ -57,6 +57,7 @@
 		"@fastify/cors": "11.3.0",
 		"@fastify/http-proxy": "11.5.0",
 		"@fastify/multipart": "10.1.0",
+		"@fastify/otel": "0.20.1",
 		"@fastify/static": "10.1.2",
 		"@kitajs/html": "4.2.13",
 		"@misskey-dev/emoji-assets": "17.0.3",
@@ -67,6 +68,15 @@
 		"@nestjs/common": "11.1.28",
 		"@nestjs/core": "11.1.28",
 		"@nestjs/testing": "11.1.28",
+		"@opentelemetry/api": "1.9.1",
+		"@opentelemetry/core": "2.9.0",
+		"@opentelemetry/exporter-trace-otlp-proto": "0.220.0",
+		"@opentelemetry/instrumentation": "0.220.0",
+		"@opentelemetry/instrumentation-pg": "0.72.0",
+		"@opentelemetry/resources": "2.9.0",
+		"@opentelemetry/sdk-trace-base": "2.9.0",
+		"@opentelemetry/sdk-trace-node": "2.9.0",
+		"@opentelemetry/semantic-conventions": "1.43.0",
 		"@oxc-project/runtime": "0.139.0",
 		"@peertube/http-signature": "1.7.0",
 		"@sentry/node": "10.65.0",

packages/backend/rolldown.config.ts

@@ -87,6 +87,7 @@ export default defineConfig((args) => {
 		'class-validator',
 		/^@sentry\/.*/,
 		/^@sentry-internal\/.*/,
+		/^@opentelemetry\/.*/,
 		'@nestjs/websockets/socket-module',
 		'@nestjs/microservices/microservices-module',
 		'@nestjs/microservices',

packages/backend/src/config.ts

@@ -27,6 +27,25 @@ type SentryBackendConfig = {
 	disabledIntegrations?: string[];
 };
 
+type SentryBackendConfigSource = Omit<SentryBackendConfig, 'options'> & {
+	options?: Partial<Sentry.NodeOptions>;
+};
+
+type OtelBackendConfig = {
+	endpoint?: string;
+	headers?: Record<string, string>;
+	sampleRate?: number;
+	capturePgSpans?: boolean;
+	capturePgStatement?: boolean;
+	capturePgConnectionSpans?: boolean;
+	captureRedisCommandSpans?: boolean;
+	captureRedisConnectionSpans?: boolean;
+	captureRedisRootSpans?: boolean;
+	resourceAttributes?: Record<string, string>;
+	propagateTraceToRemote?: boolean;
+	jobTraceContextMode?: 'link' | 'parent';
+};
+
 /**
  * 設定ファイルの型
  */
@@ -71,7 +90,8 @@ type Source = {
 		index: string;
 		scope?: 'local' | 'global' | string[];
 	};
-	sentryForBackend?: SentryBackendConfig;
+	sentryForBackend?: SentryBackendConfigSource;
+	otelForBackend?: OtelBackendConfig;
 	sentryForFrontend?: {
 		options: Partial<SentryVue.BrowserOptions> & { dsn: string };
 		vueIntegration?: SentryVue.VueIntegrationOptions | null;
@@ -217,6 +237,7 @@ export type Config = {
 	redisForTimelines: RedisOptions & RedisOptionsSource;
 	redisForReactions: RedisOptions & RedisOptionsSource;
 	sentryForBackend: SentryBackendConfig | undefined;
+	otelForBackend: OtelBackendConfig | undefined;
 	sentryForFrontend: {
 		options: Partial<SentryVue.BrowserOptions> & { dsn: string };
 		vueIntegration?: SentryVue.VueIntegrationOptions | null;
@@ -321,7 +342,8 @@ export function loadConfig(): Config {
 		redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis,
 		redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis,
 		redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis,
-		sentryForBackend: config.sentryForBackend,
+		sentryForBackend: config.sentryForBackend == null ? undefined : normalizeSentryBackendConfig(config.sentryForBackend),
+		otelForBackend: config.otelForBackend,
 		sentryForFrontend: config.sentryForFrontend,
 		id: config.id,
 		proxy: config.proxy,
@@ -358,6 +380,13 @@ export function loadConfig(): Config {
 	};
 }
 
+export function normalizeSentryBackendConfig(config: SentryBackendConfigSource): SentryBackendConfig {
+	return {
+		...config,
+		options: config.options ?? {},
+	};
+}
+
 function tryCreateUrl(url: string) {
 	try {
 		return new URL(url);

packages/backend/src/core/QueueModule.ts

@@ -9,6 +9,7 @@ import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import { baseQueueOptions, QUEUE } from '@/queue/const.js';
 import { allSettled } from '@/misc/promise-tracker.js';
+import { instrumentQueue } from '@/core/telemetry/queue-instrumentation.js';
 import {
 	DeliverJobData,
 	EndedPollNotificationJobData,
@@ -32,7 +33,12 @@ export type UserWebhookDeliverQueue = Bull.Queue<UserWebhookDeliverJobData>;
 export type SystemWebhookDeliverQueue = Bull.Queue<SystemWebhookDeliverJobData>;
 
 function createQueue<T extends object>(queueName: string, config: Config): Bull.Queue<T> {
-	return new Bull.Queue<T>(queueName, baseQueueOptions(config, queueName));
+	const queue = new Bull.Queue<T>(queueName, baseQueueOptions(config, queueName));
+	// Queue のラップは、enqueue 時に OTel context をジョブデータへ埋め込むためのもの。
+	// Sentry 単独ではジョブ間の context 伝播を使わないので、OTel 未設定時は元の Queue を返す。
+	if (config.otelForBackend == null) return queue;
+
+	return instrumentQueue(queue);
 }
 
 const $system: Provider = {

packages/backend/src/core/telemetry/TelemetryService.ts

@@ -5,7 +5,7 @@
 
 import { Injectable } from '@nestjs/common';
 import { bindThis } from '@/decorators.js';
-import { captureMessage, shutdownTelemetry, startSpan } from './telemetry-registry.js';
+import { captureMessage, shutdownTelemetry, startSpan, startSpanWithTraceContext } from './telemetry-registry.js';
 import type { OnApplicationShutdown } from '@nestjs/common';
 import type { TelemetryCaptureMessageOptions } from './adapters/TelemetryAdapter.js';
 
@@ -21,6 +21,13 @@ export class TelemetryService implements OnApplicationShutdown {
 		return startSpan(name, fn);
 	}
 
+	@bindThis
+	public startSpanWithTraceContext<T>(name: string, jobData: object, fn: () => T): T {
+		// jobData に enqueue 元の context があれば worker span へ復元する。
+		// context の無い既存ジョブは、通常の startSpan と同じ扱いになる。
+		return startSpanWithTraceContext(name, jobData, fn);
+	}
+
 	@bindThis
 	public async onApplicationShutdown(_signal?: string): Promise<void> {
 		await shutdownTelemetry();