Redesign report & Allow to use profileID and service feeds

.gitignore

@@ -27,6 +27,8 @@ wheels/
 .installed.cfg
 *.egg
 
+node_modules
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
@@ -136,6 +138,7 @@ cython_debug/
 .env
 
 *.zip
+*.db
 
 # tensorboard
 tensorboard_logs

MANIFEST.in

@@ -1,5 +1,7 @@
 recursive-include octobot_script/config *.json *.ini
 recursive-include octobot_script/resources *.js *.css *.html
+recursive-exclude octobot_script/resources/report/node_modules *
+recursive-exclude octobot_script/resources/report/src *
 
 include README.md
 include LICENSE

biome.json

@@ -0,0 +1,42 @@
+{
+  "$schema": "https://biomejs.dev/schemas/2.3.8/schema.json",
+  "assist": { "actions": { "source": { "organizeImports": "on" } } },
+  "files": {
+    "includes": [
+      "**",
+      "!**/dist/**/*",
+      "!**/node_modules/**/*",
+      "!**/src/components/ui/**/*"
+    ]
+  },
+  "linter": {
+    "enabled": true,
+    "rules": {
+      "recommended": true,
+      "suspicious": {
+        "noExplicitAny": "off",
+        "noArrayIndexKey": "off"
+      },
+      "style": {
+        "noNonNullAssertion": "off",
+        "noParameterAssign": "error",
+        "useSelfClosingElements": "error",
+        "noUselessElse": "error"
+      }
+    }
+  },
+  "formatter": {
+    "indentStyle": "space"
+  },
+  "javascript": {
+    "formatter": {
+      "quoteStyle": "double",
+      "semicolons": "asNeeded"
+    }
+  },
+  "css": {
+    "parser": {
+      "tailwindDirectives": true
+    }
+  }
+}

components.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "https://ui.shadcn.com/schema.json",
+  "style": "new-york",
+  "rsc": false,
+  "tsx": true,
+  "tailwind": {
+    "config": "",
+    "css": "octobot_script/resources/report/src/index.css",
+    "baseColor": "gray",
+    "cssVariables": true,
+    "prefix": ""
+  },
+  "iconLibrary": "lucide",
+  "rtl": false,
+  "aliases": {
+    "components": "@/components",
+    "utils": "@/lib/utils",
+    "ui": "@/components/ui",
+    "lib": "@/lib",
+    "hooks": "@/hooks"
+  },
+  "registries": {}
+}

example.py

@@ -1,30 +1,38 @@
 import asyncio
-import tulipy    # Can be any TA library.
+import tulipy  # Can be any TA library.
 import octobot_script as obs
+from octobot_script.api.ploting import generate_and_show_report
 
 
-async def rsi_test():
+async def example():
+    async def initialize(ctx):
+        # Compute entries only once per backtest.
+        closes = await obs.Close(ctx, max_history=True)
+        times = await obs.Time(ctx, max_history=True, use_close_time=True)
+        rsi_v = tulipy.rsi(closes, period=ctx.tentacle.trading_config["period"])
+        delta = len(closes) - len(rsi_v)
+        # Populate entries with timestamps of candles where RSI is
+        # bellow the "rsi_value_buy_threshold" configuration.
+        run_data["entries"] = {
+            times[index + delta]
+            for index, rsi_val in enumerate(rsi_v)
+            if rsi_val < ctx.tentacle.trading_config["rsi_value_buy_threshold"]
+        }
+        await obs.plot_indicator(ctx, "RSI", times[delta:], rsi_v, run_data["entries"])
+
     async def strategy(ctx):
-        # Will be called at each candle.
-        if run_data["entries"] is None:
-            # Compute entries only once per backtest.
-            closes = await obs.Close(ctx, max_history=True)
-            times = await obs.Time(ctx, max_history=True, use_close_time=True)
-            rsi_v = tulipy.rsi(closes, period=ctx.tentacle.trading_config["period"])
-            delta = len(closes) - len(rsi_v)
-            # Populate entries with timestamps of candles where RSI is
-            # bellow the "rsi_value_buy_threshold" configuration.
-            run_data["entries"] = {
-                times[index + delta]
-                for index, rsi_val in enumerate(rsi_v)
-                if rsi_val < ctx.tentacle.trading_config["rsi_value_buy_threshold"]
-            }
-            await obs.plot_indicator(ctx, "RSI", times[delta:], rsi_v, run_data["entries"])
+        # Called at each candle.
+        # Uses pre-computed entries times to enter positions when relevant.
+        # Also, instantly set take profits and stop losses.
+        # Position exits could also be set separately.
         if obs.current_live_time(ctx) in run_data["entries"]:
-            # Uses pre-computed entries times to enter positions when relevant.
-            # Also, instantly set take profits and stop losses.
-            # Position exists could also be set separately.
-            await obs.market(ctx, "buy", amount="10%", stop_loss_offset="-15%", take_profit_offset="25%")
+            await obs.market(
+                ctx,
+                "buy",
+                amount="10%",
+                stop_loss_offset="-15%",
+                take_profit_offset="25%",
+            )
 
     # Configuration that will be passed to each run.
     # It will be accessible under "ctx.tentacle.trading_config".
@@ -34,19 +42,23 @@ async def strategy(ctx):
     }
 
     # Read and cache candle data to make subsequent backtesting runs faster.
-    data = await obs.get_data("BTC/USDT", "1d", start_timestamp=1505606400)
+    data = await obs.get_data(
+        "BTC/USDT", "1d", start_timestamp=1505606400, social_services=[]
+    )
     run_data = {
         "entries": None,
     }
     # Run a backtest using the above data, strategy and configuration.
-    res = await obs.run(data, strategy, config)
+    res = await obs.run(
+        data, config, initialize_func=initialize, strategy_func=strategy
+    )
     print(res.describe())
     # Generate and open report including indicators plots
-    await res.plot(show=True)
+    await generate_and_show_report(res)
     # Stop data to release local databases.
     await data.stop()
 
 
 # Call the execution of the script inside "asyncio.run" as
 # OctoBot-Script runs using the python asyncio framework.
-asyncio.run(rsi_test())
+asyncio.run(example())

octobot_script/api/data_fetching.py

@@ -14,44 +14,189 @@
 #  You should have received a copy of the GNU General Public
 #  License along with OctoBot-Script. If not, see <https://www.gnu.org/licenses/>.
 
+import datetime
+
 import octobot_backtesting.api as backtesting_api
 import octobot_commons.symbols as commons_symbols
 import octobot_commons.enums as commons_enums
 import octobot_trading.enums as trading_enums
 import octobot_script.internal.octobot_mocks as octobot_mocks
 
 
+def _validate_tentacles_source(tentacles_config, profile_id):
+    if tentacles_config is not None and profile_id is not None:
+        raise ValueError("Only one of tentacles_config or profile_id can be provided.")
+
+
 def _ensure_ms_timestamp(timestamp):
     if timestamp is None:
         return timestamp
     if timestamp < 16737955050:  # Friday 28 May 2500 07:57:30
         return timestamp * 1000
 
 
-async def historical_data(symbol, timeframe, exchange="binance", exchange_type=trading_enums.ExchangeTypes.SPOT.value,
-                          start_timestamp=None, end_timestamp=None):
+def _yesterday_midnight_ms() -> int:
+    """Return today at 00:00:00 UTC (= end of yesterday) in milliseconds.
+    Used as a stable default end_timestamp so data files collected on the
+    same calendar day share an identical end boundary and can be cached."""
+    today_midnight = datetime.datetime.now(datetime.timezone.utc).replace(
+        hour=0, minute=0, second=0, microsecond=0
+    )
+    return int(today_midnight.timestamp() * 1000)
+
+
+def _resolve_end_timestamp_ms(end_timestamp) -> int:
+    if end_timestamp is None:
+        return _yesterday_midnight_ms()
+    return _ensure_ms_timestamp(end_timestamp)
+
+
+async def historical_data(
+    symbol,
+    timeframe,
+    exchange="binance",
+    exchange_type=trading_enums.ExchangeTypes.SPOT.value,
+    start_timestamp=None,
+    end_timestamp=None,
+    tentacles_config=None,
+    profile_id=None,
+):
+    _validate_tentacles_source(tentacles_config, profile_id)
     symbols = [symbol]
     time_frames = [commons_enums.TimeFrames(timeframe)]
-    data_collector_instance = backtesting_api.exchange_historical_data_collector_factory(
-        exchange,
-        trading_enums.ExchangeTypes(exchange_type),
-        octobot_mocks.get_tentacles_config(),
-        [commons_symbols.parse_symbol(symbol) for symbol in symbols],
+    start_timestamp_ms = _ensure_ms_timestamp(start_timestamp)
+    end_timestamp_ms = _resolve_end_timestamp_ms(end_timestamp)
+    existing_file = await backtesting_api.find_matching_data_file(
+        exchange_name=exchange,
+        symbols=symbols,
         time_frames=time_frames,
-        start_timestamp=_ensure_ms_timestamp(start_timestamp),
-        end_timestamp=_ensure_ms_timestamp(end_timestamp)
+        start_timestamp=start_timestamp_ms,
+        end_timestamp=end_timestamp_ms,
+    )
+    if existing_file:
+        return existing_file
+    data_collector_instance = (
+        backtesting_api.exchange_historical_data_collector_factory(
+            exchange,
+            trading_enums.ExchangeTypes(exchange_type),
+            octobot_mocks.get_tentacles_config(
+                tentacles_config, profile_id, activate_strategy_tentacles=False
+            ),
+            [commons_symbols.parse_symbol(symbol) for symbol in symbols],
+            time_frames=time_frames,
+            start_timestamp=start_timestamp_ms,
+            end_timestamp=end_timestamp_ms,
+        )
+    )
+    return await backtesting_api.initialize_and_run_data_collector(
+        data_collector_instance
+    )
+
+
+async def social_historical_data(
+    services: list[str],
+    sources: list[str] | None = None,
+    symbols: list[str] | None = None,
+    start_timestamp=None,
+    end_timestamp=None,
+    tentacles_config=None,
+    profile_id=None,
+):
+    _validate_tentacles_source(tentacles_config, profile_id)
+    start_timestamp_ms = _ensure_ms_timestamp(start_timestamp)
+    end_timestamp_ms = _resolve_end_timestamp_ms(end_timestamp)
+    existing_file = await backtesting_api.find_matching_data_file(
+        services=services,
+        symbols=symbols or [],
+        start_timestamp=start_timestamp_ms,
+        end_timestamp=end_timestamp_ms,
+    )
+    if existing_file:
+        return existing_file
+    data_collector_instance = backtesting_api.social_historical_data_collector_factory(
+        services=services,
+        tentacles_setup_config=octobot_mocks.get_tentacles_config(
+            tentacles_config, profile_id, activate_strategy_tentacles=False
+        ),
+        sources=sources,
+        symbols=[commons_symbols.parse_symbol(symbol) for symbol in symbols]
+        if symbols
+        else None,
+        start_timestamp=start_timestamp_ms,
+        end_timestamp=end_timestamp_ms,
+        config=octobot_mocks.get_config(),
+    )
+    return await backtesting_api.initialize_and_run_data_collector(
+        data_collector_instance
+    )
+
+
+async def get_data(
+    symbol,
+    time_frame,
+    exchange="binance",
+    exchange_type=trading_enums.ExchangeTypes.SPOT.value,
+    start_timestamp=None,
+    end_timestamp=None,
+    data_file=None,
+    social_data_files: list[str] | None = None,
+    social_services: list[str] | None = None,
+    social_sources: list[str] | None = None,
+    social_symbols: list[str] | None = None,
+    tentacles_config=None,
+    profile_id=None,
+):
+    _validate_tentacles_source(tentacles_config, profile_id)
+    data_files = (
+        [data_file]
+        if data_file
+        else [
+            await historical_data(
+                symbol,
+                timeframe=time_frame,
+                exchange=exchange,
+                exchange_type=exchange_type,
+                start_timestamp=start_timestamp,
+                end_timestamp=end_timestamp,
+                tentacles_config=tentacles_config,
+                profile_id=profile_id,
+            )
+        ]
     )
-    return await backtesting_api.initialize_and_run_data_collector(data_collector_instance)
 
+    if social_data_files is not None:
+        data_files.extend(social_data_files)
+    elif (
+        profile_id is not None
+        or social_sources is not None
+        or tentacles_config is not None
+    ):
+        social_services = (
+            social_services
+            if social_services is not None
+            else octobot_mocks.get_activated_social_services(
+                tentacles_config, profile_id, requested_sources=social_sources
+            )
+        )
+    if social_services:
+        for service in social_services:
+            data_files.append(
+                await social_historical_data(
+                    [service],
+                    sources=social_sources,
+                    symbols=social_symbols,
+                    start_timestamp=start_timestamp,
+                    end_timestamp=end_timestamp,
+                    tentacles_config=tentacles_config,
+                    profile_id=profile_id,
+                )
+            )
 
-async def get_data(symbol, time_frame, exchange="binance", exchange_type=trading_enums.ExchangeTypes.SPOT.value,
-                   start_timestamp=None, end_timestamp=None, data_file=None):
-    data = data_file or \
-           await historical_data(symbol, timeframe=time_frame, exchange=exchange, exchange_type=exchange_type,
-                                 start_timestamp=start_timestamp, end_timestamp=end_timestamp)
     return await backtesting_api.create_and_init_backtest_data(
-        [data],
+        data_files,
         octobot_mocks.get_config(),
-        octobot_mocks.get_tentacles_config(),
-        use_accurate_price_time_frame=True
+        octobot_mocks.get_tentacles_config(
+            tentacles_config, profile_id, activate_strategy_tentacles=False
+        ),
+        use_accurate_price_time_frame=True,
     )