From 24bff24162fd9576129ca2aae1ee2a9fcdbacca7 Mon Sep 17 00:00:00 2001 From: BORNIOS Date: Sat, 17 Oct 2026 15:55:33 -0700 Subject: [PATCH] fix: conservar el historial de migraciones de codigo al importar desde SQLite Jellyfin guarda en __EFMigrationsHistory tanto sus rutinas de codigo como las migraciones del proveedor, y ambos ids comparten la forma yyyyMMddHHmmss_nombre, asi que no se pueden distinguir por formato. Sin ese historial, Jellyfin vuelve a ejecutar rutinas antiguas que esperan library.db y el arranque falla con 'no such table: TypedBaseItems'. Ahora se copian solo los ids que corresponden a un tipo con JellyfinMigrationAttribute, identificados por el nombre de la migracion. --- .../Services/MigrationCodeMigrations.cs | 159 ++++++++++++++++-- .../Services/MigrationEngine.cs | 8 +- .../MigrationCodeMigrationsTests.cs | 145 ++++++++++++++++ 3 files changed, 292 insertions(+), 20 deletions(-) create mode 100644 Tests/Jellyfin.Database.Providers.Postgres.Tests/MigrationCodeMigrationsTests.cs diff --git a/Jellyfin.Database.Providers.Postgres/Services/MigrationCodeMigrations.cs b/Jellyfin.Database.Providers.Postgres/Services/MigrationCodeMigrations.cs index f20686f..9e8e514 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/MigrationCodeMigrations.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/MigrationCodeMigrations.cs @@ -1,4 +1,8 @@ using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; using System.Threading.Tasks; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; @@ -7,10 +11,14 @@ namespace Jellyfin.Database.Providers.Postgres.Services; /// -/// Validates the source schema and transfers only code migrations already applied to its data. +/// Validates the source schema and carries over the code migrations the source already applied. /// internal static class MigrationCodeMigrations { + private const string CodeMigrationAttributeName = "Jellyfin.Server.Migrations.JellyfinMigrationAttribute"; + + private static HashSet? _codeMigrationNames; + internal static async Task ValidateSourceAsync(SqliteConnection sqlite) { using var context = new PostgresDesignTimeDbFactory().CreateDbContext(Array.Empty()); @@ -28,31 +36,150 @@ internal static async Task ValidateSourceAsync(SqliteConnection sqlite) } } - internal static async Task CopyAppliedAsync(SqliteConnection sqlite, NpgsqlConnection pg, Action log) + /// + /// Copies the history rows of the code migrations the SQLite database already applied. + /// + /// + /// Jellyfin records both its code migrations and the provider schema migrations in + /// __EFMigrationsHistory, and every id is yyyyMMddHHmmss_name, so the two cannot be told + /// apart by shape. Re-running a routine that already succeeded is not harmless: legacy ones such as + /// RemoveDuplicateExtras open library.db, which a fresh 12.x install no longer has, and the + /// host then fails to start. Only the ids that match a known migration type are copied, which leaves the + /// provider's own history to EF. + /// + /// Source database, opened read-only. + /// Target connection, already pointing at the destination schema. + /// Migration log sink. + /// A task that completes when the history has been copied. + internal static Task CopyAppliedAsync(SqliteConnection sqlite, NpgsqlConnection pg, Action log) + => CopyAppliedAsync(sqlite, pg, DiscoverCodeMigrationNames(), log); + + /// + /// Copies the applied code migrations from the source history using an explicit set of known names. + /// + /// Source database, opened read-only. + /// Target connection, already pointing at the destination schema. + /// Names of the code migrations Jellyfin knows about. + /// Migration log sink. + /// A task that completes when the history has been copied. + internal static async Task CopyAppliedAsync( + SqliteConnection sqlite, + NpgsqlConnection pg, + IReadOnlySet codeMigrationNames, + Action log) { - using var select = sqlite.CreateCommand(); - select.CommandText = "SELECT \"MigrationId\", \"ProductVersion\" FROM \"__EFMigrationsHistory\";"; - using var reader = await select.ExecuteReaderAsync().ConfigureAwait(false); - var count = 0; - while (await reader.ReadAsync().ConfigureAwait(false)) + if (codeMigrationNames.Count == 0) { - var id = reader.GetString(0); - // Jellyfin code IDs use yyyyMMddHHmmsss; provider schema IDs use 14 digits. - if (!IsCodeMigrationId(id)) + log("[CodeMigrations] No se encontraron tipos de migracion de codigo de Jellyfin; el historial no se copia."); + return; + } + + var sourceRows = await ReadSourceHistoryAsync(sqlite).ConfigureAwait(false); + var copied = 0; + foreach (var (id, productVersion) in sourceRows) + { + if (!IsCodeMigrationId(id, codeMigrationNames)) { continue; } using var insert = new NpgsqlCommand( - "INSERT INTO \"__EFMigrationsHistory\" (\"MigrationId\", \"ProductVersion\") VALUES (@id, @version) ON CONFLICT DO NOTHING;", pg); + "INSERT INTO \"__EFMigrationsHistory\" (\"MigrationId\", \"ProductVersion\") VALUES (@id, @version) ON CONFLICT DO NOTHING;", + pg); insert.Parameters.AddWithValue("id", id); - insert.Parameters.AddWithValue("version", reader.GetString(1)); - count += await insert.ExecuteNonQueryAsync().ConfigureAwait(false); + insert.Parameters.AddWithValue("version", productVersion); + copied += await insert.ExecuteNonQueryAsync().ConfigureAwait(false); + } + + log($"[CodeMigrations] {copied} de {sourceRows.Count} filas del historial SQLite son migraciones de codigo aplicadas."); + + if (copied == 0) + { + log("[CodeMigrations] ADVERTENCIA: no se copio ninguna migracion de codigo; Jellyfin volvera a ejecutar rutinas que pueden necesitar library.db."); + } + } + + /// + /// Reports whether an id belongs to a Jellyfin code migration. + /// + /// History id, shaped as yyyyMMddHHmmss_name. + /// Names of the code migrations Jellyfin knows about. + /// true when the id is a known code migration. + internal static bool IsCodeMigrationId(string id, IReadOnlySet codeMigrationNames) + { + var separator = id.IndexOf('_', StringComparison.Ordinal); + if (separator < 14 || separator > 15 || separator == id.Length - 1) + { + return false; + } + + if (id.AsSpan(0, separator).ContainsAnyExceptInRange('0', '9')) + { + return false; + } + + return codeMigrationNames.Contains(id[(separator + 1)..]); + } + + /// + /// Collects the names of the code migrations declared by the host assemblies. + /// + /// Names read from JellyfinMigrationAttribute; empty when no host assembly is loaded. + internal static HashSet DiscoverCodeMigrationNames() + => _codeMigrationNames ??= ScanCodeMigrationNames(); + + private static HashSet ScanCodeMigrationNames() + { + var names = new HashSet(StringComparer.Ordinal); + foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) + { + foreach (var type in GetLoadableTypes(assembly)) + { + foreach (var attribute in type.GetCustomAttributes(inherit: false)) + { + if (!string.Equals(attribute.GetType().FullName, CodeMigrationAttributeName, StringComparison.Ordinal)) + { + continue; + } + + if (attribute.GetType().GetProperty("Name")?.GetValue(attribute) is string name && name.Length > 0) + { + names.Add(name); + } + } + } } - log($"[CodeMigrations] {count} migraciones aplicadas copiadas desde SQLite."); + return names; + } + + private static IEnumerable GetLoadableTypes(Assembly assembly) + { + try + { + return assembly.GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + return ex.Types.Where(static type => type is not null).Cast(); + } + catch (Exception ex) when (ex is FileNotFoundException or NotSupportedException or BadImageFormatException) + { + return Array.Empty(); + } } - internal static bool IsCodeMigrationId(string id) - => id.Length > 16 && id[15] == '_' && !id.AsSpan(0, 15).ContainsAnyExceptInRange('0', '9'); + private static async Task> ReadSourceHistoryAsync(SqliteConnection sqlite) + { + var rows = new List<(string Id, string ProductVersion)>(); + using var select = sqlite.CreateCommand(); + select.CommandText = "SELECT \"MigrationId\", \"ProductVersion\" FROM \"__EFMigrationsHistory\";"; + using var reader = await select.ExecuteReaderAsync().ConfigureAwait(false); + while (await reader.ReadAsync().ConfigureAwait(false)) + { + rows.Add((reader.GetString(0), reader.GetString(1))); + } + + return rows; + } } diff --git a/Jellyfin.Database.Providers.Postgres/Services/MigrationEngine.cs b/Jellyfin.Database.Providers.Postgres/Services/MigrationEngine.cs index 1a0fdda..03be44a 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/MigrationEngine.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/MigrationEngine.cs @@ -85,10 +85,10 @@ await MigrationFkManager throw new InvalidOperationException($"{errors.Count} tabla(s) fallaron: " + string.Join(", ", errors.Select(e => e.Table))); } - // Do not mark Jellyfin code migrations as applied merely because the source history contains - // their ids. They may create data or objects which do not exist in PostgreSQL yet; the host - // migration service must execute and record them against its active database. - svc.Log("[CodeMigrations] No se pre-marcan migraciones de código; Jellyfin las ejecutará al iniciar."); + // The source history records every routine that already ran against this server, on the very data + // that was just copied. Leaving them out makes Jellyfin run them again on the next start, and the + // legacy ones expect library.db, which a fresh 12.x install no longer has. + await MigrationCodeMigrations.CopyAppliedAsync(sqlite, pg, svc.Log).ConfigureAwait(false); svc.Log($"Migracion completada. Total de filas copiadas: {totalRows}."); PostgresLog.Info($"[Migration] Total filas migradas: {totalRows:N0}"); diff --git a/Tests/Jellyfin.Database.Providers.Postgres.Tests/MigrationCodeMigrationsTests.cs b/Tests/Jellyfin.Database.Providers.Postgres.Tests/MigrationCodeMigrationsTests.cs new file mode 100644 index 0000000..b4f499a --- /dev/null +++ b/Tests/Jellyfin.Database.Providers.Postgres.Tests/MigrationCodeMigrationsTests.cs @@ -0,0 +1,145 @@ +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Locking; +using Jellyfin.Database.Providers.Postgres.Services; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; +using Npgsql; +using Xunit; + +namespace Jellyfin.Database.Providers.Postgres.Tests; + +/// +/// Jellyfin stores its code migrations and the provider's schema migrations in the same history table, +/// with the same date-prefixed shape. Only the code migrations must be carried over to PostgreSQL. +/// +public sealed class MigrationCodeMigrationsTests +{ + private const string SkipReason = "POSTGRES_TEST_CONNECTION is not set; skipping PostgreSQL integration tests."; + + private static readonly HashSet KnownNames = new(StringComparer.Ordinal) + { + "RemoveDuplicateExtras", + "MigrateActivityLogDb", + }; + + [Theory] + [InlineData("20250420080000_RemoveDuplicateExtras", true)] + [InlineData("20250420070000_MigrateActivityLogDb", true)] + [InlineData("20240101000000_SqliteOnlyMigration", false)] + [InlineData("20250420080000_UnknownRoutine", false)] + [InlineData("no_underscore_prefix", false)] + [InlineData("20250420080000_", false)] + public void OnlyKnownCodeMigrationsAreRecognised(string id, bool expected) + => Assert.Equal(expected, MigrationCodeMigrations.IsCodeMigrationId(id, KnownNames)); + + [SkippableFact] + public async Task AppliedCodeMigrationsAreCopiedWithoutTheSchemaOnes() + { + Skip.IfNot(TestEnvironment.PostgresAvailable, SkipReason); + await using var scratch = await TestEnvironment.ScratchDatabaseScope.CreateAsync(); + + using (var context = CreateContext(scratch.Database.ConnectionString)) + { + await context.Database.MigrateAsync(); + } + + var source = Path.Combine(Path.GetTempPath(), $"jf-history-{Guid.NewGuid():N}.db"); + try + { + await CreateSourceHistoryAsync(source); + + await using var sqlite = new SqliteConnection($"Data Source={source};Mode=ReadOnly"); + await sqlite.OpenAsync(); + await using var pg = new NpgsqlConnection(scratch.Database.ConnectionString); + await pg.OpenAsync(); + + var log = new List(); + await MigrationCodeMigrations.CopyAppliedAsync(sqlite, pg, KnownNames, log.Add); + + var applied = await ReadTargetHistoryAsync(pg); + Assert.Contains("20250420080000_RemoveDuplicateExtras", applied.Keys, StringComparer.Ordinal); + Assert.Contains("20250420070000_MigrateActivityLogDb", applied.Keys, StringComparer.Ordinal); + Assert.DoesNotContain("20240101000000_SqliteOnlyMigration", applied.Keys, StringComparer.Ordinal); + Assert.Equal("9.0.0", applied["20250420080000_RemoveDuplicateExtras"]); + Assert.Contains(log, entry => entry.Contains("2 de 3", StringComparison.Ordinal)); + } + finally + { + SqliteConnection.ClearAllPools(); + File.Delete(source); + } + } + + [SkippableFact] + public async Task NoKnownMigrationsLeavesTheHistoryUntouched() + { + Skip.IfNot(TestEnvironment.PostgresAvailable, SkipReason); + await using var scratch = await TestEnvironment.ScratchDatabaseScope.CreateAsync(); + + using (var context = CreateContext(scratch.Database.ConnectionString)) + { + await context.Database.MigrateAsync(); + } + + var source = Path.Combine(Path.GetTempPath(), $"jf-history-{Guid.NewGuid():N}.db"); + try + { + await CreateSourceHistoryAsync(source); + + await using var sqlite = new SqliteConnection($"Data Source={source};Mode=ReadOnly"); + await sqlite.OpenAsync(); + await using var pg = new NpgsqlConnection(scratch.Database.ConnectionString); + await pg.OpenAsync(); + + var before = (await ReadTargetHistoryAsync(pg)).Count; + var log = new List(); + await MigrationCodeMigrations.CopyAppliedAsync(sqlite, pg, new HashSet(StringComparer.Ordinal), log.Add); + + Assert.Equal(before, (await ReadTargetHistoryAsync(pg)).Count); + Assert.Contains(log, entry => entry.Contains("No se encontraron tipos", StringComparison.Ordinal)); + } + finally + { + SqliteConnection.ClearAllPools(); + File.Delete(source); + } + } + + private static async Task CreateSourceHistoryAsync(string path) + { + await using var connection = new SqliteConnection($"Data Source={path}"); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = + """ + CREATE TABLE "__EFMigrationsHistory" ("MigrationId" TEXT NOT NULL PRIMARY KEY, "ProductVersion" TEXT NOT NULL); + INSERT INTO "__EFMigrationsHistory" VALUES ('20250420080000_RemoveDuplicateExtras', '9.0.0'); + INSERT INTO "__EFMigrationsHistory" VALUES ('20250420070000_MigrateActivityLogDb', '9.0.0'); + INSERT INTO "__EFMigrationsHistory" VALUES ('20240101000000_SqliteOnlyMigration', '9.0.0'); + """; + await command.ExecuteNonQueryAsync(); + } + + private static async Task> ReadTargetHistoryAsync(NpgsqlConnection connection) + { + var rows = new Dictionary(StringComparer.Ordinal); + await using var command = new NpgsqlCommand("SELECT \"MigrationId\", \"ProductVersion\" FROM \"__EFMigrationsHistory\";", connection); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + rows[reader.GetString(0)] = reader.GetString(1); + } + + return rows; + } + + private static JellyfinDbContext CreateContext(string connectionString) + => new( + new DbContextOptionsBuilder() + .UseNpgsql(connectionString, pg => pg.MigrationsAssembly(typeof(PostgresDatabaseProvider).Assembly.FullName)) + .Options, + NullLogger.Instance, + new PostgresDatabaseProvider(null, null), + new NoLockBehavior(NullLogger.Instance)); +}