Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 143 additions & 16 deletions Jellyfin.Database.Providers.Postgres/Services/MigrationCodeMigrations.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
Expand All @@ -7,10 +11,14 @@
namespace Jellyfin.Database.Providers.Postgres.Services;

/// <summary>
/// Validates the source schema and transfers only code migrations already applied to its data.
/// Validates the source schema and carries over the code migrations the source already applied.
/// </summary>
internal static class MigrationCodeMigrations
{
private const string CodeMigrationAttributeName = "Jellyfin.Server.Migrations.JellyfinMigrationAttribute";

private static HashSet<string>? _codeMigrationNames;

internal static async Task ValidateSourceAsync(SqliteConnection sqlite)
{
using var context = new PostgresDesignTimeDbFactory().CreateDbContext(Array.Empty<string>());
Expand All @@ -28,31 +36,150 @@ internal static async Task ValidateSourceAsync(SqliteConnection sqlite)
}
}

internal static async Task CopyAppliedAsync(SqliteConnection sqlite, NpgsqlConnection pg, Action<string> log)
/// <summary>
/// Copies the history rows of the code migrations the SQLite database already applied.
/// </summary>
/// <remarks>
/// Jellyfin records both its code migrations and the provider schema migrations in
/// <c>__EFMigrationsHistory</c>, and every id is <c>yyyyMMddHHmmss_name</c>, so the two cannot be told
/// apart by shape. Re-running a routine that already succeeded is not harmless: legacy ones such as
/// <c>RemoveDuplicateExtras</c> open <c>library.db</c>, which a fresh 12.x install no longer has, and the
/// host then fails to start. Only the ids that match a known migration type are copied, which leaves the
/// provider's own history to EF.
/// </remarks>
/// <param name="sqlite">Source database, opened read-only.</param>
/// <param name="pg">Target connection, already pointing at the destination schema.</param>
/// <param name="log">Migration log sink.</param>
/// <returns>A task that completes when the history has been copied.</returns>
internal static Task CopyAppliedAsync(SqliteConnection sqlite, NpgsqlConnection pg, Action<string> log)
=> CopyAppliedAsync(sqlite, pg, DiscoverCodeMigrationNames(), log);

/// <summary>
/// Copies the applied code migrations from the source history using an explicit set of known names.
/// </summary>
/// <param name="sqlite">Source database, opened read-only.</param>
/// <param name="pg">Target connection, already pointing at the destination schema.</param>
/// <param name="codeMigrationNames">Names of the code migrations Jellyfin knows about.</param>
/// <param name="log">Migration log sink.</param>
/// <returns>A task that completes when the history has been copied.</returns>
internal static async Task CopyAppliedAsync(
SqliteConnection sqlite,
NpgsqlConnection pg,
IReadOnlySet<string> codeMigrationNames,
Action<string> log)
{
using var select = sqlite.CreateCommand();
select.CommandText = "SELECT \"MigrationId\", \"ProductVersion\" FROM \"__EFMigrationsHistory\";";
using var reader = await select.ExecuteReaderAsync().ConfigureAwait(false);
var count = 0;
while (await reader.ReadAsync().ConfigureAwait(false))
if (codeMigrationNames.Count == 0)
{
var id = reader.GetString(0);
// Jellyfin code IDs use yyyyMMddHHmmsss; provider schema IDs use 14 digits.
if (!IsCodeMigrationId(id))
log("[CodeMigrations] No se encontraron tipos de migracion de codigo de Jellyfin; el historial no se copia.");
return;
}

var sourceRows = await ReadSourceHistoryAsync(sqlite).ConfigureAwait(false);
var copied = 0;
foreach (var (id, productVersion) in sourceRows)
{
if (!IsCodeMigrationId(id, codeMigrationNames))
{
continue;
}

using var insert = new NpgsqlCommand(
"INSERT INTO \"__EFMigrationsHistory\" (\"MigrationId\", \"ProductVersion\") VALUES (@id, @version) ON CONFLICT DO NOTHING;", pg);
"INSERT INTO \"__EFMigrationsHistory\" (\"MigrationId\", \"ProductVersion\") VALUES (@id, @version) ON CONFLICT DO NOTHING;",
pg);
insert.Parameters.AddWithValue("id", id);
insert.Parameters.AddWithValue("version", reader.GetString(1));
count += await insert.ExecuteNonQueryAsync().ConfigureAwait(false);
insert.Parameters.AddWithValue("version", productVersion);
copied += await insert.ExecuteNonQueryAsync().ConfigureAwait(false);
}

log($"[CodeMigrations] {copied} de {sourceRows.Count} filas del historial SQLite son migraciones de codigo aplicadas.");

if (copied == 0)
{
log("[CodeMigrations] ADVERTENCIA: no se copio ninguna migracion de codigo; Jellyfin volvera a ejecutar rutinas que pueden necesitar library.db.");
}
}

/// <summary>
/// Reports whether an id belongs to a Jellyfin code migration.
/// </summary>
/// <param name="id">History id, shaped as <c>yyyyMMddHHmmss_name</c>.</param>
/// <param name="codeMigrationNames">Names of the code migrations Jellyfin knows about.</param>
/// <returns><c>true</c> when the id is a known code migration.</returns>
internal static bool IsCodeMigrationId(string id, IReadOnlySet<string> codeMigrationNames)
{
var separator = id.IndexOf('_', StringComparison.Ordinal);
if (separator < 14 || separator > 15 || separator == id.Length - 1)
{
return false;
}

if (id.AsSpan(0, separator).ContainsAnyExceptInRange('0', '9'))
{
return false;
}

return codeMigrationNames.Contains(id[(separator + 1)..]);
}

/// <summary>
/// Collects the names of the code migrations declared by the host assemblies.
/// </summary>
/// <returns>Names read from <c>JellyfinMigrationAttribute</c>; empty when no host assembly is loaded.</returns>
internal static HashSet<string> DiscoverCodeMigrationNames()
=> _codeMigrationNames ??= ScanCodeMigrationNames();

private static HashSet<string> ScanCodeMigrationNames()
{
var names = new HashSet<string>(StringComparer.Ordinal);
foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies())
{
foreach (var type in GetLoadableTypes(assembly))
{
foreach (var attribute in type.GetCustomAttributes(inherit: false))
{
if (!string.Equals(attribute.GetType().FullName, CodeMigrationAttributeName, StringComparison.Ordinal))
{
continue;
}

if (attribute.GetType().GetProperty("Name")?.GetValue(attribute) is string name && name.Length > 0)
{
names.Add(name);
}
}
}
}

log($"[CodeMigrations] {count} migraciones aplicadas copiadas desde SQLite.");
return names;
}

private static IEnumerable<Type> GetLoadableTypes(Assembly assembly)
{
try
{
return assembly.GetTypes();
}
catch (ReflectionTypeLoadException ex)
{
return ex.Types.Where(static type => type is not null).Cast<Type>();
}
catch (Exception ex) when (ex is FileNotFoundException or NotSupportedException or BadImageFormatException)
{
return Array.Empty<Type>();
}
}

internal static bool IsCodeMigrationId(string id)
=> id.Length > 16 && id[15] == '_' && !id.AsSpan(0, 15).ContainsAnyExceptInRange('0', '9');
private static async Task<List<(string Id, string ProductVersion)>> ReadSourceHistoryAsync(SqliteConnection sqlite)
{
var rows = new List<(string Id, string ProductVersion)>();
using var select = sqlite.CreateCommand();
select.CommandText = "SELECT \"MigrationId\", \"ProductVersion\" FROM \"__EFMigrationsHistory\";";
using var reader = await select.ExecuteReaderAsync().ConfigureAwait(false);
while (await reader.ReadAsync().ConfigureAwait(false))
{
rows.Add((reader.GetString(0), reader.GetString(1)));
}

return rows;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,10 @@ await MigrationFkManager
throw new InvalidOperationException($"{errors.Count} tabla(s) fallaron: " + string.Join(", ", errors.Select(e => e.Table)));
}

// Do not mark Jellyfin code migrations as applied merely because the source history contains
// their ids. They may create data or objects which do not exist in PostgreSQL yet; the host
// migration service must execute and record them against its active database.
svc.Log("[CodeMigrations] No se pre-marcan migraciones de código; Jellyfin las ejecutará al iniciar.");
// The source history records every routine that already ran against this server, on the very data
// that was just copied. Leaving them out makes Jellyfin run them again on the next start, and the
// legacy ones expect library.db, which a fresh 12.x install no longer has.
await MigrationCodeMigrations.CopyAppliedAsync(sqlite, pg, svc.Log).ConfigureAwait(false);

svc.Log($"Migracion completada. Total de filas copiadas: {totalRows}.");
PostgresLog.Info($"[Migration] Total filas migradas: {totalRows:N0}");
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
using Jellyfin.Database.Implementations;
using Jellyfin.Database.Implementations.Locking;
using Jellyfin.Database.Providers.Postgres.Services;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Npgsql;
using Xunit;

namespace Jellyfin.Database.Providers.Postgres.Tests;

/// <summary>
/// Jellyfin stores its code migrations and the provider's schema migrations in the same history table,
/// with the same date-prefixed shape. Only the code migrations must be carried over to PostgreSQL.
/// </summary>
public sealed class MigrationCodeMigrationsTests
{
private const string SkipReason = "POSTGRES_TEST_CONNECTION is not set; skipping PostgreSQL integration tests.";

private static readonly HashSet<string> KnownNames = new(StringComparer.Ordinal)
{
"RemoveDuplicateExtras",
"MigrateActivityLogDb",
};

[Theory]
[InlineData("20250420080000_RemoveDuplicateExtras", true)]
[InlineData("20250420070000_MigrateActivityLogDb", true)]
[InlineData("20240101000000_SqliteOnlyMigration", false)]
[InlineData("20250420080000_UnknownRoutine", false)]
[InlineData("no_underscore_prefix", false)]
[InlineData("20250420080000_", false)]
public void OnlyKnownCodeMigrationsAreRecognised(string id, bool expected)
=> Assert.Equal(expected, MigrationCodeMigrations.IsCodeMigrationId(id, KnownNames));

[SkippableFact]
public async Task AppliedCodeMigrationsAreCopiedWithoutTheSchemaOnes()
{
Skip.IfNot(TestEnvironment.PostgresAvailable, SkipReason);
await using var scratch = await TestEnvironment.ScratchDatabaseScope.CreateAsync();

using (var context = CreateContext(scratch.Database.ConnectionString))
{
await context.Database.MigrateAsync();
}

var source = Path.Combine(Path.GetTempPath(), $"jf-history-{Guid.NewGuid():N}.db");
try
{
await CreateSourceHistoryAsync(source);

await using var sqlite = new SqliteConnection($"Data Source={source};Mode=ReadOnly");
await sqlite.OpenAsync();
await using var pg = new NpgsqlConnection(scratch.Database.ConnectionString);
await pg.OpenAsync();

var log = new List<string>();
await MigrationCodeMigrations.CopyAppliedAsync(sqlite, pg, KnownNames, log.Add);

var applied = await ReadTargetHistoryAsync(pg);
Assert.Contains("20250420080000_RemoveDuplicateExtras", applied.Keys, StringComparer.Ordinal);
Assert.Contains("20250420070000_MigrateActivityLogDb", applied.Keys, StringComparer.Ordinal);
Assert.DoesNotContain("20240101000000_SqliteOnlyMigration", applied.Keys, StringComparer.Ordinal);
Assert.Equal("9.0.0", applied["20250420080000_RemoveDuplicateExtras"]);
Assert.Contains(log, entry => entry.Contains("2 de 3", StringComparison.Ordinal));
}
finally
{
SqliteConnection.ClearAllPools();
File.Delete(source);
}
}

[SkippableFact]
public async Task NoKnownMigrationsLeavesTheHistoryUntouched()
{
Skip.IfNot(TestEnvironment.PostgresAvailable, SkipReason);
await using var scratch = await TestEnvironment.ScratchDatabaseScope.CreateAsync();

using (var context = CreateContext(scratch.Database.ConnectionString))
{
await context.Database.MigrateAsync();
}

var source = Path.Combine(Path.GetTempPath(), $"jf-history-{Guid.NewGuid():N}.db");
try
{
await CreateSourceHistoryAsync(source);

await using var sqlite = new SqliteConnection($"Data Source={source};Mode=ReadOnly");
await sqlite.OpenAsync();
await using var pg = new NpgsqlConnection(scratch.Database.ConnectionString);
await pg.OpenAsync();

var before = (await ReadTargetHistoryAsync(pg)).Count;
var log = new List<string>();
await MigrationCodeMigrations.CopyAppliedAsync(sqlite, pg, new HashSet<string>(StringComparer.Ordinal), log.Add);

Assert.Equal(before, (await ReadTargetHistoryAsync(pg)).Count);
Assert.Contains(log, entry => entry.Contains("No se encontraron tipos", StringComparison.Ordinal));
}
finally
{
SqliteConnection.ClearAllPools();
File.Delete(source);
}
}

private static async Task CreateSourceHistoryAsync(string path)
{
await using var connection = new SqliteConnection($"Data Source={path}");
await connection.OpenAsync();
await using var command = connection.CreateCommand();
command.CommandText =
"""
CREATE TABLE "__EFMigrationsHistory" ("MigrationId" TEXT NOT NULL PRIMARY KEY, "ProductVersion" TEXT NOT NULL);
INSERT INTO "__EFMigrationsHistory" VALUES ('20250420080000_RemoveDuplicateExtras', '9.0.0');
INSERT INTO "__EFMigrationsHistory" VALUES ('20250420070000_MigrateActivityLogDb', '9.0.0');
INSERT INTO "__EFMigrationsHistory" VALUES ('20240101000000_SqliteOnlyMigration', '9.0.0');
""";
await command.ExecuteNonQueryAsync();
}

private static async Task<Dictionary<string, string>> ReadTargetHistoryAsync(NpgsqlConnection connection)
{
var rows = new Dictionary<string, string>(StringComparer.Ordinal);
await using var command = new NpgsqlCommand("SELECT \"MigrationId\", \"ProductVersion\" FROM \"__EFMigrationsHistory\";", connection);
await using var reader = await command.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
rows[reader.GetString(0)] = reader.GetString(1);
}

return rows;
}

private static JellyfinDbContext CreateContext(string connectionString)
=> new(
new DbContextOptionsBuilder<JellyfinDbContext>()
.UseNpgsql(connectionString, pg => pg.MigrationsAssembly(typeof(PostgresDatabaseProvider).Assembly.FullName))
.Options,
NullLogger<JellyfinDbContext>.Instance,
new PostgresDatabaseProvider(null, null),
new NoLockBehavior(NullLogger<NoLockBehavior>.Instance));
}
Loading