From 459b2719ca265d35e2c7dd5f9362456417856bf4 Mon Sep 17 00:00:00 2001 From: albertlast Date: Sun, 2 Aug 2026 07:44:46 +0200 Subject: [PATCH 01/11] Reports maintenance tool failures on the command line Maintenance::exit() renders the tool's templates, and those are the only place errors are ever shown. On the command line it takes the fallthrough path instead and goes straight to die(), so nothing was reported and the exit status was always 0: a scripted install that died on step three looked exactly like one that had finished. ToolsBase::updateSettingsFile() made the same assumption more directly, calling die() outright when Settings.php could not be written rather than recording the error the way the web path does. Writes the warnings and errors to stderr and exits non-zero when the tool actually failed. A step that merely wants input it was not given sets neither, so pausing part way through is still a success - the installer is meant to be called more than once - and that case now says which step it stopped on instead of nothing at all. Co-Authored-By: Claude Opus 5 Signed-off-by: albertlast --- Sources/Maintenance/Maintenance.php | 56 +++++++++++++++++++++++++ Sources/Maintenance/Tools/ToolsBase.php | 9 ++-- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/Sources/Maintenance/Maintenance.php b/Sources/Maintenance/Maintenance.php index 8809ee6f5fb..1b2a1b2c04f 100644 --- a/Sources/Maintenance/Maintenance.php +++ b/Sources/Maintenance/Maintenance.php @@ -808,6 +808,46 @@ public static function setQueryString(): string */ public static function exit(bool $fallthrough = false): void { + // On the command line there is no template to render, so everything the + // tool wanted to tell us has nowhere to go: a scripted install that died + // on step three looks exactly like one that finished. Put the problems on + // stderr and leave a non-zero status behind instead. + // + // A step that simply needs more input sets neither of these, so pausing + // part way through is still a success -- the installer is meant to be + // called more than once. + if ($fallthrough && Sapi::isCLI()) { + foreach (self::$warnings as $warning) { + fwrite(STDERR, 'warning: ' . self::plainText($warning) . "\n"); + } + + $problems = self::$errors; + + if (self::$fatal_error !== '') { + array_unshift($problems, self::$fatal_error); + } + + if ($problems !== []) { + foreach ($problems as $problem) { + fwrite(STDERR, 'error: ' . self::plainText($problem) . "\n"); + } + + exit(1); + } + + // Nothing went wrong, but we are not finished either: a step wanted + // input it was not given. Say which one, so a script that has to be + // run more than once can tell where it got to. + if (isset(self::$tool) && self::getCurrentStep() <= \count(self::$tool->getSteps())) { + fwrite( + STDERR, + 'stopped at step ' . self::getCurrentStep() + . ' of ' . \count(self::$tool->getSteps()) + . ' (' . (self::$tool->getSteps()[self::getCurrentStep()]?->getName() ?? 'unknown') . ")\n", + ); + } + } + // We usually dump our templates out. if (!$fallthrough) { // Send character set. @@ -920,4 +960,20 @@ private static function setCurrentStep(?int $step = null): void { $_GET['step'] = $step ?? (self::getCurrentStep() + 1); } + + /** + * Flattens one of our messages into something worth reading in a terminal. + * + * The steps build these for a browser, so they arrive carrying markup: the + * database errors in particular wrap the driver's own message in a div. + * + * @param string $message The message, as the step wrote it. + * @return string The same message, without the markup. + */ + private static function plainText(string $message): string + { + $message = preg_replace('~~i', "\n", $message) ?? $message; + + return trim(html_entity_decode(strip_tags($message), ENT_QUOTES | ENT_HTML5, 'UTF-8')); + } } diff --git a/Sources/Maintenance/Tools/ToolsBase.php b/Sources/Maintenance/Tools/ToolsBase.php index 7193b974371..d8c0a0be4d4 100644 --- a/Sources/Maintenance/Tools/ToolsBase.php +++ b/Sources/Maintenance/Tools/ToolsBase.php @@ -612,10 +612,11 @@ public function updateSettingsFile(array $config_vars, ?bool $keep_quotes = null if (!Config::updateSettingsFile($config_vars, $keep_quotes, $rebuild)) { $this->logProgress(Lang::getTxt('log_failed_with_error', ['error' => Lang::getTxt('settings_error', file: 'Maintenance')], file: 'Maintenance')); - if (Sapi::isCLI()) { - die(); - } - + // This used to die() outright on the command line, which reported + // nothing and exited 0 -- a scripted install that could not write + // Settings.php looked exactly like one that worked. Recording the + // error and returning stops the run just as firmly, and now the + // caller gets to say why. Maintenance::$fatal_error = Lang::getTxt('settings_error', file: 'Maintenance'); return false; From 11fe20d2959d6d95e8b09a6f94f1dcc97b45458a Mon Sep 17 00:00:00 2001 From: albertlast Date: Sun, 2 Aug 2026 07:44:59 +0200 Subject: [PATCH 02/11] Stops the installer assuming there is a web request Two things in the installer only hold when a browser is on the other end, and both are reached before the forum exists, so neither could be worked around from outside. defaultHost() reads $_SERVER['SERVER_NAME'] and ['SERVER_PORT'] whenever HTTP_HOST is absent. On the command line none of the three is set, so every run began with an undefined index warning. Falls back to localhost: the value only seeds the suggested board URL on the form, and a scripted install passes its own boardurl in. forumSettings() then built the same suggestion with substr($self, 0, strrpos($self, '/')). getSelf() is $_SERVER['PHP_SELF'], which in a request is a rooted path but on the command line is whatever was typed - usually a bare 'install.php' with no directory in it. strrpos() returns false, and substr() with a false length is fatal on PHP 8, so the installer died here on every CLI run. While in there: an unrecognised database type reported Lang::getTxt('upgrade_unknown_error'), which is not a string that exists. The fatal error was therefore blank in the browser too. Names the type that was rejected and the ones that would have been accepted, which matters most on the command line where the type is typed by hand rather than picked from a list of exactly those keys. Co-Authored-By: Claude Opus 5 Signed-off-by: albertlast --- Languages/en_US/Maintenance.php | 1 + Sources/Maintenance/Tools/Install.php | 38 ++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/Languages/en_US/Maintenance.php b/Languages/en_US/Maintenance.php index 3f3f4ee4477..21dceeb9f5b 100644 --- a/Languages/en_US/Maintenance.php +++ b/Languages/en_US/Maintenance.php @@ -124,6 +124,7 @@ It is recommended that you visit the Simple Machines website to ensure you are installing the latest version.'; $txt['error_already_installed'] = 'The installer has detected that you already have SMF installed. It is strongly advised that you do not try to overwrite an existing installation, continuing with installation may result in the loss or corruption of existing data.

If you wish to upgrade please visit the Simple Machines Website and download the latest upgrade package.

If you wish to overwrite your existing installation, including all data, it is recommended that you delete the existing database tables and replace Settings.php and try again.'; $txt['error_db_missing'] = 'The installer was unable to detect any database support in PHP. Please ask your host to ensure that PHP was compiled with the desired database, or that the proper extension is being loaded.'; +$txt['error_db_type_unknown'] = '“{db_type}” is not a database type this server supports. Supported types: {supported}.'; $txt['error_session_missing'] = 'The installer was unable to detect sessions support in your server’s installation of PHP. Please ask your host to ensure that PHP was compiled with session support (which in fact is the PHP default, meaning your host currently has explicitly disabled it).'; $txt['error_missing_files'] = 'Unable to find crucial installation files in the directory of this script!

Please make sure you uploaded the entire installation package, including the sql file, and then try again.'; $txt['error_session_save_path'] = 'Please inform your host that the session.save_path specified in php.ini is not valid! It needs to be changed to a directory that exists and is writable by the user PHP is running under.
'; diff --git a/Sources/Maintenance/Tools/Install.php b/Sources/Maintenance/Tools/Install.php index 51c1879883a..3a0a526b2a1 100644 --- a/Sources/Maintenance/Tools/Install.php +++ b/Sources/Maintenance/Tools/Install.php @@ -439,7 +439,19 @@ public function databaseSettings(): bool $db_prefix = $_POST['db_prefix']; if (!isset(Maintenance::$context['databases'][$db_type])) { - Maintenance::$fatal_error = Lang::getTxt('upgrade_unknown_error', file: 'Maintenance'); + // upgrade_unknown_error, which used to be reported here, does not + // exist -- so this produced an empty fatal error and left no clue + // what had gone wrong. Naming the type and the alternatives matters + // most on the command line, where the type is typed out by hand + // rather than picked from a list of exactly these keys. + Maintenance::$fatal_error = Lang::getTxt( + 'error_db_type_unknown', + [ + 'db_type' => $db_type, + 'supported' => Lang::sentenceList(array_keys(Maintenance::$context['databases'])), + ], + file: 'Maintenance', + ); $this->logProgress(Maintenance::$fatal_error); return false; @@ -609,7 +621,15 @@ public function forumSettings(): bool Db::load(); // Now, to put what we've learned together... and add a path. - Maintenance::$context['detected_url'] = 'http' . (Sapi::httpsOn() ? 's' : '') . '://' . $this->defaultHost() . substr(Maintenance::getSelf(), 0, strrpos(Maintenance::getSelf(), '/')); + // getSelf() is $_SERVER['PHP_SELF'], which in a request is a rooted path + // but on the command line is whatever was typed -- usually a bare + // 'install.php' with no directory in it at all. strrpos() then returns + // false, and substr() with a false length is fatal on PHP 8, so the + // installer died here on every CLI run. + $self = Maintenance::getSelf(); + $last_slash = strrpos($self, '/'); + + Maintenance::$context['detected_url'] = 'http' . (Sapi::httpsOn() ? 's' : '') . '://' . $this->defaultHost() . ($last_slash === false ? '' : substr($self, 0, $last_slash)); // Check if the database sessions will even work. Maintenance::$context['test_dbsession'] = (\ini_get('session.auto_start') != 1); @@ -1416,7 +1436,19 @@ private function saveProgress(): bool */ private function defaultHost(): string { - return empty($_SERVER['HTTP_HOST']) ? $_SERVER['SERVER_NAME'] . (empty($_SERVER['SERVER_PORT']) || $_SERVER['SERVER_PORT'] == '80' ? '' : ':' . $_SERVER['SERVER_PORT']) : $_SERVER['HTTP_HOST']; + if (!empty($_SERVER['HTTP_HOST'])) { + return $_SERVER['HTTP_HOST']; + } + + // On the command line there is no request to describe, so neither of + // these is set. This value only seeds the suggested board URL on the + // form, and a scripted install passes its own boardurl in, so a + // placeholder is enough -- but reading the keys unguarded was a warning + // on every CLI run. + $host = $_SERVER['SERVER_NAME'] ?? 'localhost'; + $port = $_SERVER['SERVER_PORT'] ?? ''; + + return $host . (empty($port) || $port == '80' ? '' : ':' . $port); } /** From 7477a6c37ee14b72f68f1747bb4927f450cc560e Mon Sep 17 00:00:00 2001 From: albertlast Date: Sun, 2 Aug 2026 07:57:51 +0200 Subject: [PATCH 03/11] Skips the browser sign-in when installing from the command line finalize() ends by signing the new administrator in, so the browser that just ran the installer lands on an admin session instead of a login form. It sets a login cookie, then records the session against the user agent that asked for it. None of that has any meaning on the command line. There is no browser to hold the cookie and no user agent to key the session on, so every CLI install ended with four warnings - headers sent after output had already started, a session that could not be started, and an id that could not be regenerated - and then wrote a sessions row built from an undefined HTTP_USER_AGENT. Runs the whole block only when there is a request behind it. The stats that follow it are untouched, so an install still records latestMember, totalMessages and totalTopics either way. Co-Authored-By: Claude Opus 5 Signed-off-by: albertlast --- Sources/Maintenance/Tools/Install.php | 82 +++++++++++++++------------ 1 file changed, 46 insertions(+), 36 deletions(-) diff --git a/Sources/Maintenance/Tools/Install.php b/Sources/Maintenance/Tools/Install.php index a0c2d284a53..73d9d286228 100644 --- a/Sources/Maintenance/Tools/Install.php +++ b/Sources/Maintenance/Tools/Install.php @@ -1202,48 +1202,58 @@ public function finalize(): bool Db::$db->free_result($request); } - // Automatically log them in ;) - if (isset(Maintenance::$context['id_member'], Maintenance::$context['password_salt'])) { - Cookie::setLoginCookie(3153600 * 60, Maintenance::$context['id_member'], Cookie::encrypt($_POST['password1'], Maintenance::$context['password_salt'])); - } + // Sign the new administrator in, so the browser that just ran the + // installer lands on an admin session rather than a login form. + // + // None of that means anything on the command line: there is no browser + // to hold the cookie, and no user agent to record against the session. + // Attempting it anyway sent headers after output had already started and + // left four warnings on every run, then wrote a session row keyed on an + // undefined HTTP_USER_AGENT. + if (!Sapi::isCLI()) { + // Automatically log them in ;) + if (isset(Maintenance::$context['id_member'], Maintenance::$context['password_salt'])) { + Cookie::setLoginCookie(3153600 * 60, Maintenance::$context['id_member'], Cookie::encrypt($_POST['password1'], Maintenance::$context['password_salt'])); + } - $result = Db::$db->query( - 'SELECT value - FROM {db_prefix}settings - WHERE variable = {string:db_sessions}', - [ - 'db_sessions' => 'databaseSession_enable', - 'db_error_skip' => true, - ], - ); + $result = Db::$db->query( + 'SELECT value + FROM {db_prefix}settings + WHERE variable = {string:db_sessions}', + [ + 'db_sessions' => 'databaseSession_enable', + 'db_error_skip' => true, + ], + ); - if (Db::$db->num_rows($result) != 0) { - list($db_sessions) = Db::$db->fetch_row($result); - } - Db::$db->free_result($result); + if (Db::$db->num_rows($result) != 0) { + list($db_sessions) = Db::$db->fetch_row($result); + } + Db::$db->free_result($result); - if (empty($db_sessions)) { - $_SESSION['admin_time'] = time(); - } else { - $_SERVER['HTTP_USER_AGENT'] = substr($_SERVER['HTTP_USER_AGENT'], 0, 211); + if (empty($db_sessions)) { + $_SESSION['admin_time'] = time(); + } else { + $_SERVER['HTTP_USER_AGENT'] = substr($_SERVER['HTTP_USER_AGENT'], 0, 211); - Db::$db->insert( - 'replace', - '{db_prefix}sessions', - [ - 'session_id' => 'string', - 'last_update' => 'int', - 'data' => 'string', - ], - [ + Db::$db->insert( + 'replace', + '{db_prefix}sessions', [ - session_id(), - time(), - 'USER_AGENT|s:' . \strlen($_SERVER['HTTP_USER_AGENT']) . ':"' . $_SERVER['HTTP_USER_AGENT'] . '";admin_time|i:' . time() . ';', + 'session_id' => 'string', + 'last_update' => 'int', + 'data' => 'string', ], - ], - ['session_id'], - ); + [ + [ + session_id(), + time(), + 'USER_AGENT|s:' . \strlen($_SERVER['HTTP_USER_AGENT']) . ':"' . $_SERVER['HTTP_USER_AGENT'] . '";admin_time|i:' . time() . ';', + ], + ], + ['session_id'], + ); + } } Logging::updateStats('member'); From 89324b1d038f9238c8323bca5d78cdd119b20b58 Mon Sep 17 00:00:00 2001 From: albertlast Date: Sun, 2 Aug 2026 07:57:51 +0200 Subject: [PATCH 04/11] Reports the step a maintenance tool actually paused on Two things were wrong with the note the command line prints when a tool stops part way. It indexed the step list to get the number, which counts from zero, while every other line of output uses the step's own id, which counts from one - so it disagreed with the "Step 3: Database Settings" lines immediately above it. It also fired on a successful run. Tools deliberately return false from their last step so the web flow stops and renders its "all done" template, which means reaching that step is success rather than a pause, and a completed install claimed to have stopped at it. Co-Authored-By: Claude Opus 5 Signed-off-by: albertlast --- Sources/Maintenance/Maintenance.php | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/Sources/Maintenance/Maintenance.php b/Sources/Maintenance/Maintenance.php index 1b2a1b2c04f..8c47056f45d 100644 --- a/Sources/Maintenance/Maintenance.php +++ b/Sources/Maintenance/Maintenance.php @@ -835,15 +835,25 @@ public static function exit(bool $fallthrough = false): void exit(1); } - // Nothing went wrong, but we are not finished either: a step wanted - // input it was not given. Say which one, so a script that has to be - // run more than once can tell where it got to. - if (isset(self::$tool) && self::getCurrentStep() <= \count(self::$tool->getSteps())) { + // Nothing went wrong, but we may not be finished either: a step can + // stop because it wanted input it was not given. Say which one, so + // a script that has to be run more than once can tell where it got + // to. The step numbers its own id from one, which is what every + // other line of output uses. + // + // The last step is excluded on purpose. Tools end by returning false + // from it so that the web flow stops and renders its "all done" + // template, which means reaching it is success, not a pause. + $steps = isset(self::$tool) ? self::$tool->getSteps() : []; + + if (isset($steps[self::getCurrentStep()]) && self::getCurrentStep() < \count($steps) - 1) { + $stopped = $steps[self::getCurrentStep()]; + fwrite( STDERR, - 'stopped at step ' . self::getCurrentStep() - . ' of ' . \count(self::$tool->getSteps()) - . ' (' . (self::$tool->getSteps()[self::getCurrentStep()]?->getName() ?? 'unknown') . ")\n", + 'stopped at step ' . $stopped->getId() + . ' of ' . \count($steps) + . ' (' . $stopped->getName() . ")\n", ); } } From 7665045176b5aa7987dc16480f031b64398891cf Mon Sep 17 00:00:00 2001 From: albertlast Date: Sun, 2 Aug 2026 07:58:06 +0200 Subject: [PATCH 05/11] Installs the forum from the command line The dev environment stopped at a Settings.php and a staged install.php, leaving the actual install to a human clicking through a browser. That is the one step between a fresh clone and a running forum that could not be scripted, and everything that wants to test against a real install has to start by doing it. Adds four scripts under .docker/: install-forum.sh installs a forum, no browser involved use-engine.sh switches which installed forum is live reset.sh empties one engine's database and restages lib.sh shared settings and engine name normalisation The installer is already CLI-native - parseCliArguments() turns --name=value into $_POST and execute() runs every step in one process - so this is two passes rather than 2.1's five curl requests. The second pass carries pop_done, which is the short-circuit past the population report; passing it on the first pass would skip building the schema. --engine both installs MySQL and then PostgreSQL. It has to be sequential: Settings.php pins a single db_type and Db::load() hands back the connection it already made, so only one engine is ever live in a process. Both installs are kept, and use-engine.sh swaps between them by putting the saved Settings.php back - no restart, because the entrypoint only writes one when there is not one already. --pin-secrets fixes auth_secret and image_proxy_secret, which are generated with random_bytes() and stored nowhere but Settings.php. Without it the two installs differ by more than their database and a login cookie does not survive the switch. The cookie name needs no such help: createCookieName() is a crc32 of the database name and prefix. Co-Authored-By: Claude Opus 5 Signed-off-by: albertlast --- .docker/README.md | 55 +++++++++++- .docker/install-forum.sh | 187 +++++++++++++++++++++++++++++++++++++++ .docker/lib.sh | 123 +++++++++++++++++++++++++ .docker/reset.sh | 99 +++++++++++++++++++++ .docker/use-engine.sh | 48 ++++++++++ .gitignore | 4 + 6 files changed, 514 insertions(+), 2 deletions(-) create mode 100644 .docker/install-forum.sh create mode 100644 .docker/lib.sh create mode 100644 .docker/reset.sh create mode 100644 .docker/use-engine.sh diff --git a/.docker/README.md b/.docker/README.md index 2ba2f277fb3..111bef70986 100644 --- a/.docker/README.md +++ b/.docker/README.md @@ -53,6 +53,57 @@ forum. ## Installing the forum +```sh +.docker/install-forum.sh --engine mysql +.docker/install-forum.sh --engine postgresql +.docker/install-forum.sh --engine both +``` + +That resets the engine's database and installs a forum into it, with no browser +involved. It takes about a minute. Log in at http://localhost:8080 as +`admin` / `password`. + +SMF 3.0's installer is CLI-native: `Maintenance::parseCliArguments()` turns +`--name=value` into `$_POST`, and `Maintenance::execute()` then runs every step +in one process, stopping at the first that still needs input. The script makes +two passes, because `databasePopulation()` always stops the first time even +though it succeeded — it pauses so a human can read its "N duplicate tables +ignored" report, and the form's `pop_done` field is the short-circuit past it. +Passing `pop_done` on the first pass would skip building the schema entirely. + +Two flags worth knowing: + +- `--force` reinstalls even when a forum is already there. Without it the + script leaves an existing install alone. +- `--pin-secrets` fixes `auth_secret` and `image_proxy_secret` to known values + instead of the random ones `ForumSettings()` generates. Both installs then + differ only in their database, so a login cookie survives `use-engine.sh`. + Dev-only values for a throwaway forum: never reuse them. + +### Two forums at once + +`--engine both` installs MySQL first and PostgreSQL second, one after the other. +It has to be sequential: `Settings.php` pins a single `$db_type`, and +`Db::load()` hands back the connection it already made, so only one engine can +ever be live in a process. + +Both installs are kept. Switch between them with: + +```sh +.docker/use-engine.sh postgresql +``` + +That puts the saved `Settings.php` back and clears `cache/`. No restart is +needed — the entrypoint only writes `Settings.php` when there is not one, so it +leaves whatever is in place alone. The copies live in `.docker/settings/` and +are gitignored. + +`reset.sh` is the other half: it empties one engine's database and restages the +installer, discarding that forum. `use-engine.sh` switches between forums, +`reset.sh` throws one away. + +### Installing in a browser instead + On first boot the entrypoint writes a `Settings.php` pre-filled for the chosen engine and copies `other/install.php` to the web root, so http://localhost:8080 redirects into the installer. @@ -102,8 +153,8 @@ The repository is bind-mounted at `/var/www/html`, so edits on the host are live on the next request. Opcache is on but revalidates every request, so you never need to restart for a PHP change. -To reinstall from scratch: `docker compose down -v`, delete `Settings.php` and -`Settings_bak.php`, then `docker compose up -d`. +To reinstall from scratch: `.docker/install-forum.sh --engine mysql --force`. +To wipe everything including the volumes: `docker compose down -v`. ## Debugging SQL with the PostgreSQL log diff --git a/.docker/install-forum.sh b/.docker/install-forum.sh new file mode 100644 index 00000000000..613b604c90a --- /dev/null +++ b/.docker/install-forum.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# Installs the forum without a browser. +# +# .docker/install-forum.sh --engine mysql +# .docker/install-forum.sh --engine postgresql +# .docker/install-forum.sh --engine both +# +# SMF 3.0's installer is CLI-native: Maintenance::parseCliArguments() turns +# --name=value into $_POST, and Maintenance::execute() then runs every step in +# one process, stopping at the first that still needs input. So unlike 2.1, +# which needs a five-request curl driver, this is two invocations: +# +# pass 1 Welcome -> Writable -> Database settings -> Forum settings +# -> Database population, which builds the schema and then stops +# pass 2 the same again, plus --pop_done, which walks straight past the +# population report into the admin account and finalise +# +# databasePopulation() always stops the first time even though it succeeded: it +# pauses so a human can read its "N duplicate tables ignored" report, and the +# form's pop_done field is the short-circuit that skips it. Passing pop_done on +# pass 1 would skip building the schema altogether, which is why this is two +# passes and not one. +# +# Every step re-runs on pass 2. They are all idempotent given the same input -- +# the settings steps rewrite the same values, and adminAccount() stops if an +# administrator already exists. +# +# Runs on the host. +set -euo pipefail + +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" + +ENGINE='' +PIN_SECRETS=0 +FORCE=0 + +while [ $# -gt 0 ]; do + case "$1" in + --engine) ENGINE="$2"; shift 2 ;; + --engine=*) ENGINE="${1#*=}"; shift ;; + --pin-secrets) PIN_SECRETS=1; shift ;; + --force) FORCE=1; shift ;; + -h|--help) sed -n '2,27p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +[ -n "$ENGINE" ] || die 'need --engine mysql|postgresql|both' +ENGINES=$(engine_list "$ENGINE") || die "unknown engine: $ENGINE" + +cd "$BOARD_DIR" + +# The installer's own name for each engine, which is the key of the array it +# builds from the drivers it found. These are capitalised, and a lowercase +# db_type is rejected outright -- so they are spelled exactly as the installer +# spells them rather than reusing the SMF type. +installer_db_type() { + case "$1" in + mysql) echo 'MySQL' ;; + postgresql) echo 'PostgreSQL' ;; + *) return 1 ;; + esac +} + +install_one() { + local smf_type="$1" db_type server port args + + db_type=$(installer_db_type "$smf_type") + server=$(engine_server "$smf_type") + port=$(engine_port "$smf_type") + + if [ "$FORCE" -eq 0 ] && [ -n "$(installed_version "$smf_type" || true)" ]; then + log "${smf_type}: already installed (SMF $(installed_version "$smf_type")), nothing to do" + + return 0 + fi + + log "${smf_type}: resetting" + "$DOCKER_DIR/reset.sh" --engine "$smf_type" >/dev/null + + args=( + --contbutt=1 + --db_type="$db_type" + --db_server="$server" + --db_port="$port" + --db_name="$DB_NAME" + --db_user="$DB_USER" + --db_passwd="$DB_PASSWORD" + --db_prefix="$DB_PREFIX" + --boardurl="$SMF_BOARDURL" + --mbname="$SMF_MBNAME" + --username="$SMF_ADMIN_USER" + --email="$SMF_ADMIN_EMAIL" + --server_email="$SMF_ADMIN_EMAIL" + --password1="$SMF_ADMIN_PASS" + --password2="$SMF_ADMIN_PASS" + ) + + log "${smf_type}: building the schema" + docker compose exec -T web php install.php "${args[@]}" >/dev/null + + log "${smf_type}: creating the administrator and finalising" + docker compose exec -T web php install.php "${args[@]}" --pop_done=1 >/dev/null + + local version + version=$(installed_version "$smf_type" || true) + + [ -n "$version" ] || die "${smf_type}: the installer finished but the forum is not installed" + + log "${smf_type}: installed SMF ${version}" + + if [ "$PIN_SECRETS" -eq 1 ]; then + pin_secrets + fi + + save_settings "$smf_type" +} + +# ForumSettings() generates auth_secret and image_proxy_secret with +# random_bytes() and stores them nowhere but Settings.php, so the two engines +# end up with different ones and a login cookie stops being valid the moment +# use-engine.sh switches. Pinning them leaves the database as the only thing +# that differs between the two installs. +# +# The cookie name needs no such help: createCookieName() is a crc32 of the +# database name and prefix, which are the same on both. +# +# Dev-only values for a throwaway forum, published here deliberately. Never +# reuse them anywhere real. +pin_secrets() { + log 'pinning auth_secret and image_proxy_secret' + + # The values have to be handed over with -e. Exporting them on the host does + # nothing: docker compose exec starts a fresh environment, so getenv() came + # back empty and this wrote two empty secrets over the generated ones. + docker compose exec -T \ + -e PIN_AUTH_SECRET="$PIN_AUTH_SECRET" \ + -e PIN_IMAGE_PROXY_SECRET="$PIN_IMAGE_PROXY_SECRET" \ + web php -r ' + define("SMF", 1); + define("SMF_SETTINGS_FILE", "/var/www/html/Settings.php"); + define("SMF_SETTINGS_BACKUP_FILE", "/var/www/html/Settings_bak.php"); + require_once "/var/www/html/index.php"; + + $auth = (string) getenv("PIN_AUTH_SECRET"); + $proxy = (string) getenv("PIN_IMAGE_PROXY_SECRET"); + + if ($auth === "" || $proxy === "") { + fwrite(STDERR, "pin-secrets: the secrets did not reach the container\n"); + exit(1); + } + + exit(SMF\Config::updateSettingsFile([ + "auth_secret" => $auth, + "image_proxy_secret" => $proxy, + ]) ? 0 : 1); + ' >/dev/null +} + +# Keep each engine's Settings.php so use-engine.sh can put it back without a +# reinstall. Gitignored: generated secrets and a machine-specific board URL. +save_settings() { + local smf_type="$1" + + mkdir -p "$SETTINGS_DIR" + cp Settings.php "$SETTINGS_DIR/Settings.${smf_type}.php" + cp Settings_bak.php "$SETTINGS_DIR/Settings_bak.${smf_type}.php" + + log "${smf_type}: settings saved to .docker/settings/" +} + +PIN_AUTH_SECRET="${PIN_AUTH_SECRET:-0b6e5f3c1a94d27e8f5b0c3a76d1e94f2b8c5a03e7d146f9b2c8a501d3e7f4c69}" +PIN_IMAGE_PROXY_SECRET="${PIN_IMAGE_PROXY_SECRET:-7f2a9c4e0b6d18a35c92}" + +# Sequential on purpose. Settings.php pins one $db_type and Db::load() returns +# the connection it already made, so only one engine can be live at a time -- +# "both" is a chain, never two connections. +for smf_type in $ENGINES; do + install_one "$smf_type" +done + +# Leave the first engine of a "both" run active rather than whichever happened +# to go last, so the result does not depend on the order. +FIRST_ENGINE="${ENGINES%% *}" +"$DOCKER_DIR/use-engine.sh" "$FIRST_ENGINE" >/dev/null + +log "active engine: ${FIRST_ENGINE} -- ${SMF_BOARDURL} (${SMF_ADMIN_USER} / ${SMF_ADMIN_PASS})" diff --git a/.docker/lib.sh b/.docker/lib.sh new file mode 100644 index 00000000000..5c13d410080 --- /dev/null +++ b/.docker/lib.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Shared settings and helpers for the .docker scripts. Sourced, never run. +# +# Host-side scripts (reset.sh, install-forum.sh, use-engine.sh) source this from +# wherever the caller happens to be standing; everything below resolves paths +# for itself rather than assuming a working directory. +# +# Everything defined here is consumed by the scripts that source this file, and +# a linter reading it on its own cannot see any of those uses -- hence the +# blanket disable below. Keep it on its own, with nothing after it that starts +# with the linter's name, or the following line gets parsed as a directive too. +# +# shellcheck disable=SC2034 + +# Git Bash on Windows rewrites anything that looks like a Unix path before +# handing it to a program, so a container-side path like /var/www/html/... is +# silently turned into C:/Program Files/Git/var/www/html/... and the command +# fails with "Could not open input file". These two switch that off. They mean +# nothing on Linux and macOS. +export MSYS_NO_PATHCONV=1 +export MSYS2_ARG_CONV_EXCL='*' + +# Repository root, regardless of where the caller was standing. +DOCKER_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +BOARD_DIR=$(cd -- "$DOCKER_DIR/.." && pwd) + +# Where use-engine.sh keeps each engine's Settings.php. Gitignored: these hold +# generated secrets and a machine-specific board URL. +SETTINGS_DIR="$DOCKER_DIR/settings" + +# ---------------------------------------------------------------- credentials +# These match compose.yaml's defaults. Override them in the environment if you +# changed them in .env. +DB_NAME="${DB_NAME:-smf}" +DB_USER="${DB_USER:-smf}" +DB_PASSWORD="${DB_PASSWORD:-smf}" +DB_ROOT_PASSWORD="${DB_ROOT_PASSWORD:-smf}" +DB_PREFIX="${DB_PREFIX:-smf_}" + +WEB_PORT="${WEB_PORT:-8080}" +SMF_BOARDURL="${SMF_BOARDURL:-http://localhost:${WEB_PORT}}" +SMF_MBNAME="${SMF_MBNAME:-SMF Dev}" + +# The administrator the installer creates. Dev-only values for a throwaway +# forum; never reuse them anywhere real. +SMF_ADMIN_USER="${SMF_ADMIN_USER:-admin}" +SMF_ADMIN_PASS="${SMF_ADMIN_PASS:-password}" +# example.com is reserved by RFC 2606, so this can never reach a real inbox. +# SMF's validator rejects dotless domains, so 'admin@localhost' is not an option. +SMF_ADMIN_EMAIL="${SMF_ADMIN_EMAIL:-admin@example.com}" + +# --------------------------------------------------------------------- output +log() { printf '[smf-dev] %s\n' "$*"; } +warn() { printf '[smf-dev] %s\n' "$*" >&2; } +die() { printf '[smf-dev] error: %s\n' "$*" >&2; exit 1; } + +# Engine name normalisation. Everything downstream uses either the SMF type +# ('mysql' / 'postgresql') or the compose service name ('mysql' / 'postgres'), +# and mixing them up is an easy way to waste an afternoon. +engine_smf_type() { + case "$1" in + mysql|mysqli|mariadb) echo 'mysql' ;; + postgres|postgresql|pgsql) echo 'postgresql' ;; + *) return 1 ;; + esac +} + +engine_service() { + case "$1" in + mysql|mysqli|mariadb) echo 'mysql' ;; + postgres|postgresql|pgsql) echo 'postgres' ;; + *) return 1 ;; + esac +} + +# Container-internal host and port for an engine. Not the host-side ports in +# compose.yaml: these are what Settings.php has to contain. +engine_server() { + case "$(engine_smf_type "$1")" in + mysql) echo "${SMF_MYSQL_SERVER:-mysql}" ;; + postgresql) echo "${SMF_POSTGRES_SERVER:-postgres}" ;; + *) return 1 ;; + esac +} + +engine_port() { + case "$(engine_smf_type "$1")" in + mysql) echo "${SMF_MYSQL_PORT:-3306}" ;; + postgresql) echo "${SMF_POSTGRES_PORT:-5432}" ;; + *) return 1 ;; + esac +} + +# Expands "both" into the engines to act on, in the order they run. Only one +# engine can be live at a time -- Settings.php pins $db_type and Db::load() +# early-returns once the connection exists -- so "both" is a sequential chain, +# never two connections. +engine_list() { + case "$1" in + both|all) echo 'mysql postgresql' ;; + *) engine_smf_type "$1" ;; + esac +} + +# The installed version for one engine, empty if the forum is not installed. +# Asks the database directly rather than trusting the presence of a file: +# Settings.php exists from the moment the entrypoint writes it, long before +# there is a forum behind it. +installed_version() { + local engine service + engine=$(engine_smf_type "$1") || return 1 + service=$(engine_service "$1") + + if [ "$engine" = 'mysql' ]; then + docker compose exec -T -e MYSQL_PWD="$DB_PASSWORD" "$service" \ + mysql -u"$DB_USER" -D "$DB_NAME" -N -B -e \ + "SELECT value FROM ${DB_PREFIX}settings WHERE variable = 'smfVersion';" 2>/dev/null + else + docker compose exec -T "$service" \ + psql -U "$DB_USER" -d "$DB_NAME" -tAX -c \ + "SELECT value FROM ${DB_PREFIX}settings WHERE variable = 'smfVersion';" 2>/dev/null + fi +} diff --git a/.docker/reset.sh b/.docker/reset.sh new file mode 100644 index 00000000000..00f2de5a404 --- /dev/null +++ b/.docker/reset.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Returns the stack to "installable": no forum, an empty database for the chosen +# engine, and a Settings.php regenerated for it. +# +# .docker/reset.sh --engine mysql +# .docker/reset.sh --engine postgresql +# +# This is also how you move an install between engines. Settings.php pins one +# engine and wins over SMF_DB_TYPE, so switching means throwing it away and +# letting the entrypoint write a new one. To keep an install rather than +# discard it, use use-engine.sh instead. +# +# Only the chosen engine's database is touched. The two engines keep separate +# volumes, so a MySQL reset can never disturb a PostgreSQL install or vice +# versa. +# +# Runs on the host. +set -euo pipefail + +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" + +ENGINE='' +KEEP_FILES=0 + +while [ $# -gt 0 ]; do + case "$1" in + --engine) ENGINE="$2"; shift 2 ;; + --engine=*) ENGINE="${1#*=}"; shift ;; + --keep-files) KEEP_FILES=1; shift ;; + -h|--help) sed -n '2,17p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +[ -n "$ENGINE" ] || die 'need --engine mysql|postgresql' +SERVICE=$(engine_service "$ENGINE") || die "unknown engine: $ENGINE" +SMF_TYPE=$(engine_smf_type "$ENGINE") + +cd "$BOARD_DIR" + +log "resetting for ${SMF_TYPE}" + +# ------------------------------------------------------------------ the forum +# Stop the web container first: Apache holding a half-installed forum open while +# its database vanishes underneath produces confusing errors in the log. +docker compose stop web >/dev/null 2>&1 || true + +rm -f Settings.php Settings_bak.php install.php upgrade.php + +# SMF's cache holds a serialised copy of $modSettings, which would otherwise +# outlive the database it describes. +find cache -type f ! -name 'index.php' ! -name '.htaccess' -delete 2>/dev/null || true + +if [ "$KEEP_FILES" -eq 0 ]; then + for dir in attachments custom_avatar; do + find "$dir" -type f ! -name 'index.php' ! -name '.htaccess' ! -name 'blank.png' -delete 2>/dev/null || true + done + rm -f Packages/installed.list +fi + +# --------------------------------------------------------------- the database +docker compose up -d "$SERVICE" >/dev/null + +if [ "$SMF_TYPE" = 'mysql' ]; then + # As root: the smf user has rights on the smf database but cannot drop and + # recreate it. utf8mb4 matches what compose.yaml asks the server for and + # what SMF's own DDL emits. + docker compose exec -T -e MYSQL_PWD="$DB_ROOT_PASSWORD" "$SERVICE" mysql -uroot -e " + DROP DATABASE IF EXISTS \`${DB_NAME}\`; + CREATE DATABASE \`${DB_NAME}\` CHARACTER SET utf8mb4; + GRANT ALL ON \`${DB_NAME}\`.* TO '${DB_USER}'@'%'; + " +else + # The database itself cannot be dropped while we are connected to it, and + # dropping the schema is enough: it takes the tables, sequences, functions + # and operators with it. smf owns the database, so it may recreate public. + docker compose exec -T "$SERVICE" psql -v ON_ERROR_STOP=1 -q -U "$DB_USER" -d "$DB_NAME" -c ' + DROP SCHEMA IF EXISTS public CASCADE; + CREATE SCHEMA public; + ' >/dev/null +fi + +log "${SMF_TYPE} database ${DB_NAME} is empty" + +# Bring web back up so the entrypoint regenerates Settings.php for this engine +# and stages the installer. +SMF_DB_TYPE="$SMF_TYPE" docker compose up -d web >/dev/null + +# The entrypoint waits for the database before it writes anything, so give it a +# moment to get there rather than racing whatever runs next. +for _ in $(seq 1 60); do + if docker compose exec -T web test -f install.php 2>/dev/null; then + log 'installer staged, ready to install' + exit 0 + fi + sleep 1 +done + +die 'timed out waiting for the entrypoint to stage install.php (docker compose logs web)' diff --git a/.docker/use-engine.sh b/.docker/use-engine.sh new file mode 100644 index 00000000000..5fd2b7a96b5 --- /dev/null +++ b/.docker/use-engine.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Switches which installed forum is live, without reinstalling either. +# +# .docker/use-engine.sh mysql +# .docker/use-engine.sh postgresql +# +# Both database services always run, on separate volumes, so each keeps its own +# forum. What decides which one you get is Settings.php: it pins $db_type, and +# it wins over SMF_DB_TYPE. install-forum.sh files a copy per engine, and this +# puts one of them back. +# +# No container restart is needed. The entrypoint only writes Settings.php when +# there is not one, so it leaves whatever is in place alone. +# +# To throw an install away and start over, use reset.sh instead. +# +# Runs on the host. +set -euo pipefail + +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" + +[ $# -eq 1 ] || die 'usage: use-engine.sh mysql|postgresql' + +case "$1" in + -h|--help) sed -n '2,16p' "${BASH_SOURCE[0]}"; exit 0 ;; +esac + +SMF_TYPE=$(engine_smf_type "$1") || die "unknown engine: $1" +SAVED="$SETTINGS_DIR/Settings.${SMF_TYPE}.php" + +cd "$BOARD_DIR" + +[ -f "$SAVED" ] || die "no saved settings for ${SMF_TYPE} -- run .docker/install-forum.sh --engine ${SMF_TYPE}" + +cp "$SAVED" Settings.php +cp "$SETTINGS_DIR/Settings_bak.${SMF_TYPE}.php" Settings_bak.php + +# SMF's cache holds a serialised copy of $modSettings, which describes the +# database we are switching away from. $cache_enable defaults to 0, so there is +# usually nothing there -- but the directory also holds db_last_error.php and +# the generated CSS and JS, and clearing it costs nothing. +find cache -type f ! -name 'index.php' ! -name '.htaccess' -delete 2>/dev/null || true + +VERSION=$(installed_version "$SMF_TYPE" || true) + +[ -n "$VERSION" ] || warn "${SMF_TYPE} has no forum installed -- Settings.php now points at an empty database" + +log "active engine: ${SMF_TYPE}${VERSION:+ (SMF ${VERSION})} -- ${SMF_BOARDURL}" diff --git a/.gitignore b/.gitignore index 6b46b9c3a5c..7bcc6bba3a0 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,10 @@ Thumbs.db /.env /compose.override.yaml /compose.override.yml +# One saved Settings.php per engine, so use-engine.sh can switch between two +# installs without reinstalling. Generated secrets and a machine-specific +# board URL: local to whoever ran the installer. +/.docker/settings/ # Test / Private files # ######################## From 7876ac1fcaaa91ec74da2353967f40ecc83336f4 Mon Sep 17 00:00:00 2001 From: albertlast Date: Sun, 2 Aug 2026 07:58:28 +0200 Subject: [PATCH 06/11] Marks the dev environment scripts executable The README invokes them as .docker/install-forum.sh rather than through bash, which only works with the bit set. Windows checkouts do not carry it, so it has to be recorded in the index. lib.sh is left alone: it is sourced, never run. Co-Authored-By: Claude Opus 5 Signed-off-by: albertlast --- .docker/install-forum.sh | 0 .docker/reset.sh | 0 .docker/use-engine.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .docker/install-forum.sh mode change 100644 => 100755 .docker/reset.sh mode change 100644 => 100755 .docker/use-engine.sh diff --git a/.docker/install-forum.sh b/.docker/install-forum.sh old mode 100644 new mode 100755 diff --git a/.docker/reset.sh b/.docker/reset.sh old mode 100644 new mode 100755 diff --git a/.docker/use-engine.sh b/.docker/use-engine.sh old mode 100644 new mode 100755 From 61a35f536bf78bd2c51278ef349f0bfb25e88d0b Mon Sep 17 00:00:00 2001 From: albertlast Date: Sun, 2 Aug 2026 20:59:30 +0200 Subject: [PATCH 07/11] Removes install.php once the forum is installed The installer tells you to delete it and cannot do it itself: the ?delete link it offers is a GET, and command line arguments only ever reach $_POST, so nothing on the CLI path ever gets there. Leaving it behind is not cosmetic. Settings.php redirects every request back into the installer while the file exists, so the forum the script just built is unreachable, and SMF puts a "MAJOR SECURITY RISK: you have not removed install.php" box on every page it shows an administrator - which also lands in front of anything else a test or a person is trying to read on that page. Deleting it is safe for a reinstall because install_one() calls reset.sh first, and reset.sh clears Settings.php and then blocks until the entrypoint has staged a fresh copy. Adds a check in front of the two installer passes to say so out loud when it has not: without one, php reports "Could not open input file: install.php", which reads like a broken script rather than a stack that was never made installable. Signed-off-by: albertlast --- .docker/README.md | 8 ++++++++ .docker/install-forum.sh | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/.docker/README.md b/.docker/README.md index 111bef70986..57afbed0550 100644 --- a/.docker/README.md +++ b/.docker/README.md @@ -71,6 +71,14 @@ though it succeeded — it pauses so a human can read its "N duplicate tables ignored" report, and the form's `pop_done` field is the short-circuit past it. Passing `pop_done` on the first pass would skip building the schema entirely. +It then deletes `install.php`, which the installer asks for but cannot do +itself — its `?delete` link is a GET, and command line arguments only ever reach +`$_POST`. That matters more than it sounds: while the file is there +`Settings.php` redirects every request back into the installer, and SMF puts a +"MAJOR SECURITY RISK" box on every page it shows an administrator. Reinstalling +still works, because `reset.sh` runs first and does not return until the +entrypoint has staged a fresh copy. + Two flags worth knowing: - `--force` reinstalls even when a forum is already there. Without it the diff --git a/.docker/install-forum.sh b/.docker/install-forum.sh index 613b604c90a..8bc55700270 100755 --- a/.docker/install-forum.sh +++ b/.docker/install-forum.sh @@ -96,6 +96,13 @@ install_one() { --password2="$SMF_ADMIN_PASS" ) + # reset.sh does not return until the entrypoint has staged this, so its + # absence means something went wrong there rather than here. Worth saying so: + # without it php reports "Could not open input file: install.php", which reads + # like a broken script rather than a forum that was never made installable. + docker compose exec -T web test -f install.php \ + || die "${smf_type}: install.php is not staged, so there is nothing to run (docker compose logs web)" + log "${smf_type}: building the schema" docker compose exec -T web php install.php "${args[@]}" >/dev/null @@ -107,6 +114,17 @@ install_one() { [ -n "$version" ] || die "${smf_type}: the installer finished but the forum is not installed" + # The installer tells you to delete this and cannot do it itself: its ?delete + # link is a GET, and command line arguments only ever reach $_POST. Leaving it + # is not cosmetic - Settings.php redirects every request back into the + # installer while it is there, and SMF puts a "MAJOR SECURITY RISK: you have + # not removed install.php" box on every page it shows an administrator. + # + # Safe to delete even though a reinstall needs it again: install_one() always + # calls reset.sh first, and reset.sh clears Settings.php and waits for the + # entrypoint to put a fresh copy back before returning. + rm -f install.php + log "${smf_type}: installed SMF ${version}" if [ "$PIN_SECRETS" -eq 1 ]; then From c7b9f5a5c656685c7abddf49f8c76800067a2ea5 Mon Sep 17 00:00:00 2001 From: albertlast Date: Sun, 2 Aug 2026 21:12:46 +0200 Subject: [PATCH 08/11] Adds a script for checking and resetting account passwords Two forums side by side, each with its own administrator, and a password chosen at install time is a combination that ends in hand written SQL sooner or later - which is a poor way to answer a question as ordinary as "is this the password?". user.sh answers it. list shows the accounts, check says whether SMF would accept a password and exits 0 or 1 so it can be used in a conditional, and reset sets a new one. --engine reads the settings use-engine.sh saved for the other engine, so the forum that is not currently live can be looked at without switching to it and back. Two details that stop it being a thin wrapper around an UPDATE: - The hashing goes through Security::hashPassword() rather than being written here, so what lands in the table is by construction what Login2 reads back out. A script that hashes passwords its own way is a script that eventually disagrees with the forum. - reset clears passwd_flood too. SMF locks an account out for a while after enough wrong guesses, and a new password behind a live lockout behaves exactly like a password that did not take. check also points out an account that is not activated, which fails to log in with an entirely correct password. The password is passed to the container through the environment rather than in the argument list, which anything able to read the process table can see. Also completes the file list in the README, which still only described the image and had none of the scripts in it. Signed-off-by: albertlast --- .docker/README.md | 35 ++++++++ .docker/user.sh | 204 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100755 .docker/user.sh diff --git a/.docker/README.md b/.docker/README.md index 57afbed0550..aac1648fae7 100644 --- a/.docker/README.md +++ b/.docker/README.md @@ -110,6 +110,35 @@ are gitignored. installer, discarding that forum. `use-engine.sh` switches between forums, `reset.sh` throws one away. +## Accounts and passwords + +Two forums, each with its own administrator, and a password chosen months ago is +a recipe for an afternoon of hand written SQL. `user.sh` is there so it is not: + +```sh +.docker/user.sh list +.docker/user.sh check admin 'password' +.docker/user.sh reset admin 'a new password' +``` + +`check` exits 0 when SMF would accept the password and 1 when it would not, so +it works in a conditional as well as by eye. It also points out an account that +is not activated, which fails to log in with a correct password and looks +exactly like a wrong one. + +`--engine mysql|postgresql` reads the settings `use-engine.sh` saved for that +engine, so the *other* forum can be inspected without switching to it: + +```sh +.docker/user.sh check admin 'password' --engine mysql +``` + +The hashing goes through SMF's own `Security` class rather than being written +here, so what `reset` puts in the table is by construction what `Login2` expects +to find. It clears `passwd_flood` at the same time: SMF locks an account out for +a while after enough wrong guesses, and a fresh password behind a lockout looks +exactly like a password that did not take. + ### Installing in a browser instead On first boot the entrypoint writes a `Settings.php` pre-filled for the chosen @@ -229,4 +258,10 @@ compose.yaml the stack .docker/mysql/init/10-smf.sh runs once on first mysql database creation .docker/postgres/init/10-smf.sh runs once on first postgres database creation .docker/env.example optional overrides + +.docker/lib.sh paths, credentials and engine names, shared +.docker/install-forum.sh install a forum with no browser involved +.docker/reset.sh empty one engine and restage the installer +.docker/use-engine.sh switch which installed forum is live +.docker/user.sh inspect accounts, check and reset passwords ``` diff --git a/.docker/user.sh b/.docker/user.sh new file mode 100755 index 00000000000..eebf4d1a8e5 --- /dev/null +++ b/.docker/user.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# Looks at forum accounts and fixes their passwords, so "which password did this +# forum end up with?" does not turn into a session of hand written SQL. +# +# .docker/user.sh list +# .docker/user.sh check admin 'password' +# .docker/user.sh reset admin 'a new password' +# .docker/user.sh check admin 'password' --engine postgresql +# +# check exits 0 when SMF would accept the password and 1 when it would not, so +# it is usable in a conditional as well as by eye. +# +# Everything goes through SMF's own Security class rather than writing a hash +# from here: what this puts in the table is by construction what Login2 expects +# to find there. Nothing is ever printed that would reveal an existing password; +# hashes are one way and this does not try to be clever about that. +# +# Without --engine it acts on the forum Settings.php currently points at. With +# it, it reads the copy use-engine.sh saved for that engine instead, which means +# the other forum can be inspected without switching to it. +# +# Runs on the host. +set -euo pipefail + +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" + +ENGINE='' +ACTION='' +NAME='' +PASSWORD='' +POSITIONAL=() + +while [ $# -gt 0 ]; do + case "$1" in + --engine) ENGINE="$2"; shift 2 ;; + --engine=*) ENGINE="${1#*=}"; shift ;; + -h|--help) sed -n '2,22p' "${BASH_SOURCE[0]}"; exit 0 ;; + -*) die "unknown argument: $1" ;; + *) POSITIONAL+=("$1"); shift ;; + esac +done + +[ "${#POSITIONAL[@]}" -gt 0 ] || die "need an action: list, check or reset (see --help)" + +ACTION="${POSITIONAL[0]}" +NAME="${POSITIONAL[1]:-}" +PASSWORD="${POSITIONAL[2]:-}" + +case "$ACTION" in + list) ;; + check|reset) + [ -n "$NAME" ] || die "${ACTION}: need a member name" + [ -n "$PASSWORD" ] || die "${ACTION}: need a password" + ;; + *) die "unknown action: ${ACTION} (expected list, check or reset)" ;; +esac + +# The settings file to read, as the container sees it. Empty means "whichever +# forum is live", which is the common case and needs no explanation in the log. +SETTINGS='/var/www/html/Settings.php' + +if [ -n "$ENGINE" ]; then + SMF_TYPE=$(engine_smf_type "$ENGINE") || die "unknown engine: $ENGINE" + SAVED="$DOCKER_DIR/settings/Settings.${SMF_TYPE}.php" + + [ -f "$SAVED" ] || die "no saved settings for ${SMF_TYPE}; install it first with install-forum.sh --engine ${SMF_TYPE}" + + SETTINGS="/var/www/html/.docker/settings/Settings.${SMF_TYPE}.php" +fi + +cd "$BOARD_DIR" + +# The password goes through the environment rather than the argument list: +# arguments are visible to anything that can read the process table, and a +# password typed at a shell is quite enough exposure already. +docker compose exec -T \ + -e SMF_USER_ACTION="$ACTION" \ + -e SMF_USER_NAME="$NAME" \ + -e SMF_USER_PASSWORD="$PASSWORD" \ + -e SMF_USER_SETTINGS="$SETTINGS" \ + web php <<-'PHP' + query( + 'SELECT id_member, member_name, real_name, email_address, id_group, is_activated + FROM {db_prefix}members + ORDER BY id_member', + [], + ); + + printf("%-5s %-20s %-28s %-7s %s\n", 'id', 'member_name', 'email', 'group', 'activated'); + + while ($row = $db->fetch_assoc($request)) { + printf( + "%-5d %-20s %-28s %-7d %s\n", + $row['id_member'], + $row['member_name'], + $row['email_address'], + $row['id_group'], + // 1 is the only value that can log in; the rest are awaiting + // activation, awaiting approval, banned or deleted. + $row['is_activated'] == 1 ? 'yes' : 'no (' . $row['is_activated'] . ')', + ); + } + + $db->free_result($request); + + exit(0); + } + + $request = $db->query( + 'SELECT id_member, member_name, passwd, is_activated + FROM {db_prefix}members + WHERE member_name = {string:name} OR email_address = {string:name} + LIMIT 1', + [ + 'name' => $name, + ], + ); + + $member = $db->fetch_assoc($request); + $db->free_result($request); + + if (!is_array($member)) { + fwrite(STDERR, 'error: no member called "' . $name . '" (try: user.sh list)' . "\n"); + + exit(1); + } + + if ($action === 'check') { + $ok = SMF\Security::hashVerifyPassword($password, $member['passwd']); + + echo $member['member_name'], ': ', $ok ? 'password is correct' : 'password is WRONG', "\n"; + + // Being right about the password is not the same as being able to log + // in, and the difference is worth saying out loud before someone spends + // an afternoon on it. + if ($ok && $member['is_activated'] != 1) { + echo ' note: the account is not active (is_activated = ', $member['is_activated'], '), so it cannot log in', "\n"; + } + + exit($ok ? 0 : 1); + } + + $db->query( + 'UPDATE {db_prefix}members + SET passwd = {string:passwd}, passwd_flood = {string:empty} + WHERE id_member = {int:id}', + [ + 'passwd' => SMF\Security::hashPassword($password), + // Cleared as well: SMF locks an account out for a while after + // enough wrong guesses, and resetting the password while leaving + // the lockout in place looks exactly like the password not working. + 'empty' => '', + 'id' => (int) $member['id_member'], + ], + ); + + echo $member['member_name'], ': password changed', "\n"; + PHP From 6b8206b382d5179d2c4a054bb9dd57c791227a7d Mon Sep 17 00:00:00 2001 From: albertlast Date: Sun, 16 Aug 2026 10:54:30 +0200 Subject: [PATCH 09/11] Adds a check that an upgraded database matches a fresh install The installer builds the schema from Sources/Db/Schema/v3_0/ in one go. The upgrader arrives at the same place through a hundred-odd migrations applied to whatever 2.1 left behind. They are meant to converge, and nothing we have checks that they do -- so a column left at the wrong type, an index that was never created, or a primary key quietly dropped goes unnoticed until it is a bug report from someone whose forum upgraded years ago. .docker/schema-tool.php reads the shape of a database -- tables, columns, indexes, sequences, the names in settings -- and compares two of those readings. It talks to the engine directly rather than through SMF, because the database worth looking at is frequently one SMF would refuse to run on. .docker/compare-upgrade.sh drives it: empty, load a 2.1 dump, upgrade, read, reinstall from scratch, read again, report. --baseline takes any 2.1 dump, so this works against a real forum as well as against the synthetic baseline from the 2.1 environment. Two kinds of difference are reported but do not decide the exit code, since a real forum always has some: what is in settings, and the order the columns of a table sit in. And if the upgrade does not reach the end, it stops there and says so rather than comparing anyway -- a half-upgraded database differs from a fresh install in hundreds of places, every one of them the honest consequence of the migrations that never ran. Signed-off-by: albertlast --- .docker/README.md | 50 +++ .docker/compare-upgrade.sh | 197 +++++++++ .docker/schema-tool.php | 798 +++++++++++++++++++++++++++++++++++++ .gitignore | 3 + 4 files changed, 1048 insertions(+) create mode 100755 .docker/compare-upgrade.sh create mode 100644 .docker/schema-tool.php diff --git a/.docker/README.md b/.docker/README.md index aac1648fae7..9049d232711 100644 --- a/.docker/README.md +++ b/.docker/README.md @@ -193,6 +193,54 @@ never need to restart for a PHP change. To reinstall from scratch: `.docker/install-forum.sh --engine mysql --force`. To wipe everything including the volumes: `docker compose down -v`. +## Comparing an upgrade against a fresh install + +The installer builds the schema from `Sources/Db/Schema/v3_0/` in one go. The +upgrader arrives at the same place through a hundred-odd migrations applied to +whatever 2.1 left behind. They are meant to converge, and nothing checks that +they do: + +```bash +.docker/compare-upgrade.sh --engine mysql --baseline path/to/a-2.1-dump.sql +``` + +That empties the database, loads the dump, upgrades it, reads the schema, +reinstalls from scratch, reads that too, and reports every place the two +disagree — a column of the wrong type, an index that was never created, a +primary key quietly dropped. It ends with the fresh install in place, and takes +five to ten minutes. + +`--baseline` takes any SQL dump of a 2.1 database. A dump of a real forum is +the better test; the [2.1 development environment][baseline] builds a synthetic +one designed to hold something in every table an upgrade touches, which is +useful when you have no real forum to hand. + +Two kinds of difference are reported but do not fail the run, because a real +forum always has some: the contents of `settings`, and the order columns sit in +within a table. Everything else is a schema difference and sets the exit code. + +If the upgrade does not reach the end, the script stops there and says so +rather than comparing anyway. A half-upgraded database differs from a fresh +install in hundreds of places, all of them the honest consequence of the +migrations that never ran, and none of them worth reading. + +The tool underneath is usable on its own, against any two SMF databases on the +same engine — two forums you already have, or the same forum before and after +something you are testing: + +```bash +docker compose exec web php .docker/schema-tool.php dump --engine mysql --db smf > before.json +# ... do the thing ... +docker compose exec web php .docker/schema-tool.php dump --engine mysql --db smf > after.json +docker compose exec web php .docker/schema-tool.php diff before.json after.json +``` + +It talks to the database directly rather than through SMF, so it works on a +database SMF would refuse to run on — which is usually the one you want to look +at. + +[baseline]: https://github.com/SimpleMachines/SMF/pull/9330 + ## Debugging SQL with the PostgreSQL log The `postgres` log is the best tool in the stack for tracking down a broken @@ -264,4 +312,6 @@ compose.yaml the stack .docker/reset.sh empty one engine and restage the installer .docker/use-engine.sh switch which installed forum is live .docker/user.sh inspect accounts, check and reset passwords +.docker/compare-upgrade.sh upgrade a 2.1 dump, install 3.0, diff the two +.docker/schema-tool.php read a database's shape, and compare readings ``` diff --git a/.docker/compare-upgrade.sh b/.docker/compare-upgrade.sh new file mode 100755 index 00000000000..185d07b21a0 --- /dev/null +++ b/.docker/compare-upgrade.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Upgrades a 2.1 database to 3.0, installs 3.0 from scratch, and reports where +# the two schemas disagree. +# +# .docker/compare-upgrade.sh --engine mysql --baseline ../SMF-2.1/.docker/baseline/artifacts/2.1.7-1/small/mysql.sql +# .docker/compare-upgrade.sh --engine postgresql --baseline ../SMF-2.1/.docker/baseline/artifacts/2.1.7-1/small/postgres.sql +# +# The installer builds the schema from Sources/Db/Schema/v3_0/ in one go. The +# upgrader arrives at the same place through a hundred-odd migrations applied +# to whatever 2.1 left behind. Nothing checks that those two agree, and where +# they do not, the forum that upgraded is running on a schema that has never +# been tested against -- a column of the wrong type, an index that was never +# created, a primary key quietly dropped. +# +# --baseline takes any SQL dump of a 2.1 database. The one this was written +# against is the committed baseline from the 2.1 development environment, which +# is built to hold something in every table an upgrade touches, but a dump of a +# real forum works and is a better test. +# +# Both databases for the chosen engine are rebuilt, twice. Anything already +# installed on that engine is destroyed, and what remains at the end is the +# fresh install. The other engine is not touched. +# +# Runs on the host. Expect five to ten minutes per engine. +set -euo pipefail + +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" + +ENGINE='' +BASELINE='' +OUT="$DOCKER_DIR/compare" + +while [ $# -gt 0 ]; do + case "$1" in + --engine) ENGINE="$2"; shift 2 ;; + --engine=*) ENGINE="${1#*=}"; shift ;; + --baseline) BASELINE="$2"; shift 2 ;; + --baseline=*) BASELINE="${1#*=}"; shift ;; + --out) OUT="$2"; shift 2 ;; + --out=*) OUT="${1#*=}"; shift ;; + -h|--help) sed -n '2,25p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +[ -n "$ENGINE" ] || die 'need --engine mysql|postgresql|both' +ENGINES=$(engine_list "$ENGINE") || die "unknown engine: $ENGINE" +[ -n "$BASELINE" ] || die 'need --baseline ' +[ -f "$BASELINE" ] || die "no such file: $BASELINE" + +cd "$BOARD_DIR" +mkdir -p "$OUT" +OUT=$(cd -- "$OUT" && pwd) + +# The tool that reads the two files runs in the container, which sees the +# repository and nothing else, so the output has to live somewhere inside it. +case "$OUT" in + "$BOARD_DIR"/*) OUT_REL="${OUT#"$BOARD_DIR"/}" ;; + *) die "--out has to be somewhere inside the repository, since the container cannot see anywhere else" ;; +esac + +# Reads the shape of the database the given engine is pointed at. The tool runs +# in the container, so it uses the container-internal host and port rather than +# the ones published in compose.yaml. +snapshot() { + local smf_type="$1" label="$2" file="$3" + + docker compose exec -T web php .docker/schema-tool.php dump \ + --engine "$smf_type" \ + --db "$DB_NAME" \ + --prefix "$DB_PREFIX" \ + --host "$(engine_server "$smf_type")" \ + --port "$(engine_port "$smf_type")" \ + --user "$DB_USER" \ + --pass "$DB_PASSWORD" \ + --label "$label" > "$file" + + [ -s "$file" ] || die "${smf_type}: the ${label} reading came back empty" +} + +# The version this checkout is, which is what a finished upgrade has to leave +# in the database. +smf_version() { + sed -n "s/.*define('SMF_VERSION', '\([^']*\)').*/\1/p" index.php | head -1 +} + +load_baseline() { + local smf_type="$1" service + service=$(engine_service "$smf_type") + + if [ "$smf_type" = 'mysql' ]; then + docker compose exec -T -e MYSQL_PWD="$DB_PASSWORD" "$service" \ + mysql -u"$DB_USER" -D "$DB_NAME" < "$BASELINE" + else + # ON_ERROR_STOP so that a dump taken from a database that is not empty, + # or one taken with a different owner, stops here rather than producing + # a half-loaded forum that then fails somewhere in the upgrader and + # looks like a migration bug. + docker compose exec -T "$service" \ + psql -v ON_ERROR_STOP=1 -q -U "$DB_USER" -d "$DB_NAME" < "$BASELINE" >/dev/null + fi +} + +compare_one() { + local smf_type="$1" version + + # ---------------------------------------------------------- the upgrade + log "${smf_type}: emptying the database" + "$DOCKER_DIR/reset.sh" --engine "$smf_type" >/dev/null + + log "${smf_type}: loading ${BASELINE##*/}" + load_baseline "$smf_type" + + version=$(installed_version "$smf_type" || true) + + if [ -z "$version" ]; then + warn "${smf_type}: the dump loaded but there is no forum in it -- wrong prefix, or not an SMF dump" + + return 1 + fi + + log "${smf_type}: the dump is SMF ${version}" + + # The upgrader ships in other/ and expects to be run from the board + # directory, the same way install.php does. Nobody upgrading a real forum + # has install.php sitting there, so neither does this. + rm -f install.php + cp other/upgrade.php upgrade.php + + log "${smf_type}: upgrading" + + local upgraded=0 + docker compose exec -T web php upgrade.php > "$OUT/upgrade-${smf_type}.log" 2>&1 || upgraded=$? + + rm -f upgrade.php + + if [ "$upgraded" -ne 0 ]; then + warn "${smf_type}: the upgrader exited ${upgraded} -- ${OUT}/upgrade-${smf_type}.log" + + return 1 + fi + + # The upgrader reports a failed migration and stops, but still exits 0, so + # the version in the database is the only thing that says whether it got to + # the end. Checking it matters more than it looks: a half-upgraded database + # differs from a fresh install in hundreds of places, all of them the honest + # consequence of the migrations that never ran, and none of them the kind of + # difference this script exists to find. + version=$(installed_version "$smf_type" || true) + + if [ "$version" != "$(smf_version)" ]; then + warn "${smf_type}: the upgrade stopped at SMF ${version:-nothing}, expected $(smf_version)" + warn "${smf_type}: the last thing it said is at the end of ${OUT}/upgrade-${smf_type}.log" + + return 1 + fi + + log "${smf_type}: upgraded to SMF ${version}" + + snapshot "$smf_type" upgraded "$OUT/upgraded-${smf_type}.json" + + # ----------------------------------------------------- the fresh install + # --force because there is an installed forum now, and install-forum.sh + # leaves one alone unless told otherwise. + log "${smf_type}: installing from scratch" + "$DOCKER_DIR/install-forum.sh" --engine "$smf_type" --force >/dev/null + + snapshot "$smf_type" fresh "$OUT/fresh-${smf_type}.json" + + # ------------------------------------------------------------ the report + local status=0 + + docker compose exec -T web php .docker/schema-tool.php diff \ + "${OUT_REL}/fresh-${smf_type}.json" \ + "${OUT_REL}/upgraded-${smf_type}.json" \ + > "$OUT/report-${smf_type}.txt" || status=$? + + echo + cat "$OUT/report-${smf_type}.txt" + echo + + if [ "$status" -eq 0 ]; then + log "${smf_type}: the upgraded schema matches a fresh install" + else + log "${smf_type}: the schemas differ -- ${OUT}/report-${smf_type}.txt" + fi + + return "$status" +} + +failed=0 + +for smf_type in $ENGINES; do + compare_one "$smf_type" || failed=1 +done + +exit "$failed" diff --git a/.docker/schema-tool.php b/.docker/schema-tool.php new file mode 100644 index 00000000000..6d4f2c20d16 --- /dev/null +++ b/.docker/schema-tool.php @@ -0,0 +1,798 @@ + fresh.json + * php .docker/schema-tool.php diff fresh.json upgraded.json + * + * Runs inside the web container, and talks to the database directly rather + * than through SMF. Nothing here loads Settings.php or boots the forum: the + * database being examined is frequently one that SMF would refuse to run on, + * which is the whole point of looking at it. + * + * Simple Machines Forum (SMF) + * + * @package SMF + * @author Simple Machines https://www.simplemachines.org + * @copyright 2026 Simple Machines and individual contributors + * @license https://www.simplemachines.org/about/smf/license.php BSD + * + * @version 3.0 Alpha 4 + */ + +declare(strict_types=1); + +if (PHP_SAPI !== 'cli') { + exit("This is a command line script.\n"); +} + +$argv = $_SERVER['argv']; +$mode = $argv[1] ?? ''; + +exit(match ($mode) { + 'dump' => cmd_dump(parse_options(array_slice($argv, 2))), + 'diff' => cmd_diff(array_slice($argv, 2)), + default => usage(), +}); + +/** + * @return int + */ +function usage(): int +{ + fwrite(STDERR, <<<'TEXT' + Reads the shape of an SMF database, and compares two of those readings. + + php .docker/schema-tool.php dump --engine mysql --db smf > fresh.json + php .docker/schema-tool.php diff fresh.json upgraded.json + + dump options, with the defaults compose.yaml gives: + + --engine mysql or postgresql (required) + --db database name (smf) + --prefix table prefix (smf_) + --host database host (mysql / postgres) + --port database port (3306 / 5432) + --user database user (smf) + --pass database password (smf) + --label what to call this reading in a diff (the database name) + + diff exits 1 when the two disagree about the schema, 0 when they do not. + Settings and column order are reported but do not decide the exit code: + they are differences a forum can live with, and a real forum always has + some. + + TEXT); + + return 2; +} + +/** + * Turns --name=value and --name value into an array. Unknown names are an + * error rather than a shrug: a misspelled --prefix would otherwise read the + * wrong tables and report every one of them as missing. + * + * @param array $args + * @return array + */ +function parse_options(array $args): array +{ + $known = ['engine', 'db', 'prefix', 'host', 'port', 'user', 'pass', 'label']; + $options = []; + + while ($args !== []) { + $arg = array_shift($args); + + if (!str_starts_with($arg, '--')) { + fail('unexpected argument: ' . $arg); + } + + $name = substr($arg, 2); + $value = null; + + if (str_contains($name, '=')) { + [$name, $value] = explode('=', $name, 2); + } + + if (!in_array($name, $known, true)) { + fail('unknown option: --' . $name); + } + + $options[$name] = $value ?? (string) array_shift($args); + } + + return $options; +} + +/** + * @param string $message + * @return never + */ +function fail(string $message): never +{ + fwrite(STDERR, 'schema-tool: ' . $message . "\n"); + + exit(2); +} + +/** + * @param array $options + * @return int + */ +function cmd_dump(array $options): int +{ + $engine = match ($options['engine'] ?? '') { + 'mysql', 'mysqli', 'mariadb' => 'mysql', + 'postgresql', 'postgres', 'pgsql' => 'postgresql', + default => fail('need --engine mysql|postgresql'), + }; + + $db = $options['db'] ?? 'smf'; + + $schema = [ + 'engine' => $engine, + 'label' => $options['label'] ?? $db, + 'prefix' => $options['prefix'] ?? 'smf_', + 'database' => $db, + 'read_at' => gmdate('c'), + ]; + + $read = $engine === 'mysql' ? read_mysql(...) : read_postgresql(...); + + $schema += $read( + $options['host'] ?? ($engine === 'mysql' ? 'mysql' : 'postgres'), + (int) ($options['port'] ?? ($engine === 'mysql' ? 3306 : 5432)), + $db, + $options['user'] ?? 'smf', + $options['pass'] ?? 'smf', + $schema['prefix'], + ); + + echo json_encode($schema, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), "\n"; + + return 0; +} + +/** + * Everything is keyed by name with the prefix taken off, so that two databases + * using different prefixes still line up, and so that the report reads in the + * names people use rather than the ones the database stores. + * + * @param string $host + * @param int $port + * @param string $db + * @param string $user + * @param string $pass + * @param string $prefix + * @return array + */ +function read_mysql(string $host, int $port, string $db, string $user, string $pass, string $prefix): array +{ + mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT); + + try { + $link = new mysqli($host, $user, $pass, $db, $port); + } catch (mysqli_sql_exception $e) { + fail('cannot connect to ' . $db . ' on ' . $host . ': ' . $e->getMessage()); + } + + $tables = []; + + $rows = query_mysql($link, ' + SELECT TABLE_NAME, ENGINE, TABLE_COLLATION + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = \'BASE TABLE\'', [$db]); + + foreach ($rows as $row) { + $tables[unprefix($row['TABLE_NAME'], $prefix)] = [ + 'attributes' => [ + 'engine' => (string) $row['ENGINE'], + 'collation' => (string) $row['TABLE_COLLATION'], + ], + 'columns' => [], + 'indexes' => [], + ]; + } + + $rows = query_mysql($link, ' + SELECT TABLE_NAME, COLUMN_NAME, ORDINAL_POSITION, COLUMN_TYPE, + IS_NULLABLE, COLUMN_DEFAULT, EXTRA, COLLATION_NAME + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = ? + ORDER BY TABLE_NAME, ORDINAL_POSITION', [$db]); + + foreach ($rows as $row) { + $table = unprefix($row['TABLE_NAME'], $prefix); + + if (!isset($tables[$table])) { + continue; + } + + $tables[$table]['columns'][$row['COLUMN_NAME']] = [ + 'type' => (string) $row['COLUMN_TYPE'], + 'nullable' => $row['IS_NULLABLE'] === 'YES', + // A NULL default and no default at all are the same string here, + // so nullability above is what tells them apart. + 'default' => $row['COLUMN_DEFAULT'], + // auto_increment lives here, and so does ON UPDATE. + 'extra' => (string) $row['EXTRA'], + 'collation' => $row['COLLATION_NAME'], + 'position' => (int) $row['ORDINAL_POSITION'], + ]; + } + + // SUB_PART is the length of a prefix index, which SMF uses on a few of the + // longer text columns. A missing one is a real difference, so it is part of + // the column's name in the key rather than being dropped. + $rows = query_mysql($link, ' + SELECT TABLE_NAME, INDEX_NAME, NON_UNIQUE, SEQ_IN_INDEX, COLUMN_NAME, SUB_PART + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = ? + ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX', [$db]); + + foreach ($rows as $row) { + $table = unprefix($row['TABLE_NAME'], $prefix); + + if (!isset($tables[$table])) { + continue; + } + + $index = $row['INDEX_NAME']; + + $tables[$table]['indexes'][$index] ??= [ + 'primary' => $index === 'PRIMARY', + 'unique' => (int) $row['NON_UNIQUE'] === 0, + 'columns' => [], + ]; + + $tables[$table]['indexes'][$index]['columns'][] = $row['COLUMN_NAME'] + . ($row['SUB_PART'] === null ? '' : '(' . $row['SUB_PART'] . ')'); + } + + $settings = []; + + if (isset($tables['settings'])) { + foreach (query_mysql($link, 'SELECT variable FROM `' . $prefix . 'settings`', []) as $row) { + $settings[] = $row['variable']; + } + } + + $link->close(); + + return finish($tables, $settings, []); +} + +/** + * @param string $host + * @param int $port + * @param string $db + * @param string $user + * @param string $pass + * @param string $prefix + * @return array + */ +function read_postgresql(string $host, int $port, string $db, string $user, string $pass, string $prefix): array +{ + $link = @pg_connect(sprintf( + 'host=%s port=%d dbname=%s user=%s password=%s', + $host, + $port, + $db, + $user, + $pass, + )); + + if ($link === false) { + fail('cannot connect to ' . $db . ' on ' . $host); + } + + $tables = []; + + $rows = query_postgresql($link, ' + SELECT table_name + FROM information_schema.tables + WHERE table_schema = \'public\' AND table_type = \'BASE TABLE\'', []); + + foreach ($rows as $row) { + $tables[unprefix($row['table_name'], $prefix)] = [ + // PostgreSQL has no per-table storage engine or collation, so the + // MySQL side of this stays empty rather than inventing one. + 'attributes' => [], + 'columns' => [], + 'indexes' => [], + ]; + } + + $rows = query_postgresql($link, ' + SELECT table_name, column_name, ordinal_position, data_type, + character_maximum_length, numeric_precision, numeric_scale, + is_nullable, column_default + FROM information_schema.columns + WHERE table_schema = \'public\' + ORDER BY table_name, ordinal_position', []); + + foreach ($rows as $row) { + $table = unprefix($row['table_name'], $prefix); + + if (!isset($tables[$table])) { + continue; + } + + $default = $row['column_default']; + + $tables[$table]['columns'][$row['column_name']] = [ + 'type' => postgresql_type($row), + 'nullable' => $row['is_nullable'] === 'YES', + 'default' => $default, + // The nearest thing PostgreSQL has to MySQL's auto_increment, so + // that a column that lost its sequence reads the same way on both. + 'extra' => $default !== null && str_starts_with($default, 'nextval(') ? 'auto_increment' : '', + 'collation' => null, + 'position' => (int) $row['ordinal_position'], + ]; + } + + // unnest() with ordinality is what keeps a composite index in its own + // order; indkey is an int2vector, and reading it any other way sorts the + // columns alphabetically, which would make (id_group, id_board) and + // (id_board, id_group) look like the same index. They are not. + $rows = query_postgresql($link, ' + SELECT c.relname AS table_name, i.relname AS index_name, + ix.indisunique, ix.indisprimary, a.attname + FROM pg_index AS ix + INNER JOIN pg_class AS c ON (c.oid = ix.indrelid) + INNER JOIN pg_class AS i ON (i.oid = ix.indexrelid) + INNER JOIN pg_namespace AS n ON (n.oid = c.relnamespace) + CROSS JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, ord) + INNER JOIN pg_attribute AS a ON (a.attrelid = c.oid AND a.attnum = k.attnum) + WHERE n.nspname = \'public\' + ORDER BY c.relname, i.relname, k.ord', []); + + foreach ($rows as $row) { + $table = unprefix($row['table_name'], $prefix); + + if (!isset($tables[$table])) { + continue; + } + + $index = unprefix($row['index_name'], $prefix); + + $tables[$table]['indexes'][$index] ??= [ + 'primary' => $row['indisprimary'] === 't', + 'unique' => $row['indisunique'] === 't', + 'columns' => [], + ]; + + $tables[$table]['indexes'][$index]['columns'][] = $row['attname']; + } + + $sequences = []; + + foreach (query_postgresql($link, ' + SELECT sequence_name + FROM information_schema.sequences + WHERE sequence_schema = \'public\' + ORDER BY sequence_name', []) as $row) { + $sequences[] = unprefix($row['sequence_name'], $prefix); + } + + $settings = []; + + if (isset($tables['settings'])) { + foreach (query_postgresql($link, 'SELECT variable FROM "' . $prefix . 'settings"', []) as $row) { + $settings[] = $row['variable']; + } + } + + pg_close($link); + + return finish($tables, $settings, $sequences); +} + +/** + * Rebuilds the type as it was written, since information_schema takes it + * apart. varchar(255) arrives as 'character varying' with the length in a + * separate column, and a bare 'character varying' beside it means something + * else entirely. + * + * @param array $row + * @return string + */ +function postgresql_type(array $row): string +{ + $type = (string) $row['data_type']; + + if ($row['character_maximum_length'] !== null) { + return $type . '(' . $row['character_maximum_length'] . ')'; + } + + // Scale is null for integers, where the precision is the width in bits and + // says nothing anyone wants in a report. + if ($type === 'numeric' && $row['numeric_precision'] !== null && $row['numeric_scale'] !== null) { + return $type . '(' . $row['numeric_precision'] . ',' . $row['numeric_scale'] . ')'; + } + + return $type; +} + +/** + * @param array $tables + * @param array $settings + * @param array $sequences + * @return array + */ +function finish(array $tables, array $settings, array $sequences): array +{ + ksort($tables); + sort($settings); + + foreach ($tables as &$table) { + ksort($table['columns']); + ksort($table['indexes']); + } + + return [ + 'tables' => $tables, + 'sequences' => $sequences, + 'settings' => $settings, + ]; +} + +/** + * @param mysqli $link + * @param string $sql + * @param array $params + * @return array + */ +function query_mysql(mysqli $link, string $sql, array $params): array +{ + $statement = $link->prepare($sql); + + if ($params !== []) { + $statement->bind_param(str_repeat('s', count($params)), ...$params); + } + + $statement->execute(); + $rows = $statement->get_result()->fetch_all(MYSQLI_ASSOC); + $statement->close(); + + return $rows; +} + +/** + * @param \PgSql\Connection $link + * @param string $sql + * @param array $params + * @return array + */ +function query_postgresql(\PgSql\Connection $link, string $sql, array $params): array +{ + $result = @pg_query_params($link, $sql, $params); + + if ($result === false) { + fail('query failed: ' . pg_last_error($link)); + } + + return pg_fetch_all($result, PGSQL_ASSOC); +} + +/** + * @param string $name + * @param string $prefix + * @return string + */ +function unprefix(string $name, string $prefix): string +{ + return $prefix !== '' && str_starts_with($name, $prefix) ? substr($name, strlen($prefix)) : $name; +} + +/** + * @param array $files + * @return int + */ +function cmd_diff(array $files): int +{ + if (count($files) !== 2) { + fail('diff needs two files: the reading to measure against, then the one to judge'); + } + + [$a, $b] = array_map(read_dump(...), $files); + + if ($a['engine'] !== $b['engine']) { + fail('these are different engines (' . $a['engine'] . ' and ' . $b['engine'] . '), and their types do not correspond'); + } + + $left = $a['label']; + $right = $b['label']; + + echo 'Comparing ', $right, ' against ', $left, ' on ', $a['engine'], ".\n\n"; + + if ($a['prefix'] !== $b['prefix']) { + echo "Note: the prefixes differ, so anything naming a table inside a\n", + "default or a sequence differs with it.\n\n"; + } + + // Two reports rather than one. Everything in the first is a difference in + // the schema itself; everything in the second is a difference a forum can + // live with, and a real forum always has some, so only the first decides + // the exit code. + $schema = []; + $aside = []; + + compare_tables($a, $b, $left, $right, $schema); + + if ($a['engine'] === 'postgresql') { + compare_lists('Sequences', $a['sequences'], $b['sequences'], $right, $schema); + } + + compare_lists('Settings', $a['settings'], $b['settings'], $right, $aside); + compare_column_order($a, $b, $aside); + + if (render($schema) + render($aside) === 0) { + echo "No differences.\n"; + + return 0; + } + + echo "\n"; + + if ($schema !== []) { + echo count_entries($schema), " difference(s) in the schema.\n"; + } + + if ($aside !== []) { + echo count_entries($aside), " difference(s) outside it, which do not decide the exit code.\n"; + } + + return $schema === [] ? 0 : 1; +} + +/** + * @param string $file + * @return array + */ +function read_dump(string $file): array +{ + $raw = @file_get_contents($file); + + if ($raw === false) { + fail('cannot read ' . $file); + } + + $dump = json_decode($raw, true); + + if (!is_array($dump) || !isset($dump['tables'], $dump['engine'])) { + fail($file . ' is not something dump wrote'); + } + + return $dump; +} + +/** + * Adds one difference to a report. Grouped by section and then by whatever it + * is about, so that a table with eight changed columns is one heading and + * eight lines under it rather than eight headings. + * + * @param array $report + * @param string $section + * @param string $subject + * @param string $message + * @param array $details + */ +function note(array &$report, string $section, string $subject, string $message, array $details = []): void +{ + $report[$section][$subject][] = ['message' => $message, 'details' => $details]; +} + +/** + * @param array $report + * @return int + */ +function count_entries(array $report): int +{ + $total = 0; + + foreach ($report as $subjects) { + foreach ($subjects as $entries) { + $total += count($entries); + } + } + + return $total; +} + +/** + * @param array $report + * @return int + */ +function render(array $report): int +{ + foreach ($report as $section => $subjects) { + echo $section, "\n", str_repeat('-', strlen($section)), "\n"; + + foreach ($subjects as $subject => $entries) { + echo ' ', $subject, "\n"; + + foreach ($entries as $entry) { + echo ' ', $entry['message'], "\n"; + + foreach ($entry['details'] as $detail) { + echo ' ', $detail, "\n"; + } + } + } + + echo "\n"; + } + + return count_entries($report); +} + +/** + * @param array $a + * @param array $b + * @param string $left + * @param string $right + * @param array $report + */ +function compare_tables(array $a, array $b, string $left, string $right, array &$report): void +{ + foreach (array_diff(array_keys($a['tables']), array_keys($b['tables'])) as $table) { + note($report, 'Tables', $table, 'missing from ' . $right); + } + + foreach (array_diff(array_keys($b['tables']), array_keys($a['tables'])) as $table) { + note($report, 'Tables', $table, 'only in ' . $right); + } + + foreach (array_intersect_key($a['tables'], $b['tables']) as $name => $table) { + $other = $b['tables'][$name]; + + foreach ($table['attributes'] as $key => $value) { + if (($other['attributes'][$key] ?? null) !== $value) { + note($report, 'Tables', $name, $key . ': ' . $value . ' in ' . $left . ', ' . ($other['attributes'][$key] ?? 'nothing') . ' in ' . $right); + } + } + + compare_parts($report, $name, 'column', $table['columns'], $other['columns'], describe_column(...), $left, $right); + compare_parts($report, $name, 'index', $table['indexes'], $other['indexes'], describe_index(...), $left, $right); + } +} + +/** + * The columns of a table and the indexes of a table are compared the same way: + * what is only on one side, and what is on both but described differently. + * + * @param array $report + * @param string $table + * @param string $noun + * @param array $mine + * @param array $theirs + * @param callable $describe + * @param string $left + * @param string $right + */ +function compare_parts(array &$report, string $table, string $noun, array $mine, array $theirs, callable $describe, string $left, string $right): void +{ + foreach (array_diff(array_keys($mine), array_keys($theirs)) as $name) { + note($report, 'Tables', $table, $noun . ' ' . $name . ' is missing from ' . $right . ' (' . $describe($mine[$name]) . ')'); + } + + foreach (array_diff(array_keys($theirs), array_keys($mine)) as $name) { + note($report, 'Tables', $table, $noun . ' ' . $name . ' is only in ' . $right . ' (' . $describe($theirs[$name]) . ')'); + } + + foreach (array_intersect_key($mine, $theirs) as $name => $definition) { + $one = $describe($definition); + $two = $describe($theirs[$name]); + + if ($one !== $two) { + note($report, 'Tables', $table, $noun . ' ' . $name, [ + $left . ': ' . $one, + $right . ': ' . $two, + ]); + } + } +} + +/** + * @param array $column + * @return string + */ +function describe_column(array $column): string +{ + return implode(' ', array_filter([ + $column['type'], + $column['nullable'] ? 'NULL' : 'NOT NULL', + $column['default'] === null ? '' : 'DEFAULT ' . $column['default'], + $column['extra'], + $column['collation'] === null ? '' : 'COLLATE ' . $column['collation'], + ])); +} + +/** + * @param array $index + * @return string + */ +function describe_index(array $index): string +{ + $kind = $index['primary'] ? 'primary' : ($index['unique'] ? 'unique' : 'index'); + + return $kind . ' (' . implode(', ', $index['columns']) . ')'; +} + +/** + * @param string $title + * @param array $a + * @param array $b + * @param string $right + * @param array $report + */ +function compare_lists(string $title, array $a, array $b, string $right, array &$report): void +{ + foreach (array_diff($a, $b) as $name) { + note($report, $title, $name, 'missing from ' . $right); + } + + foreach (array_diff($b, $a) as $name) { + note($report, $title, $name, 'only in ' . $right); + } +} + +/** + * Column order is not part of the schema in any sense that matters to a query, + * but it is the one difference that is guaranteed: the upgrader appends, so + * every column any migration added sits at the end of the table rather than + * where the schema class puts it. Worth saying once per table, and never worth + * failing over. + * + * @param array $a + * @param array $b + * @param array $report + */ +function compare_column_order(array $a, array $b, array &$report): void +{ + foreach (array_intersect_key($a['tables'], $b['tables']) as $name => $table) { + $mine = order_of($table['columns']); + $theirs = order_of($b['tables'][$name]['columns']); + + // Only where both hold the same columns. A table that is missing one + // has been reported already, and would report here as well. + if ($mine !== $theirs && sort_copy($mine) === sort_copy($theirs)) { + note($report, 'Column order', $name, 'the columns are in a different order'); + } + } +} + +/** + * @param array $columns + * @return array + */ +function order_of(array $columns): array +{ + uasort($columns, fn($a, $b) => $a['position'] <=> $b['position']); + + return array_keys($columns); +} + +/** + * @param array $list + * @return array + */ +function sort_copy(array $list): array +{ + sort($list); + + return $list; +} diff --git a/.gitignore b/.gitignore index 7bcc6bba3a0..9f805340d40 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,9 @@ Thumbs.db # installs without reinstalling. Generated secrets and a machine-specific # board URL: local to whoever ran the installer. /.docker/settings/ +# Schema readings and reports from compare-upgrade.sh. They describe one +# machine's databases at one moment, and are rewritten on every run. +/.docker/compare/ # Test / Private files # ######################## From 887d73d4a61ec5b01beeadfbec4c4e1b756b353b Mon Sep 17 00:00:00 2001 From: albertlast Date: Sun, 16 Aug 2026 11:04:47 +0200 Subject: [PATCH 10/11] Reads the PostgreSQL objects pg_dump can see and the tool could not Checking the reading against pg_dump --schema-only of the same database found two gaps, both of them the kind of thing this is supposed to notice. Three indexes were missing entirely. An index on an expression stores 0 in indkey and has no pg_attribute row, so joining to that table dropped those keys, and an index every one of whose keys is an expression disappeared with them -- idx_member_name_low, idx_real_name_low and idx_birthdate2 on a stock install. pg_get_indexdef() per key renders a plain column and an expression alike, and needs no join at all. The compatibility functions were not read. find_in_set(), instr(), from_unixtime(), the group_concat aggregate and the rest are created at install; a query naming one of them fails outright where it is absent, which makes a missing one a worse problem than a missing index, not a lesser one. The two readings now agree object for object: 72 tables, 113 indexes, 69 primary keys, 41 sequences, 19 functions. All pg_dump still reports that this does not are the public schema and the comment on it. Signed-off-by: albertlast --- .docker/README.md | 10 ++++++++++ .docker/schema-tool.php | 35 +++++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.docker/README.md b/.docker/README.md index 9049d232711..46de081d17b 100644 --- a/.docker/README.md +++ b/.docker/README.md @@ -224,6 +224,16 @@ rather than comparing anyway. A half-upgraded database differs from a fresh install in hundreds of places, all of them the honest consequence of the migrations that never ran, and none of them worth reading. +On PostgreSQL the reading also covers sequences and the compatibility functions +SMF installs — `find_in_set()`, `instr()`, the `group_concat` aggregate and the +rest. A query naming one of those fails outright when it is not there, so a +missing function counts as a schema difference like a missing column does. + +The reading was checked against `pg_dump --schema-only` of the same database: +72 tables, 113 indexes, 69 primary keys, 41 sequences and 19 functions, the +same on both sides. The only things `pg_dump` reports that this does not are +the `public` schema itself and the comment on it. + The tool underneath is usable on its own, against any two SMF databases on the same engine — two forums you already have, or the same forum before and after something you are testing: diff --git a/.docker/schema-tool.php b/.docker/schema-tool.php index 6d4f2c20d16..8105f7f5b56 100644 --- a/.docker/schema-tool.php +++ b/.docker/schema-tool.php @@ -270,7 +270,7 @@ function read_mysql(string $host, int $port, string $db, string $user, string $p $link->close(); - return finish($tables, $settings, []); + return finish($tables, $settings, [], []); } /** @@ -347,15 +347,21 @@ function read_postgresql(string $host, int $port, string $db, string $user, stri // order; indkey is an int2vector, and reading it any other way sorts the // columns alphabetically, which would make (id_group, id_board) and // (id_board, id_group) look like the same index. They are not. + // + // pg_get_indexdef() per key rather than a join to pg_attribute, because an + // index on an expression stores 0 in indkey and has no pg_attribute row to + // join to. Joining loses those keys, and an index every one of whose keys + // is an expression disappears entirely -- which is three of them on a + // stock install, idx_member_name_low and idx_real_name_low among them. $rows = query_postgresql($link, ' SELECT c.relname AS table_name, i.relname AS index_name, - ix.indisunique, ix.indisprimary, a.attname + ix.indisunique, ix.indisprimary, + pg_get_indexdef(ix.indexrelid, k.ord::int, true) AS keydef FROM pg_index AS ix INNER JOIN pg_class AS c ON (c.oid = ix.indrelid) INNER JOIN pg_class AS i ON (i.oid = ix.indexrelid) INNER JOIN pg_namespace AS n ON (n.oid = c.relnamespace) CROSS JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, ord) - INNER JOIN pg_attribute AS a ON (a.attrelid = c.oid AND a.attnum = k.attnum) WHERE n.nspname = \'public\' ORDER BY c.relname, i.relname, k.ord', []); @@ -374,7 +380,22 @@ function read_postgresql(string $host, int $port, string $db, string $user, stri 'columns' => [], ]; - $tables[$table]['indexes'][$index]['columns'][] = $row['attname']; + $tables[$table]['indexes'][$index]['columns'][] = $row['keydef']; + } + + // SMF's compatibility layer: find_in_set(), instr(), from_unixtime() and + // the rest of the MySQL shims, plus the group_concat aggregate. A query + // naming one of these fails outright if the install never created it, so + // they belong in the reading as much as the tables do. + $routines = []; + + foreach (query_postgresql($link, ' + SELECT p.proname || \'(\' || pg_get_function_arguments(p.oid) || \')\' AS signature + FROM pg_proc AS p + INNER JOIN pg_namespace AS n ON (n.oid = p.pronamespace) + WHERE n.nspname = \'public\' + ORDER BY signature', []) as $row) { + $routines[] = $row['signature']; } $sequences = []; @@ -397,7 +418,7 @@ function read_postgresql(string $host, int $port, string $db, string $user, stri pg_close($link); - return finish($tables, $settings, $sequences); + return finish($tables, $settings, $sequences, $routines); } /** @@ -432,7 +453,7 @@ function postgresql_type(array $row): string * @param array $sequences * @return array */ -function finish(array $tables, array $settings, array $sequences): array +function finish(array $tables, array $settings, array $sequences, array $routines): array { ksort($tables); sort($settings); @@ -445,6 +466,7 @@ function finish(array $tables, array $settings, array $sequences): array return [ 'tables' => $tables, 'sequences' => $sequences, + 'routines' => $routines, 'settings' => $settings, ]; } @@ -534,6 +556,7 @@ function cmd_diff(array $files): int if ($a['engine'] === 'postgresql') { compare_lists('Sequences', $a['sequences'], $b['sequences'], $right, $schema); + compare_lists('Functions', $a['routines'] ?? [], $b['routines'] ?? [], $right, $schema); } compare_lists('Settings', $a['settings'], $b['settings'], $right, $aside); From 02e65ddbd6c603db80a37e138a978642d5ae6e8a Mon Sep 17 00:00:00 2001 From: albertlast Date: Sun, 16 Aug 2026 11:10:49 +0200 Subject: [PATCH 11/11] Reads the MySQL row format and what a generated column is generated from Same check as the last commit, the other way round: against mysqldump --no-data --routines --triggers --events. Column for column and index for index the two agree, 538 and 179, and SMF creates no routines, triggers or events on MySQL, so there is no equivalent of the missing functions. Two things were being read too shallowly, though. EXTRA says that a column is generated and never what from. smf_messages has three STORED columns read out of the edit_history JSON -- modified_time, modified_name and modified_reason -- and a wrong path in one of them gives a column of the right type holding quietly the wrong value. Comparing GENERATION_EXPRESSION is what tells $[0][7] from $[0][9]. ROW_FORMAT was not read at all. It is not decoration on this schema: COMPACT caps an index key at 767 bytes where DYNAMIC allows 3072, so a table left behind in the older format is one where half of SMF's indexes cannot be created at their full width -- and index width is already the second largest group of differences an upgrade produces. AUTO_INCREMENT is still deliberately not read. It measures how much a database has been used, not what shape it is. Signed-off-by: albertlast --- .docker/README.md | 11 +++++++---- .docker/schema-tool.php | 23 ++++++++++++++++++++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.docker/README.md b/.docker/README.md index 46de081d17b..bc6ff2363bd 100644 --- a/.docker/README.md +++ b/.docker/README.md @@ -229,10 +229,13 @@ SMF installs — `find_in_set()`, `instr()`, the `group_concat` aggregate and th rest. A query naming one of those fails outright when it is not there, so a missing function counts as a schema difference like a missing column does. -The reading was checked against `pg_dump --schema-only` of the same database: -72 tables, 113 indexes, 69 primary keys, 41 sequences and 19 functions, the -same on both sides. The only things `pg_dump` reports that this does not are -the `public` schema itself and the comment on it. +Both readings were checked name by name against the engine's own schema dump — +`pg_dump --schema-only` and `mysqldump --no-data --routines --triggers +--events` — and agree with them: 72 tables, 538 columns, 179 keys, and on +PostgreSQL 41 sequences and 19 functions besides. The only things either dump +reports that this does not are the `public` schema and the comment on it, and +the `AUTO_INCREMENT` counter, which measures how much a database has been used +rather than what shape it is. The tool underneath is usable on its own, against any two SMF databases on the same engine — two forums you already have, or the same forum before and after diff --git a/.docker/schema-tool.php b/.docker/schema-tool.php index 8105f7f5b56..9cca9cc3cfb 100644 --- a/.docker/schema-tool.php +++ b/.docker/schema-tool.php @@ -189,8 +189,13 @@ function read_mysql(string $host, int $port, string $db, string $user, string $p $tables = []; + // ROW_FORMAT is here because it is not decoration: COMPACT caps an index + // key at 767 bytes where DYNAMIC allows 3072, so a table left in the older + // format is a table where half of SMF's indexes cannot be created at their + // full width. AUTO_INCREMENT is deliberately not here -- it counts rows, + // and differs between any two databases that have been used differently. $rows = query_mysql($link, ' - SELECT TABLE_NAME, ENGINE, TABLE_COLLATION + SELECT TABLE_NAME, ENGINE, TABLE_COLLATION, ROW_FORMAT FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = \'BASE TABLE\'', [$db]); @@ -199,15 +204,22 @@ function read_mysql(string $host, int $port, string $db, string $user, string $p 'attributes' => [ 'engine' => (string) $row['ENGINE'], 'collation' => (string) $row['TABLE_COLLATION'], + 'row_format' => (string) $row['ROW_FORMAT'], ], 'columns' => [], 'indexes' => [], ]; } + // GENERATION_EXPRESSION because EXTRA says only that a column is generated, + // never what from. messages.modified_time and its two neighbours are STORED + // columns read out of the edit_history JSON, and a wrong path in one of + // them produces a column that is the right type and quietly the wrong + // value -- which is the failure this is least able to afford missing. $rows = query_mysql($link, ' SELECT TABLE_NAME, COLUMN_NAME, ORDINAL_POSITION, COLUMN_TYPE, - IS_NULLABLE, COLUMN_DEFAULT, EXTRA, COLLATION_NAME + IS_NULLABLE, COLUMN_DEFAULT, EXTRA, COLLATION_NAME, + GENERATION_EXPRESSION FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? ORDER BY TABLE_NAME, ORDINAL_POSITION', [$db]); @@ -228,6 +240,7 @@ function read_mysql(string $host, int $port, string $db, string $user, string $p // auto_increment lives here, and so does ON UPDATE. 'extra' => (string) $row['EXTRA'], 'collation' => $row['COLLATION_NAME'], + 'generated' => (string) $row['GENERATION_EXPRESSION'], 'position' => (int) $row['ORDINAL_POSITION'], ]; } @@ -317,7 +330,7 @@ function read_postgresql(string $host, int $port, string $db, string $user, stri $rows = query_postgresql($link, ' SELECT table_name, column_name, ordinal_position, data_type, character_maximum_length, numeric_precision, numeric_scale, - is_nullable, column_default + is_nullable, column_default, generation_expression FROM information_schema.columns WHERE table_schema = \'public\' ORDER BY table_name, ordinal_position', []); @@ -339,6 +352,9 @@ function read_postgresql(string $host, int $port, string $db, string $user, stri // that a column that lost its sequence reads the same way on both. 'extra' => $default !== null && str_starts_with($default, 'nextval(') ? 'auto_increment' : '', 'collation' => null, + // PostgreSQL has generated columns too, and SMF uses none of them + // here; reading the column keeps both engines the same shape. + 'generated' => (string) ($row['generation_expression'] ?? ''), 'position' => (int) $row['ordinal_position'], ]; } @@ -740,6 +756,7 @@ function describe_column(array $column): string $column['nullable'] ? 'NULL' : 'NOT NULL', $column['default'] === null ? '' : 'DEFAULT ' . $column['default'], $column['extra'], + empty($column['generated']) ? '' : 'AS (' . $column['generated'] . ')', $column['collation'] === null ? '' : 'COLLATE ' . $column['collation'], ])); }