diff --git a/.docker/README.md b/.docker/README.md index 2556731adf..1a90dcdfc4 100644 --- a/.docker/README.md +++ b/.docker/README.md @@ -71,6 +71,14 @@ though it succeeded — it pauses so a human can read its "N duplicate tables ignored" report, and the form's `pop_done` field is the short-circuit past it. Passing `pop_done` on the first pass would skip building the schema entirely. +It then deletes `install.php`, which the installer asks for but cannot do +itself — its `?delete` link is a GET, and command line arguments only ever reach +`$_POST`. That matters more than it sounds: while the file is there +`Settings.php` redirects every request back into the installer, and SMF puts a +"MAJOR SECURITY RISK" box on every page it shows an administrator. Reinstalling +still works, because `reset.sh` runs first and does not return until the +entrypoint has staged a fresh copy. + Two flags worth knowing: - `--force` reinstalls even when a forum is already there. Without it the @@ -102,6 +110,35 @@ are gitignored. installer, discarding that forum. `use-engine.sh` switches between forums, `reset.sh` throws one away. +## Accounts and passwords + +Two forums, each with its own administrator, and a password chosen months ago is +a recipe for an afternoon of hand written SQL. `user.sh` is there so it is not: + +```sh +.docker/user.sh list +.docker/user.sh check admin 'password' +.docker/user.sh reset admin 'a new password' +``` + +`check` exits 0 when SMF would accept the password and 1 when it would not, so +it works in a conditional as well as by eye. It also points out an account that +is not activated, which fails to log in with a correct password and looks +exactly like a wrong one. + +`--engine mysql|postgresql` reads the settings `use-engine.sh` saved for that +engine, so the *other* forum can be inspected without switching to it: + +```sh +.docker/user.sh check admin 'password' --engine mysql +``` + +The hashing goes through SMF's own `Security` class rather than being written +here, so what `reset` puts in the table is by construction what `Login2` expects +to find. It clears `passwd_flood` at the same time: SMF locks an account out for +a while after enough wrong guesses, and a fresh password behind a lockout looks +exactly like a password that did not take. + ### Installing in a browser instead On first boot the entrypoint writes a `Settings.php` pre-filled for the chosen @@ -182,6 +219,67 @@ never need to restart for a PHP change. To reinstall from scratch: `.docker/install-forum.sh --engine mysql --force`. To wipe everything including the volumes: `docker compose down -v`. +## Comparing an upgrade against a fresh install + +The installer builds the schema from `Sources/Db/Schema/v3_0/` in one go. The +upgrader arrives at the same place through a hundred-odd migrations applied to +whatever 2.1 left behind. They are meant to converge, and nothing checks that +they do: + +```bash +.docker/compare-upgrade.sh --engine mysql --baseline path/to/a-2.1-dump.sql +``` + +That empties the database, loads the dump, upgrades it, reads the schema, +reinstalls from scratch, reads that too, and reports every place the two +disagree — a column of the wrong type, an index that was never created, a +primary key quietly dropped. It ends with the fresh install in place, and takes +five to ten minutes. + +`--baseline` takes any SQL dump of a 2.1 database. A dump of a real forum is +the better test; the [2.1 development environment][baseline] builds a synthetic +one designed to hold something in every table an upgrade touches, which is +useful when you have no real forum to hand. + +Two kinds of difference are reported but do not fail the run, because a real +forum always has some: the contents of `settings`, and the order columns sit in +within a table. Everything else is a schema difference and sets the exit code. + +If the upgrade does not reach the end, the script stops there and says so +rather than comparing anyway. A half-upgraded database differs from a fresh +install in hundreds of places, all of them the honest consequence of the +migrations that never ran, and none of them worth reading. + +On PostgreSQL the reading also covers sequences and the compatibility functions +SMF installs — `find_in_set()`, `instr()`, the `group_concat` aggregate and the +rest. A query naming one of those fails outright when it is not there, so a +missing function counts as a schema difference like a missing column does. + +Both readings were checked name by name against the engine's own schema dump — +`pg_dump --schema-only` and `mysqldump --no-data --routines --triggers +--events` — and agree with them: 72 tables, 538 columns, 179 keys, and on +PostgreSQL 41 sequences and 19 functions besides. The only things either dump +reports that this does not are the `public` schema and the comment on it, and +the `AUTO_INCREMENT` counter, which measures how much a database has been used +rather than what shape it is. + +The tool underneath is usable on its own, against any two SMF databases on the +same engine — two forums you already have, or the same forum before and after +something you are testing: + +```bash +docker compose exec web php .docker/schema-tool.php dump --engine mysql --db smf > before.json +# ... do the thing ... +docker compose exec web php .docker/schema-tool.php dump --engine mysql --db smf > after.json +docker compose exec web php .docker/schema-tool.php diff before.json after.json +``` + +It talks to the database directly rather than through SMF, so it works on a +database SMF would refuse to run on — which is usually the one you want to look +at. + +[baseline]: https://github.com/SimpleMachines/SMF/pull/9330 + ## Debugging SQL with the PostgreSQL log The `postgres` log is the best tool in the stack for tracking down a broken @@ -247,4 +345,12 @@ compose.yaml the stack .docker/mysql/init/10-smf.sh runs once on first mysql database creation .docker/postgres/init/10-smf.sh runs once on first postgres database creation .docker/env.example optional overrides + +.docker/lib.sh paths, credentials and engine names, shared +.docker/install-forum.sh install a forum with no browser involved +.docker/reset.sh empty one engine and restage the installer +.docker/use-engine.sh switch which installed forum is live +.docker/user.sh inspect accounts, check and reset passwords +.docker/compare-upgrade.sh upgrade a 2.1 dump, install 3.0, diff the two +.docker/schema-tool.php read a database's shape, and compare readings ``` diff --git a/.docker/compare-upgrade.sh b/.docker/compare-upgrade.sh new file mode 100755 index 0000000000..185d07b21a --- /dev/null +++ b/.docker/compare-upgrade.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Upgrades a 2.1 database to 3.0, installs 3.0 from scratch, and reports where +# the two schemas disagree. +# +# .docker/compare-upgrade.sh --engine mysql --baseline ../SMF-2.1/.docker/baseline/artifacts/2.1.7-1/small/mysql.sql +# .docker/compare-upgrade.sh --engine postgresql --baseline ../SMF-2.1/.docker/baseline/artifacts/2.1.7-1/small/postgres.sql +# +# The installer builds the schema from Sources/Db/Schema/v3_0/ in one go. The +# upgrader arrives at the same place through a hundred-odd migrations applied +# to whatever 2.1 left behind. Nothing checks that those two agree, and where +# they do not, the forum that upgraded is running on a schema that has never +# been tested against -- a column of the wrong type, an index that was never +# created, a primary key quietly dropped. +# +# --baseline takes any SQL dump of a 2.1 database. The one this was written +# against is the committed baseline from the 2.1 development environment, which +# is built to hold something in every table an upgrade touches, but a dump of a +# real forum works and is a better test. +# +# Both databases for the chosen engine are rebuilt, twice. Anything already +# installed on that engine is destroyed, and what remains at the end is the +# fresh install. The other engine is not touched. +# +# Runs on the host. Expect five to ten minutes per engine. +set -euo pipefail + +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" + +ENGINE='' +BASELINE='' +OUT="$DOCKER_DIR/compare" + +while [ $# -gt 0 ]; do + case "$1" in + --engine) ENGINE="$2"; shift 2 ;; + --engine=*) ENGINE="${1#*=}"; shift ;; + --baseline) BASELINE="$2"; shift 2 ;; + --baseline=*) BASELINE="${1#*=}"; shift ;; + --out) OUT="$2"; shift 2 ;; + --out=*) OUT="${1#*=}"; shift ;; + -h|--help) sed -n '2,25p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +[ -n "$ENGINE" ] || die 'need --engine mysql|postgresql|both' +ENGINES=$(engine_list "$ENGINE") || die "unknown engine: $ENGINE" +[ -n "$BASELINE" ] || die 'need --baseline ' +[ -f "$BASELINE" ] || die "no such file: $BASELINE" + +cd "$BOARD_DIR" +mkdir -p "$OUT" +OUT=$(cd -- "$OUT" && pwd) + +# The tool that reads the two files runs in the container, which sees the +# repository and nothing else, so the output has to live somewhere inside it. +case "$OUT" in + "$BOARD_DIR"/*) OUT_REL="${OUT#"$BOARD_DIR"/}" ;; + *) die "--out has to be somewhere inside the repository, since the container cannot see anywhere else" ;; +esac + +# Reads the shape of the database the given engine is pointed at. The tool runs +# in the container, so it uses the container-internal host and port rather than +# the ones published in compose.yaml. +snapshot() { + local smf_type="$1" label="$2" file="$3" + + docker compose exec -T web php .docker/schema-tool.php dump \ + --engine "$smf_type" \ + --db "$DB_NAME" \ + --prefix "$DB_PREFIX" \ + --host "$(engine_server "$smf_type")" \ + --port "$(engine_port "$smf_type")" \ + --user "$DB_USER" \ + --pass "$DB_PASSWORD" \ + --label "$label" > "$file" + + [ -s "$file" ] || die "${smf_type}: the ${label} reading came back empty" +} + +# The version this checkout is, which is what a finished upgrade has to leave +# in the database. +smf_version() { + sed -n "s/.*define('SMF_VERSION', '\([^']*\)').*/\1/p" index.php | head -1 +} + +load_baseline() { + local smf_type="$1" service + service=$(engine_service "$smf_type") + + if [ "$smf_type" = 'mysql' ]; then + docker compose exec -T -e MYSQL_PWD="$DB_PASSWORD" "$service" \ + mysql -u"$DB_USER" -D "$DB_NAME" < "$BASELINE" + else + # ON_ERROR_STOP so that a dump taken from a database that is not empty, + # or one taken with a different owner, stops here rather than producing + # a half-loaded forum that then fails somewhere in the upgrader and + # looks like a migration bug. + docker compose exec -T "$service" \ + psql -v ON_ERROR_STOP=1 -q -U "$DB_USER" -d "$DB_NAME" < "$BASELINE" >/dev/null + fi +} + +compare_one() { + local smf_type="$1" version + + # ---------------------------------------------------------- the upgrade + log "${smf_type}: emptying the database" + "$DOCKER_DIR/reset.sh" --engine "$smf_type" >/dev/null + + log "${smf_type}: loading ${BASELINE##*/}" + load_baseline "$smf_type" + + version=$(installed_version "$smf_type" || true) + + if [ -z "$version" ]; then + warn "${smf_type}: the dump loaded but there is no forum in it -- wrong prefix, or not an SMF dump" + + return 1 + fi + + log "${smf_type}: the dump is SMF ${version}" + + # The upgrader ships in other/ and expects to be run from the board + # directory, the same way install.php does. Nobody upgrading a real forum + # has install.php sitting there, so neither does this. + rm -f install.php + cp other/upgrade.php upgrade.php + + log "${smf_type}: upgrading" + + local upgraded=0 + docker compose exec -T web php upgrade.php > "$OUT/upgrade-${smf_type}.log" 2>&1 || upgraded=$? + + rm -f upgrade.php + + if [ "$upgraded" -ne 0 ]; then + warn "${smf_type}: the upgrader exited ${upgraded} -- ${OUT}/upgrade-${smf_type}.log" + + return 1 + fi + + # The upgrader reports a failed migration and stops, but still exits 0, so + # the version in the database is the only thing that says whether it got to + # the end. Checking it matters more than it looks: a half-upgraded database + # differs from a fresh install in hundreds of places, all of them the honest + # consequence of the migrations that never ran, and none of them the kind of + # difference this script exists to find. + version=$(installed_version "$smf_type" || true) + + if [ "$version" != "$(smf_version)" ]; then + warn "${smf_type}: the upgrade stopped at SMF ${version:-nothing}, expected $(smf_version)" + warn "${smf_type}: the last thing it said is at the end of ${OUT}/upgrade-${smf_type}.log" + + return 1 + fi + + log "${smf_type}: upgraded to SMF ${version}" + + snapshot "$smf_type" upgraded "$OUT/upgraded-${smf_type}.json" + + # ----------------------------------------------------- the fresh install + # --force because there is an installed forum now, and install-forum.sh + # leaves one alone unless told otherwise. + log "${smf_type}: installing from scratch" + "$DOCKER_DIR/install-forum.sh" --engine "$smf_type" --force >/dev/null + + snapshot "$smf_type" fresh "$OUT/fresh-${smf_type}.json" + + # ------------------------------------------------------------ the report + local status=0 + + docker compose exec -T web php .docker/schema-tool.php diff \ + "${OUT_REL}/fresh-${smf_type}.json" \ + "${OUT_REL}/upgraded-${smf_type}.json" \ + > "$OUT/report-${smf_type}.txt" || status=$? + + echo + cat "$OUT/report-${smf_type}.txt" + echo + + if [ "$status" -eq 0 ]; then + log "${smf_type}: the upgraded schema matches a fresh install" + else + log "${smf_type}: the schemas differ -- ${OUT}/report-${smf_type}.txt" + fi + + return "$status" +} + +failed=0 + +for smf_type in $ENGINES; do + compare_one "$smf_type" || failed=1 +done + +exit "$failed" diff --git a/.docker/install-forum.sh b/.docker/install-forum.sh index 613b604c90..8bc5570027 100755 --- a/.docker/install-forum.sh +++ b/.docker/install-forum.sh @@ -96,6 +96,13 @@ install_one() { --password2="$SMF_ADMIN_PASS" ) + # reset.sh does not return until the entrypoint has staged this, so its + # absence means something went wrong there rather than here. Worth saying so: + # without it php reports "Could not open input file: install.php", which reads + # like a broken script rather than a forum that was never made installable. + docker compose exec -T web test -f install.php \ + || die "${smf_type}: install.php is not staged, so there is nothing to run (docker compose logs web)" + log "${smf_type}: building the schema" docker compose exec -T web php install.php "${args[@]}" >/dev/null @@ -107,6 +114,17 @@ install_one() { [ -n "$version" ] || die "${smf_type}: the installer finished but the forum is not installed" + # The installer tells you to delete this and cannot do it itself: its ?delete + # link is a GET, and command line arguments only ever reach $_POST. Leaving it + # is not cosmetic - Settings.php redirects every request back into the + # installer while it is there, and SMF puts a "MAJOR SECURITY RISK: you have + # not removed install.php" box on every page it shows an administrator. + # + # Safe to delete even though a reinstall needs it again: install_one() always + # calls reset.sh first, and reset.sh clears Settings.php and waits for the + # entrypoint to put a fresh copy back before returning. + rm -f install.php + log "${smf_type}: installed SMF ${version}" if [ "$PIN_SECRETS" -eq 1 ]; then diff --git a/.docker/schema-tool.php b/.docker/schema-tool.php new file mode 100644 index 0000000000..9cca9cc3cf --- /dev/null +++ b/.docker/schema-tool.php @@ -0,0 +1,838 @@ + fresh.json + * php .docker/schema-tool.php diff fresh.json upgraded.json + * + * Runs inside the web container, and talks to the database directly rather + * than through SMF. Nothing here loads Settings.php or boots the forum: the + * database being examined is frequently one that SMF would refuse to run on, + * which is the whole point of looking at it. + * + * Simple Machines Forum (SMF) + * + * @package SMF + * @author Simple Machines https://www.simplemachines.org + * @copyright 2026 Simple Machines and individual contributors + * @license https://www.simplemachines.org/about/smf/license.php BSD + * + * @version 3.0 Alpha 4 + */ + +declare(strict_types=1); + +if (PHP_SAPI !== 'cli') { + exit("This is a command line script.\n"); +} + +$argv = $_SERVER['argv']; +$mode = $argv[1] ?? ''; + +exit(match ($mode) { + 'dump' => cmd_dump(parse_options(array_slice($argv, 2))), + 'diff' => cmd_diff(array_slice($argv, 2)), + default => usage(), +}); + +/** + * @return int + */ +function usage(): int +{ + fwrite(STDERR, <<<'TEXT' + Reads the shape of an SMF database, and compares two of those readings. + + php .docker/schema-tool.php dump --engine mysql --db smf > fresh.json + php .docker/schema-tool.php diff fresh.json upgraded.json + + dump options, with the defaults compose.yaml gives: + + --engine mysql or postgresql (required) + --db database name (smf) + --prefix table prefix (smf_) + --host database host (mysql / postgres) + --port database port (3306 / 5432) + --user database user (smf) + --pass database password (smf) + --label what to call this reading in a diff (the database name) + + diff exits 1 when the two disagree about the schema, 0 when they do not. + Settings and column order are reported but do not decide the exit code: + they are differences a forum can live with, and a real forum always has + some. + + TEXT); + + return 2; +} + +/** + * Turns --name=value and --name value into an array. Unknown names are an + * error rather than a shrug: a misspelled --prefix would otherwise read the + * wrong tables and report every one of them as missing. + * + * @param array $args + * @return array + */ +function parse_options(array $args): array +{ + $known = ['engine', 'db', 'prefix', 'host', 'port', 'user', 'pass', 'label']; + $options = []; + + while ($args !== []) { + $arg = array_shift($args); + + if (!str_starts_with($arg, '--')) { + fail('unexpected argument: ' . $arg); + } + + $name = substr($arg, 2); + $value = null; + + if (str_contains($name, '=')) { + [$name, $value] = explode('=', $name, 2); + } + + if (!in_array($name, $known, true)) { + fail('unknown option: --' . $name); + } + + $options[$name] = $value ?? (string) array_shift($args); + } + + return $options; +} + +/** + * @param string $message + * @return never + */ +function fail(string $message): never +{ + fwrite(STDERR, 'schema-tool: ' . $message . "\n"); + + exit(2); +} + +/** + * @param array $options + * @return int + */ +function cmd_dump(array $options): int +{ + $engine = match ($options['engine'] ?? '') { + 'mysql', 'mysqli', 'mariadb' => 'mysql', + 'postgresql', 'postgres', 'pgsql' => 'postgresql', + default => fail('need --engine mysql|postgresql'), + }; + + $db = $options['db'] ?? 'smf'; + + $schema = [ + 'engine' => $engine, + 'label' => $options['label'] ?? $db, + 'prefix' => $options['prefix'] ?? 'smf_', + 'database' => $db, + 'read_at' => gmdate('c'), + ]; + + $read = $engine === 'mysql' ? read_mysql(...) : read_postgresql(...); + + $schema += $read( + $options['host'] ?? ($engine === 'mysql' ? 'mysql' : 'postgres'), + (int) ($options['port'] ?? ($engine === 'mysql' ? 3306 : 5432)), + $db, + $options['user'] ?? 'smf', + $options['pass'] ?? 'smf', + $schema['prefix'], + ); + + echo json_encode($schema, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), "\n"; + + return 0; +} + +/** + * Everything is keyed by name with the prefix taken off, so that two databases + * using different prefixes still line up, and so that the report reads in the + * names people use rather than the ones the database stores. + * + * @param string $host + * @param int $port + * @param string $db + * @param string $user + * @param string $pass + * @param string $prefix + * @return array + */ +function read_mysql(string $host, int $port, string $db, string $user, string $pass, string $prefix): array +{ + mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT); + + try { + $link = new mysqli($host, $user, $pass, $db, $port); + } catch (mysqli_sql_exception $e) { + fail('cannot connect to ' . $db . ' on ' . $host . ': ' . $e->getMessage()); + } + + $tables = []; + + // ROW_FORMAT is here because it is not decoration: COMPACT caps an index + // key at 767 bytes where DYNAMIC allows 3072, so a table left in the older + // format is a table where half of SMF's indexes cannot be created at their + // full width. AUTO_INCREMENT is deliberately not here -- it counts rows, + // and differs between any two databases that have been used differently. + $rows = query_mysql($link, ' + SELECT TABLE_NAME, ENGINE, TABLE_COLLATION, ROW_FORMAT + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = \'BASE TABLE\'', [$db]); + + foreach ($rows as $row) { + $tables[unprefix($row['TABLE_NAME'], $prefix)] = [ + 'attributes' => [ + 'engine' => (string) $row['ENGINE'], + 'collation' => (string) $row['TABLE_COLLATION'], + 'row_format' => (string) $row['ROW_FORMAT'], + ], + 'columns' => [], + 'indexes' => [], + ]; + } + + // GENERATION_EXPRESSION because EXTRA says only that a column is generated, + // never what from. messages.modified_time and its two neighbours are STORED + // columns read out of the edit_history JSON, and a wrong path in one of + // them produces a column that is the right type and quietly the wrong + // value -- which is the failure this is least able to afford missing. + $rows = query_mysql($link, ' + SELECT TABLE_NAME, COLUMN_NAME, ORDINAL_POSITION, COLUMN_TYPE, + IS_NULLABLE, COLUMN_DEFAULT, EXTRA, COLLATION_NAME, + GENERATION_EXPRESSION + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = ? + ORDER BY TABLE_NAME, ORDINAL_POSITION', [$db]); + + foreach ($rows as $row) { + $table = unprefix($row['TABLE_NAME'], $prefix); + + if (!isset($tables[$table])) { + continue; + } + + $tables[$table]['columns'][$row['COLUMN_NAME']] = [ + 'type' => (string) $row['COLUMN_TYPE'], + 'nullable' => $row['IS_NULLABLE'] === 'YES', + // A NULL default and no default at all are the same string here, + // so nullability above is what tells them apart. + 'default' => $row['COLUMN_DEFAULT'], + // auto_increment lives here, and so does ON UPDATE. + 'extra' => (string) $row['EXTRA'], + 'collation' => $row['COLLATION_NAME'], + 'generated' => (string) $row['GENERATION_EXPRESSION'], + 'position' => (int) $row['ORDINAL_POSITION'], + ]; + } + + // SUB_PART is the length of a prefix index, which SMF uses on a few of the + // longer text columns. A missing one is a real difference, so it is part of + // the column's name in the key rather than being dropped. + $rows = query_mysql($link, ' + SELECT TABLE_NAME, INDEX_NAME, NON_UNIQUE, SEQ_IN_INDEX, COLUMN_NAME, SUB_PART + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = ? + ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX', [$db]); + + foreach ($rows as $row) { + $table = unprefix($row['TABLE_NAME'], $prefix); + + if (!isset($tables[$table])) { + continue; + } + + $index = $row['INDEX_NAME']; + + $tables[$table]['indexes'][$index] ??= [ + 'primary' => $index === 'PRIMARY', + 'unique' => (int) $row['NON_UNIQUE'] === 0, + 'columns' => [], + ]; + + $tables[$table]['indexes'][$index]['columns'][] = $row['COLUMN_NAME'] + . ($row['SUB_PART'] === null ? '' : '(' . $row['SUB_PART'] . ')'); + } + + $settings = []; + + if (isset($tables['settings'])) { + foreach (query_mysql($link, 'SELECT variable FROM `' . $prefix . 'settings`', []) as $row) { + $settings[] = $row['variable']; + } + } + + $link->close(); + + return finish($tables, $settings, [], []); +} + +/** + * @param string $host + * @param int $port + * @param string $db + * @param string $user + * @param string $pass + * @param string $prefix + * @return array + */ +function read_postgresql(string $host, int $port, string $db, string $user, string $pass, string $prefix): array +{ + $link = @pg_connect(sprintf( + 'host=%s port=%d dbname=%s user=%s password=%s', + $host, + $port, + $db, + $user, + $pass, + )); + + if ($link === false) { + fail('cannot connect to ' . $db . ' on ' . $host); + } + + $tables = []; + + $rows = query_postgresql($link, ' + SELECT table_name + FROM information_schema.tables + WHERE table_schema = \'public\' AND table_type = \'BASE TABLE\'', []); + + foreach ($rows as $row) { + $tables[unprefix($row['table_name'], $prefix)] = [ + // PostgreSQL has no per-table storage engine or collation, so the + // MySQL side of this stays empty rather than inventing one. + 'attributes' => [], + 'columns' => [], + 'indexes' => [], + ]; + } + + $rows = query_postgresql($link, ' + SELECT table_name, column_name, ordinal_position, data_type, + character_maximum_length, numeric_precision, numeric_scale, + is_nullable, column_default, generation_expression + FROM information_schema.columns + WHERE table_schema = \'public\' + ORDER BY table_name, ordinal_position', []); + + foreach ($rows as $row) { + $table = unprefix($row['table_name'], $prefix); + + if (!isset($tables[$table])) { + continue; + } + + $default = $row['column_default']; + + $tables[$table]['columns'][$row['column_name']] = [ + 'type' => postgresql_type($row), + 'nullable' => $row['is_nullable'] === 'YES', + 'default' => $default, + // The nearest thing PostgreSQL has to MySQL's auto_increment, so + // that a column that lost its sequence reads the same way on both. + 'extra' => $default !== null && str_starts_with($default, 'nextval(') ? 'auto_increment' : '', + 'collation' => null, + // PostgreSQL has generated columns too, and SMF uses none of them + // here; reading the column keeps both engines the same shape. + 'generated' => (string) ($row['generation_expression'] ?? ''), + 'position' => (int) $row['ordinal_position'], + ]; + } + + // unnest() with ordinality is what keeps a composite index in its own + // order; indkey is an int2vector, and reading it any other way sorts the + // columns alphabetically, which would make (id_group, id_board) and + // (id_board, id_group) look like the same index. They are not. + // + // pg_get_indexdef() per key rather than a join to pg_attribute, because an + // index on an expression stores 0 in indkey and has no pg_attribute row to + // join to. Joining loses those keys, and an index every one of whose keys + // is an expression disappears entirely -- which is three of them on a + // stock install, idx_member_name_low and idx_real_name_low among them. + $rows = query_postgresql($link, ' + SELECT c.relname AS table_name, i.relname AS index_name, + ix.indisunique, ix.indisprimary, + pg_get_indexdef(ix.indexrelid, k.ord::int, true) AS keydef + FROM pg_index AS ix + INNER JOIN pg_class AS c ON (c.oid = ix.indrelid) + INNER JOIN pg_class AS i ON (i.oid = ix.indexrelid) + INNER JOIN pg_namespace AS n ON (n.oid = c.relnamespace) + CROSS JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, ord) + WHERE n.nspname = \'public\' + ORDER BY c.relname, i.relname, k.ord', []); + + foreach ($rows as $row) { + $table = unprefix($row['table_name'], $prefix); + + if (!isset($tables[$table])) { + continue; + } + + $index = unprefix($row['index_name'], $prefix); + + $tables[$table]['indexes'][$index] ??= [ + 'primary' => $row['indisprimary'] === 't', + 'unique' => $row['indisunique'] === 't', + 'columns' => [], + ]; + + $tables[$table]['indexes'][$index]['columns'][] = $row['keydef']; + } + + // SMF's compatibility layer: find_in_set(), instr(), from_unixtime() and + // the rest of the MySQL shims, plus the group_concat aggregate. A query + // naming one of these fails outright if the install never created it, so + // they belong in the reading as much as the tables do. + $routines = []; + + foreach (query_postgresql($link, ' + SELECT p.proname || \'(\' || pg_get_function_arguments(p.oid) || \')\' AS signature + FROM pg_proc AS p + INNER JOIN pg_namespace AS n ON (n.oid = p.pronamespace) + WHERE n.nspname = \'public\' + ORDER BY signature', []) as $row) { + $routines[] = $row['signature']; + } + + $sequences = []; + + foreach (query_postgresql($link, ' + SELECT sequence_name + FROM information_schema.sequences + WHERE sequence_schema = \'public\' + ORDER BY sequence_name', []) as $row) { + $sequences[] = unprefix($row['sequence_name'], $prefix); + } + + $settings = []; + + if (isset($tables['settings'])) { + foreach (query_postgresql($link, 'SELECT variable FROM "' . $prefix . 'settings"', []) as $row) { + $settings[] = $row['variable']; + } + } + + pg_close($link); + + return finish($tables, $settings, $sequences, $routines); +} + +/** + * Rebuilds the type as it was written, since information_schema takes it + * apart. varchar(255) arrives as 'character varying' with the length in a + * separate column, and a bare 'character varying' beside it means something + * else entirely. + * + * @param array $row + * @return string + */ +function postgresql_type(array $row): string +{ + $type = (string) $row['data_type']; + + if ($row['character_maximum_length'] !== null) { + return $type . '(' . $row['character_maximum_length'] . ')'; + } + + // Scale is null for integers, where the precision is the width in bits and + // says nothing anyone wants in a report. + if ($type === 'numeric' && $row['numeric_precision'] !== null && $row['numeric_scale'] !== null) { + return $type . '(' . $row['numeric_precision'] . ',' . $row['numeric_scale'] . ')'; + } + + return $type; +} + +/** + * @param array $tables + * @param array $settings + * @param array $sequences + * @return array + */ +function finish(array $tables, array $settings, array $sequences, array $routines): array +{ + ksort($tables); + sort($settings); + + foreach ($tables as &$table) { + ksort($table['columns']); + ksort($table['indexes']); + } + + return [ + 'tables' => $tables, + 'sequences' => $sequences, + 'routines' => $routines, + 'settings' => $settings, + ]; +} + +/** + * @param mysqli $link + * @param string $sql + * @param array $params + * @return array + */ +function query_mysql(mysqli $link, string $sql, array $params): array +{ + $statement = $link->prepare($sql); + + if ($params !== []) { + $statement->bind_param(str_repeat('s', count($params)), ...$params); + } + + $statement->execute(); + $rows = $statement->get_result()->fetch_all(MYSQLI_ASSOC); + $statement->close(); + + return $rows; +} + +/** + * @param \PgSql\Connection $link + * @param string $sql + * @param array $params + * @return array + */ +function query_postgresql(\PgSql\Connection $link, string $sql, array $params): array +{ + $result = @pg_query_params($link, $sql, $params); + + if ($result === false) { + fail('query failed: ' . pg_last_error($link)); + } + + return pg_fetch_all($result, PGSQL_ASSOC); +} + +/** + * @param string $name + * @param string $prefix + * @return string + */ +function unprefix(string $name, string $prefix): string +{ + return $prefix !== '' && str_starts_with($name, $prefix) ? substr($name, strlen($prefix)) : $name; +} + +/** + * @param array $files + * @return int + */ +function cmd_diff(array $files): int +{ + if (count($files) !== 2) { + fail('diff needs two files: the reading to measure against, then the one to judge'); + } + + [$a, $b] = array_map(read_dump(...), $files); + + if ($a['engine'] !== $b['engine']) { + fail('these are different engines (' . $a['engine'] . ' and ' . $b['engine'] . '), and their types do not correspond'); + } + + $left = $a['label']; + $right = $b['label']; + + echo 'Comparing ', $right, ' against ', $left, ' on ', $a['engine'], ".\n\n"; + + if ($a['prefix'] !== $b['prefix']) { + echo "Note: the prefixes differ, so anything naming a table inside a\n", + "default or a sequence differs with it.\n\n"; + } + + // Two reports rather than one. Everything in the first is a difference in + // the schema itself; everything in the second is a difference a forum can + // live with, and a real forum always has some, so only the first decides + // the exit code. + $schema = []; + $aside = []; + + compare_tables($a, $b, $left, $right, $schema); + + if ($a['engine'] === 'postgresql') { + compare_lists('Sequences', $a['sequences'], $b['sequences'], $right, $schema); + compare_lists('Functions', $a['routines'] ?? [], $b['routines'] ?? [], $right, $schema); + } + + compare_lists('Settings', $a['settings'], $b['settings'], $right, $aside); + compare_column_order($a, $b, $aside); + + if (render($schema) + render($aside) === 0) { + echo "No differences.\n"; + + return 0; + } + + echo "\n"; + + if ($schema !== []) { + echo count_entries($schema), " difference(s) in the schema.\n"; + } + + if ($aside !== []) { + echo count_entries($aside), " difference(s) outside it, which do not decide the exit code.\n"; + } + + return $schema === [] ? 0 : 1; +} + +/** + * @param string $file + * @return array + */ +function read_dump(string $file): array +{ + $raw = @file_get_contents($file); + + if ($raw === false) { + fail('cannot read ' . $file); + } + + $dump = json_decode($raw, true); + + if (!is_array($dump) || !isset($dump['tables'], $dump['engine'])) { + fail($file . ' is not something dump wrote'); + } + + return $dump; +} + +/** + * Adds one difference to a report. Grouped by section and then by whatever it + * is about, so that a table with eight changed columns is one heading and + * eight lines under it rather than eight headings. + * + * @param array $report + * @param string $section + * @param string $subject + * @param string $message + * @param array $details + */ +function note(array &$report, string $section, string $subject, string $message, array $details = []): void +{ + $report[$section][$subject][] = ['message' => $message, 'details' => $details]; +} + +/** + * @param array $report + * @return int + */ +function count_entries(array $report): int +{ + $total = 0; + + foreach ($report as $subjects) { + foreach ($subjects as $entries) { + $total += count($entries); + } + } + + return $total; +} + +/** + * @param array $report + * @return int + */ +function render(array $report): int +{ + foreach ($report as $section => $subjects) { + echo $section, "\n", str_repeat('-', strlen($section)), "\n"; + + foreach ($subjects as $subject => $entries) { + echo ' ', $subject, "\n"; + + foreach ($entries as $entry) { + echo ' ', $entry['message'], "\n"; + + foreach ($entry['details'] as $detail) { + echo ' ', $detail, "\n"; + } + } + } + + echo "\n"; + } + + return count_entries($report); +} + +/** + * @param array $a + * @param array $b + * @param string $left + * @param string $right + * @param array $report + */ +function compare_tables(array $a, array $b, string $left, string $right, array &$report): void +{ + foreach (array_diff(array_keys($a['tables']), array_keys($b['tables'])) as $table) { + note($report, 'Tables', $table, 'missing from ' . $right); + } + + foreach (array_diff(array_keys($b['tables']), array_keys($a['tables'])) as $table) { + note($report, 'Tables', $table, 'only in ' . $right); + } + + foreach (array_intersect_key($a['tables'], $b['tables']) as $name => $table) { + $other = $b['tables'][$name]; + + foreach ($table['attributes'] as $key => $value) { + if (($other['attributes'][$key] ?? null) !== $value) { + note($report, 'Tables', $name, $key . ': ' . $value . ' in ' . $left . ', ' . ($other['attributes'][$key] ?? 'nothing') . ' in ' . $right); + } + } + + compare_parts($report, $name, 'column', $table['columns'], $other['columns'], describe_column(...), $left, $right); + compare_parts($report, $name, 'index', $table['indexes'], $other['indexes'], describe_index(...), $left, $right); + } +} + +/** + * The columns of a table and the indexes of a table are compared the same way: + * what is only on one side, and what is on both but described differently. + * + * @param array $report + * @param string $table + * @param string $noun + * @param array $mine + * @param array $theirs + * @param callable $describe + * @param string $left + * @param string $right + */ +function compare_parts(array &$report, string $table, string $noun, array $mine, array $theirs, callable $describe, string $left, string $right): void +{ + foreach (array_diff(array_keys($mine), array_keys($theirs)) as $name) { + note($report, 'Tables', $table, $noun . ' ' . $name . ' is missing from ' . $right . ' (' . $describe($mine[$name]) . ')'); + } + + foreach (array_diff(array_keys($theirs), array_keys($mine)) as $name) { + note($report, 'Tables', $table, $noun . ' ' . $name . ' is only in ' . $right . ' (' . $describe($theirs[$name]) . ')'); + } + + foreach (array_intersect_key($mine, $theirs) as $name => $definition) { + $one = $describe($definition); + $two = $describe($theirs[$name]); + + if ($one !== $two) { + note($report, 'Tables', $table, $noun . ' ' . $name, [ + $left . ': ' . $one, + $right . ': ' . $two, + ]); + } + } +} + +/** + * @param array $column + * @return string + */ +function describe_column(array $column): string +{ + return implode(' ', array_filter([ + $column['type'], + $column['nullable'] ? 'NULL' : 'NOT NULL', + $column['default'] === null ? '' : 'DEFAULT ' . $column['default'], + $column['extra'], + empty($column['generated']) ? '' : 'AS (' . $column['generated'] . ')', + $column['collation'] === null ? '' : 'COLLATE ' . $column['collation'], + ])); +} + +/** + * @param array $index + * @return string + */ +function describe_index(array $index): string +{ + $kind = $index['primary'] ? 'primary' : ($index['unique'] ? 'unique' : 'index'); + + return $kind . ' (' . implode(', ', $index['columns']) . ')'; +} + +/** + * @param string $title + * @param array $a + * @param array $b + * @param string $right + * @param array $report + */ +function compare_lists(string $title, array $a, array $b, string $right, array &$report): void +{ + foreach (array_diff($a, $b) as $name) { + note($report, $title, $name, 'missing from ' . $right); + } + + foreach (array_diff($b, $a) as $name) { + note($report, $title, $name, 'only in ' . $right); + } +} + +/** + * Column order is not part of the schema in any sense that matters to a query, + * but it is the one difference that is guaranteed: the upgrader appends, so + * every column any migration added sits at the end of the table rather than + * where the schema class puts it. Worth saying once per table, and never worth + * failing over. + * + * @param array $a + * @param array $b + * @param array $report + */ +function compare_column_order(array $a, array $b, array &$report): void +{ + foreach (array_intersect_key($a['tables'], $b['tables']) as $name => $table) { + $mine = order_of($table['columns']); + $theirs = order_of($b['tables'][$name]['columns']); + + // Only where both hold the same columns. A table that is missing one + // has been reported already, and would report here as well. + if ($mine !== $theirs && sort_copy($mine) === sort_copy($theirs)) { + note($report, 'Column order', $name, 'the columns are in a different order'); + } + } +} + +/** + * @param array $columns + * @return array + */ +function order_of(array $columns): array +{ + uasort($columns, fn($a, $b) => $a['position'] <=> $b['position']); + + return array_keys($columns); +} + +/** + * @param array $list + * @return array + */ +function sort_copy(array $list): array +{ + sort($list); + + return $list; +} diff --git a/.docker/user.sh b/.docker/user.sh new file mode 100755 index 0000000000..eebf4d1a8e --- /dev/null +++ b/.docker/user.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# Looks at forum accounts and fixes their passwords, so "which password did this +# forum end up with?" does not turn into a session of hand written SQL. +# +# .docker/user.sh list +# .docker/user.sh check admin 'password' +# .docker/user.sh reset admin 'a new password' +# .docker/user.sh check admin 'password' --engine postgresql +# +# check exits 0 when SMF would accept the password and 1 when it would not, so +# it is usable in a conditional as well as by eye. +# +# Everything goes through SMF's own Security class rather than writing a hash +# from here: what this puts in the table is by construction what Login2 expects +# to find there. Nothing is ever printed that would reveal an existing password; +# hashes are one way and this does not try to be clever about that. +# +# Without --engine it acts on the forum Settings.php currently points at. With +# it, it reads the copy use-engine.sh saved for that engine instead, which means +# the other forum can be inspected without switching to it. +# +# Runs on the host. +set -euo pipefail + +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" + +ENGINE='' +ACTION='' +NAME='' +PASSWORD='' +POSITIONAL=() + +while [ $# -gt 0 ]; do + case "$1" in + --engine) ENGINE="$2"; shift 2 ;; + --engine=*) ENGINE="${1#*=}"; shift ;; + -h|--help) sed -n '2,22p' "${BASH_SOURCE[0]}"; exit 0 ;; + -*) die "unknown argument: $1" ;; + *) POSITIONAL+=("$1"); shift ;; + esac +done + +[ "${#POSITIONAL[@]}" -gt 0 ] || die "need an action: list, check or reset (see --help)" + +ACTION="${POSITIONAL[0]}" +NAME="${POSITIONAL[1]:-}" +PASSWORD="${POSITIONAL[2]:-}" + +case "$ACTION" in + list) ;; + check|reset) + [ -n "$NAME" ] || die "${ACTION}: need a member name" + [ -n "$PASSWORD" ] || die "${ACTION}: need a password" + ;; + *) die "unknown action: ${ACTION} (expected list, check or reset)" ;; +esac + +# The settings file to read, as the container sees it. Empty means "whichever +# forum is live", which is the common case and needs no explanation in the log. +SETTINGS='/var/www/html/Settings.php' + +if [ -n "$ENGINE" ]; then + SMF_TYPE=$(engine_smf_type "$ENGINE") || die "unknown engine: $ENGINE" + SAVED="$DOCKER_DIR/settings/Settings.${SMF_TYPE}.php" + + [ -f "$SAVED" ] || die "no saved settings for ${SMF_TYPE}; install it first with install-forum.sh --engine ${SMF_TYPE}" + + SETTINGS="/var/www/html/.docker/settings/Settings.${SMF_TYPE}.php" +fi + +cd "$BOARD_DIR" + +# The password goes through the environment rather than the argument list: +# arguments are visible to anything that can read the process table, and a +# password typed at a shell is quite enough exposure already. +docker compose exec -T \ + -e SMF_USER_ACTION="$ACTION" \ + -e SMF_USER_NAME="$NAME" \ + -e SMF_USER_PASSWORD="$PASSWORD" \ + -e SMF_USER_SETTINGS="$SETTINGS" \ + web php <<-'PHP' + query( + 'SELECT id_member, member_name, real_name, email_address, id_group, is_activated + FROM {db_prefix}members + ORDER BY id_member', + [], + ); + + printf("%-5s %-20s %-28s %-7s %s\n", 'id', 'member_name', 'email', 'group', 'activated'); + + while ($row = $db->fetch_assoc($request)) { + printf( + "%-5d %-20s %-28s %-7d %s\n", + $row['id_member'], + $row['member_name'], + $row['email_address'], + $row['id_group'], + // 1 is the only value that can log in; the rest are awaiting + // activation, awaiting approval, banned or deleted. + $row['is_activated'] == 1 ? 'yes' : 'no (' . $row['is_activated'] . ')', + ); + } + + $db->free_result($request); + + exit(0); + } + + $request = $db->query( + 'SELECT id_member, member_name, passwd, is_activated + FROM {db_prefix}members + WHERE member_name = {string:name} OR email_address = {string:name} + LIMIT 1', + [ + 'name' => $name, + ], + ); + + $member = $db->fetch_assoc($request); + $db->free_result($request); + + if (!is_array($member)) { + fwrite(STDERR, 'error: no member called "' . $name . '" (try: user.sh list)' . "\n"); + + exit(1); + } + + if ($action === 'check') { + $ok = SMF\Security::hashVerifyPassword($password, $member['passwd']); + + echo $member['member_name'], ': ', $ok ? 'password is correct' : 'password is WRONG', "\n"; + + // Being right about the password is not the same as being able to log + // in, and the difference is worth saying out loud before someone spends + // an afternoon on it. + if ($ok && $member['is_activated'] != 1) { + echo ' note: the account is not active (is_activated = ', $member['is_activated'], '), so it cannot log in', "\n"; + } + + exit($ok ? 0 : 1); + } + + $db->query( + 'UPDATE {db_prefix}members + SET passwd = {string:passwd}, passwd_flood = {string:empty} + WHERE id_member = {int:id}', + [ + 'passwd' => SMF\Security::hashPassword($password), + // Cleared as well: SMF locks an account out for a while after + // enough wrong guesses, and resetting the password while leaving + // the lockout in place looks exactly like the password not working. + 'empty' => '', + 'id' => (int) $member['id_member'], + ], + ); + + echo $member['member_name'], ': password changed', "\n"; + PHP diff --git a/.gitignore b/.gitignore index 23f4dc34e8..96ad70fc88 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,9 @@ Thumbs.db # installs without reinstalling. Generated secrets and a machine-specific # board URL: local to whoever ran the installer. /.docker/settings/ +# Schema readings and reports from compare-upgrade.sh. They describe one +# machine's databases at one moment, and are rewritten on every run. +/.docker/compare/ # Test / Private files # ########################