From 7f46ee53021d2e48c8a2cbec5bdfb257759c9044 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 19:59:13 +0000 Subject: [PATCH 1/6] feat(db): prototype db:mysql-optimize, to replace db:size --cleanup A prototype for #124: a MySQL/MariaDB-specific command that lists InnoDB tables whose rebuild would reclaim disk space, and rebuilds them only with --execute. - Reads the server's settings over SQL: version, innodb_file_per_table, wsrep_on and wsrep_cluster_size (Galera), tmpdir and innodb_tmpdir. On MySQL 8 it sets information_schema_stats_expiry = 0 so statistics are fresh; --analyze runs ANALYZE TABLE on the tables found. - Lists tables with at least --min-reclaimable (100M) and --min-percent (20%) of their size reclaimable, and whether each rebuild fits in the service's disk and /tmp (from the metrics API), leaving 10% free. - Flags tables that are probably in a shared tablespace (same DATA_FREE as another table, or innodb_file_per_table OFF), and tables with FULLTEXT or SPATIAL indexes, which cannot be rebuilt online. - --execute runs ALTER TABLE ... FORCE, ALGORITHM=INPLACE, LOCK=NONE (the same rebuild as OPTIMIZE TABLE, but it fails instead of locking), with escaped identifiers. It refuses when the headroom is unknown or too small, or on a Galera cluster, unless --force. - db:size --cleanup still works, with a deprecation notice, and now escapes identifiers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PnUBGycBKd23tLPMcyf4Rb --- integration-tests/db_mysql_optimize_test.go | 336 ++++++++++++ .../src/Command/Db/DbMysqlOptimizeCommand.php | 514 ++++++++++++++++++ legacy/src/Command/Db/DbSizeCommand.php | 11 +- legacy/src/Model/Mysql/DiskHeadroom.php | 86 +++ legacy/src/Model/Mysql/MysqlServerInfo.php | 57 ++ legacy/src/Model/Mysql/TableCandidate.php | 93 ++++ legacy/src/Service/Observability.php | 31 ++ .../tests/Model/Mysql/MysqlOptimizeTest.php | 79 +++ 8 files changed, 1205 insertions(+), 2 deletions(-) create mode 100644 integration-tests/db_mysql_optimize_test.go create mode 100644 legacy/src/Command/Db/DbMysqlOptimizeCommand.php create mode 100644 legacy/src/Model/Mysql/DiskHeadroom.php create mode 100644 legacy/src/Model/Mysql/MysqlServerInfo.php create mode 100644 legacy/src/Model/Mysql/TableCandidate.php create mode 100644 legacy/tests/Model/Mysql/MysqlOptimizeTest.php diff --git a/integration-tests/db_mysql_optimize_test.go b/integration-tests/db_mysql_optimize_test.go new file mode 100644 index 000000000..821dfe710 --- /dev/null +++ b/integration-tests/db_mysql_optimize_test.go @@ -0,0 +1,336 @@ +package tests + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/upsun/cli/pkg/mockapi" +) + +const mib = 1024 * 1024 + +// fakeMysqlServer is a fake "mariadb" client, which answers the queries of db:mysql-optimize. +type fakeMysqlServer struct { + vars map[string]string + tables [][]any // schema, table, data_length, index_length, data_free, offline_indexes, same_free +} + +// script returns the fake client, which logs each query to a file in dir. +func (s fakeMysqlServer) script(t *testing.T, dir string) string { + var vars, tables strings.Builder + for k, v := range s.vars { + fmt.Fprintf(&vars, "%s\t%s\n", k, v) + } + for _, row := range s.tables { + for i, v := range row { + if i > 0 { + tables.WriteString("\t") + } + fmt.Fprint(&tables, v) + } + tables.WriteString("\n") + } + require.NoError(t, os.WriteFile(filepath.Join(dir, "vars.tsv"), []byte(vars.String()), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "tables.tsv"), []byte(tables.String()), 0o600)) + return fmt.Sprintf(`dir=%q +q="$(cat)" +printf '%%s\n---\n' "$q" >> "$dir/queries.log" +case "$q" in + *"SHOW GLOBAL VARIABLES"*) cat "$dir/vars.tsv" ;; + *"ALTER TABLE"*|*"ANALYZE TABLE"*) ;; + *"FROM information_schema.TABLES t"*) + # Apply table name filters. + while IFS=$'\t' read -r schema name rest; do + case "$q" in *"t.TABLE_NAME = "*) + case "$q" in *"t.TABLE_NAME = '$name'"*) ;; *) continue ;; esac ;; + esac + printf '%%s\t%%s\t%%s\n' "$schema" "$name" "$rest" + done < "$dir/tables.tsv" ;; + *) echo "unexpected query: $q" >&2; exit 1 ;; +esac +`, dir) +} + +// setUpMysqlOptimizeProject serves an app with database relationships, with a fake MariaDB client, and disk metrics +// for the "db" service. It returns the project and a function returning the queries run so far. +func setUpMysqlOptimizeProject(t *testing.T, server fakeMysqlServer, withMetrics bool) ( + p *testProject, queries func() []string, +) { + t.Helper() + dir := t.TempDir() + logFile := filepath.Join(dir, "queries.log") + p = setUpDBProject(t, map[string]string{"mariadb": server.script(t, dir)}) + + envPath := "/projects/" + layoutProjectID + "/environments/main" + overviewPath := envPath + "/observability/resources/overview" + p.api.Get(envPath+"/observability/", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "_links": mockapi.MakeHALLinks("resources_overview=" + p.f.apiURL + overviewPath), + }) + }) + if withMetrics { + p.api.Get(overviewPath, func(w http.ResponseWriter, req *http.Request) { + assert.Equal(t, []string{"db"}, req.URL.Query()["services[0]"]) + _ = json.NewEncoder(w).Encode(map[string]any{ + "_grain": 60, "_from": 1790189400, "_to": 1790190000, + "data": []any{ + map[string]any{"timestamp": 1790189940, "services": map[string]any{"db": map[string]any{ + "mountpoints": map[string]any{ + "/mnt": mountpointMetrics(1024*mib, 4096*mib, 10, 1000), + "/tmp": mountpointMetrics(100*mib, 1024*mib, 1, 1000), + }, + }}}, + // The in-progress point has no services. + map[string]any{"timestamp": 1790190000}, + }, + }) + }) + } + + queries = func() []string { + b, err := os.ReadFile(logFile) + if os.IsNotExist(err) { + return nil + } + require.NoError(t, err) + return strings.Split(strings.TrimSuffix(string(b), "\n---\n"), "\n---\n") + } + return p, queries +} + +var mariadbVars = map[string]string{ + "version": "11.4.2-MariaDB-log", + "innodb_file_per_table": "ON", + "wsrep_on": "OFF", + "tmpdir": "/tmp", +} + +// optimizeTables are the tables returned by the fake server. Thresholds are applied by the query, so they are not +// tested here. +var optimizeTables = [][]any{ + {"main", "logs", 200 * mib, 50 * mib, 300 * mib, 0, 0}, + {"main", "cache", 2048 * mib, 1024 * mib, 1024 * mib, 0, 0}, + {"main", "search", 100 * mib, 100 * mib, 150 * mib, 1, 0}, + {"main", "we`ird", 100 * mib, 0, 120 * mib, 0, 1}, +} + +func TestDbMysqlOptimizeReport(t *testing.T) { + t.Parallel() + p, queries := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: optimizeTables}, true) + args := func(extra ...string) []string { + return append([]string{"db:mysql-optimize", "-p", layoutProjectID, "-e", "main", "-r", "database"}, extra...) + } + + output, stdErr, err := p.f.RunCombinedOutput(args("--format", "csv", + "--columns", "schema,table,size,reclaimable,percent,disk_needed,disk_fits,tmp_needed,tmp_fits")...) + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "Checking database service db...") + assert.Contains(t, stdErr, "Disk: 3.0 GiB free of 4.0 GiB. /tmp: 924.0 MiB free of 1.0 GiB.") + assert.Equal(t, `Schema,Table,Size,Reclaimable,Reclaimable %,Disk needed,Fits disk,/tmp needed,Fits /tmp +main,logs,262144000,314572800,120%,262144000,true,52428800,true +main,cache,3221225472,1073741824,33%,3221225472,false,1073741824,false +main,search,209715200,157286400,75%,209715200,true,104857600,true +main,we`+"`"+`ird,104857600,125829120,120%,104857600,true,0,true +`, output) + + output = p.f.Run(args("--format", "tsv", "--no-header", "--columns", "table,notes")...) + assert.Equal(t, "logs\t\ncache\t\n"+ + "search\thas a FULLTEXT or SPATIAL index: it cannot be rebuilt without blocking writes\n"+ + "we`ird\tprobably in a shared tablespace: a rebuild would not reclaim space\n", output) + + q := queries() + require.Len(t, q, 4) // Two runs. + assert.Contains(t, q[0], "SHOW GLOBAL VARIABLES WHERE Variable_name IN (") + assert.Contains(t, q[0], "SHOW GLOBAL STATUS WHERE Variable_name IN ('wsrep_cluster_size')") + // MariaDB has no information_schema_stats_expiry. + assert.NotContains(t, q[1], "information_schema_stats_expiry") + assert.Contains(t, q[1], "t.DATA_FREE >= 104857600 AND t.DATA_FREE * 100 >= 20 * (t.DATA_LENGTH + t.INDEX_LENGTH)") + + // Thresholds and table filters, with escaped strings. + _, stdErr, err = p.f.RunCombinedOutput(args("--min-reclaimable", "1G", "--min-percent", "50", + "--table", "main.logs", "-t", "o'ther")...) + require.NoError(t, err, stdErr) + q = queries() + assert.Contains(t, q[len(q)-1], "t.DATA_FREE >= 1073741824 AND t.DATA_FREE * 100 >= 50 *") + assert.Contains(t, q[len(q)-1], "((t.TABLE_SCHEMA = 'main' AND t.TABLE_NAME = 'logs') OR t.TABLE_NAME = 'o''ther')") + assert.Contains(t, stdErr, "To rebuild these tables, run this command again with --execute.") + + // The default table output. + output = p.f.Run(args("--columns", "table,size,disk_fits")...) + assert.Contains(t, output, "| logs | 250.0 MiB | yes |") + assert.Contains(t, output, "| cache | 3.0 GiB | no |") + + _, stdErr, err = p.f.RunCombinedOutput(args("--min-percent", "x")...) + assert.Error(t, err) + assert.Contains(t, stdErr, "Invalid --min-percent value: x") + _, stdErr, err = p.f.RunCombinedOutput(args("--min-reclaimable", "lots")...) + assert.Error(t, err) + assert.Contains(t, stdErr, "Invalid size: lots") +} + +func TestDbMysqlOptimizeExecute(t *testing.T) { + t.Parallel() + p, queries := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: optimizeTables}, true) + args := []string{"db:mysql-optimize", "-p", layoutProjectID, "-e", "main", "-r", "database", "--execute"} + + alters := func() []string { + var a []string + for _, q := range queries() { + if strings.HasPrefix(q, "ALTER TABLE") { + a = append(a, q) + } + } + return a + } + + // "cache" does not fit, so nothing is rebuilt. + _, stdErr, err := p.f.RunCombinedOutput(args...) + assertExitCode(t, 1, err) + assert.Contains(t, stdErr, "Skipping main.search: has a FULLTEXT or SPATIAL index") + assert.Contains(t, stdErr, "Skipping main.we`ird: probably in a shared tablespace") + assert.Contains(t, stdErr, + "Error: There is not enough free disk space to rebuild main.cache (it needs about 3.0 GiB).") + assert.Contains(t, stdErr, "Error: There is not enough free space in /tmp to rebuild main.cache") + assert.Contains(t, stdErr, "The tables were not rebuilt.") + assert.Empty(t, alters()) + + // A table that fits is rebuilt. + _, stdErr, err = p.f.RunCombinedOutput(append(args, "--table", "main.logs")...) + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "Rebuilding main.logs... [OK]") + assert.Equal(t, []string{"ALTER TABLE `main`.`logs` FORCE, ALGORITHM=INPLACE, LOCK=NONE;"}, alters()) + + // With --force, tables that do not fit are rebuilt too, with warnings. + _, stdErr, err = p.f.RunCombinedOutput(append(args, "--force")...) + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "Warning: There is not enough free disk space to rebuild main.cache") + assert.Equal(t, []string{ + "ALTER TABLE `main`.`logs` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", + "ALTER TABLE `main`.`logs` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", + "ALTER TABLE `main`.`cache` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", + }, alters()) + + _, stdErr, err = p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", "--force") + assert.Error(t, err) + assert.Contains(t, stdErr, "The --force option requires --execute.") +} + +func TestDbMysqlOptimizeEscaping(t *testing.T) { + t.Parallel() + tables := [][]any{{"my`db", "we`ird", 100 * mib, 0, 120 * mib, 0, 0}} + p, queries := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: tables}, true) + + _, stdErr, err := p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", + "-r", "database", "--execute", "--analyze") + require.NoError(t, err, stdErr) + q := queries() + require.Len(t, q, 5) + assert.Equal(t, "ANALYZE TABLE `my``db`.`we``ird`;", q[2]) + assert.Equal(t, "ALTER TABLE `my``db`.`we``ird` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", q[4]) +} + +func TestDbMysqlOptimizeGalera(t *testing.T) { + t.Parallel() + vars := map[string]string{ + "version": "10.11.8-MariaDB-log", + "innodb_file_per_table": "ON", + "wsrep_on": "ON", + "wsrep_cluster_size": "3", + } + p, queries := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: vars, tables: optimizeTables[:1]}, true) + args := []string{"db:mysql-optimize", "-p", layoutProjectID, "-e", "main", "-r", "database"} + + _, stdErr, err := p.f.RunCombinedOutput(args...) + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "Warning: the database is a Galera cluster (3 nodes)") + + _, stdErr, err = p.f.RunCombinedOutput(append(args, "--execute")...) + assertExitCode(t, 1, err) + assert.Contains(t, stdErr, "Error: The database is a Galera cluster") + + _, stdErr, err = p.f.RunCombinedOutput(append(args, "--execute", "--force")...) + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "Warning: The database is a Galera cluster") + q := queries() + assert.Equal(t, "ALTER TABLE `main`.`logs` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", q[len(q)-1]) +} + +func TestDbMysqlOptimizeHeadroomUnknown(t *testing.T) { + t.Parallel() + // MySQL 8 caches statistics, and innodb_file_per_table is OFF. + vars := map[string]string{ + "version": "8.0.36", + "version_comment": "MySQL Community Server - GPL", + "innodb_file_per_table": "OFF", + "information_schema_stats_expiry": "86400", + } + tables := [][]any{ + {"main", "logs", 200 * mib, 50 * mib, 300 * mib, 0, 0}, + {"main", "other", 200 * mib, 50 * mib, 300 * mib, 0, 0}, + } + for _, metrics := range []string{"unavailable", "disabled"} { + t.Run(metrics, func(t *testing.T) { + t.Parallel() + p, queries := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: vars, tables: tables}, false) + if metrics == "disabled" { + p.f.extraEnv = append(p.f.extraEnv, EnvPrefix+"API_METRICS=0") + } + args := []string{"db:mysql-optimize", "-p", layoutProjectID, "-e", "main", "-r", "database"} + + output, stdErr, err := p.f.RunCombinedOutput(append(args, "--format", "csv", "--no-header", + "--columns", "table,disk_fits,tmp_fits")...) + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "Warning: innodb_file_per_table is OFF") + assert.Contains(t, stdErr, "Warning: the disk usage of the service is unknown") + assert.Equal(t, "logs,,\nother,,\n", output) + q := queries() + assert.True(t, strings.HasPrefix(q[1], "SET SESSION information_schema_stats_expiry = 0;\n"), q[1]) + + // All tables are skipped, as they are in the shared tablespace. + _, stdErr, err = p.f.RunCombinedOutput(append(args, "--execute", "--force")...) + assertExitCode(t, 1, err) + assert.Contains(t, stdErr, "Skipping main.logs: probably in a shared tablespace") + assert.Contains(t, stdErr, "No tables can be rebuilt.") + }) + } + + // With file-per-table, an unknown headroom prevents a rebuild without --force. + p, _ := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: tables[:1]}, false) + _, stdErr, err := p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", + "-r", "database", "--execute") + assertExitCode(t, 1, err) + assert.Contains(t, stdErr, "Error: The disk usage of the service is unknown.") +} + +func TestDbMysqlOptimizeOtherEngines(t *testing.T) { + t.Parallel() + p, queries := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars}, true) + + _, stdErr, err := p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", "-r", "pg") + assertExitCode(t, 1, err) + assert.Contains(t, stdErr, "The relationship pg is not a MySQL or MariaDB database (its scheme is: pgsql).") + assert.Empty(t, queries()) + + // Without --relationship, the MySQL relationship is chosen. + _, stdErr, err = p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main") + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "No tables were found with at least 100.0 MiB and 20% of their size reclaimable.") +} + +func TestDbSizeCleanupDeprecated(t *testing.T) { + t.Parallel() + p := setUpDBProject(t, map[string]string{"mariadb": "cat > /dev/null; echo 52428800"}) + _, stdErr, err := p.f.RunCombinedOutput("db:size", "-p", layoutProjectID, "-e", "main", "-r", "database", + "--cleanup") + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "The --cleanup option is deprecated") + assert.Contains(t, stdErr, "platform-test db:mysql-optimize") +} diff --git a/legacy/src/Command/Db/DbMysqlOptimizeCommand.php b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php new file mode 100644 index 000000000..b56668efb --- /dev/null +++ b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php @@ -0,0 +1,514 @@ + 'Schema', + 'table' => 'Table', + 'size' => 'Size', + 'reclaimable' => 'Reclaimable', + 'percent' => 'Reclaimable %', + 'disk_needed' => 'Disk needed', + 'disk_fits' => 'Fits disk', + 'tmp_needed' => '/tmp needed', + 'tmp_fits' => 'Fits /tmp', + 'notes' => 'Notes', + ]; + private const DEFAULT_COLUMNS = ['schema', 'table', 'size', 'reclaimable', 'disk_fits', 'tmp_fits', 'notes']; + + private const DEFAULT_MIN_RECLAIMABLE = '100M'; + private const DEFAULT_MIN_PERCENT = 20; + + /** The percentage of a volume that must stay free after a rebuild. */ + public const RESERVE_PERCENT = 10; + + private const SYSTEM_SCHEMAS = ['mysql', 'information_schema', 'performance_schema', 'sys']; + + public function __construct( + private readonly Config $config, + private readonly Io $io, + private readonly Observability $observability, + private readonly QuestionHelper $questionHelper, + private readonly Relationships $relationships, + private readonly Selector $selector, + private readonly Table $table, + ) { + parent::__construct(); + } + + protected function configure(): void + { + $this + ->addOption('min-reclaimable', null, InputOption::VALUE_REQUIRED, 'The minimum reclaimable space of a table, in bytes or with a unit (K, M, G)', self::DEFAULT_MIN_RECLAIMABLE) + ->addOption('min-percent', null, InputOption::VALUE_REQUIRED, 'The minimum reclaimable space of a table, as a percentage of its size', (string) self::DEFAULT_MIN_PERCENT) + ->addOption('table', 't', InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Only check these tables (as "table" or "schema.table")') + ->addOption('analyze', null, InputOption::VALUE_NONE, 'Refresh statistics with ANALYZE TABLE on the tables found, and check them again (useful on MariaDB and MySQL 5.7)') + ->addOption('execute', null, InputOption::VALUE_NONE, 'Rebuild the tables found (by default, they are only reported)') + ->addOption('force', null, InputOption::VALUE_NONE, 'With --execute: rebuild even if the disk headroom is unknown or too small, or on a cluster (Galera)') + ->addOption('bytes', 'B', InputOption::VALUE_NONE, 'Show sizes in bytes'); + $executable = $this->config->getStr('application.executable'); + $reserve = self::RESERVE_PERCENT; + $this->setHelp(<<--min-reclaimable and --min-percent of its size. + + By default, this only reports. With --execute, each table listed is rebuilt with: + ALTER TABLE ... FORCE, ALGORITHM=INPLACE, LOCK=NONE + which is the same rebuild as OPTIMIZE TABLE, but fails instead of locking the table if it cannot be done online. + + A rebuild copies the table: it needs free disk space for the table's size, and temporary space for its indexes. + The disk and /tmp usage are read from the metrics API: if they are unknown, or a table does not fit + (leaving at least {$reserve}% of the volume free), the tables are not rebuilt, unless --force is used. + + On a cluster (Galera), a rebuild blocks writes on all nodes while it runs, so it is refused unless --force is used. + + The sizes are estimates from the database's statistics. To check the disk usage of the service, run: $executable disk + EOF); + $this->addExample('Report tables that could be optimized'); + $this->addExample('Rebuild the tables found, if there is enough disk space', '--execute'); + $this->addExample('Rebuild one table', '--execute --table main.cache_items'); + $this->selector->addProjectOption($this->getDefinition()); + $this->selector->addEnvironmentOption($this->getDefinition()); + $this->selector->addAppOption($this->getDefinition()); + $this->addCompleter($this->selector); + Relationships::configureInput($this->getDefinition()); + Table::configureInput($this->getDefinition(), self::TABLE_HEADER, self::DEFAULT_COLUMNS); + Ssh::configureInput($this->getDefinition()); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $minReclaimable = self::parseSize(Option::string($input, 'min-reclaimable')); + $minPercent = Option::string($input, 'min-percent'); + if (!is_numeric($minPercent) || (float) $minPercent < 0) { + throw new InvalidArgumentException('Invalid --min-percent value: ' . $minPercent); + } + $minPercent = (float) $minPercent; + $tableFilter = ArrayArgument::getOption($input, 'table'); + $execute = Option::bool($input, 'execute'); + $force = Option::bool($input, 'force'); + if ($force && !$execute) { + throw new InvalidArgumentException('The --force option requires --execute.'); + } + + $selection = $this->selector->getSelection($input, new SelectorConfig(allowLocalHost: $this->relationships->hasLocalEnvVar(), chooseEnvFilter: SelectorConfig::filterEnvsMaybeActive())); + $host = $this->selector->getHostFromSelection($input, $selection); + + $database = $this->chooseMysqlDatabase($host, $input, $output); + if ($database === null) { + return 1; + } + $mysql = $this->getMysqlCommand($database); + $serviceName = isset($database['service']) && is_string($database['service']) ? $database['service'] : null; + + $this->stdErr->writeln($serviceName !== null ? \sprintf('Checking database service %s...', $serviceName) : 'Checking database...'); + + // Read the server's settings. + $vars = $this->parsePairs((string) $host->runCommand($mysql, true, true, $this->variablesQuery())); + $server = new MysqlServerInfo($vars); + $this->io->debug('Server variables: ' . json_encode($vars)); + + // Read the disk usage of the service. + $headroom = null; + if (!$selection->hasEnvironment() || $serviceName === null) { + $this->io->debug('No environment or service name: the disk headroom is unknown'); + } elseif (!$this->config->getBool('api.metrics')) { + $this->io->debug('Metrics are not enabled: the disk headroom is unknown'); + } else { + try { + $point = $this->observability->latestServiceMetrics($selection->getEnvironment(), $serviceName, [MetricKind::API_TYPE_DISK], [MetricKind::API_AGG_AVG]); + $headroom = $point !== null ? DiskHeadroom::fromMetrics($point) : null; + } catch (\Exception $e) { + $this->io->debug('Failed to read metrics: ' . $e->getMessage()); + } + } + + // Find the tables. + $tables = $this->findTables($host, $mysql, $server, $minReclaimable, $minPercent, $tableFilter); + if ($tables && Option::bool($input, 'analyze')) { + $this->stdErr->writeln(\sprintf('Analyzing %d table(s)...', \count($tables))); + $host->runCommand($mysql, true, true, 'ANALYZE TABLE ' . implode(', ', array_map(fn(TableCandidate $t): string => $t->quotedName(), $tables)) . ';'); + $tables = $this->findTables($host, $mysql, $server, $minReclaimable, $minPercent, $tableFilter); + } + + $this->stdErr->writeln(''); + $this->showServerWarnings($server, $headroom); + + if (!$tables) { + $this->stdErr->writeln(\sprintf( + 'No tables were found with at least %s and %s%% of their size reclaimable.', + Helper::formatMemory($minReclaimable), + $minPercent, + )); + return 0; + } + + $this->renderTables($tables, $headroom, $server, Option::bool($input, 'bytes')); + + if (!$execute) { + if (!$this->table->formatIsMachineReadable()) { + $this->stdErr->writeln(''); + $this->stdErr->writeln('The reclaimable space is an estimate. Rebuilding a table copies it, and may take a long time.'); + $this->stdErr->writeln(\sprintf('To rebuild these tables, run this command again with --execute%s.', $tableFilter ? '' : ' (and --table to choose tables)')); + } + return 0; + } + + return $this->rebuild($host, $mysql, $tables, $server, $headroom, $force) ? 0 : 1; + } + + /** + * Chooses a MySQL or MariaDB relationship. + * + * @return array|null + */ + private function chooseMysqlDatabase(HostInterface $host, InputInterface $input, OutputInterface $output): ?array + { + // Check an explicit relationship's scheme, for a clear error. + if (Option::stringOrNull($input, 'relationship') !== null) { + $database = $this->relationships->chooseService($host, $input, $output); + if (!$database) { + return null; + } + if ($database['scheme'] !== 'mysql') { + $this->stdErr->writeln(\sprintf( + 'The relationship %s is not a MySQL or MariaDB database (its scheme is: %s).', + $database['_relationship_name'], + $database['scheme'], + )); + $this->stdErr->writeln('This command only supports MySQL and MariaDB.'); + return null; + } + return $database; + } + $database = $this->relationships->chooseDatabase($host, $input, $output, ['mysql']); + if (!$database) { + $this->stdErr->writeln('This command only supports MySQL and MariaDB.'); + return null; + } + + return $database; + } + + /** + * @param array $database + */ + private function getMysqlCommand(array $database): string + { + $cmdName = $this->relationships->isMariaDB($database) ? 'mariadb' : 'mysql'; + + return \sprintf( + '%s %s --no-auto-rehash --batch --raw --skip-column-names', + $this->relationships->mariaDbCommandWithFallback($cmdName), + $this->relationships->getDbCommandArgs($cmdName, $database, ''), + ); + } + + private function variablesQuery(): string + { + $vars = ['version', 'version_comment', 'innodb_file_per_table', 'wsrep_on', 'tmpdir', 'innodb_tmpdir', 'information_schema_stats_expiry']; + + return \sprintf( + "SHOW GLOBAL VARIABLES WHERE Variable_name IN (%s);\nSHOW GLOBAL STATUS WHERE Variable_name IN ('wsrep_cluster_size');\n", + implode(', ', array_map(fn(string $v): string => "'" . $v . "'", $vars)), + ); + } + + /** + * Parses tab-separated name/value lines. + * + * @return array + */ + private function parsePairs(string $output): array + { + $pairs = []; + foreach (explode("\n", $output) as $line) { + $parts = explode("\t", rtrim($line, "\r"), 2); + if (\count($parts) === 2) { + $pairs[strtolower($parts[0])] = $parts[1]; + } + } + + return $pairs; + } + + /** + * Finds InnoDB tables with reclaimable space. + * + * @param string[] $filter + * + * @return TableCandidate[] + */ + private function findTables(HostInterface $host, string $mysql, MysqlServerInfo $server, int $minReclaimable, float $minPercent, array $filter): array + { + $query = ''; + if ($server->hasStatsExpiry) { + // MySQL 8 caches statistics in information_schema for up to 24 hours by default. + $query .= "SET SESSION information_schema_stats_expiry = 0;\n"; + } + $query .= self::tablesQuery($minReclaimable, $minPercent, $filter) . "\n"; + $this->io->debug('Query: ' . $query); + $result = (string) $host->runCommand($mysql, true, true, $query); + $tables = []; + foreach (explode("\n", $result) as $line) { + $fields = explode("\t", rtrim($line, "\r")); + if (\count($fields) === 7) { + $tables[] = TableCandidate::fromRow($fields); + } + } + + return $tables; + } + + /** + * Returns the query to find InnoDB tables with reclaimable space. + * + * Free space in a shared tablespace (the system tablespace, if + * innodb_file_per_table is OFF, or a general tablespace) is reported for + * each of its tables: the "same_free" column counts the other tables with + * the same free space, to detect them. + * + * @param string[] $filter + */ + public static function tablesQuery(int $minReclaimable, float $minPercent, array $filter = []): string + { + $conditions = [ + "t.ENGINE = 'InnoDB'", + "t.TABLE_TYPE = 'BASE TABLE'", + 't.TABLE_SCHEMA NOT IN (' . implode(', ', array_map(self::quoteString(...), self::SYSTEM_SCHEMAS)) . ')', + 't.DATA_FREE >= ' . $minReclaimable, + 't.DATA_FREE * 100 >= ' . self::formatNumber($minPercent) . ' * (t.DATA_LENGTH + t.INDEX_LENGTH)', + ]; + if ($filter) { + $or = []; + foreach ($filter as $name) { + if (str_contains($name, '.')) { + [$schema, $table] = explode('.', $name, 2); + $or[] = \sprintf('(t.TABLE_SCHEMA = %s AND t.TABLE_NAME = %s)', self::quoteString($schema), self::quoteString($table)); + } else { + $or[] = \sprintf('t.TABLE_NAME = %s', self::quoteString($name)); + } + } + $conditions[] = '(' . implode(' OR ', $or) . ')'; + } + + return 'SELECT t.TABLE_SCHEMA, t.TABLE_NAME, t.DATA_LENGTH, t.INDEX_LENGTH, t.DATA_FREE,' + . " (SELECT COUNT(*) FROM information_schema.STATISTICS s WHERE s.TABLE_SCHEMA = t.TABLE_SCHEMA AND s.TABLE_NAME = t.TABLE_NAME AND s.INDEX_TYPE IN ('FULLTEXT', 'SPATIAL')) AS offline_indexes," + . ' COALESCE(f.n, 1) - 1 AS same_free' + . ' FROM information_schema.TABLES t' + . " LEFT JOIN (SELECT DATA_FREE AS free, COUNT(*) AS n FROM information_schema.TABLES WHERE ENGINE = 'InnoDB' AND DATA_FREE >= " . $minReclaimable . ' GROUP BY DATA_FREE) f ON f.free = t.DATA_FREE' + . ' WHERE ' . implode(' AND ', $conditions) + . ' ORDER BY t.DATA_FREE DESC;'; + } + + /** + * Shows warnings about the server's configuration. + */ + private function showServerWarnings(MysqlServerInfo $server, ?DiskHeadroom $headroom): void + { + if ($server->filePerTable === false) { + $this->stdErr->writeln('Warning: innodb_file_per_table is OFF: tables are stored in the shared system tablespace, which does not shrink when they are rebuilt.'); + } + if ($server->isGaleraCluster()) { + $this->stdErr->writeln(\sprintf('Warning: the database is a Galera cluster (%d nodes): rebuilding a table blocks writes on all nodes while it runs.', $server->clusterSize)); + } + if ($headroom === null) { + $this->stdErr->writeln('Warning: the disk usage of the service is unknown (metrics are not available), so it is not known whether a rebuild would fit.'); + } else { + $this->stdErr->writeln(\sprintf( + 'Disk: %s free of %s. /tmp: %s.', + Helper::formatMemory((int) $headroom->diskFree()), + Helper::formatMemory((int) $headroom->diskLimit), + $headroom->hasTmp() ? '' . Helper::formatMemory((int) $headroom->tmpFree()) . ' free of ' . Helper::formatMemory((int) $headroom->tmpLimit) : 'unknown', + )); + if ($server->tmpdir() !== null && !str_starts_with($server->tmpdir(), '/tmp')) { + $this->stdErr->writeln(\sprintf('Note: the server\'s temporary directory is %s, not /tmp.', $server->tmpdir())); + } + } + $this->stdErr->writeln(''); + } + + /** + * @param TableCandidate[] $tables + */ + private function renderTables(array $tables, ?DiskHeadroom $headroom, MysqlServerInfo $server, bool $bytes): void + { + $machineReadable = $this->table->formatIsMachineReadable(); + $formatSize = fn(int $size): string => $bytes || $machineReadable ? (string) $size : Helper::formatMemory($size); + $formatFits = function (?bool $fits) use ($machineReadable): string { + if ($fits === null) { + return $machineReadable ? '' : 'unknown'; + } + if ($machineReadable) { + return $fits ? 'true' : 'false'; + } + return $fits ? 'yes' : 'no'; + }; + $rows = []; + foreach ($tables as $t) { + $rows[] = [ + 'schema' => $t->schema, + 'table' => $t->name, + 'size' => $formatSize($t->size()), + 'reclaimable' => $formatSize($t->dataFree), + 'percent' => $t->size() > 0 ? \sprintf('%.0f%%', $t->dataFree / $t->size() * 100) : '', + 'disk_needed' => $formatSize($t->diskNeeded()), + 'disk_fits' => $formatFits($headroom?->diskFits($t->diskNeeded())), + 'tmp_needed' => $formatSize($t->tmpNeeded()), + 'tmp_fits' => $formatFits($headroom?->tmpFits($t->tmpNeeded())), + 'notes' => (string) $t->skipReason($server), + ]; + } + $this->table->render($rows, self::TABLE_HEADER, self::DEFAULT_COLUMNS); + } + + /** + * Rebuilds tables, after checking that it is safe. + * + * @param TableCandidate[] $tables + * + * @return bool Whether the rebuilds succeeded, or there were none to run. + */ + private function rebuild(HostInterface $host, string $mysql, array $tables, MysqlServerInfo $server, ?DiskHeadroom $headroom, bool $force): bool + { + $this->stdErr->writeln(''); + + $toRebuild = []; + foreach ($tables as $t) { + if ($reason = $t->skipReason($server)) { + $this->stdErr->writeln(\sprintf('Skipping %s: %s', $t->displayName(), $reason)); + continue; + } + $toRebuild[] = $t; + } + if (!$toRebuild) { + $this->stdErr->writeln('No tables can be rebuilt.'); + return false; + } + + $problems = []; + if ($headroom === null) { + $problems[] = 'The disk usage of the service is unknown.'; + } else { + foreach ($toRebuild as $t) { + if (!$headroom->diskFits($t->diskNeeded())) { + $problems[] = \sprintf('There is not enough free disk space to rebuild %s (it needs about %s).', $t->displayName(), Helper::formatMemory($t->diskNeeded())); + } + if ($headroom->tmpFits($t->tmpNeeded()) === false) { + $problems[] = \sprintf('There is not enough free space in /tmp to rebuild %s (it may need up to %s).', $t->displayName(), Helper::formatMemory($t->tmpNeeded())); + } + } + } + if ($server->isGaleraCluster()) { + $problems[] = 'The database is a Galera cluster: a rebuild blocks writes on all nodes while it runs.'; + } + if ($problems) { + foreach ($problems as $problem) { + $this->stdErr->writeln(($force ? 'Warning: ' : 'Error: ') . $problem); + } + if (!$force) { + $this->stdErr->writeln(''); + $this->stdErr->writeln('The tables were not rebuilt. Use --force to rebuild them anyway, at your own risk: a rebuild that fills the disk can stop the database.'); + return false; + } + $this->stdErr->writeln(''); + } + + $this->stdErr->writeln(\sprintf('Rebuilding %d table(s), one at a time. Each rebuild copies the table, and may take a long time.', \count($toRebuild))); + if (!$this->questionHelper->confirm('Are you sure you want to continue?')) { + return false; + } + + foreach ($toRebuild as $t) { + $sql = $t->rebuildQuery(); + $this->stdErr->write(\sprintf('Rebuilding %s...', $t->displayName())); + $this->io->debug('Query: ' . $sql); + $start = microtime(true); + try { + $host->runCommand($mysql, true, true, $sql); + } catch (ProcessRuntimeException $e) { + $this->stdErr->writeln(' [failed]'); + $this->stdErr->writeln($e->getMessage()); + return false; + } + $this->stdErr->writeln(\sprintf(' [OK] (%ds)', round(microtime(true) - $start))); + } + + $this->stdErr->writeln(''); + $this->stdErr->writeln('Run this command again (without --execute) to check the result.'); + + return true; + } + + /** + * Parses a size, in bytes or with a unit (K, M, G or T, in powers of 1024). + */ + public static function parseSize(string $value): int + { + if (!preg_match('/^\s*(\d+(?:\.\d+)?)\s*([kmgt])?i?b?\s*$/i', $value, $matches)) { + throw new InvalidArgumentException('Invalid size: ' . $value); + } + $exponent = isset($matches[2]) ? (int) strpos('kmgt', strtolower($matches[2])) + 1 : 0; + + return (int) round((float) $matches[1] * (1024 ** $exponent)); + } + + /** + * Quotes a MySQL identifier. + */ + public static function quoteIdentifier(string $name): string + { + return '`' . str_replace('`', '``', $name) . '`'; + } + + /** + * Quotes a MySQL string literal. + */ + public static function quoteString(string $value): string + { + return "'" . str_replace(['\\', "'"], ['\\\\', "''"], $value) . "'"; + } + + private static function formatNumber(float $number): string + { + return rtrim(rtrim(\sprintf('%.4F', $number), '0'), '.'); + } +} diff --git a/legacy/src/Command/Db/DbSizeCommand.php b/legacy/src/Command/Db/DbSizeCommand.php index 41ffae353..5152990ba 100644 --- a/legacy/src/Command/Db/DbSizeCommand.php +++ b/legacy/src/Command/Db/DbSizeCommand.php @@ -46,7 +46,7 @@ protected function configure(): void { $this ->addOption('bytes', 'B', InputOption::VALUE_NONE, 'Show sizes in bytes.') - ->addOption('cleanup', 'C', InputOption::VALUE_NONE, 'Check if tables can be cleaned up and show me recommendations (InnoDb only).'); + ->addOption('cleanup', 'C', InputOption::VALUE_NONE, 'Deprecated: use the db:mysql-optimize command instead.'); $help = self::ESTIMATE_WARNING; if ($this->config->getBool('api.metrics')) { $this->stability = self::STABILITY_DEPRECATED; @@ -136,7 +136,7 @@ private function getCleanupQueries(array $rows): array } [$schema, $table] = explode("\t", $row); - return sprintf('ALTER TABLE `%s`.`%s` ENGINE="InnoDB";', $schema, $table); + return sprintf('ALTER TABLE %s.%s ENGINE="InnoDB";', DbMysqlOptimizeCommand::quoteIdentifier($schema), DbMysqlOptimizeCommand::quoteIdentifier($table)); }, $rows), ); } @@ -148,6 +148,13 @@ private function getCleanupQueries(array $rows): array */ private function checkInnoDbTablesInNeedOfOptimizing(HostInterface $host, array $database, InputInterface $input): void { + $this->stdErr->writeln(''); + $this->stdErr->writeln('Deprecated: The --cleanup option is deprecated and will be removed in a future version.'); + $this->stdErr->writeln(\sprintf( + 'To find tables with reclaimable space, and check whether they can be rebuilt safely, run: %s db:mysql-optimize', + $this->config->getStr('application.executable'), + )); + $tablesNeedingCleanup = $host->runCommand($this->getMysqlCommand($database), true, true, $this->mysqlTablesInNeedOfOptimizing()); $queries = []; if (is_string($tablesNeedingCleanup)) { diff --git a/legacy/src/Model/Mysql/DiskHeadroom.php b/legacy/src/Model/Mysql/DiskHeadroom.php new file mode 100644 index 000000000..600cf6944 --- /dev/null +++ b/legacy/src/Model/Mysql/DiskHeadroom.php @@ -0,0 +1,86 @@ + $point + */ + public static function fromMetrics(array $point): ?self + { + $value = function (string $mountpoint, string $metric, string $agg) use ($point): ?float { + $v = Observability::nested($point, 'mountpoints', $mountpoint, $metric, $agg); + return is_numeric($v) ? (float) $v : null; + }; + $diskUsed = $value('/mnt', 'disk_used', 'avg'); + $diskLimit = $value('/mnt', 'disk_limit', 'max'); + if ($diskUsed === null || !$diskLimit) { + return null; + } + $tmpUsed = $value('/tmp', 'disk_used', 'avg'); + $tmpLimit = $value('/tmp', 'disk_limit', 'max'); + if ($tmpUsed === null || !$tmpLimit) { + $tmpUsed = $tmpLimit = null; + } + + return new self($diskUsed, $diskLimit, $tmpUsed, $tmpLimit); + } + + public function diskFree(): float + { + return max(0, $this->diskLimit - $this->diskUsed); + } + + public function hasTmp(): bool + { + return $this->tmpLimit !== null; + } + + public function tmpFree(): ?float + { + return $this->tmpLimit !== null ? max(0, $this->tmpLimit - (float) $this->tmpUsed) : null; + } + + /** + * Returns whether the disk has room for a rebuild, keeping a reserve free. + */ + public function diskFits(int $needed): bool + { + return self::fits($this->diskUsed, $this->diskLimit, $needed); + } + + /** + * Returns whether /tmp has room for a rebuild, or null if its usage is unknown. + */ + public function tmpFits(int $needed): ?bool + { + if ($this->tmpLimit === null) { + return null; + } + + return self::fits((float) $this->tmpUsed, $this->tmpLimit, $needed); + } + + private static function fits(float $used, float $limit, int $needed): bool + { + return $used + $needed <= $limit * (100 - DbMysqlOptimizeCommand::RESERVE_PERCENT) / 100; + } +} diff --git a/legacy/src/Model/Mysql/MysqlServerInfo.php b/legacy/src/Model/Mysql/MysqlServerInfo.php new file mode 100644 index 000000000..b303dda91 --- /dev/null +++ b/legacy/src/Model/Mysql/MysqlServerInfo.php @@ -0,0 +1,57 @@ + $vars Variables and status values, by lowercase name. + */ + public function __construct(array $vars) + { + $this->version = $vars['version'] ?? ''; + $this->isMariaDB = stripos($this->version . ' ' . ($vars['version_comment'] ?? ''), 'mariadb') !== false; + $this->filePerTable = isset($vars['innodb_file_per_table']) ? self::bool($vars['innodb_file_per_table']) : null; + $this->wsrepOn = isset($vars['wsrep_on']) && self::bool($vars['wsrep_on']); + $this->clusterSize = (int) ($vars['wsrep_cluster_size'] ?? 0); + $this->hasStatsExpiry = isset($vars['information_schema_stats_expiry']); + $tmpdir = ($vars['innodb_tmpdir'] ?? '') !== '' ? $vars['innodb_tmpdir'] : ($vars['tmpdir'] ?? ''); + $this->tmpdir = $tmpdir !== '' ? $tmpdir : null; + } + + /** + * Returns whether the server is a node of a Galera cluster with more than one node. + */ + public function isGaleraCluster(): bool + { + return $this->wsrepOn && $this->clusterSize > 1; + } + + /** + * Returns the directory for temporary files of online DDL (innodb_tmpdir, or tmpdir). + */ + public function tmpdir(): ?string + { + return $this->tmpdir; + } + + private static function bool(string $value): bool + { + return \in_array(strtoupper($value), ['ON', '1', 'TRUE', 'YES'], true); + } +} diff --git a/legacy/src/Model/Mysql/TableCandidate.php b/legacy/src/Model/Mysql/TableCandidate.php new file mode 100644 index 000000000..7fd2ef13b --- /dev/null +++ b/legacy/src/Model/Mysql/TableCandidate.php @@ -0,0 +1,93 @@ +dataLength + $this->indexLength; + } + + /** + * The disk space needed for the rebuilt copy of the table. + */ + public function diskNeeded(): int + { + return $this->size(); + } + + /** + * The temporary space that may be needed, to sort the secondary indexes. + * + * Concurrent writes during the rebuild are also logged to a temporary + * file (up to innodb_online_alter_log_max_size), which is not counted. + */ + public function tmpNeeded(): int + { + return $this->indexLength; + } + + public function displayName(): string + { + return $this->schema . '.' . $this->name; + } + + public function quotedName(): string + { + return DbMysqlOptimizeCommand::quoteIdentifier($this->schema) . '.' . DbMysqlOptimizeCommand::quoteIdentifier($this->name); + } + + /** + * Returns the query that rebuilds the table online, or fails. + */ + public function rebuildQuery(): string + { + return 'ALTER TABLE ' . $this->quotedName() . ' FORCE, ALGORITHM=INPLACE, LOCK=NONE;'; + } + + /** + * Returns why the table would not be rebuilt, or null. + */ + public function skipReason(MysqlServerInfo $server): ?string + { + if ($this->sameFree > 0 || $server->filePerTable === false) { + return 'probably in a shared tablespace: a rebuild would not reclaim space'; + } + if ($this->offlineIndexes > 0) { + return 'has a FULLTEXT or SPATIAL index: it cannot be rebuilt without blocking writes'; + } + + return null; + } +} diff --git a/legacy/src/Service/Observability.php b/legacy/src/Service/Observability.php index 0c0158aea..895e4bd59 100644 --- a/legacy/src/Service/Observability.php +++ b/legacy/src/Service/Observability.php @@ -7,6 +7,7 @@ use GuzzleHttp\Exception\BadResponseException; use GuzzleHttp\Exception\RequestException; use GuzzleHttp\Psr7\Request; +use Platformsh\Cli\Model\Metrics\Query; use Platformsh\Client\Exception\ApiResponseException; use Platformsh\Client\Model\Environment; @@ -65,6 +66,36 @@ public static function nested(mixed $data, string ...$keys): mixed return $data; } + /** + * Returns the latest metrics reported by a service, from the resources overview. + * + * @param string[] $types The metric types (e.g. "disk"). + * @param string[] $aggs The aggregations (e.g. "avg"). + * + * @return array|null The service's latest data point, or null if metrics are not available. + */ + public function latestServiceMetrics(Environment $environment, string $service, array $types, array $aggs, int $range = 600): ?array + { + if (!$link = self::getLink($this->getEntrypoint($environment), 'resources_overview')) { + return null; + } + $now = time(); + $query = (new Query($now - $range, $now, null)) + ->setServices([$service]) + ->setTypes($types) + ->setAggs($aggs); + $data = $this->get($link . $query->asString()); + $points = self::nested($data, 'data'); + foreach (array_reverse(is_array($points) ? $points : []) as $point) { + $metrics = self::nested($point, 'services', $service); + if (is_array($metrics)) { + return $metrics; + } + } + + return null; + } + /** * Sends a GET request and decodes the JSON response. * diff --git a/legacy/tests/Model/Mysql/MysqlOptimizeTest.php b/legacy/tests/Model/Mysql/MysqlOptimizeTest.php new file mode 100644 index 000000000..1ad2b6019 --- /dev/null +++ b/legacy/tests/Model/Mysql/MysqlOptimizeTest.php @@ -0,0 +1,79 @@ +assertSame($expected, DbMysqlOptimizeCommand::parseSize($input), $input); + } + $this->expectException(InvalidArgumentException::class); + DbMysqlOptimizeCommand::parseSize('-1M'); + } + + public function testQuoting(): void + { + $this->assertSame('`a``b`', DbMysqlOptimizeCommand::quoteIdentifier('a`b')); + $this->assertSame("'a''b\\\\c'", DbMysqlOptimizeCommand::quoteString("a'b\\c")); + $table = new TableCandidate('my`db', 't', 1, 2, 3); + $this->assertSame('ALTER TABLE `my``db`.`t` FORCE, ALGORITHM=INPLACE, LOCK=NONE;', $table->rebuildQuery()); + } + + public function testServerInfo(): void + { + $server = new MysqlServerInfo(['version' => '10.11.8-MariaDB', 'wsrep_on' => 'ON', 'wsrep_cluster_size' => '3', 'innodb_tmpdir' => '', 'tmpdir' => '/tmp']); + $this->assertTrue($server->isMariaDB); + $this->assertTrue($server->isGaleraCluster()); + $this->assertNull($server->filePerTable); + $this->assertFalse($server->hasStatsExpiry); + $this->assertSame('/tmp', $server->tmpdir()); + + // A single-node cluster does not block other nodes. + $server = new MysqlServerInfo(['version' => '8.0.36', 'wsrep_on' => 'ON', 'wsrep_cluster_size' => '1', 'innodb_file_per_table' => 'OFF', 'information_schema_stats_expiry' => '86400']); + $this->assertFalse($server->isMariaDB); + $this->assertFalse($server->isGaleraCluster()); + $this->assertFalse($server->filePerTable); + $this->assertTrue($server->hasStatsExpiry); + } + + public function testHeadroom(): void + { + $this->assertNull(DiskHeadroom::fromMetrics([])); + $headroom = DiskHeadroom::fromMetrics(['mountpoints' => [ + '/mnt' => ['disk_used' => ['avg' => 500], 'disk_limit' => ['max' => 1000]], + ]]); + $this->assertNotNull($headroom); + $this->assertFalse($headroom->hasTmp()); + $this->assertNull($headroom->tmpFits(1)); + // At least 10% of the volume stays free. + $this->assertTrue($headroom->diskFits(400)); + $this->assertFalse($headroom->diskFits(401)); + } + + public function testSkipReason(): void + { + $server = new MysqlServerInfo(['innodb_file_per_table' => 'ON']); + $this->assertNull((new TableCandidate('s', 't', 1, 1, 1))->skipReason($server)); + $this->assertStringContainsString('shared tablespace', (string) (new TableCandidate('s', 't', 1, 1, 1, 0, 2))->skipReason($server)); + $this->assertStringContainsString('FULLTEXT', (string) (new TableCandidate('s', 't', 1, 1, 1, 1))->skipReason($server)); + $this->assertStringContainsString('shared tablespace', (string) (new TableCandidate('s', 't', 1, 1, 1))->skipReason(new MysqlServerInfo(['innodb_file_per_table' => 'OFF']))); + } +} From 9128d6e7965ad8f27a941de801cf06a5d2ee17df Mon Sep 17 00:00:00 2001 From: Patrick Dawkins Date: Mon, 2 Nov 2026 21:48:17 +0000 Subject: [PATCH 2/6] feat(db): estimate reclaimable space from a sample of rows DATA_FREE only counts free extents, so it misses space freed inside pages, e.g. after scattered deletes or updates to shorter values. On MySQL 8.0 and MariaDB 11.4, a table with 9 of 10 rows deleted reported 5 MiB in DATA_FREE, and a rebuild freed 62-67 MiB. db:mysql-optimize now samples about 1000 rows of each table that could reach --min-reclaimable, and estimates the rebuilt size: - Row bytes are measured with LENGTH() for variable-size columns, with fixed sizes for other types and 20 bytes per row for InnoDB headers. - With a single-column integer primary key, rows are read from 50 ranges of 20 rows, starting at uneven intervals of the key (a golden ratio sequence), so that they do not align with a pattern that repeats in the data. Each range is an index lookup. Otherwise, the first 1000 rows are read, and the table is noted as such. - The rebuilt data size is the row bytes x TABLE_ROWS x 1.5. Secondary indexes are scaled by the same ratio as the data, capped at their current size. - The reclaimable space is the larger of DATA_FREE and the current size (with free extents) minus the rebuilt size. - The disk needed for a rebuild is the rebuilt size, plus 10% and 8 MiB, instead of the table's current size. The thresholds are now applied after sampling. Tables in a shared tablespace are not sampled. If the sample cannot be read, only DATA_FREE is used, with a warning. Checked against real servers with a user with privileges on its database only: the estimates were within 8 MiB of the space freed by rebuilds of six test tables, and the disk needed was above the size of each rebuilt file. Co-Authored-By: Claude Opus 5.5 --- integration-tests/db_mysql_optimize_test.go | 156 ++++++++---- .../src/Command/Db/DbMysqlOptimizeCommand.php | 225 ++++++++++++++++-- legacy/src/Model/Mysql/TableCandidate.php | 72 +++++- .../tests/Model/Mysql/MysqlOptimizeTest.php | 62 ++++- 4 files changed, 447 insertions(+), 68 deletions(-) diff --git a/integration-tests/db_mysql_optimize_test.go b/integration-tests/db_mysql_optimize_test.go index 821dfe710..e6d2ed2cf 100644 --- a/integration-tests/db_mysql_optimize_test.go +++ b/integration-tests/db_mysql_optimize_test.go @@ -17,30 +17,50 @@ import ( const mib = 1024 * 1024 +// fakeTable is a table returned by the fake server. +type fakeTable struct { + schema, name string + dataLength, indexLength, dataFree int + rows, offlineIndexes, sameFree int + // Whether the primary key is not a single integer column, so the first rows are sampled. + noIntegerKey bool + // The average bytes of the sampled rows. + avgRowBytes int +} + // fakeMysqlServer is a fake "mariadb" client, which answers the queries of db:mysql-optimize. type fakeMysqlServer struct { - vars map[string]string - tables [][]any // schema, table, data_length, index_length, data_free, offline_indexes, same_free + vars map[string]string + tables []fakeTable + failSample bool } // script returns the fake client, which logs each query to a file in dir. func (s fakeMysqlServer) script(t *testing.T, dir string) string { - var vars, tables strings.Builder + var vars, tables, columns, samples strings.Builder for k, v := range s.vars { fmt.Fprintf(&vars, "%s\t%s\n", k, v) } - for _, row := range s.tables { - for i, v := range row { - if i > 0 { - tables.WriteString("\t") - } - fmt.Fprint(&tables, v) + quote := func(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" } + for _, tbl := range s.tables { + fmt.Fprintf(&tables, "%s\t%s\t%d\t%d\t%d\t%d\t%d\t%d\n", tbl.schema, tbl.name, + tbl.dataLength, tbl.indexLength, tbl.dataFree, tbl.rows, tbl.offlineIndexes, tbl.sameFree) + // Columns are matched by the table's name in the query, as "('schema', 'name')". + key := "PRI" + if tbl.noIntegerKey { + key = "" } - tables.WriteString("\n") + match := fmt.Sprintf("('%s', '%s')", tbl.schema, tbl.name) + fmt.Fprintf(&columns, "%s\t%s\t%s\tid\tint\t%s\n", match, tbl.schema, tbl.name, key) + fmt.Fprintf(&columns, "%s\t%s\t%s\tpayload\tvarchar\t\n", match, tbl.schema, tbl.name) + fmt.Fprintf(&samples, "%s.%s\t1000\t%d\n", quote(tbl.schema), quote(tbl.name), 1000*tbl.avgRowBytes) + } + for name, b := range map[string]*strings.Builder{"vars": &vars, "tables": &tables, "columns": &columns, + "samples": &samples} { + require.NoError(t, os.WriteFile(filepath.Join(dir, name+".tsv"), []byte(b.String()), 0o600)) } - require.NoError(t, os.WriteFile(filepath.Join(dir, "vars.tsv"), []byte(vars.String()), 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "tables.tsv"), []byte(tables.String()), 0o600)) return fmt.Sprintf(`dir=%q +failSample=%t q="$(cat)" printf '%%s\n---\n' "$q" >> "$dir/queries.log" case "$q" in @@ -54,9 +74,23 @@ case "$q" in esac printf '%%s\t%%s\t%%s\n' "$schema" "$name" "$rest" done < "$dir/tables.tsv" ;; + *"FROM information_schema.COLUMNS"*) + # The last field may be empty, so the lines are not split with read. + while IFS= read -r line; do + case "$q" in *"${line%%%%$'\t'*}"*) printf '%%s\n' "${line#*$'\t'}" ;; esac + done < "$dir/columns.tsv" ;; + *"COALESCE(SUM(b), 0)"*) + if $failSample; then echo "ERROR 1142 (42000): SELECT command denied" >&2; exit 1; fi + # One line per table: answer with its index. + while IFS= read -r line; do + [[ $line =~ SELECT\ ([0-9]+),\ COUNT ]] || continue + while IFS=$'\t' read -r quoted n bytes; do + case "$line" in *"FROM $quoted "*) printf '%%s\t%%s\t%%s\n' "${BASH_REMATCH[1]}" "$n" "$bytes" ;; esac + done < "$dir/samples.tsv" + done <<< "$q" ;; *) echo "unexpected query: $q" >&2; exit 1 ;; esac -`, dir) +`, dir, s.failSample) } // setUpMysqlOptimizeProject serves an app with database relationships, with a fake MariaDB client, and disk metrics @@ -113,13 +147,22 @@ var mariadbVars = map[string]string{ "tmpdir": "/tmp", } -// optimizeTables are the tables returned by the fake server. Thresholds are applied by the query, so they are not -// tested here. -var optimizeTables = [][]any{ - {"main", "logs", 200 * mib, 50 * mib, 300 * mib, 0, 0}, - {"main", "cache", 2048 * mib, 1024 * mib, 1024 * mib, 0, 0}, - {"main", "search", 100 * mib, 100 * mib, 150 * mib, 1, 0}, - {"main", "we`ird", 100 * mib, 0, 120 * mib, 0, 1}, +// optimizeTables are the tables returned by the fake server. The size threshold is applied by the query, so the fake +// server ignores it. +var optimizeTables = []fakeTable{ + // Rebuilt: 75 MiB of data, and 18.75 MiB of indexes. + {schema: "main", name: "logs", dataLength: 200 * mib, indexLength: 50 * mib, dataFree: 12 * mib, + rows: 102400, avgRowBytes: 512}, + // Rebuilt: 3 GiB of data, and the indexes do not grow: only the free extents are reclaimable. + {schema: "main", name: "cache", dataLength: 2048 * mib, indexLength: 1024 * mib, dataFree: 1024 * mib, + rows: 1048576, avgRowBytes: 2048}, + // Rebuilt: 3 MiB of data, and 3 MiB of indexes. + {schema: "main", name: "search", dataLength: 100 * mib, indexLength: 100 * mib, dataFree: 150 * mib, + rows: 2048, avgRowBytes: 1024, offlineIndexes: 1, noIntegerKey: true}, + // Not sampled. + {schema: "main", name: "we`ird", dataLength: 100 * mib, dataFree: 120 * mib, sameFree: 1}, + // Rebuilt: 112.5 MiB of data, so less than 100 MiB is reclaimable. + {schema: "main", name: "small", dataLength: 150 * mib, rows: 153600, avgRowBytes: 512}, } func TestDbMysqlOptimizeReport(t *testing.T) { @@ -133,36 +176,51 @@ func TestDbMysqlOptimizeReport(t *testing.T) { "--columns", "schema,table,size,reclaimable,percent,disk_needed,disk_fits,tmp_needed,tmp_fits")...) require.NoError(t, err, stdErr) assert.Contains(t, stdErr, "Checking database service db...") + assert.Contains(t, stdErr, "Sampling rows from 4 table(s)...") assert.Contains(t, stdErr, "Disk: 3.0 GiB free of 4.0 GiB. /tmp: 924.0 MiB free of 1.0 GiB.") + // Sorted by reclaimable space. The disk needed is the rebuilt size, plus 10% and 8 MiB. assert.Equal(t, `Schema,Table,Size,Reclaimable,Reclaimable %,Disk needed,Fits disk,/tmp needed,Fits /tmp -main,logs,262144000,314572800,120%,262144000,true,52428800,true -main,cache,3221225472,1073741824,33%,3221225472,false,1073741824,false -main,search,209715200,157286400,75%,209715200,true,104857600,true +main,cache,3221225472,1073741824,33%,4732852633,false,1073741824,false +main,search,209715200,360710144,172%,15309209,true,104857600,true +main,logs,262144000,176422912,67%,116523008,true,52428800,true main,we`+"`"+`ird,104857600,125829120,120%,104857600,true,0,true `, output) output = p.f.Run(args("--format", "tsv", "--no-header", "--columns", "table,notes")...) - assert.Equal(t, "logs\t\ncache\t\n"+ - "search\thas a FULLTEXT or SPATIAL index: it cannot be rebuilt without blocking writes\n"+ + assert.Equal(t, "cache\t\n"+ + "search\thas a FULLTEXT or SPATIAL index: it cannot be rebuilt without blocking writes;"+ + " estimated from the first rows only\n"+ + "logs\t\n"+ "we`ird\tprobably in a shared tablespace: a rebuild would not reclaim space\n", output) q := queries() - require.Len(t, q, 4) // Two runs. + require.Len(t, q, 8) // Two runs. assert.Contains(t, q[0], "SHOW GLOBAL VARIABLES WHERE Variable_name IN (") assert.Contains(t, q[0], "SHOW GLOBAL STATUS WHERE Variable_name IN ('wsrep_cluster_size')") // MariaDB has no information_schema_stats_expiry. assert.NotContains(t, q[1], "information_schema_stats_expiry") - assert.Contains(t, q[1], "t.DATA_FREE >= 104857600 AND t.DATA_FREE * 100 >= 20 * (t.DATA_LENGTH + t.INDEX_LENGTH)") + assert.Contains(t, q[1], "t.DATA_LENGTH + t.INDEX_LENGTH + t.DATA_FREE >= 104857600") + // Tables in a shared tablespace are not sampled. + assert.Contains(t, q[2], "FROM information_schema.COLUMNS WHERE (TABLE_SCHEMA, TABLE_NAME) IN (('main', 'logs'), ") + assert.NotContains(t, q[2], "we`ird") + assert.Contains(t, q[3], "SELECT MIN(`id`), MAX(`id`) FROM `main`.`logs` INTO @lo, @hi;") + assert.Contains(t, q[3], "FROM `main`.`search` LIMIT 1000) s;") + assert.NotContains(t, q[3], "we``ird") // Thresholds and table filters, with escaped strings. - _, stdErr, err = p.f.RunCombinedOutput(args("--min-reclaimable", "1G", "--min-percent", "50", + _, stdErr, err = p.f.RunCombinedOutput(args("--min-reclaimable", "150M", "--min-percent", "50", "--table", "main.logs", "-t", "o'ther")...) require.NoError(t, err, stdErr) q = queries() - assert.Contains(t, q[len(q)-1], "t.DATA_FREE >= 1073741824 AND t.DATA_FREE * 100 >= 50 *") - assert.Contains(t, q[len(q)-1], "((t.TABLE_SCHEMA = 'main' AND t.TABLE_NAME = 'logs') OR t.TABLE_NAME = 'o''ther')") + assert.Contains(t, q[len(q)-3], "t.DATA_LENGTH + t.INDEX_LENGTH + t.DATA_FREE >= 157286400") + assert.Contains(t, q[len(q)-3], "((t.TABLE_SCHEMA = 'main' AND t.TABLE_NAME = 'logs') OR t.TABLE_NAME = 'o''ther')") assert.Contains(t, stdErr, "To rebuild these tables, run this command again with --execute.") + // The percentage is applied after sampling. + _, stdErr, err = p.f.RunCombinedOutput(args("--min-percent", "80", "--table", "main.logs")...) + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "No tables were found with at least 100.0 MiB and 80% of their size reclaimable.") + // The default table output. output = p.f.Run(args("--columns", "table,size,disk_fits")...) assert.Contains(t, output, "| logs | 250.0 MiB | yes |") @@ -176,6 +234,19 @@ main,we`+"`"+`ird,104857600,125829120,120%,104857600,true,0,true assert.Contains(t, stdErr, "Invalid size: lots") } +func TestDbMysqlOptimizeSampleFailure(t *testing.T) { + t.Parallel() + p, _ := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: optimizeTables, failSample: true}, + true) + + // Only the free extents are reclaimable, and a rebuild needs the table's size. + output, stdErr, err := p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", + "-r", "database", "--format", "csv", "--no-header", "--columns", "table,reclaimable,disk_needed") + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "Warning: failed to sample rows: only free extents are counted as reclaimable.") + assert.Equal(t, "cache,1073741824,3221225472\nsearch,157286400,209715200\nwe`ird,125829120,104857600\n", output) +} + func TestDbMysqlOptimizeExecute(t *testing.T) { t.Parallel() p, queries := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: optimizeTables}, true) @@ -197,7 +268,7 @@ func TestDbMysqlOptimizeExecute(t *testing.T) { assert.Contains(t, stdErr, "Skipping main.search: has a FULLTEXT or SPATIAL index") assert.Contains(t, stdErr, "Skipping main.we`ird: probably in a shared tablespace") assert.Contains(t, stdErr, - "Error: There is not enough free disk space to rebuild main.cache (it needs about 3.0 GiB).") + "Error: There is not enough free disk space to rebuild main.cache (it needs about 4.4 GiB).") assert.Contains(t, stdErr, "Error: There is not enough free space in /tmp to rebuild main.cache") assert.Contains(t, stdErr, "The tables were not rebuilt.") assert.Empty(t, alters()) @@ -213,9 +284,9 @@ func TestDbMysqlOptimizeExecute(t *testing.T) { require.NoError(t, err, stdErr) assert.Contains(t, stdErr, "Warning: There is not enough free disk space to rebuild main.cache") assert.Equal(t, []string{ - "ALTER TABLE `main`.`logs` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", "ALTER TABLE `main`.`logs` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", "ALTER TABLE `main`.`cache` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", + "ALTER TABLE `main`.`logs` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", }, alters()) _, stdErr, err = p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", "--force") @@ -225,16 +296,19 @@ func TestDbMysqlOptimizeExecute(t *testing.T) { func TestDbMysqlOptimizeEscaping(t *testing.T) { t.Parallel() - tables := [][]any{{"my`db", "we`ird", 100 * mib, 0, 120 * mib, 0, 0}} + tables := []fakeTable{{schema: "my`db", name: "we`ird", dataLength: 100 * mib, dataFree: 120 * mib, + rows: 1024, avgRowBytes: 100}} p, queries := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: tables}, true) _, stdErr, err := p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", "-r", "database", "--execute", "--analyze") require.NoError(t, err, stdErr) q := queries() - require.Len(t, q, 5) - assert.Equal(t, "ANALYZE TABLE `my``db`.`we``ird`;", q[2]) - assert.Equal(t, "ALTER TABLE `my``db`.`we``ird` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", q[4]) + require.Len(t, q, 9) + assert.Contains(t, q[2], "IN (('my`db', 'we`ird'))") + assert.Contains(t, q[3], "FROM `my``db`.`we``ird` INTO @lo, @hi;") + assert.Equal(t, "ANALYZE TABLE `my``db`.`we``ird`;", q[4]) + assert.Equal(t, "ALTER TABLE `my``db`.`we``ird` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", q[8]) } func TestDbMysqlOptimizeGalera(t *testing.T) { @@ -272,9 +346,9 @@ func TestDbMysqlOptimizeHeadroomUnknown(t *testing.T) { "innodb_file_per_table": "OFF", "information_schema_stats_expiry": "86400", } - tables := [][]any{ - {"main", "logs", 200 * mib, 50 * mib, 300 * mib, 0, 0}, - {"main", "other", 200 * mib, 50 * mib, 300 * mib, 0, 0}, + tables := []fakeTable{ + {schema: "main", name: "logs", dataLength: 200 * mib, indexLength: 50 * mib, dataFree: 300 * mib}, + {schema: "main", name: "other", dataLength: 200 * mib, indexLength: 50 * mib, dataFree: 300 * mib}, } for _, metrics := range []string{"unavailable", "disabled"} { t.Run(metrics, func(t *testing.T) { @@ -291,7 +365,9 @@ func TestDbMysqlOptimizeHeadroomUnknown(t *testing.T) { assert.Contains(t, stdErr, "Warning: innodb_file_per_table is OFF") assert.Contains(t, stdErr, "Warning: the disk usage of the service is unknown") assert.Equal(t, "logs,,\nother,,\n", output) + // Tables in the shared tablespace are not sampled. q := queries() + require.Len(t, q, 2) assert.True(t, strings.HasPrefix(q[1], "SET SESSION information_schema_stats_expiry = 0;\n"), q[1]) // All tables are skipped, as they are in the shared tablespace. @@ -303,7 +379,7 @@ func TestDbMysqlOptimizeHeadroomUnknown(t *testing.T) { } // With file-per-table, an unknown headroom prevents a rebuild without --force. - p, _ := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: tables[:1]}, false) + p, _ := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: optimizeTables[:1]}, false) _, stdErr, err := p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", "-r", "database", "--execute") assertExitCode(t, 1, err) diff --git a/legacy/src/Command/Db/DbMysqlOptimizeCommand.php b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php index b56668efb..bed40f73d 100644 --- a/legacy/src/Command/Db/DbMysqlOptimizeCommand.php +++ b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php @@ -61,6 +61,25 @@ class DbMysqlOptimizeCommand extends CommandBase private const SYSTEM_SCHEMAS = ['mysql', 'information_schema', 'performance_schema', 'sys']; + /** Rows are sampled in this many ranges of the primary key, of this many rows each. */ + public const SAMPLE_RANGES = 50; + public const SAMPLE_RANGE_ROWS = 20; + + /** The number of rows sampled from the start of a table, if its primary key is not a single integer column. */ + public const SAMPLE_FIRST_ROWS = 1000; + + /** The bytes per row stored by InnoDB besides the columns (record header, transaction ID and roll pointer). */ + private const ROW_HEADER_BYTES = 20; + + /** The bytes stored for fixed-size column types (at most). Other types are measured with LENGTH(). */ + private const FIXED_SIZES = [ + 'tinyint' => 1, 'smallint' => 2, 'mediumint' => 3, 'int' => 4, 'integer' => 4, 'bigint' => 8, + 'float' => 4, 'double' => 8, 'real' => 8, 'bit' => 8, + 'date' => 3, 'time' => 6, 'datetime' => 8, 'timestamp' => 7, 'year' => 1, + 'enum' => 2, 'set' => 8, + ]; + private const INTEGER_TYPES = ['tinyint', 'smallint', 'mediumint', 'int', 'integer', 'bigint']; + public function __construct( private readonly Config $config, private readonly Io $io, @@ -88,19 +107,21 @@ protected function configure(): void $this->setHelp(<<--min-reclaimable. A table is listed if its free space is at least --min-reclaimable and --min-percent of its size. By default, this only reports. With --execute, each table listed is rebuilt with: ALTER TABLE ... FORCE, ALGORITHM=INPLACE, LOCK=NONE which is the same rebuild as OPTIMIZE TABLE, but fails instead of locking the table if it cannot be done online. - A rebuild copies the table: it needs free disk space for the table's size, and temporary space for its indexes. + A rebuild copies the table: it needs free disk space for the rebuilt table, and temporary space for its indexes. The disk and /tmp usage are read from the metrics API: if they are unknown, or a table does not fit (leaving at least {$reserve}% of the volume free), the tables are not rebuilt, unless --force is used. On a cluster (Galera), a rebuild blocks writes on all nodes while it runs, so it is refused unless --force is used. - The sizes are estimates from the database's statistics. To check the disk usage of the service, run: $executable disk + The sizes are estimates, from the database's statistics and the sample of rows. To check the disk usage of the service, run: $executable disk EOF); $this->addExample('Report tables that could be optimized'); $this->addExample('Rebuild the tables found, if there is enough disk space', '--execute'); @@ -284,14 +305,71 @@ private function findTables(HostInterface $host, string $mysql, MysqlServerInfo // MySQL 8 caches statistics in information_schema for up to 24 hours by default. $query .= "SET SESSION information_schema_stats_expiry = 0;\n"; } - $query .= self::tablesQuery($minReclaimable, $minPercent, $filter) . "\n"; + $query .= self::tablesQuery($minReclaimable, $filter) . "\n"; $this->io->debug('Query: ' . $query); - $result = (string) $host->runCommand($mysql, true, true, $query); $tables = []; - foreach (explode("\n", $result) as $line) { - $fields = explode("\t", rtrim($line, "\r")); - if (\count($fields) === 7) { - $tables[] = TableCandidate::fromRow($fields); + foreach ($this->parseRows((string) $host->runCommand($mysql, true, true, $query), 8) as $fields) { + $tables[] = TableCandidate::fromRow($fields); + } + + $tables = $this->sampleRows($host, $mysql, $server, $tables); + + $tables = array_values(array_filter($tables, fn(TableCandidate $t): bool => $t->reclaimable() >= $minReclaimable + && $t->reclaimable() * 100 >= $minPercent * $t->size())); + usort($tables, fn(TableCandidate $a, TableCandidate $b): int => $b->reclaimable() <=> $a->reclaimable()); + + return $tables; + } + + /** + * Estimates the size of the rows of each table, from a sample. + * + * Tables in a shared tablespace are not sampled, as a rebuild would not shrink it. + * If the sample cannot be read, the tables are returned unchanged. + * + * @param TableCandidate[] $tables + * + * @return TableCandidate[] + */ + private function sampleRows(HostInterface $host, string $mysql, MysqlServerInfo $server, array $tables): array + { + $toSample = array_filter($tables, fn(TableCandidate $t): bool => !$t->inSharedTablespace($server)); + if (!$toSample) { + return $tables; + } + + try { + $query = self::columnsQuery($toSample); + $this->io->debug('Query: ' . $query); + $columns = []; + foreach ($this->parseRows((string) $host->runCommand($mysql, true, true, $query), 5) as $fields) { + $columns[$fields[0] . "\0" . $fields[1]][] = ['name' => $fields[2], 'type' => strtolower($fields[3]), 'primary' => $fields[4] === 'PRI']; + } + + $query = ''; + $firstRows = []; + foreach ($toSample as $i => $t) { + $tableColumns = $columns[$t->schema . "\0" . $t->name] ?? []; + if ($tableColumns) { + $query .= self::sampleQuery($i, $t, $tableColumns) . "\n"; + $firstRows[$i] = self::integerPrimaryKey($tableColumns) === null; + } + } + if ($query === '') { + return $tables; + } + $this->stdErr->writeln(\sprintf('Sampling rows from %d table(s)...', \count($firstRows))); + $this->io->debug('Query: ' . $query); + $result = (string) $host->runCommand($mysql, true, true, $query); + } catch (ProcessRuntimeException $e) { + $this->stdErr->writeln('Warning: failed to sample rows: only free extents are counted as reclaimable.'); + $this->io->debug($e->getMessage()); + return $tables; + } + + foreach ($this->parseRows($result, 3) as [$i, $count, $bytes]) { + if (isset($firstRows[(int) $i])) { + $tables[(int) $i] = $tables[(int) $i]->withSample((int) $count > 0 ? (int) $bytes / (int) $count : 0.0, $firstRows[(int) $i]); } } @@ -299,7 +377,28 @@ private function findTables(HostInterface $host, string $mysql, MysqlServerInfo } /** - * Returns the query to find InnoDB tables with reclaimable space. + * Parses tab-separated lines, with the given number of fields. + * + * @return list + */ + private function parseRows(string $output, int $numFields): array + { + $rows = []; + foreach (explode("\n", $output) as $line) { + $fields = explode("\t", rtrim($line, "\r")); + if (\count($fields) === $numFields) { + $rows[] = $fields; + } + } + + return $rows; + } + + /** + * Returns the query to find InnoDB tables that may have reclaimable space. + * + * The thresholds are applied after rows are sampled: here, a table is + * only excluded if its total size is below the minimum. * * Free space in a shared tablespace (the system tablespace, if * innodb_file_per_table is OFF, or a general tablespace) is reported for @@ -308,14 +407,13 @@ private function findTables(HostInterface $host, string $mysql, MysqlServerInfo * * @param string[] $filter */ - public static function tablesQuery(int $minReclaimable, float $minPercent, array $filter = []): string + public static function tablesQuery(int $minReclaimable, array $filter = []): string { $conditions = [ "t.ENGINE = 'InnoDB'", "t.TABLE_TYPE = 'BASE TABLE'", 't.TABLE_SCHEMA NOT IN (' . implode(', ', array_map(self::quoteString(...), self::SYSTEM_SCHEMAS)) . ')', - 't.DATA_FREE >= ' . $minReclaimable, - 't.DATA_FREE * 100 >= ' . self::formatNumber($minPercent) . ' * (t.DATA_LENGTH + t.INDEX_LENGTH)', + 't.DATA_LENGTH + t.INDEX_LENGTH + t.DATA_FREE >= ' . $minReclaimable, ]; if ($filter) { $or = []; @@ -330,13 +428,96 @@ public static function tablesQuery(int $minReclaimable, float $minPercent, array $conditions[] = '(' . implode(' OR ', $or) . ')'; } - return 'SELECT t.TABLE_SCHEMA, t.TABLE_NAME, t.DATA_LENGTH, t.INDEX_LENGTH, t.DATA_FREE,' + return 'SELECT t.TABLE_SCHEMA, t.TABLE_NAME, t.DATA_LENGTH, t.INDEX_LENGTH, t.DATA_FREE, t.TABLE_ROWS,' . " (SELECT COUNT(*) FROM information_schema.STATISTICS s WHERE s.TABLE_SCHEMA = t.TABLE_SCHEMA AND s.TABLE_NAME = t.TABLE_NAME AND s.INDEX_TYPE IN ('FULLTEXT', 'SPATIAL')) AS offline_indexes," . ' COALESCE(f.n, 1) - 1 AS same_free' . ' FROM information_schema.TABLES t' . " LEFT JOIN (SELECT DATA_FREE AS free, COUNT(*) AS n FROM information_schema.TABLES WHERE ENGINE = 'InnoDB' AND DATA_FREE >= " . $minReclaimable . ' GROUP BY DATA_FREE) f ON f.free = t.DATA_FREE' - . ' WHERE ' . implode(' AND ', $conditions) - . ' ORDER BY t.DATA_FREE DESC;'; + . ' WHERE ' . implode(' AND ', $conditions) . ';'; + } + + /** + * Returns the query to find the stored columns of tables. + * + * @param TableCandidate[] $tables + */ + public static function columnsQuery(array $tables): string + { + $names = array_map(fn(TableCandidate $t): string => \sprintf('(%s, %s)', self::quoteString($t->schema), self::quoteString($t->name)), $tables); + + return 'SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, DATA_TYPE, COLUMN_KEY FROM information_schema.COLUMNS' + . ' WHERE (TABLE_SCHEMA, TABLE_NAME) IN (' . implode(', ', $names) . ")" + . " AND EXTRA NOT LIKE '%VIRTUAL%'" + . ' ORDER BY TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION;'; + } + + /** + * Returns the query to sample the rows of a table: it outputs the index, the number of rows sampled, and their bytes. + * + * With a single-column integer primary key, rows are read from ranges spread across the key's values: each range + * is an index lookup. The ranges start at uneven intervals (a golden ratio sequence), so that they do not all + * fall at the same point of a pattern that repeats in the data. Otherwise, the first rows are read. + * + * @param array{name: string, type: string, primary: bool}[] $columns + */ + public static function sampleQuery(int $index, TableCandidate $table, array $columns): string + { + $expr = self::rowBytesExpression($columns); + $from = $table->quotedName(); + $pk = self::integerPrimaryKey($columns); + if ($pk === null) { + return \sprintf('SELECT %d, COUNT(*), COALESCE(SUM(b), 0) FROM (SELECT %s AS b FROM %s LIMIT %d) s;', $index, $expr, $from, self::SAMPLE_FIRST_ROWS); + } + $pk = self::quoteIdentifier($pk); + $ranges = []; + for ($i = 0; $i < self::SAMPLE_RANGES; $i++) { + $ranges[] = \sprintf( + '(SELECT %s AS b FROM %s WHERE %s >= @lo + FLOOR((CAST(@hi AS DECIMAL(65)) - @lo) * %.6F) ORDER BY %s LIMIT %d)', + $expr, + $from, + $pk, + fmod($i * 0.6180339887, 1), + $pk, + self::SAMPLE_RANGE_ROWS, + ); + } + + return \sprintf('SELECT MIN(%s), MAX(%s) FROM %s INTO @lo, @hi;', $pk, $pk, $from) + . \sprintf(' SELECT %d, COUNT(*), COALESCE(SUM(b), 0) FROM (%s) s;', $index, implode(' UNION ALL ', $ranges)); + } + + /** + * Returns an expression for the approximate bytes that InnoDB stores for a row. + * + * @param array{name: string, type: string, primary: bool}[] $columns + */ + public static function rowBytesExpression(array $columns): string + { + $fixed = self::ROW_HEADER_BYTES; + $variable = []; + foreach ($columns as $column) { + if (isset(self::FIXED_SIZES[$column['type']])) { + $fixed += self::FIXED_SIZES[$column['type']]; + } else { + // Plus a byte for the length. + $fixed++; + $variable[] = \sprintf('COALESCE(LENGTH(%s), 0)', self::quoteIdentifier($column['name'])); + } + } + + return implode(' + ', array_merge([(string) $fixed], $variable)); + } + + /** + * Returns the name of the primary key column, if the key is a single integer column. + * + * @param array{name: string, type: string, primary: bool}[] $columns + */ + private static function integerPrimaryKey(array $columns): ?string + { + $primary = array_values(array_filter($columns, fn(array $c): bool => $c['primary'])); + + return \count($primary) === 1 && \in_array($primary[0]['type'], self::INTEGER_TYPES, true) ? $primary[0]['name'] : null; } /** @@ -388,13 +569,16 @@ private function renderTables(array $tables, ?DiskHeadroom $headroom, MysqlServe 'schema' => $t->schema, 'table' => $t->name, 'size' => $formatSize($t->size()), - 'reclaimable' => $formatSize($t->dataFree), - 'percent' => $t->size() > 0 ? \sprintf('%.0f%%', $t->dataFree / $t->size() * 100) : '', + 'reclaimable' => $formatSize($t->reclaimable()), + 'percent' => $t->size() > 0 ? \sprintf('%.0f%%', $t->reclaimable() / $t->size() * 100) : '', 'disk_needed' => $formatSize($t->diskNeeded()), 'disk_fits' => $formatFits($headroom?->diskFits($t->diskNeeded())), 'tmp_needed' => $formatSize($t->tmpNeeded()), 'tmp_fits' => $formatFits($headroom?->tmpFits($t->tmpNeeded())), - 'notes' => (string) $t->skipReason($server), + 'notes' => implode('; ', array_filter([ + $t->skipReason($server), + $t->sampledFirstRows ? 'estimated from the first rows only' : null, + ])), ]; } $this->table->render($rows, self::TABLE_HEADER, self::DEFAULT_COLUMNS); @@ -506,9 +690,4 @@ public static function quoteString(string $value): string { return "'" . str_replace(['\\', "'"], ['\\\\', "''"], $value) . "'"; } - - private static function formatNumber(float $number): string - { - return rtrim(rtrim(\sprintf('%.4F', $number), '0'), '.'); - } } diff --git a/legacy/src/Model/Mysql/TableCandidate.php b/legacy/src/Model/Mysql/TableCandidate.php index 7fd2ef13b..f049e521a 100644 --- a/legacy/src/Model/Mysql/TableCandidate.php +++ b/legacy/src/Model/Mysql/TableCandidate.php @@ -11,16 +11,29 @@ */ readonly class TableCandidate { + /** The ratio of a rebuilt table's data size to the bytes in its rows (for headers and free space in pages). */ + public const PAGE_OVERHEAD = 1.5; + + /** The disk space needed for a rebuild, beyond the estimated size, as a percentage and in bytes (for the file's free extents). */ + public const DISK_MARGIN_PERCENT = 10; + public const DISK_MARGIN_BYTES = 8 * 1024 * 1024; + public function __construct( public string $schema, public string $name, public int $dataLength, public int $indexLength, public int $dataFree, + /** The estimated number of rows (TABLE_ROWS). */ + public int $rows = 0, /** The number of FULLTEXT or SPATIAL indexes, which prevent an online rebuild. */ public int $offlineIndexes = 0, /** The number of other tables reporting the same free space (a sign of a shared tablespace). */ public int $sameFree = 0, + /** The average bytes per row, from a sample of rows, or null if unknown. */ + public ?float $avgRowBytes = null, + /** Whether the sample is the first rows only, rather than spread across the table. */ + public bool $sampledFirstRows = false, ) {} /** @@ -28,7 +41,15 @@ public function __construct( */ public static function fromRow(array $row): self { - return new self($row[0], $row[1], (int) $row[2], (int) $row[3], (int) $row[4], (int) $row[5], (int) $row[6]); + return new self($row[0], $row[1], (int) $row[2], (int) $row[3], (int) $row[4], (int) $row[5], (int) $row[6], (int) $row[7]); + } + + /** + * Returns a copy with the result of a sample of rows. + */ + public function withSample(float $avgRowBytes, bool $firstRows): self + { + return new self($this->schema, $this->name, $this->dataLength, $this->indexLength, $this->dataFree, $this->rows, $this->offlineIndexes, $this->sameFree, $avgRowBytes, $firstRows); } /** @@ -39,12 +60,49 @@ public function size(): int return $this->dataLength + $this->indexLength; } + /** + * The estimated size of the table's data and indexes after a rebuild, or null if unknown. + * + * Secondary indexes are assumed to shrink in the same ratio as the data. + */ + public function rebuiltSize(): ?int + { + if ($this->avgRowBytes === null) { + return null; + } + $data = (int) ceil($this->avgRowBytes * $this->rows * self::PAGE_OVERHEAD); + $index = $this->dataLength > 0 ? (int) round($this->indexLength * min(1, $data / $this->dataLength)) : $this->indexLength; + + return $data + $index; + } + + /** + * The estimated disk space that a rebuild would return. + * + * Free extents (DATA_FREE) are always reclaimable. Free space inside + * pages is only counted if rows were sampled. + */ + public function reclaimable(): int + { + $rebuilt = $this->rebuiltSize(); + if ($rebuilt === null) { + return $this->dataFree; + } + + return max($this->dataFree, $this->size() + $this->dataFree - $rebuilt); + } + /** * The disk space needed for the rebuilt copy of the table. */ public function diskNeeded(): int { - return $this->size(); + $rebuilt = $this->rebuiltSize(); + if ($rebuilt === null) { + return $this->size(); + } + + return $rebuilt + intdiv($rebuilt * self::DISK_MARGIN_PERCENT, 100) + self::DISK_MARGIN_BYTES; } /** @@ -76,12 +134,20 @@ public function rebuildQuery(): string return 'ALTER TABLE ' . $this->quotedName() . ' FORCE, ALGORITHM=INPLACE, LOCK=NONE;'; } + /** + * Whether the table is probably in a shared tablespace, which a rebuild does not shrink. + */ + public function inSharedTablespace(MysqlServerInfo $server): bool + { + return $this->sameFree > 0 || $server->filePerTable === false; + } + /** * Returns why the table would not be rebuilt, or null. */ public function skipReason(MysqlServerInfo $server): ?string { - if ($this->sameFree > 0 || $server->filePerTable === false) { + if ($this->inSharedTablespace($server)) { return 'probably in a shared tablespace: a rebuild would not reclaim space'; } if ($this->offlineIndexes > 0) { diff --git a/legacy/tests/Model/Mysql/MysqlOptimizeTest.php b/legacy/tests/Model/Mysql/MysqlOptimizeTest.php index 1ad2b6019..e1c67eeff 100644 --- a/legacy/tests/Model/Mysql/MysqlOptimizeTest.php +++ b/legacy/tests/Model/Mysql/MysqlOptimizeTest.php @@ -68,12 +68,70 @@ public function testHeadroom(): void $this->assertFalse($headroom->diskFits(401)); } + public function testEstimates(): void + { + $mib = 1024 * 1024; + // 90 MiB of data and 10 MiB of indexes, with 5 MiB of free extents. + $table = new TableCandidate('s', 't', 90 * $mib, 10 * $mib, 5 * $mib, rows: 10240); + + // Without a sample, only the free extents are reclaimable, and a rebuild needs the current size. + $this->assertNull($table->rebuiltSize()); + $this->assertSame(5 * $mib, $table->reclaimable()); + $this->assertSame(100 * $mib, $table->diskNeeded()); + + // 10240 rows of 1 KiB: 15 MiB of data, and the indexes shrink in the same ratio (to 1/6). + $sampled = $table->withSample(1024, false); + $rebuilt = 15 * $mib + (int) round(10 * $mib / 6); + $this->assertSame($rebuilt, $sampled->rebuiltSize()); + $this->assertSame(105 * $mib - $rebuilt, $sampled->reclaimable()); + $this->assertSame($rebuilt + intdiv($rebuilt, 10) + 8 * $mib, $sampled->diskNeeded()); + + // A table that would grow: the indexes are not scaled up, and the free extents are still reclaimable. + $growing = $table->withSample(10240, false); + $this->assertSame(160 * $mib, $growing->rebuiltSize()); + $this->assertSame(5 * $mib, $growing->reclaimable()); + + // An empty table. + $this->assertSame(105 * $mib, (new TableCandidate('s', 't', 90 * $mib, 10 * $mib, 5 * $mib))->withSample(0, false)->reclaimable()); + } + + public function testSampleQuery(): void + { + $columns = [ + ['name' => 'id', 'type' => 'bigint', 'primary' => true], + ['name' => 'na`me', 'type' => 'varchar', 'primary' => false], + ['name' => 'created', 'type' => 'datetime', 'primary' => false], + ['name' => 'body', 'type' => 'text', 'primary' => false], + ]; + // 20 bytes of header, 8 + 8 for the fixed columns, and a byte for each variable length. + $this->assertSame('38 + COALESCE(LENGTH(`na``me`), 0) + COALESCE(LENGTH(`body`), 0)', DbMysqlOptimizeCommand::rowBytesExpression($columns)); + + $table = new TableCandidate('my`db', 't', 1, 2, 3); + $query = DbMysqlOptimizeCommand::sampleQuery(3, $table, $columns); + $this->assertStringStartsWith('SELECT MIN(`id`), MAX(`id`) FROM `my``db`.`t` INTO @lo, @hi; SELECT 3, COUNT(*), COALESCE(SUM(b), 0) FROM ((SELECT 38 + ', $query); + $this->assertSame(DbMysqlOptimizeCommand::SAMPLE_RANGES, substr_count($query, 'ORDER BY `id` LIMIT 20)')); + $this->assertStringContainsString('WHERE `id` >= @lo + FLOOR((CAST(@hi AS DECIMAL(65)) - @lo) * 0.000000) ORDER BY', $query); + $this->assertStringContainsString('* 0.618034) ORDER BY', $query); + $this->assertStringContainsString('* 0.236068) ORDER BY', $query); + + // Without a single-column integer primary key, the first rows are read. + $expected = 'SELECT 0, COUNT(*), COALESCE(SUM(b), 0) FROM (SELECT 23 + COALESCE(LENGTH(`code`), 0) AS b FROM `my``db`.`t` LIMIT 1000) s;'; + $this->assertSame($expected, DbMysqlOptimizeCommand::sampleQuery(0, $table, [ + ['name' => 'code', 'type' => 'varchar', 'primary' => true], + ['name' => 'n', 'type' => 'smallint', 'primary' => false], + ])); + $this->assertStringEndsWith('LIMIT 1000) s;', DbMysqlOptimizeCommand::sampleQuery(0, $table, [ + ['name' => 'a', 'type' => 'int', 'primary' => true], + ['name' => 'b', 'type' => 'int', 'primary' => true], + ])); + } + public function testSkipReason(): void { $server = new MysqlServerInfo(['innodb_file_per_table' => 'ON']); $this->assertNull((new TableCandidate('s', 't', 1, 1, 1))->skipReason($server)); - $this->assertStringContainsString('shared tablespace', (string) (new TableCandidate('s', 't', 1, 1, 1, 0, 2))->skipReason($server)); - $this->assertStringContainsString('FULLTEXT', (string) (new TableCandidate('s', 't', 1, 1, 1, 1))->skipReason($server)); + $this->assertStringContainsString('shared tablespace', (string) (new TableCandidate('s', 't', 1, 1, 1, sameFree: 2))->skipReason($server)); + $this->assertStringContainsString('FULLTEXT', (string) (new TableCandidate('s', 't', 1, 1, 1, offlineIndexes: 1))->skipReason($server)); $this->assertStringContainsString('shared tablespace', (string) (new TableCandidate('s', 't', 1, 1, 1))->skipReason(new MysqlServerInfo(['innodb_file_per_table' => 'OFF']))); } } From b06e5a0c4c8ce472bfda7509abb2360e1c0b9a74 Mon Sep 17 00:00:00 2001 From: Patrick Dawkins Date: Mon, 2 Nov 2026 22:00:28 +0000 Subject: [PATCH 3/6] fix(db): address review of db:mysql-optimize - Show only the database's error when a rebuild fails. The exception's message contains the command line, with the database password, and on remote hosts it leaves out the error output. - Count tables as probably in a shared tablespace when they share at least 8 MiB of free space, instead of --min-reclaimable. With row sampling, tables with less free space than the threshold can be candidates, and they were never flagged. Tested with two tables created while innodb_file_per_table was OFF on MySQL 8.0 and MariaDB 11.4: both report 18 MiB, and are now flagged. - Do not use the sample if TABLE_ROWS is lower than the number of rows sampled: statistics are out of date, and the estimate (proportional to TABLE_ROWS) could let a rebuild through the disk check. - With --analyze, analyze every table checked, before sampling, and sample once. MariaDB's default ANALYZE TABLE does not scan tables for column statistics (use_stat_tables=PREFERABLY_FOR_QUERIES). - Show the deprecation notice of db:size --cleanup for any database. Co-Authored-By: Claude Opus 5.5 --- integration-tests/db_mysql_optimize_test.go | 65 +++++++++++++--- .../src/Command/Db/DbMysqlOptimizeCommand.php | 75 ++++++++++++++----- legacy/src/Command/Db/DbSizeCommand.php | 14 ++-- 3 files changed, 118 insertions(+), 36 deletions(-) diff --git a/integration-tests/db_mysql_optimize_test.go b/integration-tests/db_mysql_optimize_test.go index e6d2ed2cf..dd65e2ccf 100644 --- a/integration-tests/db_mysql_optimize_test.go +++ b/integration-tests/db_mysql_optimize_test.go @@ -65,6 +65,9 @@ q="$(cat)" printf '%%s\n---\n' "$q" >> "$dir/queries.log" case "$q" in *"SHOW GLOBAL VARIABLES"*) cat "$dir/vars.tsv" ;; + *"ALTER TABLE "*"broken"*) + echo "ERROR 1846 (0A000) at line 1: LOCK=NONE is not supported. Reason: try LOCK=SHARED" >&2 + exit 1 ;; *"ALTER TABLE"*|*"ANALYZE TABLE"*) ;; *"FROM information_schema.TABLES t"*) # Apply table name filters. @@ -200,6 +203,8 @@ main,we`+"`"+`ird,104857600,125829120,120%,104857600,true,0,true // MariaDB has no information_schema_stats_expiry. assert.NotContains(t, q[1], "information_schema_stats_expiry") assert.Contains(t, q[1], "t.DATA_LENGTH + t.INDEX_LENGTH + t.DATA_FREE >= 104857600") + // Tables sharing at least 8 MiB of free space are probably in a shared tablespace, whatever the threshold. + assert.Contains(t, q[1], "DATA_FREE >= 8388608 GROUP BY DATA_FREE") // Tables in a shared tablespace are not sampled. assert.Contains(t, q[2], "FROM information_schema.COLUMNS WHERE (TABLE_SCHEMA, TABLE_NAME) IN (('main', 'logs'), ") assert.NotContains(t, q[2], "we`ird") @@ -247,6 +252,36 @@ func TestDbMysqlOptimizeSampleFailure(t *testing.T) { assert.Equal(t, "cache,1073741824,3221225472\nsearch,157286400,209715200\nwe`ird,125829120,104857600\n", output) } +func TestDbMysqlOptimizeStaleRowCount(t *testing.T) { + t.Parallel() + // 1000 rows are sampled, but TABLE_ROWS is 10: the estimate would be far too small. + tables := []fakeTable{{schema: "main", name: "loaded", dataLength: 2048 * mib, dataFree: 600 * mib, + rows: 10, avgRowBytes: 1024}} + p, _ := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: tables}, true) + + output, stdErr, err := p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", + "-r", "database", "--format", "csv", "--no-header", "--columns", "table,reclaimable,disk_needed") + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "Warning: the row count of main.loaded is out of date") + assert.Equal(t, "loaded,629145600,2147483648\n", output) +} + +func TestDbMysqlOptimizeRebuildFailure(t *testing.T) { + t.Parallel() + tables := []fakeTable{{schema: "main", name: "broken", dataLength: 200 * mib, dataFree: 150 * mib, + rows: 1024, avgRowBytes: 100}} + p, _ := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: tables}, true) + + // The database's error is shown, without the command line, which includes the password. + _, stdErr, err := p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", + "-r", "database", "--execute") + assertExitCode(t, 1, err) + assert.Contains(t, stdErr, "Rebuilding main.broken... [failed]\n"+ + "ERROR 1846 (0A000) at line 1: LOCK=NONE is not supported.") + assert.NotContains(t, stdErr, "--password") + assert.NotContains(t, stdErr, "Full command") +} + func TestDbMysqlOptimizeExecute(t *testing.T) { t.Parallel() p, queries := setUpMysqlOptimizeProject(t, fakeMysqlServer{vars: mariadbVars, tables: optimizeTables}, true) @@ -303,12 +338,14 @@ func TestDbMysqlOptimizeEscaping(t *testing.T) { _, stdErr, err := p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", "-r", "database", "--execute", "--analyze") require.NoError(t, err, stdErr) + // The tables are analyzed before they are sampled. q := queries() - require.Len(t, q, 9) - assert.Contains(t, q[2], "IN (('my`db', 'we`ird'))") - assert.Contains(t, q[3], "FROM `my``db`.`we``ird` INTO @lo, @hi;") - assert.Equal(t, "ANALYZE TABLE `my``db`.`we``ird`;", q[4]) - assert.Equal(t, "ALTER TABLE `my``db`.`we``ird` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", q[8]) + require.Len(t, q, 7) + assert.Equal(t, "ANALYZE TABLE `my``db`.`we``ird`;", q[2]) + assert.Contains(t, q[3], "FROM information_schema.TABLES t") + assert.Contains(t, q[4], "IN (('my`db', 'we`ird'))") + assert.Contains(t, q[5], "FROM `my``db`.`we``ird` INTO @lo, @hi;") + assert.Equal(t, "ALTER TABLE `my``db`.`we``ird` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", q[6]) } func TestDbMysqlOptimizeGalera(t *testing.T) { @@ -403,10 +440,16 @@ func TestDbMysqlOptimizeOtherEngines(t *testing.T) { func TestDbSizeCleanupDeprecated(t *testing.T) { t.Parallel() - p := setUpDBProject(t, map[string]string{"mariadb": "cat > /dev/null; echo 52428800"}) - _, stdErr, err := p.f.RunCombinedOutput("db:size", "-p", layoutProjectID, "-e", "main", "-r", "database", - "--cleanup") - require.NoError(t, err, stdErr) - assert.Contains(t, stdErr, "The --cleanup option is deprecated") - assert.Contains(t, stdErr, "platform-test db:mysql-optimize") + p := setUpDBProject(t, map[string]string{ + "mariadb": "cat > /dev/null; echo 52428800", + "psql": "cat > /dev/null; echo 52428800", + }) + // The notice is shown for any database, as the option is ignored for other types. + for _, rel := range []string{"database", "pg"} { + _, stdErr, err := p.f.RunCombinedOutput("db:size", "-p", layoutProjectID, "-e", "main", "-r", rel, + "--cleanup") + require.NoError(t, err, stdErr) + assert.Contains(t, stdErr, "The --cleanup option is deprecated", rel) + assert.Contains(t, stdErr, "platform-test db:mysql-optimize", rel) + } } diff --git a/legacy/src/Command/Db/DbMysqlOptimizeCommand.php b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php index bed40f73d..38fb49ab0 100644 --- a/legacy/src/Command/Db/DbMysqlOptimizeCommand.php +++ b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php @@ -27,6 +27,8 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Process\Exception\ProcessFailedException; +use Symfony\Component\Process\Exception\ProcessTimedOutException; use Symfony\Component\Process\Exception\RuntimeException as ProcessRuntimeException; /** @@ -61,6 +63,9 @@ class DbMysqlOptimizeCommand extends CommandBase private const SYSTEM_SCHEMAS = ['mysql', 'information_schema', 'performance_schema', 'sys']; + /** The minimum free space shared by tables for them to count as probably in a shared tablespace (smaller amounts are common in separate files). */ + private const SHARED_FREE_MIN = 8 * 1024 * 1024; + /** Rows are sampled in this many ranges of the primary key, of this many rows each. */ public const SAMPLE_RANGES = 50; public const SAMPLE_RANGE_ROWS = 20; @@ -98,7 +103,7 @@ protected function configure(): void ->addOption('min-reclaimable', null, InputOption::VALUE_REQUIRED, 'The minimum reclaimable space of a table, in bytes or with a unit (K, M, G)', self::DEFAULT_MIN_RECLAIMABLE) ->addOption('min-percent', null, InputOption::VALUE_REQUIRED, 'The minimum reclaimable space of a table, as a percentage of its size', (string) self::DEFAULT_MIN_PERCENT) ->addOption('table', 't', InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Only check these tables (as "table" or "schema.table")') - ->addOption('analyze', null, InputOption::VALUE_NONE, 'Refresh statistics with ANALYZE TABLE on the tables found, and check them again (useful on MariaDB and MySQL 5.7)') + ->addOption('analyze', null, InputOption::VALUE_NONE, 'Refresh statistics with ANALYZE TABLE on the tables checked, before estimating their free space') ->addOption('execute', null, InputOption::VALUE_NONE, 'Rebuild the tables found (by default, they are only reported)') ->addOption('force', null, InputOption::VALUE_NONE, 'With --execute: rebuild even if the disk headroom is unknown or too small, or on a cluster (Galera)') ->addOption('bytes', 'B', InputOption::VALUE_NONE, 'Show sizes in bytes'); @@ -183,12 +188,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int } // Find the tables. - $tables = $this->findTables($host, $mysql, $server, $minReclaimable, $minPercent, $tableFilter); - if ($tables && Option::bool($input, 'analyze')) { - $this->stdErr->writeln(\sprintf('Analyzing %d table(s)...', \count($tables))); - $host->runCommand($mysql, true, true, 'ANALYZE TABLE ' . implode(', ', array_map(fn(TableCandidate $t): string => $t->quotedName(), $tables)) . ';'); - $tables = $this->findTables($host, $mysql, $server, $minReclaimable, $minPercent, $tableFilter); - } + $tables = $this->findTables($host, $mysql, $server, $minReclaimable, $minPercent, $tableFilter, Option::bool($input, 'analyze')); $this->stdErr->writeln(''); $this->showServerWarnings($server, $headroom); @@ -298,7 +298,7 @@ private function parsePairs(string $output): array * * @return TableCandidate[] */ - private function findTables(HostInterface $host, string $mysql, MysqlServerInfo $server, int $minReclaimable, float $minPercent, array $filter): array + private function findTables(HostInterface $host, string $mysql, MysqlServerInfo $server, int $minReclaimable, float $minPercent, array $filter, bool $analyze): array { $query = ''; if ($server->hasStatsExpiry) { @@ -306,10 +306,15 @@ private function findTables(HostInterface $host, string $mysql, MysqlServerInfo $query .= "SET SESSION information_schema_stats_expiry = 0;\n"; } $query .= self::tablesQuery($minReclaimable, $filter) . "\n"; - $this->io->debug('Query: ' . $query); - $tables = []; - foreach ($this->parseRows((string) $host->runCommand($mysql, true, true, $query), 8) as $fields) { - $tables[] = TableCandidate::fromRow($fields); + $readTables = function () use ($host, $mysql, $query): array { + $this->io->debug('Query: ' . $query); + return array_map(TableCandidate::fromRow(...), $this->parseRows((string) $host->runCommand($mysql, true, true, $query), 8)); + }; + $tables = $readTables(); + if ($tables && $analyze) { + $this->stdErr->writeln(\sprintf('Analyzing %d table(s)...', \count($tables))); + $host->runCommand($mysql, true, true, 'ANALYZE TABLE ' . implode(', ', array_map(fn(TableCandidate $t): string => $t->quotedName(), $tables)) . ';'); + $tables = $readTables(); } $tables = $this->sampleRows($host, $mysql, $server, $tables); @@ -363,14 +368,24 @@ private function sampleRows(HostInterface $host, string $mysql, MysqlServerInfo $result = (string) $host->runCommand($mysql, true, true, $query); } catch (ProcessRuntimeException $e) { $this->stdErr->writeln('Warning: failed to sample rows: only free extents are counted as reclaimable.'); - $this->io->debug($e->getMessage()); + $this->io->debug(self::processError($e)); return $tables; } foreach ($this->parseRows($result, 3) as [$i, $count, $bytes]) { - if (isset($firstRows[(int) $i])) { - $tables[(int) $i] = $tables[(int) $i]->withSample((int) $count > 0 ? (int) $bytes / (int) $count : 0.0, $firstRows[(int) $i]); + $i = (int) $i; + if (!isset($firstRows[$i])) { + continue; + } + // The estimate is proportional to the row count, so it cannot be used if the count is out of date. + if ($tables[$i]->rows < (int) $count) { + $this->stdErr->writeln(\sprintf( + 'Warning: the row count of %s is out of date: only its free extents are counted as reclaimable. Use --analyze to refresh it.', + $tables[$i]->displayName(), + )); + continue; } + $tables[$i] = $tables[$i]->withSample((int) $count > 0 ? (int) $bytes / (int) $count : 0.0, $firstRows[$i]); } return $tables; @@ -401,9 +416,11 @@ private function parseRows(string $output, int $numFields): array * only excluded if its total size is below the minimum. * * Free space in a shared tablespace (the system tablespace, if - * innodb_file_per_table is OFF, or a general tablespace) is reported for - * each of its tables: the "same_free" column counts the other tables with - * the same free space, to detect them. + * innodb_file_per_table is or was OFF, or a general tablespace) is + * reported for each of its tables: the "same_free" column counts the + * other tables with the same free space, to detect them. A table that is + * alone in a shared tablespace, among the tables the user can see, is not + * detected. * * @param string[] $filter */ @@ -432,7 +449,7 @@ public static function tablesQuery(int $minReclaimable, array $filter = []): str . " (SELECT COUNT(*) FROM information_schema.STATISTICS s WHERE s.TABLE_SCHEMA = t.TABLE_SCHEMA AND s.TABLE_NAME = t.TABLE_NAME AND s.INDEX_TYPE IN ('FULLTEXT', 'SPATIAL')) AS offline_indexes," . ' COALESCE(f.n, 1) - 1 AS same_free' . ' FROM information_schema.TABLES t' - . " LEFT JOIN (SELECT DATA_FREE AS free, COUNT(*) AS n FROM information_schema.TABLES WHERE ENGINE = 'InnoDB' AND DATA_FREE >= " . $minReclaimable . ' GROUP BY DATA_FREE) f ON f.free = t.DATA_FREE' + . " LEFT JOIN (SELECT DATA_FREE AS free, COUNT(*) AS n FROM information_schema.TABLES WHERE ENGINE = 'InnoDB' AND DATA_FREE >= " . self::SHARED_FREE_MIN . ' GROUP BY DATA_FREE) f ON f.free = t.DATA_FREE' . ' WHERE ' . implode(' AND ', $conditions) . ';'; } @@ -650,7 +667,7 @@ private function rebuild(HostInterface $host, string $mysql, array $tables, Mysq $host->runCommand($mysql, true, true, $sql); } catch (ProcessRuntimeException $e) { $this->stdErr->writeln(' [failed]'); - $this->stdErr->writeln($e->getMessage()); + $this->stdErr->writeln(self::processError($e)); return false; } $this->stdErr->writeln(\sprintf(' [OK] (%ds)', round(microtime(true) - $start))); @@ -662,6 +679,24 @@ private function rebuild(HostInterface $host, string $mysql, array $tables, Mysq return true; } + /** + * Returns the error output of a failed database command. + * + * The exception's message is not used, as it contains the command line, with the database password. + */ + private static function processError(ProcessRuntimeException $e): string + { + if (!$e instanceof ProcessFailedException) { + return $e instanceof ProcessTimedOutException ? 'The command timed out.' : 'The command failed.'; + } + $lines = array_filter( + explode("\n", trim($e->getProcess()->getErrorOutput())), + fn(string $line): bool => !str_contains($line, 'Using a password on the command line'), + ); + + return $lines ? implode("\n", $lines) : 'The command failed with the exit code: ' . $e->getProcess()->getExitCode(); + } + /** * Parses a size, in bytes or with a unit (K, M, G or T, in powers of 1024). */ diff --git a/legacy/src/Command/Db/DbSizeCommand.php b/legacy/src/Command/Db/DbSizeCommand.php index 5152990ba..ee374131d 100644 --- a/legacy/src/Command/Db/DbSizeCommand.php +++ b/legacy/src/Command/Db/DbSizeCommand.php @@ -70,6 +70,15 @@ protected function configure(): void */ protected function execute(InputInterface $input, OutputInterface $output): int { + if (Option::bool($input, 'cleanup')) { + $this->stdErr->writeln('Deprecated: The --cleanup option is deprecated and will be removed in a future version.'); + $this->stdErr->writeln(\sprintf( + 'To find tables with reclaimable space, and check whether they can be rebuilt safely, run: %s db:mysql-optimize', + $this->config->getStr('application.executable'), + )); + $this->stdErr->writeln(''); + } + $selection = $this->selector->getSelection($input, new SelectorConfig(allowLocalHost: $this->relationships->hasLocalEnvVar(), chooseEnvFilter: SelectorConfig::filterEnvsMaybeActive())); $host = $this->selector->getHostFromSelection($input, $selection); @@ -149,11 +158,6 @@ private function getCleanupQueries(array $rows): array private function checkInnoDbTablesInNeedOfOptimizing(HostInterface $host, array $database, InputInterface $input): void { $this->stdErr->writeln(''); - $this->stdErr->writeln('Deprecated: The --cleanup option is deprecated and will be removed in a future version.'); - $this->stdErr->writeln(\sprintf( - 'To find tables with reclaimable space, and check whether they can be rebuilt safely, run: %s db:mysql-optimize', - $this->config->getStr('application.executable'), - )); $tablesNeedingCleanup = $host->runCommand($this->getMysqlCommand($database), true, true, $this->mysqlTablesInNeedOfOptimizing()); $queries = []; From 6e0062f0d6ba2455bde498c19da30f6913d8f6ea Mon Sep 17 00:00:00 2001 From: Patrick Dawkins Date: Mon, 2 Nov 2026 23:49:15 +0000 Subject: [PATCH 4/6] fix(db): include free extents in the table size of db:mysql-optimize The Size column was the data and indexes only, so the reclaimable space (which includes DATA_FREE) could be more than the size, and the reclaimable percentage more than 100%. Size now includes DATA_FREE (about the size of the table's file), and --min-percent is relative to it. Without a row sample, the disk needed for a rebuild is still the data and indexes, as free extents are not copied. Co-Authored-By: Claude Opus 5.5 --- integration-tests/db_mysql_optimize_test.go | 12 ++++++------ legacy/src/Command/Db/DbMysqlOptimizeCommand.php | 3 ++- legacy/src/Model/Mysql/TableCandidate.php | 8 ++++---- legacy/tests/Model/Mysql/MysqlOptimizeTest.php | 5 ++++- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/integration-tests/db_mysql_optimize_test.go b/integration-tests/db_mysql_optimize_test.go index dd65e2ccf..d1a307a73 100644 --- a/integration-tests/db_mysql_optimize_test.go +++ b/integration-tests/db_mysql_optimize_test.go @@ -183,10 +183,10 @@ func TestDbMysqlOptimizeReport(t *testing.T) { assert.Contains(t, stdErr, "Disk: 3.0 GiB free of 4.0 GiB. /tmp: 924.0 MiB free of 1.0 GiB.") // Sorted by reclaimable space. The disk needed is the rebuilt size, plus 10% and 8 MiB. assert.Equal(t, `Schema,Table,Size,Reclaimable,Reclaimable %,Disk needed,Fits disk,/tmp needed,Fits /tmp -main,cache,3221225472,1073741824,33%,4732852633,false,1073741824,false -main,search,209715200,360710144,172%,15309209,true,104857600,true -main,logs,262144000,176422912,67%,116523008,true,52428800,true -main,we`+"`"+`ird,104857600,125829120,120%,104857600,true,0,true +main,cache,4294967296,1073741824,25%,4732852633,false,1073741824,false +main,search,367001600,360710144,98%,15309209,true,104857600,true +main,logs,274726912,176422912,64%,116523008,true,52428800,true +main,we`+"`"+`ird,230686720,125829120,55%,104857600,true,0,true `, output) output = p.f.Run(args("--format", "tsv", "--no-header", "--columns", "table,notes")...) @@ -228,8 +228,8 @@ main,we`+"`"+`ird,104857600,125829120,120%,104857600,true,0,true // The default table output. output = p.f.Run(args("--columns", "table,size,disk_fits")...) - assert.Contains(t, output, "| logs | 250.0 MiB | yes |") - assert.Contains(t, output, "| cache | 3.0 GiB | no |") + assert.Contains(t, output, "| logs | 262.0 MiB | yes |") + assert.Contains(t, output, "| cache | 4.0 GiB | no |") _, stdErr, err = p.f.RunCombinedOutput(args("--min-percent", "x")...) assert.Error(t, err) diff --git a/legacy/src/Command/Db/DbMysqlOptimizeCommand.php b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php index 38fb49ab0..e7ed9b0c3 100644 --- a/legacy/src/Command/Db/DbMysqlOptimizeCommand.php +++ b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php @@ -114,7 +114,8 @@ protected function configure(): void The free space is estimated from the free extents reported by the database (DATA_FREE), and from the size of the rows: this reads a sample of about 1000 rows from each table larger than --min-reclaimable. - A table is listed if its free space is at least --min-reclaimable and --min-percent of its size. + A table is listed if its free space is at least --min-reclaimable and --min-percent of its size + (its data, indexes and free extents). By default, this only reports. With --execute, each table listed is rebuilt with: ALTER TABLE ... FORCE, ALGORITHM=INPLACE, LOCK=NONE diff --git a/legacy/src/Model/Mysql/TableCandidate.php b/legacy/src/Model/Mysql/TableCandidate.php index f049e521a..26b975672 100644 --- a/legacy/src/Model/Mysql/TableCandidate.php +++ b/legacy/src/Model/Mysql/TableCandidate.php @@ -53,11 +53,11 @@ public function withSample(float $avgRowBytes, bool $firstRows): self } /** - * The size of the table's data and indexes, excluding free space. + * The size of the table's data, indexes and free extents (about the size of its file). */ public function size(): int { - return $this->dataLength + $this->indexLength; + return $this->dataLength + $this->indexLength + $this->dataFree; } /** @@ -89,7 +89,7 @@ public function reclaimable(): int return $this->dataFree; } - return max($this->dataFree, $this->size() + $this->dataFree - $rebuilt); + return max($this->dataFree, $this->size() - $rebuilt); } /** @@ -99,7 +99,7 @@ public function diskNeeded(): int { $rebuilt = $this->rebuiltSize(); if ($rebuilt === null) { - return $this->size(); + return $this->dataLength + $this->indexLength; } return $rebuilt + intdiv($rebuilt * self::DISK_MARGIN_PERCENT, 100) + self::DISK_MARGIN_BYTES; diff --git a/legacy/tests/Model/Mysql/MysqlOptimizeTest.php b/legacy/tests/Model/Mysql/MysqlOptimizeTest.php index e1c67eeff..71c384c77 100644 --- a/legacy/tests/Model/Mysql/MysqlOptimizeTest.php +++ b/legacy/tests/Model/Mysql/MysqlOptimizeTest.php @@ -74,7 +74,10 @@ public function testEstimates(): void // 90 MiB of data and 10 MiB of indexes, with 5 MiB of free extents. $table = new TableCandidate('s', 't', 90 * $mib, 10 * $mib, 5 * $mib, rows: 10240); - // Without a sample, only the free extents are reclaimable, and a rebuild needs the current size. + // The size includes the free extents. + $this->assertSame(105 * $mib, $table->size()); + + // Without a sample, only the free extents are reclaimable, and a rebuild needs the current data and indexes. $this->assertNull($table->rebuiltSize()); $this->assertSame(5 * $mib, $table->reclaimable()); $this->assertSame(100 * $mib, $table->diskNeeded()); From 037d752c33c9c2f2191e31c78ecbf013fbb73bc2 Mon Sep 17 00:00:00 2001 From: Patrick Dawkins Date: Mon, 2 Nov 2026 23:54:47 +0000 Subject: [PATCH 5/6] fix(db): exclude a shared tablespace's free space from table sizes In a shared tablespace, DATA_FREE is the free space of the whole tablespace, reported on every table in it. Adding it to each table's size made small tables look as large as the tablespace's free space. For tables that are probably in a shared tablespace, the size is now the data and indexes only, and the reclaimable percentage is left empty, as a rebuild would not reclaim that space. Co-Authored-By: Claude Opus 5.5 --- integration-tests/db_mysql_optimize_test.go | 2 +- legacy/src/Command/Db/DbMysqlOptimizeCommand.php | 6 +++--- legacy/src/Model/Mysql/TableCandidate.php | 10 ++++++++-- legacy/tests/Model/Mysql/MysqlOptimizeTest.php | 7 +++++-- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/integration-tests/db_mysql_optimize_test.go b/integration-tests/db_mysql_optimize_test.go index d1a307a73..a0de6431a 100644 --- a/integration-tests/db_mysql_optimize_test.go +++ b/integration-tests/db_mysql_optimize_test.go @@ -186,7 +186,7 @@ func TestDbMysqlOptimizeReport(t *testing.T) { main,cache,4294967296,1073741824,25%,4732852633,false,1073741824,false main,search,367001600,360710144,98%,15309209,true,104857600,true main,logs,274726912,176422912,64%,116523008,true,52428800,true -main,we`+"`"+`ird,230686720,125829120,55%,104857600,true,0,true +main,we`+"`"+`ird,104857600,125829120,,104857600,true,0,true `, output) output = p.f.Run(args("--format", "tsv", "--no-header", "--columns", "table,notes")...) diff --git a/legacy/src/Command/Db/DbMysqlOptimizeCommand.php b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php index e7ed9b0c3..f42abd625 100644 --- a/legacy/src/Command/Db/DbMysqlOptimizeCommand.php +++ b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php @@ -321,7 +321,7 @@ private function findTables(HostInterface $host, string $mysql, MysqlServerInfo $tables = $this->sampleRows($host, $mysql, $server, $tables); $tables = array_values(array_filter($tables, fn(TableCandidate $t): bool => $t->reclaimable() >= $minReclaimable - && $t->reclaimable() * 100 >= $minPercent * $t->size())); + && $t->reclaimable() * 100 >= $minPercent * $t->size($server))); usort($tables, fn(TableCandidate $a, TableCandidate $b): int => $b->reclaimable() <=> $a->reclaimable()); return $tables; @@ -586,9 +586,9 @@ private function renderTables(array $tables, ?DiskHeadroom $headroom, MysqlServe $rows[] = [ 'schema' => $t->schema, 'table' => $t->name, - 'size' => $formatSize($t->size()), + 'size' => $formatSize($t->size($server)), 'reclaimable' => $formatSize($t->reclaimable()), - 'percent' => $t->size() > 0 ? \sprintf('%.0f%%', $t->reclaimable() / $t->size() * 100) : '', + 'percent' => $t->size($server) > 0 && !$t->inSharedTablespace($server) ? \sprintf('%.0f%%', $t->reclaimable() / $t->size($server) * 100) : '', 'disk_needed' => $formatSize($t->diskNeeded()), 'disk_fits' => $formatFits($headroom?->diskFits($t->diskNeeded())), 'tmp_needed' => $formatSize($t->tmpNeeded()), diff --git a/legacy/src/Model/Mysql/TableCandidate.php b/legacy/src/Model/Mysql/TableCandidate.php index 26b975672..18f7a2b3c 100644 --- a/legacy/src/Model/Mysql/TableCandidate.php +++ b/legacy/src/Model/Mysql/TableCandidate.php @@ -54,9 +54,15 @@ public function withSample(float $avgRowBytes, bool $firstRows): self /** * The size of the table's data, indexes and free extents (about the size of its file). + * + * In a shared tablespace, the free extents are the tablespace's, so they are not counted. */ - public function size(): int + public function size(MysqlServerInfo $server): int { + if ($this->inSharedTablespace($server)) { + return $this->dataLength + $this->indexLength; + } + return $this->dataLength + $this->indexLength + $this->dataFree; } @@ -89,7 +95,7 @@ public function reclaimable(): int return $this->dataFree; } - return max($this->dataFree, $this->size() - $rebuilt); + return max($this->dataFree, $this->dataLength + $this->indexLength + $this->dataFree - $rebuilt); } /** diff --git a/legacy/tests/Model/Mysql/MysqlOptimizeTest.php b/legacy/tests/Model/Mysql/MysqlOptimizeTest.php index 71c384c77..ce2359014 100644 --- a/legacy/tests/Model/Mysql/MysqlOptimizeTest.php +++ b/legacy/tests/Model/Mysql/MysqlOptimizeTest.php @@ -74,8 +74,11 @@ public function testEstimates(): void // 90 MiB of data and 10 MiB of indexes, with 5 MiB of free extents. $table = new TableCandidate('s', 't', 90 * $mib, 10 * $mib, 5 * $mib, rows: 10240); - // The size includes the free extents. - $this->assertSame(105 * $mib, $table->size()); + // The size includes the free extents, unless they are a shared tablespace's. + $server = new MysqlServerInfo(['version' => '8.0.36', 'innodb_file_per_table' => 'ON']); + $this->assertSame(105 * $mib, $table->size($server)); + $this->assertSame(100 * $mib, (new TableCandidate('s', 't', 90 * $mib, 10 * $mib, 5 * $mib, sameFree: 1))->size($server)); + $this->assertSame(100 * $mib, $table->size(new MysqlServerInfo(['version' => '8.0.36', 'innodb_file_per_table' => 'OFF']))); // Without a sample, only the free extents are reclaimable, and a rebuild needs the current data and indexes. $this->assertNull($table->rebuiltSize()); From 95c521348a1278013934fc36469efb85b0ac5442 Mon Sep 17 00:00:00 2001 From: Patrick Dawkins Date: Tue, 3 Nov 2026 09:45:07 +0000 Subject: [PATCH 6/6] fix(db): rebuild the smallest tables first in db:mysql-optimize Tables were rebuilt in order of reclaimable space, largest first. A rebuild copies the table, so on a nearly full disk the largest rebuild is the most likely to fail. Rebuilding the tables that need the least disk space first frees space for the larger ones, as db:size --cleanup did. Co-Authored-By: Claude Opus 5.5 --- integration-tests/db_mysql_optimize_test.go | 4 ++-- legacy/src/Command/Db/DbMysqlOptimizeCommand.php | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/integration-tests/db_mysql_optimize_test.go b/integration-tests/db_mysql_optimize_test.go index a0de6431a..2c00161c4 100644 --- a/integration-tests/db_mysql_optimize_test.go +++ b/integration-tests/db_mysql_optimize_test.go @@ -314,14 +314,14 @@ func TestDbMysqlOptimizeExecute(t *testing.T) { assert.Contains(t, stdErr, "Rebuilding main.logs... [OK]") assert.Equal(t, []string{"ALTER TABLE `main`.`logs` FORCE, ALGORITHM=INPLACE, LOCK=NONE;"}, alters()) - // With --force, tables that do not fit are rebuilt too, with warnings. + // With --force, tables that do not fit are rebuilt too, with warnings, and the smallest rebuild runs first. _, stdErr, err = p.f.RunCombinedOutput(append(args, "--force")...) require.NoError(t, err, stdErr) assert.Contains(t, stdErr, "Warning: There is not enough free disk space to rebuild main.cache") assert.Equal(t, []string{ "ALTER TABLE `main`.`logs` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", - "ALTER TABLE `main`.`cache` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", "ALTER TABLE `main`.`logs` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", + "ALTER TABLE `main`.`cache` FORCE, ALGORITHM=INPLACE, LOCK=NONE;", }, alters()) _, stdErr, err = p.f.RunCombinedOutput("db:mysql-optimize", "-p", layoutProjectID, "-e", "main", "--force") diff --git a/legacy/src/Command/Db/DbMysqlOptimizeCommand.php b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php index f42abd625..ec045da6d 100644 --- a/legacy/src/Command/Db/DbMysqlOptimizeCommand.php +++ b/legacy/src/Command/Db/DbMysqlOptimizeCommand.php @@ -122,6 +122,7 @@ protected function configure(): void which is the same rebuild as OPTIMIZE TABLE, but fails instead of locking the table if it cannot be done online. A rebuild copies the table: it needs free disk space for the rebuilt table, and temporary space for its indexes. + The tables are rebuilt one at a time, the smallest first, so that the space freed by each rebuild is available for the next. The disk and /tmp usage are read from the metrics API: if they are unknown, or a table does not fit (leaving at least {$reserve}% of the volume free), the tables are not rebuilt, unless --force is used. @@ -625,6 +626,8 @@ private function rebuild(HostInterface $host, string $mysql, array $tables, Mysq $this->stdErr->writeln('No tables can be rebuilt.'); return false; } + // The smallest rebuilds run first, so that the space they free is available for the larger ones. + usort($toRebuild, fn(TableCandidate $a, TableCandidate $b): int => $a->diskNeeded() <=> $b->diskNeeded()); $problems = []; if ($headroom === null) {