diff --git a/.dredd/hooks/capabilities.go b/.dredd/hooks/capabilities.go index d3b6a92ec0..869dda4653 100644 --- a/.dredd/hooks/capabilities.go +++ b/.dredd/hooks/capabilities.go @@ -234,7 +234,12 @@ var pathSubPatterns = []func() string{ // hooks, so leave the path segment untouched (kept here only to preserve // the positional mapping of the entries that follow). func() string { return strconv.Itoa(15) }, - func() string { return strconv.Itoa(runner.ID) }, // runner_id (project), x-example: 16 + func() string { + if runner == nil { + return "0" + } + return strconv.Itoa(runner.ID) + }, // runner_id (project), x-example: 16 func() string { return strconv.Itoa(globalRunner.ID) }, // global runner_id, x-example: 17 func() string { if workflow == nil { diff --git a/.dredd/hooks/main.go b/.dredd/hooks/main.go index ff93b1adee..566126eb72 100644 --- a/.dredd/hooks/main.go +++ b/.dredd/hooks/main.go @@ -145,12 +145,21 @@ func main() { h.Before("workflow > /api/project/{project_id}/workflows/{workflow_id}/runs/{run_id}/approvals/{node_id} > Resolve workflow approval > 200 > application/json", func(t *trans.Transaction) { t.Request.Body = "{\"status\":\"approved\"}" }) + + // project runners + h.Before("runner > /api/project/{project_id}/runners > Get project runners > 200 > application/json", capabilityWrapper("project")) + //h.Before("runner > /api/project/{project_id}/runners > Add project runner > 201 > application/json", capabilityWrapper("project")) + h.Before("runner > /api/project/{project_id}/runner_tags > Get project runner tags > 200 > application/json", capabilityWrapper("project")) + h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Get project runner > 200 > application/json", capabilityWrapper("runner")) + h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Update project runner > 204 > application/json", capabilityWrapper("runner")) + h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Delete project runner > 204 > application/json", capabilityWrapper("runner")) + h.Before("runner > /api/project/{project_id}/runners/{runner_id}/active > Set project runner active state > 204 > application/json", capabilityWrapper("runner")) + h.Before("runner > /api/project/{project_id}/runners/{runner_id}/cache > Clear project runner cache > 204 > application/json", capabilityWrapper("runner")) } else { - // The workflow API is implemented by the PRO module. In a non-PRO - // build (e.g. PR builds without access to pro_impl) the workflow - // store methods are no-ops, so the fixtures cannot be set up and - // these tests must be skipped. - workflowTests := []string{ + // Workflows and project runners are implemented by the PRO module. + // In a non-PRO build (e.g. PR builds without access to pro_impl) + // these endpoints are not functional, so their tests must be skipped. + proOnlyTests := []string{ "workflow > /api/project/{project_id}/workflows > Get workflows > 200 > application/json", "workflow > /api/project/{project_id}/workflows > Add workflow > 201 > application/json", "workflow > /api/project/{project_id}/workflows/{workflow_id} > Get workflow > 200 > application/json", @@ -161,8 +170,17 @@ func main() { "workflow > /api/project/{project_id}/workflows/{workflow_id}/runs/{run_id} > Get workflow run details > 200 > application/json", "workflow > /api/project/{project_id}/workflows/{workflow_id}/runs/{run_id}/approvals > Get workflow run approvals > 200 > application/json", "workflow > /api/project/{project_id}/workflows/{workflow_id}/runs/{run_id}/approvals/{node_id} > Resolve workflow approval > 200 > application/json", + "workflow > /api/project/{project_id}/workflows/{workflow_id}/runs/{run_id}/artifacts > Get merged workflow run artifacts > 200 > application/json", + "runner > /api/project/{project_id}/runners > Get project runners > 200 > application/json", + "runner > /api/project/{project_id}/runner_tags > Get project runner tags > 200 > application/json", + "runner > /api/project/{project_id}/runners/{runner_id} > Get project runner > 200 > application/json", + "runner > /api/project/{project_id}/runners/{runner_id} > Update project runner > 204 > application/json", + "runner > /api/project/{project_id}/runners/{runner_id} > Delete project runner > 204 > application/json", + "runner > /api/project/{project_id}/runners/{runner_id}/active > Set project runner active state > 204 > application/json", + "runner > /api/project/{project_id}/runners/{runner_id}/registration-token > Regenerate the one-time registration token of an unregistered project runner > 200 > application/json", + "runner > /api/project/{project_id}/runners/{runner_id}/cache > Clear project runner cache > 204 > application/json", } - for _, v := range workflowTests { + for _, v := range proOnlyTests { h.Before(v, skipTest) } } @@ -188,16 +206,6 @@ func main() { addCapabilities([]string{"repository", "inventory", "environment", "view", "template"}) }) - // project runners - h.Before("runner > /api/project/{project_id}/runners > Get project runners > 200 > application/json", capabilityWrapper("project")) - //h.Before("runner > /api/project/{project_id}/runners > Add project runner > 201 > application/json", capabilityWrapper("project")) - h.Before("runner > /api/project/{project_id}/runner_tags > Get project runner tags > 200 > application/json", capabilityWrapper("project")) - h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Get project runner > 200 > application/json", capabilityWrapper("runner")) - h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Update project runner > 204 > application/json", capabilityWrapper("runner")) - h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Delete project runner > 204 > application/json", capabilityWrapper("runner")) - h.Before("runner > /api/project/{project_id}/runners/{runner_id}/active > Set project runner active state > 204 > application/json", capabilityWrapper("runner")) - h.Before("runner > /api/project/{project_id}/runners/{runner_id}/cache > Clear project runner cache > 204 > application/json", capabilityWrapper("runner")) - // global runners (admin) h.Before("runner > /api/runners/{runner_id} > Get global runner > 200 > application/json", capabilityWrapper("global_runner")) h.Before("runner > /api/runners/{runner_id} > Update global runner > 204 > application/json", capabilityWrapper("global_runner")) diff --git a/api/projects/repository.go b/api/projects/repository.go index bd94f4c62d..4829cc7ea8 100644 --- a/api/projects/repository.go +++ b/api/projects/repository.go @@ -1,16 +1,23 @@ package projects import ( + "crypto/sha1" "errors" "fmt" "net/http" + "sync" "github.com/semaphoreui/semaphore/api/helpers" "github.com/semaphoreui/semaphore/db" "github.com/semaphoreui/semaphore/db_lib" + "github.com/semaphoreui/semaphore/pkg/task_logger" "github.com/semaphoreui/semaphore/util" ) +// repoBrowseMu protects per-repo+branch scratch checkouts from concurrent +// git operations that would race on .git/index.lock. +var repoBrowseMu sync.Map // key: string (TmpDirName) → value: *sync.Mutex + // RepositoryMiddleware ensures a repository exists and loads it to the context func RepositoryMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -76,6 +83,75 @@ func (c *RepositoryController) GetRepositoryBranches(w http.ResponseWriter, r *h helpers.WriteJSON(w, http.StatusOK, branches) } +// GetRepositoryPlaybooks returns the list of playbook (.yml/.yaml) file paths, +// relative to the repository root, found in the repository. For git/ssh/https +// repositories it checks out the requested branch (defaulting to the +// repository's configured branch) into a scratch directory before scanning it. +func (c *RepositoryController) GetRepositoryPlaybooks(w http.ResponseWriter, r *http.Request) { + repo := helpers.GetFromContext(r, "repository").(db.Repository) + + var rootDir string + + if repo.GetType() == db.RepositoryLocal || repo.GetType() == db.RepositoryFile { + rootDir = repo.GetFullPath(0) + } else { + branch := r.URL.Query().Get("branch") + if branch == "" { + branch = repo.GitBranch + } + + if err := db.ValidateGitBranch(branch, "repository"); err != nil { + helpers.WriteError(w, err) + return + } + + repoCopy := repo + repoCopy.GitBranch = branch + // Clone() does a single-branch clone (git clone --branch ), so a + // scratch checkout can only ever serve the branch it was first cloned + // with. Key the scratch dir by branch (hashed, since branch names can + // contain slashes) so each branch gets its own cached checkout instead + // of failing to check out a branch that was never fetched. + branchHash := sha1.Sum([]byte(branch)) + git := db_lib.GitRepository{ + Repository: repoCopy, + TmpDirName: fmt.Sprintf("repository_%d_browse_%x", repo.ID, branchHash[:4]), + Client: db_lib.CreateDefaultGitClient(c.keyInstaller), + Logger: task_logger.NopLogger{}, + } + + // Serialize concurrent requests for the same repo+branch to avoid + // git lock conflicts on the shared scratch checkout. + lockKey := git.TmpDirName + mu, _ := repoBrowseMu.LoadOrStore(lockKey, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + defer mu.(*sync.Mutex).Unlock() + + var err error + if err = git.ValidateRepo(); err != nil { + err = git.Clone() + } else { + err = git.Pull() + } + + if err != nil { + helpers.WriteError(w, err) + return + } + + rootDir = git.GetFullPath() + } + + playbooks, err := db_lib.FindPlaybooks(rootDir) + + if err != nil { + helpers.WriteError(w, err) + return + } + + helpers.WriteJSON(w, http.StatusOK, playbooks) +} + // GetRepositories returns all repositories in a project sorted by type func GetRepositories(w http.ResponseWriter, r *http.Request) { if repo := helpers.GetFromContext(r, "repository"); repo != nil { diff --git a/api/router.go b/api/router.go index 34d081a21b..9a1d0b672a 100644 --- a/api/router.go +++ b/api/router.go @@ -419,6 +419,7 @@ func Route( projectRepoManagement.HandleFunc("/{repository_id}", projects.UpdateRepository).Methods("PUT") projectRepoManagement.HandleFunc("/{repository_id}", projects.RemoveRepository).Methods("DELETE") projectRepoManagement.HandleFunc("/{repository_id}/branches", repositoryController.GetRepositoryBranches).Methods("GET", "HEAD") + projectRepoManagement.HandleFunc("/{repository_id}/playbooks", repositoryController.GetRepositoryPlaybooks).Methods("GET", "HEAD") projectInventoryManagement := projectUserAPI.PathPrefix("/inventory").Subrouter() projectInventoryManagement.Use(projects.InventoryMiddleware) diff --git a/db_lib/CmdGitClient_injection_test.go b/db_lib/CmdGitClient_injection_test.go index 3fcaad9aa7..9cd2db6016 100644 --- a/db_lib/CmdGitClient_injection_test.go +++ b/db_lib/CmdGitClient_injection_test.go @@ -5,7 +5,6 @@ import ( "os/exec" "path/filepath" "testing" - "time" "github.com/semaphoreui/semaphore/db" "github.com/semaphoreui/semaphore/pkg/ssh" @@ -22,20 +21,6 @@ func (nopKeyInstaller) Install(key db.AccessKey, usage db.AccessKeyRole, logger return ssh.AccessKeyInstallation{}, nil } -// nopLogger is a no-op task_logger.Logger for tests. -type nopLogger struct{} - -func (nopLogger) Log(string) {} -func (nopLogger) Logf(string, ...any) {} -func (nopLogger) LogWithTime(time.Time, string) {} -func (nopLogger) LogfWithTime(time.Time, string, ...any) {} -func (nopLogger) LogCmd(*exec.Cmd) {} -func (nopLogger) SetStatus(task_logger.TaskStatus) {} -func (nopLogger) AddStatusListener(task_logger.StatusListener) {} -func (nopLogger) AddLogListener(task_logger.LogListener) {} -func (nopLogger) SetCommit(string, string) {} -func (nopLogger) WaitLog() {} - func gitInit(t *testing.T, dir string) { t.Helper() run := func(args ...string) { @@ -61,7 +46,7 @@ func newTestGitRepo(t *testing.T, gitURL, gitBranch string) GitRepository { GitBranch: gitBranch, SSHKey: db.AccessKey{Type: db.AccessKeyNone}, }, - Logger: nopLogger{}, + Logger: task_logger.NopLogger{}, } } diff --git a/db_lib/GitRepository.go b/db_lib/GitRepository.go index 3b37378dc3..d7f7ed4091 100644 --- a/db_lib/GitRepository.go +++ b/db_lib/GitRepository.go @@ -49,7 +49,13 @@ func (r GitRepository) ValidateRepo() error { } func (r GitRepository) Clone() error { - return r.Client.Clone(r) + err := r.Client.Clone(r) + if err != nil { + // Remove any partial/corrupt clone so the next attempt starts fresh + // instead of getting stuck on ValidateRepo() passing against a broken dir. + os.RemoveAll(r.GetFullPath()) + } + return err } func (r GitRepository) Pull() error { diff --git a/db_lib/PlaybookFiles.go b/db_lib/PlaybookFiles.go new file mode 100644 index 0000000000..1c8ab6eb8b --- /dev/null +++ b/db_lib/PlaybookFiles.go @@ -0,0 +1,81 @@ +package db_lib + +import ( + "io/fs" + "path/filepath" + "sort" + "strings" +) + +// maxPlaybookFiles caps the number of playbook paths returned by FindPlaybooks +// as a safety net against huge repositories. +const maxPlaybookFiles = 1000 + +// excludedPlaybookDirs are conventional Ansible directories that never contain +// top-level playbooks (roles, variable files, templates, etc). They are skipped +// while walking the repository to avoid flooding the result with non-playbook +// yml/yaml files. +var excludedPlaybookDirs = map[string]bool{ + ".git": true, + "roles": true, + "group_vars": true, + "host_vars": true, + "files": true, + "templates": true, + "library": true, + "filter_plugins": true, + "module_utils": true, + "meta": true, + "handlers": true, + "defaults": true, + "vars": true, + "molecule": true, + "tests": true, +} + +// FindPlaybooks walks rootDir and returns a sorted slice of paths (relative to +// rootDir) of files with a .yml or .yaml extension (case-insensitive), +// skipping .git and conventional non-playbook Ansible directories. +// The result is capped at maxPlaybookFiles entries. +func FindPlaybooks(rootDir string) ([]string, error) { + var result []string + + err := filepath.WalkDir(rootDir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if len(result) >= maxPlaybookFiles { + return filepath.SkipAll + } + + if d.IsDir() { + if p != rootDir && excludedPlaybookDirs[d.Name()] { + return filepath.SkipDir + } + return nil + } + + ext := strings.ToLower(filepath.Ext(d.Name())) + if ext != ".yml" && ext != ".yaml" { + return nil + } + + rel, err := filepath.Rel(rootDir, p) + if err != nil { + return err + } + + result = append(result, filepath.ToSlash(rel)) + + return nil + }) + + if err != nil { + return nil, err + } + + sort.Strings(result) + + return result, nil +} diff --git a/db_lib/PlaybookFiles_test.go b/db_lib/PlaybookFiles_test.go new file mode 100644 index 0000000000..4af68a2501 --- /dev/null +++ b/db_lib/PlaybookFiles_test.go @@ -0,0 +1,77 @@ +package db_lib + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeFile creates a file (and its parent directories) with some content. +func writeFile(t *testing.T, root string, relPath string) { + t.Helper() + full := filepath.Join(root, relPath) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0755)) + require.NoError(t, os.WriteFile(full, []byte("---\n"), 0644)) +} + +func TestFindPlaybooks(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T, root string) + expected []string + }{ + { + name: "nested directories", + setup: func(t *testing.T, root string) { + writeFile(t, root, "site.yml") + writeFile(t, root, "playbooks/deploy.yml") + writeFile(t, root, "playbooks/nested/inner.yaml") + }, + expected: []string{"playbooks/deploy.yml", "playbooks/nested/inner.yaml", "site.yml"}, + }, + { + name: "excluded directories are skipped", + setup: func(t *testing.T, root string) { + writeFile(t, root, "site.yml") + writeFile(t, root, "roles/common/tasks/main.yml") + writeFile(t, root, "group_vars/all.yml") + writeFile(t, root, "host_vars/host1.yml") + writeFile(t, root, ".git/config.yml") + writeFile(t, root, "templates/config.yml") + writeFile(t, root, "molecule/default/molecule.yml") + }, + expected: []string{"site.yml"}, + }, + { + name: "mixed extensions only yml and yaml counted", + setup: func(t *testing.T, root string) { + writeFile(t, root, "site.yml") + writeFile(t, root, "readme.txt") + writeFile(t, root, "vars.YAML") + writeFile(t, root, "script.sh") + writeFile(t, root, "inventory.ini") + }, + expected: []string{"site.yml", "vars.YAML"}, + }, + { + name: "empty directory", + setup: func(t *testing.T, root string) {}, + expected: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + tt.setup(t, root) + + result, err := FindPlaybooks(root) + + require.NoError(t, err) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/go.sum b/go.sum index 64b5dbbca3..5b4e6f9a5e 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,6 @@ github.com/boombuler/barcode v1.1.0 h1:ChaYjBR63fr4LFyGn8E8nt7dBSt3MiU3zMOZqFvVk github.com/boombuler/barcode v1.1.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= -github.com/coreos/go-oidc/v3 v3.18.0 h1:V9orjXynvu5wiC9SemFTWnG4F45v403aIcjWo0d41+A= -github.com/coreos/go-oidc/v3 v3.18.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= github.com/coreos/go-oidc/v3 v3.19.0 h1:F/xyOi3x1UnG1U27YVnM1N6bHiL1K2upi6U/0qr8r+I= github.com/coreos/go-oidc/v3 v3.19.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= diff --git a/pkg/task_logger/task_logger.go b/pkg/task_logger/task_logger.go index 4d21da7d68..258b0a65fa 100644 --- a/pkg/task_logger/task_logger.go +++ b/pkg/task_logger/task_logger.go @@ -141,3 +141,19 @@ type Logger interface { WaitLog() } + +// NopLogger is a no-op Logger, useful for git operations triggered outside of +// a task run (e.g. browsing repository files, or tests) where there is no +// task log to write to. +type NopLogger struct{} + +func (NopLogger) Log(string) {} +func (NopLogger) Logf(string, ...any) {} +func (NopLogger) LogWithTime(time.Time, string) {} +func (NopLogger) LogfWithTime(time.Time, string, ...any) {} +func (NopLogger) LogCmd(*exec.Cmd) {} +func (NopLogger) SetStatus(TaskStatus) {} +func (NopLogger) AddStatusListener(StatusListener) {} +func (NopLogger) AddLogListener(LogListener) {} +func (NopLogger) SetCommit(string, string) {} +func (NopLogger) WaitLog() {} diff --git a/pro/go.mod b/pro/go.mod index d9b76613e9..5f7bda05fa 100644 --- a/pro/go.mod +++ b/pro/go.mod @@ -19,6 +19,7 @@ require ( github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-git/go-git/v5 v5.19.1 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-sql-driver/mysql v1.10.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/go-github v17.0.0+incompatible // indirect diff --git a/pro/go.sum b/pro/go.sum index 7997ae4c8b..935d0dccce 100644 --- a/pro/go.sum +++ b/pro/go.sum @@ -41,6 +41,8 @@ github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= diff --git a/web/src/components/TemplateForm.vue b/web/src/components/TemplateForm.vue index 545cec210b..86117b9a9f 100644 --- a/web/src/components/TemplateForm.vue +++ b/web/src/components/TemplateForm.vue @@ -140,19 +140,38 @@ :disabled="formSaving" > - +
+
+ +
+
+ +
+