fix: add per-repo+branch mutex to prevent concurrent git lock errors

.dredd/hooks/capabilities.go

@@ -234,7 +234,12 @@ var pathSubPatterns = []func() string{
 	// hooks, so leave the path segment untouched (kept here only to preserve
 	// the positional mapping of the entries that follow).
 	func() string { return strconv.Itoa(15) },
-	func() string { return strconv.Itoa(runner.ID) },       // runner_id (project), x-example: 16
+	func() string {
+		if runner == nil {
+			return "0"
+		}
+		return strconv.Itoa(runner.ID)
+	}, // runner_id (project), x-example: 16
 	func() string { return strconv.Itoa(globalRunner.ID) }, // global runner_id, x-example: 17
 	func() string {
 		if workflow == nil {

.dredd/hooks/main.go

@@ -145,12 +145,21 @@ func main() {
 		h.Before("workflow > /api/project/{project_id}/workflows/{workflow_id}/runs/{run_id}/approvals/{node_id} > Resolve workflow approval > 200 > application/json", func(t *trans.Transaction) {
 			t.Request.Body = "{\"status\":\"approved\"}"
 		})
+
+		// project runners
+		h.Before("runner > /api/project/{project_id}/runners > Get project runners > 200 > application/json", capabilityWrapper("project"))
+		//h.Before("runner > /api/project/{project_id}/runners > Add project runner > 201 > application/json", capabilityWrapper("project"))
+		h.Before("runner > /api/project/{project_id}/runner_tags > Get project runner tags > 200 > application/json", capabilityWrapper("project"))
+		h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Get project runner > 200 > application/json", capabilityWrapper("runner"))
+		h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Update project runner > 204 > application/json", capabilityWrapper("runner"))
+		h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Delete project runner > 204 > application/json", capabilityWrapper("runner"))
+		h.Before("runner > /api/project/{project_id}/runners/{runner_id}/active > Set project runner active state > 204 > application/json", capabilityWrapper("runner"))
+		h.Before("runner > /api/project/{project_id}/runners/{runner_id}/cache > Clear project runner cache > 204 > application/json", capabilityWrapper("runner"))
 	} else {
-		// The workflow API is implemented by the PRO module. In a non-PRO
-		// build (e.g. PR builds without access to pro_impl) the workflow
-		// store methods are no-ops, so the fixtures cannot be set up and
-		// these tests must be skipped.
-		workflowTests := []string{
+		// Workflows and project runners are implemented by the PRO module.
+		// In a non-PRO build (e.g. PR builds without access to pro_impl)
+		// these endpoints are not functional, so their tests must be skipped.
+		proOnlyTests := []string{
 			"workflow > /api/project/{project_id}/workflows > Get workflows > 200 > application/json",
 			"workflow > /api/project/{project_id}/workflows > Add workflow > 201 > application/json",
 			"workflow > /api/project/{project_id}/workflows/{workflow_id} > Get workflow > 200 > application/json",
@@ -161,8 +170,17 @@ func main() {
 			"workflow > /api/project/{project_id}/workflows/{workflow_id}/runs/{run_id} > Get workflow run details > 200 > application/json",
 			"workflow > /api/project/{project_id}/workflows/{workflow_id}/runs/{run_id}/approvals > Get workflow run approvals > 200 > application/json",
 			"workflow > /api/project/{project_id}/workflows/{workflow_id}/runs/{run_id}/approvals/{node_id} > Resolve workflow approval > 200 > application/json",
+			"workflow > /api/project/{project_id}/workflows/{workflow_id}/runs/{run_id}/artifacts > Get merged workflow run artifacts > 200 > application/json",
+			"runner > /api/project/{project_id}/runners > Get project runners > 200 > application/json",
+			"runner > /api/project/{project_id}/runner_tags > Get project runner tags > 200 > application/json",
+			"runner > /api/project/{project_id}/runners/{runner_id} > Get project runner > 200 > application/json",
+			"runner > /api/project/{project_id}/runners/{runner_id} > Update project runner > 204 > application/json",
+			"runner > /api/project/{project_id}/runners/{runner_id} > Delete project runner > 204 > application/json",
+			"runner > /api/project/{project_id}/runners/{runner_id}/active > Set project runner active state > 204 > application/json",
+			"runner > /api/project/{project_id}/runners/{runner_id}/registration-token > Regenerate the one-time registration token of an unregistered project runner > 200 > application/json",
+			"runner > /api/project/{project_id}/runners/{runner_id}/cache > Clear project runner cache > 204 > application/json",
 		}
-		for _, v := range workflowTests {
+		for _, v := range proOnlyTests {
 			h.Before(v, skipTest)
 		}
 	}
@@ -188,16 +206,6 @@ func main() {
 		addCapabilities([]string{"repository", "inventory", "environment", "view", "template"})
 	})
 
-	// project runners
-	h.Before("runner > /api/project/{project_id}/runners > Get project runners > 200 > application/json", capabilityWrapper("project"))
-	//h.Before("runner > /api/project/{project_id}/runners > Add project runner > 201 > application/json", capabilityWrapper("project"))
-	h.Before("runner > /api/project/{project_id}/runner_tags > Get project runner tags > 200 > application/json", capabilityWrapper("project"))
-	h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Get project runner > 200 > application/json", capabilityWrapper("runner"))
-	h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Update project runner > 204 > application/json", capabilityWrapper("runner"))
-	h.Before("runner > /api/project/{project_id}/runners/{runner_id} > Delete project runner > 204 > application/json", capabilityWrapper("runner"))
-	h.Before("runner > /api/project/{project_id}/runners/{runner_id}/active > Set project runner active state > 204 > application/json", capabilityWrapper("runner"))
-	h.Before("runner > /api/project/{project_id}/runners/{runner_id}/cache > Clear project runner cache > 204 > application/json", capabilityWrapper("runner"))
-
 	// global runners (admin)
 	h.Before("runner > /api/runners/{runner_id} > Get global runner > 200 > application/json", capabilityWrapper("global_runner"))
 	h.Before("runner > /api/runners/{runner_id} > Update global runner > 204 > application/json", capabilityWrapper("global_runner"))

api/projects/repository.go

@@ -1,16 +1,23 @@
 package projects
 
 import (
+	"crypto/sha1"
 	"errors"
 	"fmt"
 	"net/http"
+	"sync"
 
 	"github.com/semaphoreui/semaphore/api/helpers"
 	"github.com/semaphoreui/semaphore/db"
 	"github.com/semaphoreui/semaphore/db_lib"
+	"github.com/semaphoreui/semaphore/pkg/task_logger"
 	"github.com/semaphoreui/semaphore/util"
 )
 
+// repoBrowseMu protects per-repo+branch scratch checkouts from concurrent
+// git operations that would race on .git/index.lock.
+var repoBrowseMu sync.Map // key: string (TmpDirName) → value: *sync.Mutex
+
 // RepositoryMiddleware ensures a repository exists and loads it to the context
 func RepositoryMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -76,6 +83,75 @@ func (c *RepositoryController) GetRepositoryBranches(w http.ResponseWriter, r *h
 	helpers.WriteJSON(w, http.StatusOK, branches)
 }
 
+// GetRepositoryPlaybooks returns the list of playbook (.yml/.yaml) file paths,
+// relative to the repository root, found in the repository. For git/ssh/https
+// repositories it checks out the requested branch (defaulting to the
+// repository's configured branch) into a scratch directory before scanning it.
+func (c *RepositoryController) GetRepositoryPlaybooks(w http.ResponseWriter, r *http.Request) {
+	repo := helpers.GetFromContext(r, "repository").(db.Repository)
+
+	var rootDir string
+
+	if repo.GetType() == db.RepositoryLocal || repo.GetType() == db.RepositoryFile {
+		rootDir = repo.GetFullPath(0)
+	} else {
+		branch := r.URL.Query().Get("branch")
+		if branch == "" {
+			branch = repo.GitBranch
+		}
+
+		if err := db.ValidateGitBranch(branch, "repository"); err != nil {
+			helpers.WriteError(w, err)
+			return
+		}
+
+		repoCopy := repo
+		repoCopy.GitBranch = branch
+		// Clone() does a single-branch clone (git clone --branch <branch>), so a
+		// scratch checkout can only ever serve the branch it was first cloned
+		// with. Key the scratch dir by branch (hashed, since branch names can
+		// contain slashes) so each branch gets its own cached checkout instead
+		// of failing to check out a branch that was never fetched.
+		branchHash := sha1.Sum([]byte(branch))
+		git := db_lib.GitRepository{
+			Repository: repoCopy,
+			TmpDirName: fmt.Sprintf("repository_%d_browse_%x", repo.ID, branchHash[:4]),
+			Client:     db_lib.CreateDefaultGitClient(c.keyInstaller),
+			Logger:     task_logger.NopLogger{},
+		}
+
+		// Serialize concurrent requests for the same repo+branch to avoid
+		// git lock conflicts on the shared scratch checkout.
+		lockKey := git.TmpDirName
+		mu, _ := repoBrowseMu.LoadOrStore(lockKey, &sync.Mutex{})
+		mu.(*sync.Mutex).Lock()
+		defer mu.(*sync.Mutex).Unlock()
+
+		var err error
+		if err = git.ValidateRepo(); err != nil {
+			err = git.Clone()
+		} else {
+			err = git.Pull()
+		}
+
+		if err != nil {
+			helpers.WriteError(w, err)
+			return
+		}
+
+		rootDir = git.GetFullPath()
+	}
+
+	playbooks, err := db_lib.FindPlaybooks(rootDir)
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, playbooks)
+}
+
 // GetRepositories returns all repositories in a project sorted by type
 func GetRepositories(w http.ResponseWriter, r *http.Request) {
 	if repo := helpers.GetFromContext(r, "repository"); repo != nil {

api/router.go

@@ -419,6 +419,7 @@ func Route(
 	projectRepoManagement.HandleFunc("/{repository_id}", projects.UpdateRepository).Methods("PUT")
 	projectRepoManagement.HandleFunc("/{repository_id}", projects.RemoveRepository).Methods("DELETE")
 	projectRepoManagement.HandleFunc("/{repository_id}/branches", repositoryController.GetRepositoryBranches).Methods("GET", "HEAD")
+	projectRepoManagement.HandleFunc("/{repository_id}/playbooks", repositoryController.GetRepositoryPlaybooks).Methods("GET", "HEAD")
 
 	projectInventoryManagement := projectUserAPI.PathPrefix("/inventory").Subrouter()
 	projectInventoryManagement.Use(projects.InventoryMiddleware)

db_lib/CmdGitClient_injection_test.go

@@ -5,7 +5,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"testing"
-	"time"
 
 	"github.com/semaphoreui/semaphore/db"
 	"github.com/semaphoreui/semaphore/pkg/ssh"
@@ -22,20 +21,6 @@ func (nopKeyInstaller) Install(key db.AccessKey, usage db.AccessKeyRole, logger
 	return ssh.AccessKeyInstallation{}, nil
 }
 
-// nopLogger is a no-op task_logger.Logger for tests.
-type nopLogger struct{}
-
-func (nopLogger) Log(string)                                   {}
-func (nopLogger) Logf(string, ...any)                          {}
-func (nopLogger) LogWithTime(time.Time, string)                {}
-func (nopLogger) LogfWithTime(time.Time, string, ...any)       {}
-func (nopLogger) LogCmd(*exec.Cmd)                             {}
-func (nopLogger) SetStatus(task_logger.TaskStatus)             {}
-func (nopLogger) AddStatusListener(task_logger.StatusListener) {}
-func (nopLogger) AddLogListener(task_logger.LogListener)       {}
-func (nopLogger) SetCommit(string, string)                     {}
-func (nopLogger) WaitLog()                                     {}
-
 func gitInit(t *testing.T, dir string) {
 	t.Helper()
 	run := func(args ...string) {
@@ -61,7 +46,7 @@ func newTestGitRepo(t *testing.T, gitURL, gitBranch string) GitRepository {
 			GitBranch: gitBranch,
 			SSHKey:    db.AccessKey{Type: db.AccessKeyNone},
 		},
-		Logger: nopLogger{},
+		Logger: task_logger.NopLogger{},
 	}
 }
 

db_lib/GitRepository.go

@@ -49,7 +49,13 @@ func (r GitRepository) ValidateRepo() error {
 }
 
 func (r GitRepository) Clone() error {
-	return r.Client.Clone(r)
+	err := r.Client.Clone(r)
+	if err != nil {
+		// Remove any partial/corrupt clone so the next attempt starts fresh
+		// instead of getting stuck on ValidateRepo() passing against a broken dir.
+		os.RemoveAll(r.GetFullPath())
+	}
+	return err
 }
 
 func (r GitRepository) Pull() error {

db_lib/PlaybookFiles.go

@@ -0,0 +1,81 @@
+package db_lib
+
+import (
+	"io/fs"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// maxPlaybookFiles caps the number of playbook paths returned by FindPlaybooks
+// as a safety net against huge repositories.
+const maxPlaybookFiles = 1000
+
+// excludedPlaybookDirs are conventional Ansible directories that never contain
+// top-level playbooks (roles, variable files, templates, etc). They are skipped
+// while walking the repository to avoid flooding the result with non-playbook
+// yml/yaml files.
+var excludedPlaybookDirs = map[string]bool{
+	".git":           true,
+	"roles":          true,
+	"group_vars":     true,
+	"host_vars":      true,
+	"files":          true,
+	"templates":      true,
+	"library":        true,
+	"filter_plugins": true,
+	"module_utils":   true,
+	"meta":           true,
+	"handlers":       true,
+	"defaults":       true,
+	"vars":           true,
+	"molecule":       true,
+	"tests":          true,
+}
+
+// FindPlaybooks walks rootDir and returns a sorted slice of paths (relative to
+// rootDir) of files with a .yml or .yaml extension (case-insensitive),
+// skipping .git and conventional non-playbook Ansible directories.
+// The result is capped at maxPlaybookFiles entries.
+func FindPlaybooks(rootDir string) ([]string, error) {
+	var result []string
+
+	err := filepath.WalkDir(rootDir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if len(result) >= maxPlaybookFiles {
+			return filepath.SkipAll
+		}
+
+		if d.IsDir() {
+			if p != rootDir && excludedPlaybookDirs[d.Name()] {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		ext := strings.ToLower(filepath.Ext(d.Name()))
+		if ext != ".yml" && ext != ".yaml" {
+			return nil
+		}
+
+		rel, err := filepath.Rel(rootDir, p)
+		if err != nil {
+			return err
+		}
+
+		result = append(result, filepath.ToSlash(rel))
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	sort.Strings(result)
+
+	return result, nil
+}

db_lib/PlaybookFiles_test.go

@@ -0,0 +1,77 @@
+package db_lib
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// writeFile creates a file (and its parent directories) with some content.
+func writeFile(t *testing.T, root string, relPath string) {
+	t.Helper()
+	full := filepath.Join(root, relPath)
+	require.NoError(t, os.MkdirAll(filepath.Dir(full), 0755))
+	require.NoError(t, os.WriteFile(full, []byte("---\n"), 0644))
+}
+
+func TestFindPlaybooks(t *testing.T) {
+	tests := []struct {
+		name     string
+		setup    func(t *testing.T, root string)
+		expected []string
+	}{
+		{
+			name: "nested directories",
+			setup: func(t *testing.T, root string) {
+				writeFile(t, root, "site.yml")
+				writeFile(t, root, "playbooks/deploy.yml")
+				writeFile(t, root, "playbooks/nested/inner.yaml")
+			},
+			expected: []string{"playbooks/deploy.yml", "playbooks/nested/inner.yaml", "site.yml"},
+		},
+		{
+			name: "excluded directories are skipped",
+			setup: func(t *testing.T, root string) {
+				writeFile(t, root, "site.yml")
+				writeFile(t, root, "roles/common/tasks/main.yml")
+				writeFile(t, root, "group_vars/all.yml")
+				writeFile(t, root, "host_vars/host1.yml")
+				writeFile(t, root, ".git/config.yml")
+				writeFile(t, root, "templates/config.yml")
+				writeFile(t, root, "molecule/default/molecule.yml")
+			},
+			expected: []string{"site.yml"},
+		},
+		{
+			name: "mixed extensions only yml and yaml counted",
+			setup: func(t *testing.T, root string) {
+				writeFile(t, root, "site.yml")
+				writeFile(t, root, "readme.txt")
+				writeFile(t, root, "vars.YAML")
+				writeFile(t, root, "script.sh")
+				writeFile(t, root, "inventory.ini")
+			},
+			expected: []string{"site.yml", "vars.YAML"},
+		},
+		{
+			name:     "empty directory",
+			setup:    func(t *testing.T, root string) {},
+			expected: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			root := t.TempDir()
+			tt.setup(t, root)
+
+			result, err := FindPlaybooks(root)
+
+			require.NoError(t, err)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}