diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..b52f544 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,20 @@ +name: Test + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version: 24 + - run: npm test diff --git a/.gitignore b/.gitignore index 872d5f6..bf80477 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,8 @@ out # Nuxt.js build / generate output .nuxt dist +!dist/ +!dist/index.mjs .output # Gatsby files diff --git a/README.md b/README.md index 97e2750..8448d6a 100644 --- a/README.md +++ b/README.md @@ -1 +1,50 @@ -# contributor-trust-action \ No newline at end of file +# Contributor Detection Action + +Detects likely automated contributors from public GitHub evidence for a pull request, issue, or issue comment. It reads public profile and activity data, optionally asks GitHub Models for a second opinion, and updates one detection comment and label. + +The action does not infer identity from writing style or treat AI disclosure as misconduct. Organization blocking is opt-in and limited to high-risk user accounts that GitHub Models classifies as likely automated with at least 90% confidence. GitHub App bot accounts are reported but never organization-blocked by this workflow. + +## Usage + +```yaml +name: Contributor detection + +on: + issues: + types: [opened] + issue_comment: + types: [created] + pull_request_target: + types: [opened] + +permissions: + contents: read + issues: write + pull-requests: write + models: read + +jobs: + detection: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: Open-Source-Bazaar/contributor-trust-action@v1 + with: + github-token: ${{ github.token }} + organization-token: ${{ secrets.PAT }} + block-high-confidence-automation: 'true' +``` + +The action never checks out or executes code from an external pull request. GitHub Models is best-effort: if Models is disabled, the public-evidence report still completes. + +## Outputs + +- `author` +- `risk-level`: `low`, `medium`, or `high` +- `risk-score`: `0` to `100` +- `report-json` +- `blocked`: `true` when the organization block request succeeded + +Set `fail-on-high-risk: 'true'` only after reviewing the action against your community's contribution patterns. + +`organization-token` must be a dedicated fine-grained PAT or GitHub App token with organization `Blocking users: write`. Keep blocking disabled when that permission is not intentionally configured. diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..96850c8 --- /dev/null +++ b/action.yml @@ -0,0 +1,51 @@ +name: Contributor Detection +description: Detect high-confidence automated contributors from public GitHub evidence. +author: Open-Source-Bazaar contributors + +inputs: + github-token: + description: GitHub token with read access and permission to comment or label. + required: true + organization-token: + description: Optional organization token with Blocking users write permission. + required: false + block-high-confidence-automation: + description: Block high-confidence automated user accounts when an organization token is provided. + required: false + default: 'false' + ai-review: + description: Use GitHub Models for a second, evidence-grounded review. + required: false + default: 'true' + model: + description: GitHub Models model identifier. + required: false + default: openai/gpt-4.1 + comment: + description: Create or update the contributor report comment. + required: false + default: 'true' + fail-on-high-risk: + description: Fail the workflow when the report is high risk. + required: false + default: 'false' + +outputs: + author: + description: Contributor login that was inspected. + risk-level: + description: low, medium, or high. + risk-score: + description: Numeric risk score from 0 to 100. + report-json: + description: Machine-readable report. + blocked: + description: Whether the contributor was blocked from the organization. + +runs: + using: node24 + main: dist/index.mjs + +branding: + icon: shield + color: blue diff --git a/dist/index.mjs b/dist/index.mjs new file mode 100644 index 0000000..73ec5d3 --- /dev/null +++ b/dist/index.mjs @@ -0,0 +1,298 @@ +import { appendFile, readFile } from 'node:fs/promises'; + +import { + analyzeContributor, + mergeAiReview, + shouldBlockContributor, + shouldRetainReviewLabel, +} from '../src/analyze.mjs'; + +const token = input('github-token'); +const organizationToken = input('organization-token'); +const blockHighConfidenceAutomation = input('block-high-confidence-automation', 'false') === 'true'; +const useAi = input('ai-review', 'true') === 'true'; +const model = input('model', 'openai/gpt-4.1'); +const shouldComment = input('comment', 'true') === 'true'; +const failOnHighRisk = input('fail-on-high-risk', 'false') === 'true'; +const [owner, repo] = (process.env.GITHUB_REPOSITORY ?? '').split('/'); + +if (!token || !owner || !repo || !process.env.GITHUB_EVENT_PATH) { + throw new Error('github-token, GITHUB_REPOSITORY and GITHUB_EVENT_PATH are required'); +} + +const payload = JSON.parse(await readFile(process.env.GITHUB_EVENT_PATH, 'utf8')); +const target = resolveTarget(payload); +const encodedLogin = encodeURIComponent(target.login); + +const [profile, events, pullSearch, issueSearch, organizationSearch] = await Promise.all([ + github(`/users/${encodedLogin}`), + github(`/users/${encodedLogin}/events/public?per_page=100`), + github(`/search/issues?q=${encodeURIComponent(`type:pr author:${target.login}`)}&per_page=1`), + github(`/search/issues?q=${encodeURIComponent(`type:issue author:${target.login}`)}&per_page=1`), + github(`/search/issues?q=${encodeURIComponent(`type:pr org:${owner} author:${target.login}`)}&per_page=1`), +]); + +let report = analyzeContributor({ + profile, + events, + authoredPullRequests: pullSearch.total_count, + authoredIssues: issueSearch.total_count, + organizationPullRequests: organizationSearch.total_count, + association: target.association, + content: target.content, +}); + +let aiError = ''; +if (useAi && !report.trusted && profile.type !== 'Bot') { + try { + report = mergeAiReview(report, await reviewWithGitHubModels({ profile, report, target })); + } catch (error) { + aiError = error.message; + console.warning?.(`GitHub Models review unavailable: ${aiError}`); + console.log(`GitHub Models review unavailable: ${aiError}`); + } +} + +const blocked = await blockContributorIfConfigured({ profile, report, target }); +const finalReport = { + author: target.login, + subject: target.kind, + number: target.number, + ...report, + blocked, + aiError: aiError || undefined, +}; + +if (shouldComment) await syncRepositoryState(finalReport); +await writeSummary(finalReport); +await output('author', target.login); +await output('risk-level', report.level); +await output('risk-score', String(report.score)); +await output('report-json', JSON.stringify(finalReport)); +await output('blocked', String(blocked)); + +if (failOnHighRisk && report.level === 'high') { + process.exitCode = 1; + console.error(`Contributor detection for @${target.login} is high risk (${report.score}/100).`); +} + +function input(name, fallback = '') { + return process.env[`INPUT_${name.toUpperCase()}`] ?? fallback; +} + +function resolveTarget(event) { + if (event.pull_request) { + return { + kind: 'pull request', + number: event.pull_request.number, + login: event.pull_request.user.login, + association: event.pull_request.author_association ?? 'NONE', + content: `${event.pull_request.title ?? ''}\n${event.pull_request.body ?? ''}`, + }; + } + if (event.comment && event.issue) { + return { + kind: 'issue comment', + number: event.issue.number, + login: event.comment.user.login, + association: event.comment.author_association ?? 'NONE', + content: event.comment.body ?? '', + }; + } + if (event.issue) { + return { + kind: 'issue', + number: event.issue.number, + login: event.issue.user.login, + association: event.issue.author_association ?? 'NONE', + content: `${event.issue.title ?? ''}\n${event.issue.body ?? ''}`, + }; + } + throw new Error(`Unsupported event payload: ${process.env.GITHUB_EVENT_NAME ?? 'unknown'}`); +} + +async function github(path, options = {}, allowNotFound = false) { + const response = await fetch(`https://api.github.com${path}`, { + ...options, + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2026-03-10', + ...options.headers, + }, + }); + if (allowNotFound && response.status === 404) return null; + if (!response.ok) throw new Error(`${options.method ?? 'GET'} ${path}: ${response.status}`); + return response.status === 204 ? null : response.json(); +} + +async function organizationGithub(path, options = {}) { + const response = await fetch(`https://api.github.com${path}`, { + ...options, + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${organizationToken}`, + 'X-GitHub-Api-Version': '2026-03-10', + ...options.headers, + }, + }); + if (!response.ok) throw new Error(`${options.method ?? 'GET'} ${path}: ${response.status}`); + return response.status === 204 ? null : response.json(); +} + +async function blockContributorIfConfigured({ profile, report, target }) { + if (!blockHighConfidenceAutomation || !organizationToken) return false; + if (!shouldBlockContributor({ profile, report })) return false; + + await organizationGithub(`/orgs/${encodeURIComponent(owner)}/blocks/${encodeURIComponent(target.login)}`, { + method: 'PUT', + }); + return true; +} + +async function reviewWithGitHubModels({ profile, report, target }) { + const response = await fetch('https://models.github.ai/inference/chat/completions', { + method: 'POST', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2026-03-10', + }, + body: JSON.stringify({ + model, + temperature: 0, + max_tokens: 400, + response_format: { type: 'json_object' }, + messages: [ + { + role: 'system', + content: [ + 'Assess whether a public GitHub contribution warrants maintainer review for likely automation or bounty spam.', + 'Use only supplied facts. Never infer identity from writing style, language, nationality, or AI-tool disclosure.', + 'A new or sparse account is not proof. Prefer inconclusive when evidence is weak.', + 'Return JSON: {"classification":"likely-human|inconclusive|likely-automated","confidence":0..1,"reasons":[...],"recommendation":"..."}.', + ].join(' '), + }, + { + role: 'user', + content: JSON.stringify({ + profile: { + login: profile.login, + type: profile.type, + created_at: profile.created_at, + public_repos: profile.public_repos, + followers: profile.followers, + profile_complete: Boolean(profile.name || profile.bio || profile.company || profile.blog), + }, + facts: report.facts, + heuristicReasons: report.reasons, + contributionKind: target.kind, + contributionText: target.content.slice(0, 4000), + }), + }, + ], + }), + }); + if (!response.ok) throw new Error(`GitHub Models returned ${response.status}`); + const data = await response.json(); + const text = data.choices?.[0]?.message?.content ?? ''; + const parsed = JSON.parse(text.replace(/^```json\s*|\s*```$/g, '')); + const classifications = new Set(['likely-human', 'inconclusive', 'likely-automated']); + if (!classifications.has(parsed.classification)) throw new Error('GitHub Models returned an invalid classification'); + return { + classification: parsed.classification, + confidence: Math.min(1, Math.max(0, Number(parsed.confidence) || 0)), + reasons: Array.isArray(parsed.reasons) ? parsed.reasons.slice(0, 5).map(String) : [], + recommendation: String(parsed.recommendation ?? ''), + }; +} + +async function syncRepositoryState(report) { + const label = 'needs-contributor-review'; + const marker = ``; + const comments = await github(`/repos/${owner}/${repo}/issues/${report.number}/comments?per_page=100`); + const existingLabel = await github(`/repos/${owner}/${repo}/labels/${encodeURIComponent(label)}`, {}, true); + if (!existingLabel) { + await github(`/repos/${owner}/${repo}/labels`, { + method: 'POST', + body: JSON.stringify({ + name: label, + color: 'bf8700', + description: 'Public account signals need human review', + }), + }); + } + + const needsReview = shouldRetainReviewLabel(report, comments); + if (needsReview) { + await github(`/repos/${owner}/${repo}/issues/${report.number}/labels`, { + method: 'POST', + body: JSON.stringify({ labels: [label] }), + }); + } else { + await github( + `/repos/${owner}/${repo}/issues/${report.number}/labels/${encodeURIComponent(label)}`, + { method: 'DELETE' }, + true, + ); + } + + const existing = comments.find(comment => comment.body?.includes(marker)); + const body = renderComment(report, marker); + if (existing) { + await github(`/repos/${owner}/${repo}/issues/comments/${existing.id}`, { + method: 'PATCH', + body: JSON.stringify({ body }), + }); + } else { + await github(`/repos/${owner}/${repo}/issues/${report.number}/comments`, { + method: 'POST', + body: JSON.stringify({ body }), + }); + } +} + +function renderComment(report, marker) { + const facts = report.facts; + const reasons = report.reasons.length ? report.reasons.map(reason => `- ${reason}`).join('\n') : '- No heuristic warnings.'; + const ai = report.aiReview + ? `\n### AI review\n\n- Classification: **${report.aiReview.classification}** (${Math.round(report.aiReview.confidence * 100)}% confidence)\n- Recommendation: ${report.aiReview.recommendation || 'No recommendation.'}\n${report.aiReview.reasons.map(reason => `- ${reason}`).join('\n')}` + : report.aiError + ? '\n### AI review\n\nUnavailable; the evidence-only report remains valid.' + : ''; + return `${marker} +## Contributor detection + +**@${report.author}: ${report.level.toUpperCase()} (${report.score}/100)** + +| Public signal | Value | +| --- | ---: | +| Account age | ${facts.accountAgeDays} days | +| Public repositories | ${facts.publicRepositories} | +| Recent public events | ${facts.recentPublicEvents} | +| Public pull requests | ${facts.authoredPullRequests} | +| Earlier PRs in this organization | ${facts.organizationPullRequests} | + +### Evidence + +${reasons}${ai} + +Blocking result: **${report.blocked ? 'blocked from the organization' : 'not blocked'}**. + +This detection uses public evidence to prioritize human review. Sparse-account signals and GitHub App bot status alone never trigger blocking; blocking requires a high-confidence likely-automated classification for a user account.`; +} + +async function writeSummary(report) { + if (!process.env.GITHUB_STEP_SUMMARY) return; + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + `## Contributor detection\n\n- Author: @${report.author}\n- Risk: ${report.level} (${report.score}/100)\n- Subject: ${report.subject} #${report.number}\n- Blocked: ${report.blocked}\n`, + ); +} + +async function output(name, value) { + if (!process.env.GITHUB_OUTPUT) return; + const delimiter = `EOF_${Date.now()}_${Math.random().toString(16).slice(2)}`; + await appendFile(process.env.GITHUB_OUTPUT, `${name}<<${delimiter}\n${value}\n${delimiter}\n`); +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..e3ebaf2 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "contributor-trust-action", + "version": "1.0.0", + "private": true, + "type": "module", + "license": "LGPL-2.1-only", + "scripts": { + "test": "node --test" + }, + "engines": { + "node": ">=24" + } +} diff --git a/src/analyze.mjs b/src/analyze.mjs new file mode 100644 index 0000000..e4c9a9e --- /dev/null +++ b/src/analyze.mjs @@ -0,0 +1,142 @@ +const trustedAssociations = new Set(['OWNER', 'MEMBER', 'COLLABORATOR']); + +function clamp(value, minimum, maximum) { + return Math.min(maximum, Math.max(minimum, value)); +} + +function accountAgeDays(createdAt, now) { + const created = new Date(createdAt).getTime(); + return Number.isFinite(created) ? Math.max(0, Math.floor((now.getTime() - created) / 86_400_000)) : 0; +} + +export function analyzeContributor({ + profile, + events = [], + authoredPullRequests = 0, + authoredIssues = 0, + organizationPullRequests = 0, + association = 'NONE', + content = '', + now = new Date(), +}) { + if (trustedAssociations.has(association)) { + return { + score: 0, + level: 'low', + trusted: true, + reasons: [`Repository association is ${association}.`], + facts: buildFacts(profile, events, authoredPullRequests, authoredIssues, organizationPullRequests, now), + }; + } + + const reasons = []; + let score = 0; + const ageDays = accountAgeDays(profile.created_at, now); + const isBot = profile.type === 'Bot' || /\[bot\]$/i.test(profile.login ?? ''); + + if (isBot) { + score = 100; + reasons.push('GitHub identifies the account as a bot.'); + } else { + if (ageDays < 7) { + score += 35; + reasons.push(`Account is only ${ageDays} days old.`); + } else if (ageDays < 30) { + score += 24; + reasons.push(`Account is ${ageDays} days old.`); + } else if (ageDays < 90) { + score += 10; + reasons.push(`Account is relatively new (${ageDays} days).`); + } + + if (profile.public_repos === 0) { + score += 15; + reasons.push('Account has no public repositories.'); + } else if (profile.public_repos <= 2) { + score += 7; + reasons.push(`Account has ${profile.public_repos} public repositories.`); + } + + const profileFields = [profile.name, profile.bio, profile.company, profile.blog, profile.location]; + if (profileFields.every(value => !String(value ?? '').trim())) { + score += 10; + reasons.push('Public profile has no identifying or project context.'); + } + + if ((profile.followers ?? 0) === 0) { + score += 4; + reasons.push('Account has no followers.'); + } + if (events.length === 0) { + score += 12; + reasons.push('No recent public activity is visible through the GitHub API.'); + } + if (authoredPullRequests === 0) { + score += 10; + reasons.push('No public pull requests were found.'); + } + if (organizationPullRequests > 0) { + score -= Math.min(15, 5 + organizationPullRequests * 2); + reasons.push(`Found ${organizationPullRequests} earlier pull request(s) in this organization.`); + } + if (content.trim().length < 20) { + score += 5; + reasons.push('Current contribution contains very little context.'); + } + } + + score = clamp(score, 0, 100); + return { + score, + level: score >= 55 ? 'high' : score >= 30 ? 'medium' : 'low', + trusted: false, + reasons, + facts: buildFacts(profile, events, authoredPullRequests, authoredIssues, organizationPullRequests, now), + }; +} + +export function mergeAiReview(report, aiReview) { + if (!aiReview) return report; + + let score = report.score; + if (aiReview.classification === 'likely-automated' && aiReview.confidence >= 0.75) score += 20; + if (aiReview.classification === 'likely-human' && aiReview.confidence >= 0.75) score -= 10; + score = clamp(score, 0, 100); + + return { + ...report, + score, + level: score >= 55 ? 'high' : score >= 30 ? 'medium' : 'low', + aiReview, + }; +} + +export function shouldRetainReviewLabel(report, comments = []) { + if (report.level === 'medium' || report.level === 'high') return true; + + const ownMarker = ``; + return comments.some(comment => { + const body = String(comment.body ?? ''); + if (!body.includes('\n**@risky-user: HIGH (80/100)**', + }, + { + body: '\n**@current-user: HIGH (70/100)**', + }, + ]; + + assert.equal( + shouldRetainReviewLabel({ author: 'current-user', level: 'low' }, comments), + true, + ); +}); + +test('clears review label when only the current contributor is now low risk', () => { + const comments = [ + { + body: '\n**@current-user: HIGH (70/100)**', + }, + ]; + + assert.equal( + shouldRetainReviewLabel({ author: 'current-user', level: 'low' }, comments), + false, + ); +}); diff --git a/test/fixtures/pull_request.json b/test/fixtures/pull_request.json new file mode 100644 index 0000000..4af005e --- /dev/null +++ b/test/fixtures/pull_request.json @@ -0,0 +1,11 @@ +{ + "pull_request": { + "number": 91, + "title": "Contributor trust action smoke test", + "body": "Exercises evidence collection without writing to the repository.", + "author_association": "NONE", + "user": { + "login": "Neroxsh" + } + } +}