diff --git a/.github/scripts/generate-demo-matrix.sh b/.github/scripts/generate-demo-matrix.sh new file mode 100755 index 00000000000..e584a41092b --- /dev/null +++ b/.github/scripts/generate-demo-matrix.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Generate a matrix of demo directories for GitHub Actions +# Usage: generate-demo-matrix.sh [base_sha] +# Example: generate-demo-matrix.sh android pull_request abc123 + +set -e +shopt -s nullglob # Make globs expand to nothing if no matches + +PLATFORM="$1" +EVENT_NAME="$2" +BASE_SHA="$3" + +if [ -z "$PLATFORM" ] || [ -z "$EVENT_NAME" ]; then + echo "Usage: $0 [base_sha]" >&2 + exit 1 +fi + +DEMO_PATH="demos/${PLATFORM}" + +if [ "$EVENT_NAME" = "pull_request" ]; then + if [ -z "$BASE_SHA" ]; then + echo "Error: base_sha is required for pull_request events" >&2 + exit 1 + fi + + # Get list of changed files in demos// directory + changed_files=$(git diff --name-only "$BASE_SHA" HEAD -- "${DEMO_PATH}/*") + + echo "Changed files:" >&2 + echo "$changed_files" >&2 + + # Extract unique demo directories + matrix=$(echo "$changed_files" | grep -oE "${DEMO_PATH}/[^/]*/MASTG-DEMO-[^/]+" | sort -u | head -c -1 | tr '\n' ' ' | sed 's/ /","/g') + + # If no changes, set empty matrix + if [ -z "$matrix" ]; then + echo '{"demo":[]}' + else + echo "{\"demo\":[\"$matrix\"]}" + fi +else + # Default behavior: include all demos for master branch + demos=(${DEMO_PATH}/*/MASTG-DEMO-*) + if [ ${#demos[@]} -eq 0 ]; then + echo '{"demo":[]}' + else + matrix=$(printf '%s\n' "${demos[@]}" | sed 's/.*/"&"/' | paste -sd, -) + echo "{\"demo\":[$matrix]}" + fi +fi diff --git a/.github/workflows/build-android-demos.yml b/.github/workflows/build-android-demos.yml index 3d443f00234..5c114d31006 100644 --- a/.github/workflows/build-android-demos.yml +++ b/.github/workflows/build-android-demos.yml @@ -12,7 +12,6 @@ on: - master paths: - 'demos/**' - - '.github/workflows/build-android-demos.yml' jobs: generate-matrix: @@ -23,36 +22,17 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: - sparse-checkout: demos/android + sparse-checkout: | + demos/android + .github/scripts fetch-depth: 2 # Required for git diff in PRs - name: Generate matrix id: set-matrix run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - # Get list of changed files in demos/android/ directory - changed_files=$(git diff --name-only ${{ github.event.pull_request.base.sha }} HEAD -- 'demos/android/*') - - echo "Changed files:" - echo "$changed_files" - - # Extract unique demo directories - matrix=$(echo "$changed_files" | grep -oE 'demos/android/[^/]*/MASTG-DEMO-[^/]+' | sort -u | head -c -1 | tr '\n' ' ' | sed 's/ /","/g') - - # If no changes, set empty matrix - if [ -z "$matrix" ]; then - echo "matrix={\"demo\":[]}" >> $GITHUB_OUTPUT - else - echo "matrix={\"demo\":[\"$matrix\"]}" >> $GITHUB_OUTPUT - fi - else - # Default behavior: include all demos for master branch - matrix=$(echo demos/android/*/MASTG-DEMO-* | sed 's/ /","/g') - echo "matrix={\"demo\":[\"$matrix\"]}" >> $GITHUB_OUTPUT - fi - echo "Print matrix: $matrix" - - name: Print matrix - run: echo "${{ steps.set-matrix.outputs.matrix }}" + matrix=$(.github/scripts/generate-demo-matrix.sh android "${{ github.event_name }}" "${{ github.event.pull_request.base.sha }}") + echo "matrix=$matrix" >> $GITHUB_OUTPUT + echo "Generated matrix: $matrix" build-base-app: runs-on: ubuntu-latest diff --git a/.github/workflows/build-ios-demos.yml b/.github/workflows/build-ios-demos.yml index 8122f013b26..44649db49d5 100644 --- a/.github/workflows/build-ios-demos.yml +++ b/.github/workflows/build-ios-demos.yml @@ -12,7 +12,7 @@ on: - master paths: - 'demos/**' - - '.github/workflows/build-ios-demos.yml' + jobs: generate-matrix: runs-on: ubuntu-latest @@ -22,34 +22,17 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: - sparse-checkout: demos/ios + sparse-checkout: | + demos/ios + .github/scripts fetch-depth: 2 # Required for git diff in PRs - name: Generate matrix id: set-matrix run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - # Get list of changed files in demos/ios/ directory - changed_files=$(git diff --name-only ${{ github.event.pull_request.base.sha }} HEAD -- 'demos/ios/*') - - echo "Changed files:" - echo "$changed_files" - - # Extract unique demo directories - matrix=$(echo "$changed_files" | grep -oE 'demos/ios/[^/]*/MASTG-DEMO-[^/]+' | sort -u | head -c -1 | tr '\n' ' ' | sed 's/ /","/g') - - # If no changes, set empty matrix - if [ -z "$matrix" ]; then - echo "matrix={\"demo\":[]}" >> $GITHUB_OUTPUT - else - echo "matrix={\"demo\":[\"$matrix\"]}" >> $GITHUB_OUTPUT - fi - else - # Default behavior: include all demos for master branch - matrix=$(echo demos/ios/*/MASTG-DEMO-* | sed 's/ /","/g') - echo "matrix={\"demo\":[\"$matrix\"]}" >> $GITHUB_OUTPUT - fi - echo "Print matrix: $matrix" + matrix=$(.github/scripts/generate-demo-matrix.sh ios "${{ github.event_name }}" "${{ github.event.pull_request.base.sha }}") + echo "matrix=$matrix" >> $GITHUB_OUTPUT + echo "Generated matrix: $matrix" build: needs: generate-matrix diff --git a/.github/workflows/check-duplicate-ids.yml b/.github/workflows/check-duplicate-ids.yml index cc61fb9e653..56b6cb68987 100644 --- a/.github/workflows/check-duplicate-ids.yml +++ b/.github/workflows/check-duplicate-ids.yml @@ -13,67 +13,67 @@ on: jobs: check-duplicates: runs-on: ubuntu-latest - + steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 1 - + - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' - + - name: Get new files in PR id: get-new-files run: | # Get the base branch ref from the PR event BASE_REF="${{ github.event.pull_request.base.ref }}" echo "Base branch is: $BASE_REF" - + # Fetch the base branch to ensure it's available locally git fetch origin $BASE_REF - + # Create a file with the list of new files in this PR git diff --name-status --diff-filter=A origin/$BASE_REF..HEAD | grep -E "^A\s+(apps/|best-practices/|demos/|tests-beta/|tools/|techniques/)" | cut -f2 > new_files_in_pr.txt || echo "No new files matching the pattern" - + echo "New files in PR:" cat new_files_in_pr.txt - + # Ensure the file exists even if empty touch new_files_in_pr.txt - + - name: Check for duplicate file IDs id: check-duplicates run: | python .github/scripts/check_duplicate_ids.py - + - name: Delete previous comments and comment on PR if duplicates found if: always() - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const fs = require('fs'); - + // Helper function to format a notification message const formatDuplicateMessage = (duplicate, isReviewComment = true) => { // Extract the ID prefix for a more specific message let idType = duplicate.file_id.split('-').slice(0, 2).join('-'); - + // Choose the appropriate header style based on comment type const headerPrefix = isReviewComment ? '###' : '##'; - + return ` ${headerPrefix} ⚠️ Duplicate \`${idType}\` ID Detected - + ${isReviewComment ? 'This file' : `File \`${duplicate.file_path}\``} has the ID \`${duplicate.file_id}\` which already exists in \`${duplicate.existing_path}\`. - + **IMPORTANT:** Please use the next available ID: \`${duplicate.suggested_id}\` `; }; - + // Step 1: Try to clean up existing comments try { const comments = await github.rest.pulls.listReviewComments({ @@ -81,15 +81,15 @@ jobs: repo: context.repo.repo, pull_number: context.issue.number }); - + // Filter for comments from this action - const ourComments = comments.data.filter(comment => - comment.body.includes('⚠️ Duplicate') && + const ourComments = comments.data.filter(comment => + comment.body.includes('⚠️ Duplicate') && comment.body.includes('ID Detected') ); - + console.log(`Found ${ourComments.length} previous comments from this action.`); - + // Delete the comments for (const comment of ourComments) { try { @@ -107,18 +107,18 @@ jobs: console.log(`Error handling existing comments: ${error.message}`); // Continue with the workflow regardless of errors with existing comments } - + // Step 2: Check if we need to post new notifications const hasDuplicatesOutput = '${{ steps.check-duplicates.outputs.has_duplicates }}'; - + if (hasDuplicatesOutput !== 'true' || !fs.existsSync('duplicate_files.json')) { console.log('No duplicates to report.'); return; } - + // Step 3: Post notifications for each duplicate const duplicatesData = JSON.parse(fs.readFileSync('duplicate_files.json', 'utf8')); - + for (const duplicate of duplicatesData) { // First try posting a review comment (preferred method) try { @@ -135,7 +135,7 @@ jobs: console.log(`Successfully posted review comment for ${duplicate.file_path}`); } catch (error) { console.log(`Error posting review comment: ${error.message}`); - + // Fallback: post a comment on the PR thread instead of as a review comment try { await github.rest.issues.createComment({ @@ -149,4 +149,4 @@ jobs: console.log(`Error posting fallback PR thread comment: ${fallbackError.message}`); } } - } \ No newline at end of file + } diff --git a/.github/workflows/check-website-build.yml b/.github/workflows/check-website-build.yml deleted file mode 100644 index e99e8605e80..00000000000 --- a/.github/workflows/check-website-build.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Check Website Build -on: - pull_request: - branches: - - "*" - -jobs: - build: - uses: OWASP/mas-website/.github/workflows/build-website-reusable.yml@main - with: - deploy: false - sources_override_json: ${{ format('{{"OWASP/mastg":"refs/pull/{0}/head"}}', github.event.pull_request.number) }} diff --git a/.github/workflows/main-markdown-lint.yml b/.github/workflows/main-markdown-lint.yml new file mode 100644 index 00000000000..ddc2e100294 --- /dev/null +++ b/.github/workflows/main-markdown-lint.yml @@ -0,0 +1,17 @@ +name: Markdown Lint + +on: + workflow_dispatch: + push: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + markdown-lint: + name: Markdown Lint + uses: ./.github/workflows/reusable-markdown-linter.yml + secrets: inherit diff --git a/.github/workflows/main-spell-check.yml b/.github/workflows/main-spell-check.yml new file mode 100644 index 00000000000..1a176107643 --- /dev/null +++ b/.github/workflows/main-spell-check.yml @@ -0,0 +1,17 @@ +name: Spell Check + +on: + workflow_dispatch: + push: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + spell-check: + name: Spell Check (Full) + uses: ./.github/workflows/reusable-spell-checker.yml + secrets: inherit diff --git a/.github/workflows/main-url-check.yml b/.github/workflows/main-url-check.yml new file mode 100644 index 00000000000..82ef80e36ec --- /dev/null +++ b/.github/workflows/main-url-check.yml @@ -0,0 +1,17 @@ +name: URL Check + +on: + workflow_dispatch: + push: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + url-check: + name: URL Check (Full) + uses: ./.github/workflows/reusable-url-checker.yml + secrets: inherit diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml new file mode 100644 index 00000000000..3f934c3ef8f --- /dev/null +++ b/.github/workflows/pr-checks.yml @@ -0,0 +1,36 @@ +name: PR Checks + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + spell-check: + name: Spell Check (Changed Files) + uses: ./.github/workflows/reusable-spell-checker.yml + secrets: inherit + with: + check_modified_files_only: true + pr_number: ${{ github.event.pull_request.number }} + + url-check: + name: URL Check (Changed Files) + uses: ./.github/workflows/reusable-url-checker.yml + secrets: inherit + with: + check_modified_files_only: true + + markdown-lint: + name: Markdown Lint + uses: ./.github/workflows/reusable-markdown-linter.yml + secrets: inherit + + website-build: + name: Website Build Check + uses: ./.github/workflows/reusable-website-build-checker.yml + secrets: inherit + with: + pr_number: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/markdown-linter.yml b/.github/workflows/reusable-markdown-linter.yml similarity index 81% rename from .github/workflows/markdown-linter.yml rename to .github/workflows/reusable-markdown-linter.yml index 776ab3ead88..5b69f38d3db 100644 --- a/.github/workflows/markdown-linter.yml +++ b/.github/workflows/reusable-markdown-linter.yml @@ -1,11 +1,7 @@ -name: Markdown Linter +name: Markdown Linter (Reusable) on: - workflow_dispatch: - pull_request: - push: - branches: - - master + workflow_call: jobs: markdown-lint-check: @@ -15,10 +11,11 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Install markdownlint-rule-search-replace plugin - run: npm install markdownlint-rule-search-replace --save-dev + run: npm install markdownlint-rule-search-replace --save-dev + - name: markdownlint-cli2 uses: DavidAnson/markdownlint-cli2-action@v16.0.0 with: config: ".markdownlint.jsonc" - diff --git a/.github/workflows/reusable-spell-checker.yml b/.github/workflows/reusable-spell-checker.yml new file mode 100644 index 00000000000..41bc61731fe --- /dev/null +++ b/.github/workflows/reusable-spell-checker.yml @@ -0,0 +1,45 @@ +name: Spell Checker (Reusable) + +on: + workflow_call: + inputs: + check_modified_files_only: + description: 'Only check files changed in the PR' + required: false + default: false + type: boolean + pr_number: + description: 'PR number (required when check_modified_files_only is true)' + required: false + type: number + +jobs: + codespell: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Get Changed Files from Pull Request + if: inputs.check_modified_files_only + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + files=$(gh pr diff ${{ inputs.pr_number }} --name-only | xargs -I {} sh -c 'echo "./{}"' | tr '\n' ' ') + echo "CHANGED_FILES=$files" >> "$GITHUB_ENV" + + - name: Check Spelling (Changed Files) + if: inputs.check_modified_files_only + uses: codespell-project/actions-codespell@master + with: + check_filenames: true + skip: "*.json,*.yml,*.apk,*.ipa,*.svg,*.txt,*.kt,*.swift,*.xml,*.java" + ignore_words_list: "aas,aaS,ba,bund,compliancy,firt,ist,keypair,ligh,Manuel,Manual,ro,ser,synopsys,theses,zuser,lief,EDE" + path: ${{ env.CHANGED_FILES }} + + - name: Check Spelling (All Files) + if: ${{ !inputs.check_modified_files_only }} + uses: codespell-project/actions-codespell@master + with: + check_filenames: true + skip: "*.json,*.yml,*.apk,*.ipa,*.svg,*.txt,*.kt,*.swift,*.xml,*.java" + ignore_words_list: "aas,aaS,ba,bund,compliancy,firt,ist,keypair,ligh,Manuel,Manual,ro,ser,synopsys,theses,zuser,lief,EDE" diff --git a/.github/workflows/url-checker.yml b/.github/workflows/reusable-url-checker.yml similarity index 51% rename from .github/workflows/url-checker.yml rename to .github/workflows/reusable-url-checker.yml index 62037cf994d..b927eec49a8 100644 --- a/.github/workflows/url-checker.yml +++ b/.github/workflows/reusable-url-checker.yml @@ -1,10 +1,13 @@ -name: URL Checker +name: URL Checker (Reusable) on: - workflow_dispatch: - push: - branches: - - master + workflow_call: + inputs: + check_modified_files_only: + description: 'Only check files modified in the PR' + required: false + default: false + type: boolean jobs: markdown-link-check: @@ -12,8 +15,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: URL Link Check uses: Diolor/github-action-markdown-link-check@1.3 with: use-quiet-mode: 'yes' config-file: '.github/workflows/config/url-checker-config.json' + check-modified-files-only: ${{ inputs.check_modified_files_only && 'yes' || 'no' }} diff --git a/.github/workflows/reusable-website-build-checker.yml b/.github/workflows/reusable-website-build-checker.yml new file mode 100644 index 00000000000..7adb0b05b55 --- /dev/null +++ b/.github/workflows/reusable-website-build-checker.yml @@ -0,0 +1,16 @@ +name: Website Build Checker (Reusable) + +on: + workflow_call: + inputs: + pr_number: + description: 'Pull request number' + required: true + type: number + +jobs: + website-build-check: + uses: OWASP/mas-website/.github/workflows/build-website-reusable.yml@main + with: + deploy: false + sources_override_json: ${{ format('{{"OWASP/mastg":"refs/pull/{0}/head"}}', inputs.pr_number) }} diff --git a/.github/workflows/spell-checker-pr.yml b/.github/workflows/spell-checker-pr.yml deleted file mode 100644 index f107ebbd71d..00000000000 --- a/.github/workflows/spell-checker-pr.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Spell Checker (PR) - -on: - pull_request: - -jobs: - codespell: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Get Changed Files from Pull Request - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # get file names and seperate them with space ' ' - files=$(gh pr diff ${{ github.event.pull_request.number }} --name-only | xargs -I {} sh -c 'echo "./{}"' | tr '\n' ' ') - echo "CHANGED_FILES=$files" >> "$GITHUB_ENV" - - - name: Output Changed Files - run: echo ${{ env.CHANGED_FILES }} - - - name: Check for Spelling Errors for Changed Files - uses: codespell-project/actions-codespell@master - with: - check_filenames: true - skip: "*.json,*.yml,*.apk,*.ipa,*.svg,*.txt,*.kt,*.swift,*.xml,*.java" - ignore_words_list: "aas,aaS,ba,bund,compliancy,firt,ist,keypair,ligh,Manuel,Manual,ro,ser,synopsys,theses,zuser,lief,EDE" - path: ${{ env.CHANGED_FILES }} diff --git a/.github/workflows/spell-checker.yml b/.github/workflows/spell-checker.yml deleted file mode 100644 index a85d6b48f80..00000000000 --- a/.github/workflows/spell-checker.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Spell Checker - -on: - workflow_dispatch: - push: - branches: - - master - -jobs: - codespell: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: codespell-project/actions-codespell@master - with: - ignore_words_list: "aas,aaS,ba,bund,compliancy,firt,ist,keypair,ligh,Manuel,Manual,ro,ser,synopsys,theses,zuser,lief,EDE" - skip: "*.json,*.yml,*.apk,*.ipa,*.svg,*.txt,*.kt,*.swift,*.xml,*.java" diff --git a/.github/workflows/url-checker-pr.yml b/.github/workflows/url-checker-pr.yml deleted file mode 100644 index 80543406382..00000000000 --- a/.github/workflows/url-checker-pr.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: URL Checker (PR) - -on: [pull_request] - -jobs: - markdown-link-check: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: URL Link Check - uses: Diolor/github-action-markdown-link-check@1.3 - with: - use-quiet-mode: 'yes' - config-file: '.github/workflows/config/url-checker-config.json' - check-modified-files-only: 'yes' diff --git a/README.md b/README.md index 2933ecd3b72..43096a90a86 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ [![OWASP Flagship](https://img.shields.io/badge/owasp-flagship%20project-48A646.svg)](https://owasp.org/projects/) [![Creative Commons License](https://img.shields.io/github/license/OWASP/mastg?color=48A646)](https://creativecommons.org/licenses/by-sa/4.0/ "CC BY-SA 4.0") -[![Markdown Linter](https://github.com/OWASP/mastg/actions/workflows/markdown-linter.yml/badge.svg?branch=master)](https://github.com/OWASP/mastg/actions/workflows/markdown-linter.yml) -[![URL Checker](https://github.com/OWASP/mastg/actions/workflows/url-checker.yml/badge.svg?branch=master)](https://github.com/OWASP/mastg/actions/workflows/url-checker.yml) +[![Markdown Lint](https://github.com/OWASP/mastg/actions/workflows/main-markdown-lint.yml/badge.svg?branch=master)](https://github.com/OWASP/mastg/actions/workflows/main-markdown-lint.yml) +[![URL Check](https://github.com/OWASP/mastg/actions/workflows/main-url-check.yml/badge.svg?branch=master)](https://github.com/OWASP/mastg/actions/workflows/main-url-check.yml) The **OWASP Mobile Application Security Testing Guide (MASTG)** is a comprehensive manual for mobile app security testing and reverse engineering. It describes technical processes for verifying the [OWASP Mobile Security Weakness Enumeration (MASWE)](https://github.com/OWASP/maswe "MASWE") weaknesses, which are in alignment with the controls listed in the [OWASP Mobile Application Verification Standard (MASVS)](https://github.com/OWASP/masvs "MASVS").