diff --git a/backend/src/apps/github/common.py b/backend/src/apps/github/common.py index 1371fcf782..56f1248ddc 100644 --- a/backend/src/apps/github/common.py +++ b/backend/src/apps/github/common.py @@ -23,6 +23,7 @@ from apps.github.models.repository_contributor import RepositoryContributor from apps.github.models.user import User from apps.github.utils import check_owasp_site_repository +from apps.owasp.models.activity_event import ActivityEvent logger: logging.Logger = logging.getLogger(__name__) @@ -115,6 +116,7 @@ def sync_repository( if (latest_updated_issue := repository.latest_updated_issue) else month_ago ) + issues = [] for gh_issue in gh_repository.get_issues(**kwargs): if gh_issue.pull_request: # Skip pull requests. continue @@ -138,6 +140,7 @@ def sync_repository( milestone=milestone, repository=repository, ) + issues.append(issue) # Assignees. issue.assignees.clear() @@ -152,6 +155,7 @@ def sync_repository( issue.labels.add(Label.update_data(gh_issue_label)) except UnknownObjectException: logger.exception("Couldn't get GitHub issue label %s", issue.url) + ActivityEvent.bulk_save_for_sources(issues) else: logger.info("Skipping issues sync for %s", repository.name) @@ -166,6 +170,7 @@ def sync_repository( if (latest_updated_pull_request := repository.latest_updated_pull_request) else month_ago ) + pull_requests = [] for gh_pull_request in gh_repository.get_pulls(**kwargs): if gh_pull_request.updated_at < until: break @@ -186,6 +191,7 @@ def sync_repository( milestone=milestone, repository=repository, ) + pull_requests.append(pull_request) # Assignees. pull_request.assignees.clear() @@ -200,12 +206,16 @@ def sync_repository( pull_request.labels.add(Label.update_data(gh_pull_request_label)) except UnknownObjectException: logger.exception("Couldn't get GitHub pull request label %s", pull_request.url) + ActivityEvent.bulk_save_for_sources(pull_requests) # GitHub repository releases. releases = [] if not is_owasp_site_repository: existing_release_node_ids = set( - Release.objects.filter(repository=repository).values_list("node_id", flat=True) + Release.objects.filter( + repository=repository, + published_at__isnull=False, + ).values_list("node_id", flat=True) if repository.id else () ) @@ -215,8 +225,11 @@ def sync_repository( break author = User.update_data(gh_release.author) - releases.append(Release.update_data(gh_release, author=author, repository=repository)) + release = Release.update_data(gh_release, author=author, repository=repository) + releases.append(release) + releases_for_events = list(releases) Release.bulk_save(releases) + ActivityEvent.bulk_save_for_sources(releases_for_events) # GitHub repository contributors. RepositoryContributor.bulk_save( diff --git a/backend/src/apps/owasp/admin/__init__.py b/backend/src/apps/owasp/admin/__init__.py index 261225145a..524c70a757 100644 --- a/backend/src/apps/owasp/admin/__init__.py +++ b/backend/src/apps/owasp/admin/__init__.py @@ -4,6 +4,7 @@ from apps.owasp.models.project_health_requirements import ProjectHealthRequirements +from .activity_event import ActivityEventAdmin from .board_of_directors import BoardOfDirectorsAdmin from .chapter import ChapterAdmin from .committee import CommitteeAdmin diff --git a/backend/src/apps/owasp/admin/activity_event.py b/backend/src/apps/owasp/admin/activity_event.py new file mode 100644 index 0000000000..6a16479878 --- /dev/null +++ b/backend/src/apps/owasp/admin/activity_event.py @@ -0,0 +1,32 @@ +"""OWASP app ActivityEvent model admin.""" + +from django.contrib import admin + +from apps.owasp.models.activity_event import ActivityEvent + + +class ActivityEventAdmin(admin.ModelAdmin): + """Admin for ActivityEvent model.""" + + autocomplete_fields = ( + "github_user", + "github_repository", + ) + list_display = ( + "activity_type", + "github_repository", + "github_user", + "occurred_at", + ) + list_filter = ( + "activity_type", + "occurred_at", + ) + search_fields = ( + "activity_type", + "github_repository__name", + "github_user__login", + ) + + +admin.site.register(ActivityEvent, ActivityEventAdmin) diff --git a/backend/src/apps/owasp/migrations/0073_activityevent.py b/backend/src/apps/owasp/migrations/0073_activityevent.py new file mode 100644 index 0000000000..ba75297650 --- /dev/null +++ b/backend/src/apps/owasp/migrations/0073_activityevent.py @@ -0,0 +1,98 @@ +# Generated by Django 6.0.7 on 2026-07-28 07:54 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("contenttypes", "0002_remove_content_type_name"), + ("github", "0044_user_indexes"), + ("owasp", "0072_project_project_name_gin_idx_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="ActivityEvent", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ( + "activity_type", + models.CharField( + choices=[ + ("issue_closed", "Issue Closed"), + ("issue_opened", "Issue Opened"), + ("pr_closed", "PR Closed"), + ("pr_merged", "PR Merged"), + ("pr_opened", "PR Opened"), + ("release_published", "Release Published"), + ], + max_length=32, + verbose_name="Activity Type", + ), + ), + ("object_id", models.PositiveBigIntegerField()), + ( + "occurred_at", + models.DateTimeField( + help_text="Timestamp when the activity event occurred on GitHub", + verbose_name="Occurred at", + ), + ), + ( + "content_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="contenttypes.contenttype" + ), + ), + ( + "github_repository", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="activity_events", + to="github.repository", + verbose_name="GitHub Repository", + ), + ), + ( + "github_user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="activity_events", + to="github.user", + verbose_name="GitHub User", + ), + ), + ], + options={ + "verbose_name_plural": "Activity Events", + "db_table": "github_activity_events", + "indexes": [ + models.Index(fields=["activity_type"], name="activity_event_type_idx"), + models.Index(fields=["github_user"], name="activity_event_github_user_idx"), + models.Index( + fields=["content_type", "object_id"], name="activity_event_source_idx" + ), + models.Index(fields=["occurred_at"], name="activity_event_occurred_at_idx"), + models.Index( + fields=["github_repository"], name="activity_event_github_repo_idx" + ), + ], + "constraints": [ + models.UniqueConstraint( + fields=("activity_type", "content_type", "object_id", "occurred_at"), + name="unique_activity_event", + ) + ], + }, + ), + ] diff --git a/backend/src/apps/owasp/models/__init__.py b/backend/src/apps/owasp/models/__init__.py index 3cbb120b8b..3bbb4a7832 100644 --- a/backend/src/apps/owasp/models/__init__.py +++ b/backend/src/apps/owasp/models/__init__.py @@ -1,3 +1,4 @@ +from .activity_event import ActivityEvent from .board_of_directors import BoardOfDirectors from .chapter import Chapter from .committee import Committee diff --git a/backend/src/apps/owasp/models/activity_event.py b/backend/src/apps/owasp/models/activity_event.py new file mode 100644 index 0000000000..ed5d307bca --- /dev/null +++ b/backend/src/apps/owasp/models/activity_event.py @@ -0,0 +1,164 @@ +"""OWASP app activity event model.""" + +import logging + +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.db import models + +from apps.common.models import BulkSaveModel, TimestampedModel +from apps.github.models.generic_issue_model import GenericIssueModel + +logger = logging.getLogger(__name__) + + +class ActivityEvent(BulkSaveModel, TimestampedModel): + """Represents a discrete GitHub activity event linked to a single source object. + + Uses a polymorphic GenericForeignKey to reference the source object. + """ + + class Meta: + """Model options.""" + + db_table = "github_activity_events" + verbose_name_plural = "Activity Events" + + constraints = [ + models.UniqueConstraint( + fields=[ + "activity_type", + "content_type", + "object_id", + "occurred_at", + ], + name="unique_activity_event", + ), + ] + + indexes = [ + models.Index(fields=["activity_type"], name="activity_event_type_idx"), + models.Index(fields=["github_user"], name="activity_event_github_user_idx"), + models.Index( + fields=["content_type", "object_id"], + name="activity_event_source_idx", + ), + models.Index(fields=["occurred_at"], name="activity_event_occurred_at_idx"), + models.Index(fields=["github_repository"], name="activity_event_github_repo_idx"), + ] + + class ActivityType(models.TextChoices): + """Activity type choices.""" + + ISSUE_CLOSED = "issue_closed", "Issue Closed" + ISSUE_OPENED = "issue_opened", "Issue Opened" + PR_CLOSED = "pr_closed", "PR Closed" + PR_MERGED = "pr_merged", "PR Merged" + PR_OPENED = "pr_opened", "PR Opened" + RELEASE_PUBLISHED = "release_published", "Release Published" + + activity_type = models.CharField( + verbose_name="Activity Type", + max_length=32, + choices=ActivityType.choices, + ) + github_user = models.ForeignKey( + "github.User", + verbose_name="GitHub User", + on_delete=models.SET_NULL, + blank=True, + null=True, + related_name="activity_events", + ) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) + object_id = models.PositiveBigIntegerField() + occurred_at = models.DateTimeField( + verbose_name="Occurred at", + help_text="Timestamp when the activity event occurred on GitHub", + ) + github_repository = models.ForeignKey( + "github.Repository", + verbose_name="GitHub Repository", + on_delete=models.CASCADE, + related_name="activity_events", + ) + source_object = GenericForeignKey("content_type", "object_id") + + HANDLERS: dict[str, str] = { + "Issue": "build_for_issue", + "PullRequest": "build_for_pull_request", + "Release": "build_for_release", + } + + def __str__(self) -> str: + """Return human-readable representation.""" + return f"{self.activity_type} by {self.github_user} in {self.github_repository}" + + @staticmethod + def bulk_save(activity_events, fields=None) -> None: # type: ignore[override] + """Bulk save activity events.""" + BulkSaveModel.bulk_save(ActivityEvent, activity_events, fields=fields) + + @staticmethod + def build_for_issue(issue) -> list[tuple]: + """Return event tuples for an Issue.""" + events = [(ActivityEvent.ActivityType.ISSUE_OPENED, issue.created_at, issue.author)] + if issue.state == GenericIssueModel.IssueState.CLOSED and issue.closed_at: + events.append((ActivityEvent.ActivityType.ISSUE_CLOSED, issue.closed_at, issue.author)) + return events + + @staticmethod + def build_for_pull_request(pr) -> list[tuple]: + """Return event tuples for a PullRequest.""" + events = [(ActivityEvent.ActivityType.PR_OPENED, pr.created_at, pr.author)] + if pr.merged_at: + events.append((ActivityEvent.ActivityType.PR_MERGED, pr.merged_at, pr.author)) + elif pr.state == GenericIssueModel.IssueState.CLOSED and pr.closed_at: + events.append((ActivityEvent.ActivityType.PR_CLOSED, pr.closed_at, pr.author)) + return events + + @staticmethod + def build_for_release(release) -> list[tuple]: + """Return event tuples for a Release.""" + if release.published_at is None: + return [] + + return [ + (ActivityEvent.ActivityType.RELEASE_PUBLISHED, release.published_at, release.author) + ] + + @staticmethod + def update_data(source) -> list["ActivityEvent"]: + """Return unsaved ActivityEvent instances for a GitHub model object.""" + handler_name = ActivityEvent.HANDLERS.get(type(source).__name__) + if handler_name is None: + logger.error( + "ActivityEvent.update_data received unsupported model type: %s", + type(source).__name__, + ) + message = f"Unsupported model type: {type(source)}" + raise TypeError(message) + + handler = getattr(ActivityEvent, handler_name) + events = handler(source) + content_type = ContentType.objects.get_for_model(source) + + return [ + ActivityEvent( + activity_type=activity_type, + content_type=content_type, + object_id=source.pk, + occurred_at=occurred_at, + github_user=github_user, + github_repository=source.repository, + ) + for activity_type, occurred_at, github_user in events + if occurred_at is not None + ] + + @staticmethod + def bulk_save_for_sources(sources: list) -> None: + """Bulk-insert ActivityEvent rows for source objects, skipping duplicates.""" + events = [event for source in sources for event in ActivityEvent.update_data(source)] + if events: + ActivityEvent.objects.bulk_create(events, ignore_conflicts=True) diff --git a/backend/tests/unit/apps/github/common_test.py b/backend/tests/unit/apps/github/common_test.py index a5501eed1c..8106f85584 100644 --- a/backend/tests/unit/apps/github/common_test.py +++ b/backend/tests/unit/apps/github/common_test.py @@ -21,6 +21,7 @@ def mock_common_deps(mocker): "Label": mocker.patch("apps.github.common.Label"), "Release": mocker.patch("apps.github.common.Release"), "RepositoryContributor": mocker.patch("apps.github.common.RepositoryContributor"), + "ActivityEvent": mocker.patch("apps.github.common.ActivityEvent"), "check_owasp": mocker.patch( "apps.github.common.check_owasp_site_repository", return_value=False ), diff --git a/docker-compose/local/compose.override.yaml b/docker-compose/local/compose.override.yaml index e69de29bb2..25ce0eb8cb 100644 --- a/docker-compose/local/compose.override.yaml +++ b/docker-compose/local/compose.override.yaml @@ -0,0 +1,13 @@ +volumes: + backend-venv: + name: backend-venv-pulse + cache-data: + name: cache-data-pulse + db-data: + name: db-data-pulse + docs-venv: + name: docs-venv-pulse + frontend-next: + name: frontend-next-pulse + frontend-node-modules: + name: frontend-node-modules-pulse