diff --git a/apps/jobs/utils.py b/apps/jobs/utils.py index eb2e4a9e94..f3efeb1b15 100644 --- a/apps/jobs/utils.py +++ b/apps/jobs/utils.py @@ -426,70 +426,27 @@ def calculate_distinct_sorted_leaderboard_data( submission__is_public=True ) - if challenge_phase_split.show_execution_time: - time_diff_expression = ExpressionWrapper( - F("submission__completed_at") - F("submission__started_at"), - output_field=fields.DurationField(), - ) - leaderboard_data = leaderboard_data.annotate( - filtering_score=RawSQL( - "result->>%s", (default_order_by,), output_field=FloatField() - ), - filtering_error=RawSQL( - "error->>%s", - ("error_{0}".format(default_order_by),), - output_field=FloatField(), - ), - submission__execution_time=time_diff_expression, - ).values( - "id", - "submission__participant_team", - "submission__participant_team__team_name", - "submission__participant_team__team_url", - "submission__is_baseline", - "submission__is_public", - "challenge_phase_split", - "result", - "error", - "filtering_score", - "filtering_error", - "leaderboard__schema", - "submission__submitted_at", - "submission__method_name", - "submission__id", - "submission__submission_metadata", - "submission__execution_time", - "submission__is_verified_by_host", - ) - else: - leaderboard_data = leaderboard_data.annotate( - filtering_score=RawSQL( - "result->>%s", (default_order_by,), output_field=FloatField() - ), - filtering_error=RawSQL( - "error->>%s", - ("error_{0}".format(default_order_by),), - output_field=FloatField(), - ), - ).values( - "id", - "submission__participant_team", - "submission__participant_team__team_name", - "submission__participant_team__team_url", - "submission__is_baseline", - "submission__is_public", - "challenge_phase_split", - "result", - "error", - "filtering_score", - "filtering_error", - "leaderboard__schema", - "submission__submitted_at", - "submission__method_name", - "submission__id", - "submission__submission_metadata", - "submission__is_verified_by_host", - ) + # Stage 1: fetch a lightweight projection (no heavy JSON columns) for + # sort + dedup. This avoids transferring result/error/submission_metadata + # for rows that will be discarded by per-team deduplication. + leaderboard_data_light = leaderboard_data.annotate( + filtering_score=RawSQL( + "result->>%s", (default_order_by,), output_field=FloatField() + ), + filtering_error=RawSQL( + "error->>%s", + ("error_{0}".format(default_order_by),), + output_field=FloatField(), + ), + ).values( + "id", + "submission__participant_team", + "submission__participant_team__team_name", + "submission__is_baseline", + "error", + "filtering_score", + "filtering_error", + ) all_banned_participant_team = set() all_banned_email_ids_set = ( @@ -498,15 +455,12 @@ def calculate_distinct_sorted_leaderboard_data( # Apply query limit to prevent slow queries on popular challenges max_limit = getattr(settings, "MAX_LEADERBOARD_QUERY_LIMIT", 10000) - leaderboard_data = leaderboard_data[:max_limit] - - # Convert to list to allow multiple iterations - leaderboard_data = list(leaderboard_data) + leaderboard_data_light = list(leaderboard_data_light[:max_limit]) # Prefetch all participant teams and their participants' emails in bulk # (fixes N+1 query) unique_team_ids = set( - item["submission__participant_team"] for item in leaderboard_data + item["submission__participant_team"] for item in leaderboard_data_light ) participant_teams = ParticipantTeam.objects.filter( id__in=unique_team_ids @@ -517,7 +471,7 @@ def calculate_distinct_sorted_leaderboard_data( for team in participant_teams } - for leaderboard_item in leaderboard_data: + for leaderboard_item in leaderboard_data_light: participant_team_id = leaderboard_item["submission__participant_team"] all_participants_email_ids = team_emails_lookup.get( participant_team_id, [] @@ -527,21 +481,23 @@ def calculate_distinct_sorted_leaderboard_data( all_banned_participant_team.add(participant_team_id) break if leaderboard_item["error"] is None: - leaderboard_item.update(filtering_error=0) + leaderboard_item["filtering_error"] = 0 if leaderboard_item["filtering_score"] is None: - leaderboard_item.update(filtering_score=0) + leaderboard_item["filtering_score"] = 0 + if challenge_phase_split.show_leaderboard_by_latest_submission: - sorted_leaderboard_data = leaderboard_data + sorted_leaderboard_data = leaderboard_data_light else: sorted_leaderboard_data = sorted( - leaderboard_data, + leaderboard_data_light, key=lambda k: ( float(k["filtering_score"]), float(-k["filtering_error"]), ), reverse=True if is_leaderboard_order_descending else False, ) - distinct_sorted_leaderboard_data = [] + + retained_light = [] team_list = set() for data in sorted_leaderboard_data: if ( @@ -551,11 +507,86 @@ def calculate_distinct_sorted_leaderboard_data( ): continue elif data["submission__is_baseline"] is True: - distinct_sorted_leaderboard_data.append(data) + retained_light.append(data) else: - distinct_sorted_leaderboard_data.append(data) + retained_light.append(data) team_list.add(data["submission__participant_team__team_name"]) + # Stage 2: fetch full row data (with heavy JSON columns) only for the + # retained leaderboard rows, then reapply the order from stage 1. + retained_ids = [item["id"] for item in retained_light] + heavy_qs = LeaderboardData.objects.filter(id__in=retained_ids).annotate( + filtering_score=RawSQL( + "result->>%s", (default_order_by,), output_field=FloatField() + ), + filtering_error=RawSQL( + "error->>%s", + ("error_{0}".format(default_order_by),), + output_field=FloatField(), + ), + ) + if challenge_phase_split.show_execution_time: + time_diff_expression = ExpressionWrapper( + F("submission__completed_at") - F("submission__started_at"), + output_field=fields.DurationField(), + ) + heavy_qs = heavy_qs.annotate( + submission__execution_time=time_diff_expression, + ).values( + "id", + "submission__participant_team", + "submission__participant_team__team_name", + "submission__participant_team__team_url", + "submission__is_baseline", + "submission__is_public", + "challenge_phase_split", + "result", + "error", + "filtering_score", + "filtering_error", + "leaderboard__schema", + "submission__submitted_at", + "submission__method_name", + "submission__id", + "submission__submission_metadata", + "submission__execution_time", + "submission__is_verified_by_host", + ) + else: + heavy_qs = heavy_qs.values( + "id", + "submission__participant_team", + "submission__participant_team__team_name", + "submission__participant_team__team_url", + "submission__is_baseline", + "submission__is_public", + "challenge_phase_split", + "result", + "error", + "filtering_score", + "filtering_error", + "leaderboard__schema", + "submission__submitted_at", + "submission__method_name", + "submission__id", + "submission__submission_metadata", + "submission__is_verified_by_host", + ) + + heavy_by_id = {row["id"]: row for row in heavy_qs} + + distinct_sorted_leaderboard_data = [] + for light_item in retained_light: + full = heavy_by_id.get(light_item["id"]) + if full is None: + # Row removed between stage 1 and stage 2; skip. + continue + if full["error"] is None: + full["filtering_error"] = 0 + if full["filtering_score"] is None: + full["filtering_score"] = 0 + distinct_sorted_leaderboard_data.append(full) + leaderboard_labels = challenge_phase_split.leaderboard.schema["labels"] show_scores = challenge_phase_split.show_scores_on_leaderboard for item in distinct_sorted_leaderboard_data: diff --git a/tests/unit/jobs/test_utils.py b/tests/unit/jobs/test_utils.py index 4e8c84a7e7..915de0e41c 100644 --- a/tests/unit/jobs/test_utils.py +++ b/tests/unit/jobs/test_utils.py @@ -731,6 +731,7 @@ def test_bulk_prefetch_avoids_n_plus_one_queries( # Test data for leaderboard entries test_data = [ { + "id": 1, "submission__participant_team": 1, "submission__participant_team__team_name": "Team1", "submission__is_baseline": False, @@ -739,6 +740,7 @@ def test_bulk_prefetch_avoids_n_plus_one_queries( "result": {"score": 10, "time": 5}, }, { + "id": 2, "submission__participant_team": 1, "submission__participant_team__team_name": "Team1", "submission__is_baseline": False, @@ -747,6 +749,7 @@ def test_bulk_prefetch_avoids_n_plus_one_queries( "result": {"score": 8, "time": 6}, }, { + "id": 3, "submission__participant_team": 2, "submission__participant_team__team_name": "Team2", "submission__is_baseline": False, @@ -756,12 +759,14 @@ def test_bulk_prefetch_avoids_n_plus_one_queries( }, ] - # Set up chainable mock for: - # .filter().exclude().filter().filter().order_by().annotate().values() + # Stage 1 chain: .filter().exclude().filter().filter().order_by() + # .annotate().values() + # Stage 2 chain: .filter(id__in=...).annotate().values() mock_qs = MagicMock() mock_filter_result = MagicMock() mock_leaderboard_data_objects.filter.return_value = mock_filter_result mock_filter_result.exclude.return_value = mock_qs + mock_filter_result.annotate.return_value = mock_qs mock_qs.filter.return_value = mock_qs mock_qs.order_by.return_value = mock_qs mock_qs.annotate.return_value = mock_qs @@ -823,6 +828,7 @@ def test_multiple_banned_participants_in_team( # Test data for leaderboard entries test_data = [ { + "id": 1, "submission__participant_team": 1, "submission__participant_team__team_name": "Team1", "submission__is_baseline": False, @@ -831,6 +837,7 @@ def test_multiple_banned_participants_in_team( "result": {"score": 10, "time": 5}, }, { + "id": 2, "submission__participant_team": 2, "submission__participant_team__team_name": "Team2", "submission__is_baseline": False, @@ -840,16 +847,9 @@ def test_multiple_banned_participants_in_team( }, ] - # Set up chainable mock for: - # .filter().exclude().filter().filter().order_by().annotate().values() - mock_qs = MagicMock() - mock_filter_result = MagicMock() - mock_leaderboard_data_objects.filter.return_value = mock_filter_result - mock_filter_result.exclude.return_value = mock_qs - mock_qs.filter.return_value = mock_qs - mock_qs.order_by.return_value = mock_qs - mock_qs.annotate.return_value = mock_qs - mock_qs.values.return_value = test_data + self._create_mock_leaderboard_chain( + mock_leaderboard_data_objects, test_data + ) # Team 1 has one banned participant among multiple mock_team1 = Mock() @@ -905,6 +905,7 @@ def test_empty_banned_email_ids_includes_all_teams( test_data = [ { + "id": 1, "submission__participant_team": 1, "submission__participant_team__team_name": "Team1", "submission__is_baseline": False, @@ -913,6 +914,7 @@ def test_empty_banned_email_ids_includes_all_teams( "result": {"score": 10, "time": 5}, }, { + "id": 2, "submission__participant_team": 2, "submission__participant_team__team_name": "Team2", "submission__is_baseline": False, @@ -974,6 +976,7 @@ def test_none_banned_email_ids_includes_all_teams( test_data = [ { + "id": 1, "submission__participant_team": 1, "submission__participant_team__team_name": "Team1", "submission__is_baseline": False, @@ -1010,6 +1013,7 @@ def test_none_banned_email_ids_includes_all_teams( test_data = [ { + "id": 1, "submission__participant_team": 1, "submission__participant_team__team_name": "Team1", "submission__is_baseline": False, @@ -1018,6 +1022,7 @@ def test_none_banned_email_ids_includes_all_teams( "result": {"score": 10, "time": 5}, }, { + "id": 2, "submission__participant_team": 2, "submission__participant_team__team_name": "Team2", "submission__is_baseline": False, @@ -1079,6 +1084,7 @@ def test_distinct_team_list_with_many_duplicates( # Many entries from same teams - Team1 appears 5x, Team2 appears 3x test_data = [ { + "id": i, "submission__participant_team": 1, "submission__participant_team__team_name": "Team1", "submission__is_baseline": False, @@ -1089,6 +1095,7 @@ def test_distinct_team_list_with_many_duplicates( for i in range(10, 5, -1) ] + [ { + "id": 100 + i, "submission__participant_team": 2, "submission__participant_team__team_name": "Team2", "submission__is_baseline": False, @@ -1157,6 +1164,7 @@ def test_baseline_entries_always_included( test_data = [ { + "id": 1, "submission__participant_team": 1, "submission__participant_team__team_name": "Team1", "submission__is_baseline": False, @@ -1165,6 +1173,7 @@ def test_baseline_entries_always_included( "result": {"score": "10", "time": "0"}, }, { + "id": 2, "submission__participant_team": 2, "submission__participant_team__team_name": "Baseline", "submission__is_baseline": True, @@ -1173,6 +1182,7 @@ def test_baseline_entries_always_included( "result": {"score": "5", "time": "0"}, }, { + "id": 3, "submission__participant_team": 2, "submission__participant_team__team_name": "Baseline", "submission__is_baseline": True, @@ -1218,11 +1228,17 @@ def test_baseline_entries_always_included( def _create_mock_leaderboard_chain( self, mock_leaderboard_data_objects, test_data ): - """Helper to create mock chain for LeaderboardData queryset.""" + """Helper to create mock chain for LeaderboardData queryset. + + Stage 1: .filter().exclude().filter().filter().order_by() + .annotate().values() -> test_data + Stage 2: .filter(id__in=...).annotate().values() -> test_data + """ mock_qs = MagicMock() mock_filter_result = MagicMock() mock_leaderboard_data_objects.filter.return_value = mock_filter_result mock_filter_result.exclude.return_value = mock_qs + mock_filter_result.annotate.return_value = mock_qs mock_qs.filter.return_value = mock_qs mock_qs.order_by.return_value = mock_qs mock_qs.annotate.return_value = mock_qs