Scope wildcard content removal

CHANGES/+scoped-wildcard-removal.feature

@@ -0,0 +1 @@
+Added scoped wildcard removal for APT repositories. Using `remove_content_units` with `distribution` and `component` can now empty a single component or a whole distribution, removing the covered packages along with their `ReleaseComponent` and, for a whole distribution, its `Release` and `ReleaseArchitectures`. Setting both selectors to `*` removes all content units from the repository. Since `*` is now a selector, it is rejected as a `distribution` or `component` name.

pulp_deb/app/serializers/content_serializers.py

@@ -57,6 +57,21 @@
 log = logging.getLogger(__name__)
 
 
+def validate_no_wildcards(**fields):
+    """Reject '*' as a name, since it is reserved as a selector when modifying repositories.
+
+    Each keyword argument maps a field name onto a single name or a list of names.
+    """
+    message = "This field does not accept the special value '*'!"
+    errors = {
+        name: _(message)
+        for name, values in fields.items()
+        if "*" in ([values] if isinstance(values, str) else values or [])
+    }
+    if errors:
+        raise ValidationError(errors)
+
+
 class YesNoField(Field):
     """
     A serializer field that accepts 'yes' or 'no' as boolean.
@@ -259,6 +274,16 @@ class SinglePackageUploadSerializer(SingleArtifactContentUploadSerializer):
     distribution = CharField(help_text="Name of the distribution.", required=False)
     component = CharField(help_text="Name of the component.", required=False)
 
+    def validate(self, data):
+        """
+        Ensure the upload does not create structure content named '*'.
+        """
+        data = super().validate(data)
+        validate_no_wildcards(
+            distribution=data.get("distribution"), component=data.get("component")
+        )
+        return data
+
     @staticmethod
     def _get_or_create_content_and_qs(model, **data):
         content, created = model.objects.get_or_create(**data)
@@ -830,6 +855,16 @@ def get_unique_together_validators(self):
     architectures = ListField(child=CharField(), required=False)
     components = ListField(child=CharField(), required=False)
 
+    def validate(self, data):
+        """
+        Ensure we do not create a Release or ReleaseComponent object named '*'.
+        """
+        data = super().validate(data)
+        validate_no_wildcards(
+            distribution=data.get("distribution"), components=data.get("components")
+        )
+        return data
+
     @staticmethod
     def _get_or_create_content_pk(model, **data):
         content, created = model.objects.get_or_create(**data)
@@ -921,6 +956,7 @@ def validate(self, data):
         if data.get("architecture") == "all":
             message = "This field does not accept the special value 'all'!"
             raise ValidationError({"architecture": _(message)})
+        validate_no_wildcards(distribution=data.get("distribution"))
         return data
 
     def get_unique_together_validators(self):
@@ -954,6 +990,16 @@ class ReleaseComponentSerializer(NoArtifactContentSerializer):
     A Serializer for ReleaseComponent.
     """
 
+    def validate(self, data):
+        """
+        Ensure we do not create a ReleaseComponent object named '*'.
+        """
+        data = super().validate(data)
+        validate_no_wildcards(
+            distribution=data.get("distribution"), component=data.get("component")
+        )
+        return data
+
     def get_unique_together_validators(self):
         """
         We do not want UniqueTogetherValidator since we have retrieve logic!

pulp_deb/app/serializers/repository_serializers.py

@@ -25,26 +25,44 @@
     AptRepositoryReleaseServiceOverride,
 )
 from pulp_deb.app.schema import COPY_CONFIG_SCHEMA
+from pulp_deb.app.serializers.content_serializers import validate_no_wildcards
 
 
 class AptRepositoryAddRemoveContentSerializer(RepositoryAddRemoveContentSerializer):
     distribution = serializers.CharField(
         help_text=_(
             "Name of the distribution any packages from add_content_units or remove_content_units "
             "should be added to or removed from. Defaults to DEFAULT_DISTRIBUTION if only a "
-            "component is provided."
+            "component is provided. When remove_content_units is ['*'], a distribution limits "
+            "the removal to packages in that distribution, along with its Release and "
+            "ReleaseArchitectures. Set both distribution and component to '*' to remove all "
+            "content units from the repository."
         ),
         required=False,
     )
     component = serializers.CharField(
         help_text=_(
             "Name of the component any packages from add_content_units or remove_content_units "
             "should be added to or removed from. Defaults to DEFAULT_COMPONENT if only a "
-            "distribution is provided.."
+            "distribution is provided. When remove_content_units is ['*'], a component limits "
+            "the removal to packages in that component, along with its ReleaseComponent. Set "
+            "component to '*' to remove packages from every component in the selected "
+            "distribution."
         ),
         required=False,
     )
 
+    def validate(self, data):
+        """
+        Ensure adding content does not create structure content named '*'.
+        """
+        data = super().validate(data)
+        if data.get("add_content_units"):
+            validate_no_wildcards(
+                distribution=data.get("distribution"), component=data.get("component")
+            )
+        return data
+
     class Meta(RepositoryAddRemoveContentSerializer.Meta):
         fields = RepositoryAddRemoveContentSerializer.Meta.fields + ["distribution", "component"]
 

pulp_deb/app/tasks/signing.py

@@ -29,6 +29,7 @@
     AptRepository,
     Package,
     PackageReleaseComponent,
+    Release,
     ReleaseArchitecture,
     ReleaseComponent,
     SourcePackage,
@@ -42,16 +43,54 @@
 log = logging.getLogger(__name__)
 
 
+def _filter_by_scope(queryset, distribution, component, prefix=""):
+    """Narrow a queryset to a distribution/component, where "*" matches every value."""
+    if distribution != "*":
+        queryset = queryset.filter(**{f"{prefix}distribution": distribution})
+    if component != "*":
+        queryset = queryset.filter(**{f"{prefix}component": component})
+    return queryset
+
+
+def _prepare_release_removals(repository_version, remove_content_units, distribution, component):
+    """Expand a scoped wildcard removal to the release metadata it covers.
+
+    Emptying a single component drops only that ReleaseComponent, while emptying a whole
+    distribution ("*" component) also drops its Release and ReleaseArchitectures.
+    """
+    release_components = _filter_by_scope(
+        ReleaseComponent.objects.filter(pk__in=repository_version.content),
+        distribution,
+        component,
+    )
+    remove_content_units.extend(str(pk) for pk in release_components.values_list("pk", flat=True))
+    if component != "*":
+        return
+
+    for model in (Release, ReleaseArchitecture):
+        units = model.objects.filter(pk__in=repository_version.content)
+        if distribution != "*":
+            units = units.filter(distribution=distribution)
+        remove_content_units.extend(str(pk) for pk in units.values_list("pk", flat=True))
+
+
 def _prepare_package_removals(repo, remove_content_units, base_version_pk, distribution, component):
     """Expand the removal list to include the release component relationships of each package.
 
     Removing a (source) package also requires removing its PackageReleaseComponent /
     SourcePackageReleaseComponent links. When a distribution/component is given, the removal is
     scoped to that component: a package is only removed from the repository if the scope held its
     last relationship, so packages linked elsewhere or not linked at all are kept.
+
+    A "*" removal names every package in scope rather than an explicit list, and additionally
+    removes the release metadata that scope covers.
     """
-    # "*" removes all content, so there is nothing to resolve here.
-    if not remove_content_units or "*" in remove_content_units:
+    if not remove_content_units:
+        return
+
+    wildcard_removal = "*" in remove_content_units
+    # An unscoped wildcard removes all repository content through pulpcore.
+    if wildcard_removal and distribution in (None, "*") and component in (None, "*"):
         return
 
     repository_version = (
@@ -64,23 +103,35 @@ def _prepare_package_removals(repo, remove_content_units, base_version_pk, distr
     if scoped:
         distribution = distribution or DEFAULT_DISTRIBUTION
         component = component or DEFAULT_COMPONENT
+    if wildcard_removal:
+        remove_content_units.clear()
 
     for model, relationship_model, relationship_field in (
         (Package, PackageReleaseComponent, "package"),
         (SourcePackage, SourcePackageReleaseComponent, "source_package"),
     ):
-        units = model.objects.filter(pk__in=remove_content_units)
-        relationships = relationship_model.objects.filter(
-            **{
-                f"{relationship_field}__in": units,
-                "pk__in": repository_version.content,
-            }
-        )
-        if scoped:
-            scoped_relationships = relationships.filter(
-                release_component__distribution=distribution,
-                release_component__component=component,
+        if wildcard_removal:
+            relationships = relationship_model.objects.filter(pk__in=repository_version.content)
+            scoped_relationships = _filter_by_scope(
+                relationships, distribution, component, "release_component__"
+            )
+            units = model.objects.filter(
+                pk__in=scoped_relationships.values_list(f"{relationship_field}_id", flat=True)
             )
+            remove_content_units.extend(str(pk) for pk in units.values_list("pk", flat=True))
+        else:
+            units = model.objects.filter(pk__in=remove_content_units)
+            relationships = relationship_model.objects.filter(
+                **{
+                    f"{relationship_field}__in": units,
+                    "pk__in": repository_version.content,
+                }
+            )
+            if scoped:
+                scoped_relationships = _filter_by_scope(
+                    relationships, distribution, component, "release_component__"
+                )
+        if scoped:
             # Relationships named in the request are removed alongside the scoped ones.
             removed_relationship_ids = set(
                 relationship_model.objects.filter(pk__in=remove_content_units).values_list(
@@ -102,6 +153,9 @@ def _prepare_package_removals(repo, remove_content_units, base_version_pk, distr
             relationships = scoped_relationships
         remove_content_units.extend(str(pk) for pk in relationships.values_list("pk", flat=True))
 
+    if wildcard_removal:
+        _prepare_release_removals(repository_version, remove_content_units, distribution, component)
+
 
 def _prepare_package_additions(add_content_units, distribution, component):
     """Expand the addition list with the metadata needed to publish the packages in a component.