Add admin CSV uploader for alumni imports

.github/workflows/main.yml

@@ -9,11 +9,11 @@ jobs:
         strategy:
             matrix:
                 python: ["3.13"]
-                node: ["18"]
+                node: ["24.21.0"]
                 browser: ["chrome", "firefox"]
 
         steps:
-        - uses: actions/checkout@v3
+        - uses: actions/checkout@v4
 
         - name: Install Python ${{ matrix.python }}
           uses: actions/setup-python@v5
@@ -60,14 +60,14 @@ jobs:
 
 
         - name: Install Node ${{ matrix.node }}
-          uses: actions/setup-node@v1
+          uses: actions/setup-node@v4
           with:
             node-version: ${{ matrix.node }}
         - name: Run 'yarn install'
-          run: yarn install
+          run: corepack yarn install --frozen-lockfile
 
         - name: Run 'yarn build'
-          run: yarn build
+          run: corepack yarn build
         - name: Check if 'black' has been run
           run:
             poetry run black --exclude 'migrations' --check .
@@ -82,7 +82,7 @@ jobs:
         name: Docker Smoke Test
         runs-on: ubuntu-latest
         steps:
-        - uses: actions/checkout@v3
+        - uses: actions/checkout@v4
 
         - name: 'Build Docker Container'
           run: docker build -t jacobsalumni/membermanagement .

.tool-versions

@@ -1,2 +1,2 @@
-nodejs 16.16.0
+nodejs 24.21.0
 python 3.10.5

Dockerfile

@@ -5,7 +5,7 @@ RUN git describe --always > /PORTAL_VERSION
 RUN echo "Saved version file containing '$(cat /PORTAL_VERSION)'"
 
 # image for building node dependencies
-FROM node:18-alpine AS frontend
+FROM node:24-alpine AS frontend
 
 RUN apk add --no-cache \
     git
@@ -14,13 +14,13 @@ RUN apk add --no-cache \
 ADD package.json /app/package.json
 ADD yarn.lock /app/yarn.lock
 WORKDIR /app/
-RUN yarn install
+RUN corepack yarn install --frozen-lockfile
 
 # install frontend scripts + build
 ADD tsconfig.json /app/tsconfig.json
 ADD webpack.config.js /app/webpack.config.js
 ADD assets/ /app/assets/
-RUN yarn build
+RUN corepack yarn build
 
 # image for python
 FROM python:3.13-alpine

MemberManagement/asgi.py

@@ -16,7 +16,6 @@
 
 from django.core.asgi import get_asgi_application
 
-
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "MemberManagement.settings")
 
 # Have to call before importing consumers as they depend on models
@@ -25,7 +24,6 @@
 
 from donations.consumers import DonationUpdateConsumer
 
-
 application = ProtocolTypeRouter(
     {
         # Django's ASGI application to handle traditional HTTP requests

MemberManagement/tests/test_access.py

@@ -113,13 +113,13 @@ def test_setup_skills(self) -> None:
                 url, "setup_skills", "{} is protected by setup".format(url)
             )
 
-    def test_setup_atlas(self) -> None:
+    def test_setup_membership_without_atlas(self) -> None:
         self.load_fixture("registry/tests/fixtures/signup_05_skills.json")
         self.login("Mounfem")
 
         for url in SETUP_PROTECTED_URLS:
             self.assert_url_follow(
-                url, "setup_atlas", "{} is protected by setup".format(url)
+                url, "setup_membership", "{} is protected by setup".format(url)
             )
 
     def test_setup_tier(self) -> None:

README.md

@@ -44,7 +44,7 @@ python manage.py migrate
 Afterwards install the frontend dependencies using [yarn](https://yarnpkg.com/):
 
 ```bash
-yarn
+corepack yarn install
 ```
 
 Note that this is only tested using the NodeJS version specified in `.tool-versions`.
@@ -54,7 +54,7 @@ By default a local instance is then configured to store data in a local `db.sqli
 One needs to start the django app in two parts, one part building the frontend dependencies using:
 
 ```bash
-yarn dev
+corepack yarn dev
 ```
 
 and in a seperate terminal the normal django development server:
@@ -140,23 +140,23 @@ The integration tests run headless by default and support the following browsers
 To run tests make sure that development dependencies are installed and then run:
 
 ```
-yarn build # ensure that static assets have been built
+corepack yarn build # ensure that static assets have been built
 pytest # to run the tests
 ```
 
 By default, the tests are running in headless mode.
 To enforce a visible browser, instead use:
 
 ```bash
-yarn build # ensure that static assets have been built
+corepack yarn build # ensure that static assets have been built
 SELENIUM_HEADLESS=0 pytest # to run headless
 ```
 
 When offline Stripe frontend tests might fail because they require a connection to Stripe Servers.
 To work around this, you can set the `SKIP_STRIPE_TESTS` variable as follows:
 
 ```bash
-yarn build
+corepack yarn build
 SKIP_STRIPE_TESTS=1 pytest
 ```
 

alumni/admin/__init__.py

@@ -8,9 +8,11 @@
 from .inlines import AlumniAdminInlines
 from .list import AlumniListDisplay, AlumniListFilter
 from .actions import AlumniAdminActions
+from .import_csv import AlumniCsvImport
 
 
 class AlumniAdmin(
+    AlumniCsvImport,
     AlumniSearch,
     AlumniListFilter,
     AlumniAdminInlines,

alumni/admin/import_csv.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import csv
+from io import TextIOWrapper
+
+from django.contrib import messages
+from django.http import HttpResponse
+from django.shortcuts import redirect, render
+from django.urls import path
+
+from registry.management.commands.import_csv import DEFAULT_COLUMNS, import_csv_rows
+
+
+class AlumniCsvImport:
+    change_list_template = "admin/alumni/alumni/change_list.html"
+
+    def get_urls(self):
+        return [
+            path(
+                "import-csv/",
+                self.admin_site.admin_view(self.import_csv_view),
+                name="alumni_alumni_import_csv",
+            )
+        ] + super().get_urls()
+
+    def import_csv_view(self, request):
+        if request.method == "POST":
+            upload = request.FILES.get("csv")
+            columns = request.POST.get("columns", DEFAULT_COLUMNS).split(",")
+            no_stripe = request.POST.get("no_stripe") == "on"
+
+            if upload is None:
+                messages.error(request, "Choose a CSV file to import.")
+            else:
+                try:
+                    return self.import_csv_upload(request, upload, columns, no_stripe)
+                except Exception as e:
+                    messages.error(request, "CSV import failed: {}".format(e))
+
+        context = {
+            **self.admin_site.each_context(request),
+            "opts": self.model._meta,
+            "title": "Import alumni CSV",
+            "default_columns": DEFAULT_COLUMNS,
+        }
+        return render(request, "admin/alumni/alumni/import_csv.html", context)
+
+    def import_csv_upload(self, request, upload, columns, no_stripe):
+        reader = csv.reader(
+            TextIOWrapper(upload.file, encoding="utf-8-sig", newline="")
+        )
+        header = next(reader, [])
+        rows = list(reader)
+
+        check = import_csv_rows(rows, columns, no_stripe=True, simulate=True)
+        if check.failures:
+            return self.error_csv_response(header or columns, check)
+
+        result = import_csv_rows(rows, columns, no_stripe=no_stripe)
+        if result.failures:
+            return self.error_csv_response(header or columns, result)
+
+        self.message_user(request, "Imported {} user(s).".format(len(result.created)))
+        return redirect("admin:alumni_alumni_changelist")
+
+    def error_csv_response(self, header, result):
+        response = HttpResponse(content_type="text/csv")
+        response["Content-Disposition"] = 'attachment; filename="import_errors.csv"'
+
+        writer = csv.writer(response)
+        writer.writerow(["row_number", "error"] + list(header))
+        for failure in result.failures:
+            writer.writerow([failure.row_number, failure.error] + failure.row)
+
+        return response

alumni/templates/admin/alumni/alumni/change_list.html

@@ -0,0 +1,8 @@
+{% extends "admin/change_list.html" %}
+
+{% block object-tools-items %}
+    <li>
+        <a href="import-csv/" class="addlink">Import CSV</a>
+    </li>
+    {{ block.super }}
+{% endblock %}

alumni/templates/admin/alumni/alumni/import_csv.html

@@ -0,0 +1,42 @@
+{% extends "admin/base_site.html" %}
+{% load i18n admin_urls static %}
+
+{% block breadcrumbs %}
+    <div class="breadcrumbs">
+        <a href="{% url 'admin:index' %}">{% translate 'Home' %}</a>
+        &rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
+        &rsaquo; <a href="{% url 'admin:alumni_alumni_changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
+        &rsaquo; {{ title }}
+    </div>
+{% endblock %}
+
+{% block content %}
+    <div id="content-main">
+        <form method="post" enctype="multipart/form-data">
+            {% csrf_token %}
+            <fieldset class="module aligned">
+                <div class="form-row">
+                    <label class="required" for="id_csv">CSV file:</label>
+                    <input type="file" name="csv" id="id_csv" accept=".csv,text/csv" required>
+                </div>
+                <div class="form-row">
+                    <label for="id_columns">Columns:</label>
+                    <input type="text" name="columns" id="id_columns" value="{{ default_columns }}" class="vTextField">
+                    <div class="help">
+                        The first CSV row is treated as a header. Use an empty column name to ignore a CSV column.
+                    </div>
+                </div>
+                <div class="form-row">
+                    <label for="id_no_stripe">Skip Stripe:</label>
+                    <input type="checkbox" name="no_stripe" id="id_no_stripe">
+                    <div class="help">
+                        Validate first, then import. If validation fails, no users are imported and an error CSV is downloaded.
+                    </div>
+                </div>
+            </fieldset>
+            <div class="submit-row">
+                <input type="submit" value="Upload and import" class="default">
+            </div>
+        </form>
+    </div>
+{% endblock %}

alumni/templates/admin/index.html

@@ -0,0 +1,63 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block extrastyle %}{{ block.super }}<link rel="stylesheet" href="{% static "admin/css/dashboard.css" %}">{% endblock %}
+
+{% block coltype %}colMS{% endblock %}
+
+{% block bodyclass %}{{ block.super }} dashboard{% endblock %}
+
+{% block nav-breadcrumbs %}{% endblock %}
+
+{% block nav-sidebar %}{% endblock %}
+
+{% block content %}
+<div id="content-main">
+  {% include "admin/app_list.html" with app_list=app_list show_changelinks=True %}
+
+  <div class="module">
+    <table>
+      <caption>
+        <span class="section">Board tools</span>
+      </caption>
+      <tr>
+        <th scope="row"><a href="{% url 'admin:alumni_alumni_import_csv' %}">Import alumni CSV</a></th>
+        <td></td>
+        <td><a href="{% url 'admin:alumni_alumni_import_csv' %}" class="addlink">Import</a></td>
+      </tr>
+    </table>
+  </div>
+</div>
+{% endblock %}
+
+{% block sidebar %}
+<div id="content-related">
+    <div class="module" id="recent-actions-module">
+        <h2>{% translate 'Recent actions' %}</h2>
+        <h3>{% translate 'My actions' %}</h3>
+            {% load log %}
+            {% get_admin_log 10 as admin_log for_user user %}
+            {% if not admin_log %}
+            <p>{% translate 'None available' %}</p>
+            {% else %}
+            <ul class="actionlist">
+            {% for entry in admin_log %}
+            <li class="{% if entry.is_addition %}addlink{% endif %}{% if entry.is_change %}changelink{% endif %}{% if entry.is_deletion %}deletelink{% endif %}">
+                {% if entry.is_deletion or not entry.get_admin_url %}
+                    {{ entry.object_repr }}
+                {% else %}
+                    <a href="{{ entry.get_admin_url }}">{{ entry.object_repr }}</a>
+                {% endif %}
+                <br>
+                {% if entry.content_type %}
+                    <span class="mini quiet">{% filter capfirst %}{{ entry.content_type.name }}{% endfilter %}</span>
+                {% else %}
+                    <span class="mini quiet">{% translate 'Unknown content' %}</span>
+                {% endif %}
+            </li>
+            {% endfor %}
+            </ul>
+            {% endif %}
+    </div>
+</div>
+{% endblock %}

atlas/admin.py

@@ -4,7 +4,6 @@
 
 from .models import GeoLocation, AtlasSettings
 
-
 # Register your models here.
 
 

atlas/models.py

@@ -50,6 +50,10 @@ class AtlasSettings(AlumniComponentMixin, models.Model):
         help_text="Secret Search Text that the member can be found with",
     )
 
+    @classmethod
+    def component_exists(cls, alumni: Alumni) -> bool:
+        return True
+
 
 class GeoLocation(models.Model):
     """Represents a (cached) GeoLocation"""

atlas/tests/test_01_geocache.py

@@ -6,11 +6,14 @@
 from django.core import management
 import os
 
+import pytest
 import unittest
 
 GEOCACHE_TEST_ENABLED = os.environ.get("ENABLE_GEOCACHE_TEST", "0") == "1"
+pytestmark = pytest.mark.skip(reason="Atlas is no longer used.")
 
 
+@unittest.skip("Atlas is no longer used.")
 class GeoCacheTest(TestCase):
     fixtures = ["registry/tests/fixtures/integration.json"]