Prep for VM deployment

.github/workflows/docker-build.yml

@@ -80,43 +80,6 @@ jobs:
         docker image tag ghcr.io/savageaim/app/ws-backend:${{ steps.get_release.outputs.tag_name }} freyamade/savageaim:websockets-${{ steps.get_release.outputs.tag_name }}
         docker push --all-tags freyamade/savageaim
 
-  build-task-backend:
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: backend
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Log in to Docker Hub
-      uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
-    - name: Log in to the Container registry
-      uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Get Release Info
-      id: get_release
-      uses: bruceadams/get-release@v1.3.2
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-
-    - name: Build and Push Docker Images
-      run: |
-        docker build . --file deployment/tasks.Dockerfile -t ghcr.io/savageaim/app/task-backend:latest -t ghcr.io/savageaim/app/task-backend:${{ steps.get_release.outputs.tag_name }}
-        docker push --all-tags ghcr.io/savageaim/app/task-backend
-        # Rename images to dockerhub and push there too
-        docker image tag ghcr.io/savageaim/app/task-backend:latest freyamade/savageaim:tasks
-        docker image tag ghcr.io/savageaim/app/task-backend:${{ steps.get_release.outputs.tag_name }} freyamade/savageaim:tasks-${{ steps.get_release.outputs.tag_name }}
-        docker push --all-tags freyamade/savageaim
-
   build-frontend:
     runs-on: ubuntu-latest
     defaults:

backend/api/apps.py

@@ -4,15 +4,3 @@
 class ApiConfig(AppConfig):
     name = 'api'
     default_auto_field = 'django.db.models.AutoField'
-
-    def ready(self):
-        super().ready()
-        # Import all our cloud tasks in here so they can be discovered
-        from api.tasks import (  # noqa
-            CheckGameVersionTask,
-            DBCleanupTask,
-            RefreshTokensTask,
-            SeedTask,
-            VerificationReminderTask,
-            VerifyCharacterTask,
-        )

backend/api/management/commands/seed.py

@@ -1,11 +1,81 @@
+# stdlib
+from os import scandir
+from pathlib import Path
 # lib
+from django.db import IntegrityError
+import yaml
+from django.conf import settings
 from django.core.management.base import BaseCommand
 # local
-from api.tasks import SeedTask
+from api import models
 
 
 class Command(BaseCommand):
     help = 'Seed the DB with static data for Gear, Tier and Job information.'
 
     def handle(self, *args, **options):
-        SeedTask.sync({}, {'attributes': 'required'})
+        self.stdout.write(self.style.HTTP_REDIRECT('Beginning Seed of DB'))
+        seed_data_dir = settings.BASE_DIR / 'seed_data'
+        gear_data_dir = seed_data_dir / 'gear'
+
+        # Get the Tier and Gear data and import them
+        with open(seed_data_dir / 'tiers.yml', 'r') as f:
+            self.stdout.write(self.style.HTTP_REDIRECT('Seeding Tiers'))
+            self.import_file(f, models.Tier)
+
+        with scandir(gear_data_dir) as expac_dirs:
+            for expac_dir in expac_dirs:
+                if not expac_dir.is_dir():
+                    continue
+
+                with scandir(expac_dir.path) as gear_files:
+                    for file in gear_files:
+                        version = Path(file.path).stem
+                        self.stdout.write(self.style.HTTP_REDIRECT(f'Seeding Gear from {version}'))
+
+                        # Store the version for the file in the DB
+                        try:
+                            models.XIVVersion.objects.create(version=version)
+                        except IntegrityError:
+                            pass
+
+                        with open(file.path, 'r') as f:
+                            self.import_file(f, models.Gear)
+
+        # Lastly we import the Job data.
+        # This is handled *slightly* differently because the 'ordering' key in this file will most likely change
+        # between expansions, especially for dps
+        # So this Integrity Error will be handled slightly differently
+
+        with open(seed_data_dir / 'jobs.yml', 'r') as f:
+            self.stdout.write(self.style.HTTP_REDIRECT('Seeding Jobs'))
+            self.import_jobs(f)
+
+    def import_file(self, file, model):
+        data = yaml.safe_load(file)
+        for item in data:
+            self.stdout.write(f'\t{item["name"]}')
+            _, created = model.objects.get_or_create(**item)
+            if not created:
+                self.stdout.write('\t\tSkipping, as it is already in the DB.')
+
+    def import_jobs(self, file):
+        """
+        Import Job data.
+        If Job exists, ensure the ordering value is up to date
+        """
+        data = yaml.safe_load(file)
+        for job in data:
+            self.stdout.write(f'\t{job["id"]}')
+
+            # Check if the Job is already in the Database
+            try:
+                obj = models.Job.objects.get(pk=job['id'])
+                self.stdout.write(
+                    f'\t\tAlready exists, ensuring correct ordering ({obj.ordering} -> {job["ordering"]})',
+                )
+                obj.ordering = job['ordering']
+                obj.save()
+            except models.Job.DoesNotExist:
+                # If it doesn't exist, just create it!
+                models.Job.objects.create(**job)

backend/api/tasks.py

@@ -0,0 +1,152 @@
+"""
+Set up our tasks for celery to run
+
+Task to verify accounts on XIVAPI.
+"""
+# stdlib
+from datetime import timedelta
+# lib
+from asgiref.sync import async_to_sync
+from celery import shared_task
+from celery.utils.log import get_task_logger
+from channels.layers import get_channel_layer
+from django.core.management import call_command
+from django.db.models import Q
+from django.utils import timezone
+# local
+from . import notifier
+from .lodestone_scraper import LodestoneScraper
+from .models import Character, Notification, Team
+
+logger = get_task_logger(__name__)
+
+
+def assimilate_proxies(real_char: Character):
+    # Find all Proxy characters that have the same lodestone ID as this one
+    proxies = Character.objects.filter(
+        Q(user__isnull=True) | Q(user_id=real_char.user_id),
+        lodestone_id=real_char.lodestone_id,
+    )
+
+    # For each Character (which should only ever be in one team each);
+    #   - Notify the Team Leader that the claim has happened
+    #   - Move the BIS List to the real Character, name it using the Team's name
+    #   - Update the TeamMember object to point to this character
+    for char in proxies:
+        for tm in char.teammember_set.all():
+            if char.user is None:
+                notifier.team_proxy_claim(tm)
+
+            bis = tm.bis_list
+            bis.owner = real_char
+            bis.name = f'BIS From {tm.team.name}'
+            bis.save()
+
+            tm.character = real_char
+            tm.save()
+
+
+@shared_task(name='verify_character')
+def verify_character(pk: int):
+    """
+    Verify the character has the expected token in the bio on xivapi.
+
+    If so, update the flag to True, and delete all other unverified characters with the same lodestone id
+    """
+    # Check that the character is unverified and exists
+    logger.info(f'Commencing verification attempt for Character #{pk}.')
+    try:
+        obj = Character.objects.get(pk=pk, verified=False)
+    except Character.DoesNotExist:
+        logger.warn(f'Character #{pk} either does not exist or is verified. Exiting.')
+        return
+
+    # Call the xivapi function in a sync context
+    logger.debug('calling lookup function')
+    err = LodestoneScraper.get_instance().check_token(obj.lodestone_id, obj.token)
+    logger.debug('finished lookup function')
+
+    if err is not None:
+        notifier.verify_fail(obj, err)
+        logger.info(f'Character #{pk} could not be verified. Exiting. ({err})')
+        return
+
+    logger.info(f'Character #{pk} verified. Updating DB.')
+    # First we update the flag on the object specified
+    obj.verified = True
+    obj.save()
+
+    # Before we go deleting any Characters, we need to sort all the Proxies that share the lodestone ID
+    assimilate_proxies(obj)
+
+    # Next delete all unverified instances of the character (this includes proxies)
+    logger.info(f'Deleting unverified instances of Character #{obj.lodestone_id} (#{pk}) owned by {obj.user_id}.')
+    objs = Character.objects.filter(
+        Q(user__isnull=True) | Q(user_id=obj.user_id),
+        verified=False,
+        lodestone_id=obj.lodestone_id,
+    ).exclude(pk=pk)
+    ids_to_delete = [o.pk for o in objs]
+    logger.info(f'Found {objs.count()} instances of Character #{obj.lodestone_id} to delete.\n{ids_to_delete}')
+    objs.delete()
+    # Then we're done!
+    notifier.verify_success(obj)
+    # Also send websocket details
+    channel_layer = get_channel_layer()
+    if channel_layer is not None:
+        async_to_sync(channel_layer.group_send)(f'user-updates-{obj.user.id}', {'type': 'character', 'id': obj.pk})
+
+
+@shared_task(name='verify_reminder')
+def remind_users_to_verify():
+    """
+    Find non-verified Characters that are 5 days old.
+    Send Notifications to remind the User to verify.
+    """
+    logger.debug(f'Running at: {timezone.now()}')
+    older_than = timezone.now() - timedelta(days=5)
+    logger.debug(f'Reminding unverified characters older than {older_than}.')
+
+    characters = Character.objects.filter(verified=False, user__isnull=False, created__lt=older_than)
+    logger.debug(f'Found {characters.count()} characters. Reminding their Users.')
+    for char in characters:
+        # Check that there wasn't already a reminder sent about this Character
+        if not Notification.objects.filter(type='verify_reminder', link=f'/characters/{char.id}/').exists():
+            notifier.verify_reminder(char)
+
+
+@shared_task(name='cleanup')
+def cleanup():
+    """
+    Cleanup the DB of all unverified (non-proxy) characters made more than 7 days ago
+    """
+    logger.debug(f'Running at: {timezone.now()}')
+    older_than = timezone.now() - timedelta(days=7)
+    logger.debug(f'Deleting unverified characters older than {older_than}.')
+
+    objs = Character.objects.filter(verified=False, user__isnull=False, created__lt=older_than)
+    logger.debug(f'Found {objs.count()} characters. Deleting them.')
+    for char in objs:
+        # Remove them from every team they are a member of
+        teams = Team.objects.filter(members__character=char).distinct()
+        for team in teams:
+            team.remove_character(char, False)
+
+        char.bis_lists.all().delete()
+        char.delete()
+
+
+@shared_task(name='refresh_tokens')
+def refresh_tokens():
+    """
+    Refresh any tokens that are about to expire
+    """
+    call_command('refresh_tokens')
+
+
+@shared_task(name='check_game_version')
+def check_game_version():
+    """
+    Check if a new, unseeded, game version has been added to xivapi that we don't have yet
+    """
+    call_command('check_game_version', '--latest', '--notify')