Move sponsorship emails to asynchronous Celery tasks

compose.yml

@@ -12,7 +12,10 @@ services:
       POSTGRES_FSYNC: null
     healthcheck:
       test: ["CMD", "pg_isready", "-U", "pyladiescon", "-d", "pyladiescon"]
-      interval: 1s
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
     volumes:
       - ./docker-compose/postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
       - pgdata:/var/lib/postgresql/data
@@ -23,7 +26,10 @@ services:
       - "6379:6379"
     healthcheck:
      test: ["CMD", "redis-cli","ping"]
-     interval: 1s
+     interval: 10s
+     timeout: 5s
+     retries: 5
+     start_period: 10s
 
   web:
     build:
@@ -43,11 +49,35 @@ services:
       DJANGO_DEFAULT_FROM_EMAIL: PyLadiesCon <[email protected]>
       DJANGO_EMAIL_HOST: maildev
       DJANGO_EMAIL_PORT: 1025
+      CELERY_BROKER_URL: redis://redis:6379/0
     depends_on:
       redis:
         condition: service_healthy
       postgres:
         condition: service_healthy
+
+  celery:
+    build:
+      target: dev
+      args:
+        USER_ID: ${USER_ID:-1000}
+        GROUP_ID: ${GROUP_ID:-1000}
+    image: pyladiescon-portal-celery:docker-compose
+    command: celery -A portal worker --loglevel=info
+    working_dir: /code
+    volumes:
+      - .:/code
+    environment:
+      CELERY_BROKER_URL: redis://redis:6379/0
+      DATABASE_URL: postgresql://pyladiescon:pyladiescon@postgres:5432/pyladiescon
+      SECRET_KEY: verysecure
+      DEBUG: True
+    depends_on:
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+    restart: unless-stopped
 
   maildev:
     image: maildev/maildev:2.2.1

portal/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

portal/celery.py

@@ -0,0 +1,23 @@
+"""
+Celery configuration for PyLadies Portal.
+
+This module sets up Celery for handling asynchronous tasks,
+particularly for sending emails in the background.
+"""
+import os
+
+from celery import Celery
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'portal.settings')
+
+app = Celery('portal')
+
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+app.autodiscover_tasks()
+
+
+@app.task(bind=True, ignore_result=True)
+def debug_task(self):
+    """Debug task for testing Celery setup."""
+    print(f'Request: {self.request!r}')

portal/settings.py

@@ -18,6 +18,8 @@
 import sentry_sdk
 from sentry_sdk.integrations.django import DjangoIntegration
 
+import sys
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
@@ -52,9 +54,12 @@
             ),
         ],
     )
-ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS")
-if ALLOWED_HOSTS:
-    ALLOWED_HOSTS = ALLOWED_HOSTS.split(",")
+# When DJANGO_ALLOWED_HOSTS is not set, Django requires ALLOWED_HOSTS
+# to still be a list or tuple. This default prevents Celery and other
+# background processes from failing during settings initialization.
+ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS", "")
+ALLOWED_HOSTS = ALLOWED_HOSTS.split(",") if ALLOWED_HOSTS else []
+
 
 # Application definition
 
@@ -335,3 +340,12 @@
 
 PRETIX_API_TOKEN = os.getenv("PRETIX_API_TOKEN")
 PRETIX_WEBHOOK_SECRET = os.getenv("PRETIX_WEBHOOK_SECRET")
+
+# Celery settings - using Redis
+CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL')
+
+# This makes Celery run tasks synchronously during tests
+if 'test' in sys.argv or 'pytest' in sys.modules:
+    CELERY_TASK_ALWAYS_EAGER = True
+    CELERY_TASK_EAGER_PROPAGATES = True
+

requirements-dev.txt

@@ -10,3 +10,5 @@ pytest-django==4.8.0
 pytest==8.3.5
 pytest-cov==6.1.1
 coverage==7.7.0
+celery==5.6.0
+redis==6.4.0

sponsorship/signals.py

@@ -1,96 +1,27 @@
-from django.conf import settings
-from django.contrib.auth.models import User
-from django.db.models import Q
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 
-from common.send_emails import send_email
-from volunteer.models import RoleTypes, VolunteerProfile
-
 from .models import SponsorshipProfile
-
-
-def _send_internal_email(
-    subject,
-    *,
-    markdown_template,
-    context=None,
-):
-    """Helper function to send an internal email.
-
-    Lookup who the internal team members who should receive the email and then send the emails individually.
-    Send the email to staff, admin, and sponsorship team members
-
-    Only supports Markdown templates going forward.
-    """
-
-    recipients = User.objects.filter(
-        Q(
-            id__in=VolunteerProfile.objects.prefetch_related("roles")
-            .filter(roles__short_name__in=[RoleTypes.ADMIN, RoleTypes.STAFF])
-            .values_list("id", flat=True)
-        )
-        | Q(is_superuser=True)
-        | Q(is_staff=True)
-    ).distinct()
-
-    if not recipients.exists():
-        return
-
-    # send each email individually to each recipient, for privacy reasons
-    for recipient in recipients:
-        context["recipient_name"] = recipient.get_full_name() or recipient.username
-
-        send_email(
-            subject,
-            [recipient.email],
-            markdown_template=markdown_template,
-            context=context,
-        )
-
-
-def send_internal_sponsor_onboarding_email(instance):
-    """Send email to team whenever a new sponsor is created.
-
-    Emails will be sent to team members with the role type Staff or Admin, and to sponsorship team.
-    Emails will also be sent to users with is_superuser or is_staff set to True.
-    """
-    context = {"profile": instance}
-    subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} New Sponsorship Tracking: {instance.organization_name}"
-
-    markdown_template = "emails/sponsorship/internal_sponsor_onboarding.md"
-
-    _send_internal_email(
-        subject,
-        markdown_template=markdown_template,
-        context=context,
-    )
-
-
-def send_internal_sponsor_progress_update_email(instance):
-    """Send email to team whenever there is a change in sponsorship progress."""
-
-    context = {"profile": instance}
-    subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} Update in Sponsorship Tracking for {instance.organization_name}"
-
-    markdown_template = "emails/sponsorship/internal_sponsor_updated.md"
-
-    _send_internal_email(
-        subject,
-        markdown_template=markdown_template,
-        context=context,
-    )
-
+from .tasks import (
+    send_internal_sponsor_onboarding_email_task,
+    send_internal_sponsor_progress_update_email_task,
+)
 
 @receiver(post_save, sender=SponsorshipProfile)
 def sponsorship_profile_signal(sender, instance, created, **kwargs):
     """Send emails when sponsorship profile is created or updated.
+
+    Emails are sent asynchronously using Celery tasks to avoid blocking
+    the request/response cycle.
+
     Do not send emails if the instance was created via import/export. (too noisy).
     """
     if hasattr(instance, "from_import_export"):
         return
+
+    if created:
+        # Send onboarding email asynchronously
+        send_internal_sponsor_onboarding_email_task.delay(instance.id)
     else:
-        if created:
-            send_internal_sponsor_onboarding_email(instance)
-        else:
-            send_internal_sponsor_progress_update_email(instance)
+        # Send progress update email asynchronously
+        send_internal_sponsor_progress_update_email_task.delay(instance.id)

sponsorship/tasks.py

@@ -0,0 +1,80 @@
+from celery import shared_task
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.db.models import Q
+
+from common.send_emails import send_email
+from volunteer.models import RoleTypes, VolunteerProfile
+from .models import SponsorshipProfile
+
+@shared_task(bind=True, max_retries=3, default_retry_delay=60)
+def send_internal_email_task(self, subject, markdown_template, context):
+    """
+    Send internal notification emails to team members.
+
+    This task looks up admin/staff users and sends individual emails
+    to each recipient for privacy.
+    """
+    recipients = User.objects.filter(
+        Q(
+            id__in=VolunteerProfile.objects.prefetch_related("roles")
+            .filter(roles__short_name__in=[RoleTypes.ADMIN, RoleTypes.STAFF])
+            .values_list("id", flat=True)
+        )
+        | Q(is_superuser=True)
+        | Q(is_staff=True)
+    ).distinct()
+
+    if not recipients.exists():
+        return f"No recipients found for: {subject}"
+
+    sent_count = 0
+    # Send each email individually to each recipient, for privacy reasons
+    for recipient in recipients:
+        context["recipient_name"] = recipient.get_full_name() or recipient.username
+
+        send_email(
+            subject,
+            [recipient.email],
+            markdown_template=markdown_template,
+            context=context,
+        )
+        sent_count += 1
+
+    return f"Sent {sent_count} internal emails: {subject}"
+
+@shared_task(bind=True, max_retries=3, default_retry_delay=60)
+def send_internal_sponsor_onboarding_email_task(self, profile_id):
+    """
+    Send onboarding notification to internal team when new sponsor is created.
+    """
+
+    try:
+        profile = SponsorshipProfile.objects.get(id=profile_id)
+
+        context = {"profile": profile}
+        subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} New Sponsorship Tracking: {profile.organization_name}"
+        markdown_template = "emails/sponsorship/internal_sponsor_onboarding.md"
+
+        return send_internal_email_task(subject, markdown_template, context)
+
+    except SponsorshipProfile.DoesNotExist:
+        return f"SponsorshipProfile with id {profile_id} not found"
+
+@shared_task(bind=True, max_retries=3, default_retry_delay=60)
+def send_internal_sponsor_progress_update_email_task(self, profile_id):
+    """
+    Send progress update notification to internal team.
+    """
+
+    try:
+        profile = SponsorshipProfile.objects.get(id=profile_id)
+
+        context = {"profile": profile}
+        subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} Update in Sponsorship Tracking for {profile.organization_name}"
+        markdown_template = "emails/sponsorship/internal_sponsor_updated.md"
+
+        return send_internal_email_task(subject, markdown_template, context)
+
+    except SponsorshipProfile.DoesNotExist:
+        return f"SponsorshipProfile with id {profile_id} not found"

tests/portal/test_celery.py

@@ -0,0 +1,10 @@
+import pytest
+from portal.celery import debug_task
+
+
+def test_debug_task(capsys):
+    """Test the debug task execution."""
+    debug_task()
+
+    captured = capsys.readouterr()
+    assert 'Request:' in captured.out