Purge caches after publishing stembureaus

.gitignore

@@ -26,6 +26,7 @@ app/data/gemeenten.json
 files/deels_vooringevuld/*
 app/data/nieuwe-gemeenten.json
 config.py
+caching_setup
 backup.sh
 last_processed_line.csv
 update_bag.sh

README.md

@@ -10,11 +10,14 @@ Collecting and presenting stembureaus: [WaarIsMijnStemlokaal.nl](https://waarism
    - Fill in a password at `<DB_PASSWORD>`
 - Copy `config.py.example` to `config.py` and edit it
    - Create a SECRET_KEY as per the instructions in the file
+   - Create a CACHE_PURGE_KEY as per the instructions in the file
    - Fill in your CKAN URL and CKAN API key
    - Fill in the same `<DB_PASSWORD>` as used in `docker/docker-compose.yml`
    - Specify the name(s) of the election(s) and its corresponding CKAN draft and publish resource IDs
       - NOTE: Use the exact same `<name of election>` values in the 'verkiezingen' field in 'app/data/gemeenten.json'
    - Specify email related information in order for the application to send emails
+- Copy `docker/nginx/caching_setup.example` to `docker/nginx/caching_setup` and edit it
+   - Fill in the same `<CACHE_PURGE_KEY>` as used in `config.py`
 - Copy `app/data/gemeenten.json.example` to `app/data/gemeenten.json` and edit it
    - Fill in the email addresses of the gemeenten
    - Add the name(s) of the election(s) for each gemeenten in which it participates. NOTE: make sure that these names are exactly the same as the name(s) of the election(s) in `app/config.py`

app/cache_purger.py

@@ -0,0 +1,72 @@
+import threading
+import requests
+
+from flask import url_for
+
+class CachePurger:
+  # Uses a background daemon to purge the nginx caches for
+  #   - the home page /
+  #   - the gemeente page
+  #   - all stembureau pages for the gemeente
+  # Note that when logged in, `Cache-Control` is set to `private` in `routes.py` meaning pages
+  # you request that are not yet cached will not be cached.
+  #
+  # Development
+  #   - When you are using the production environment on your development laptop, you are using
+  #     a dedicated local hostname such as `dev.waarismijnstemlokaal.nl`. To test the purging of
+  #     nginx caches, pass the IP address (stm network) of the nginx container via `extra_hosts`
+  #     so that cache purging actually works. On the production server `waarismijnstemlokaal.nl`
+  #     will be resolved without additional measures. See `docker-compose.yml.example`.
+
+
+  def __init__(self, current_app, join_thread = False):
+    self.current_app = current_app
+    self.join_thread = join_thread
+
+
+  def purge(self, gemeente, gemeente_records):
+    self.current_app.logger.info(f"Purging nginx caches started for {gemeente.gemeente_code}")
+
+    try:
+        uuids = map(lambda record: record['UUID'], gemeente_records)
+        args = [self.current_app._get_current_object(), gemeente, uuids]
+        thread = threading.Thread(target=self.purge_target, args=args, daemon=True)
+        thread.start()
+
+        self.current_app.logger.info(f"Purging nginx caches sent to background for {gemeente.gemeente_code}")
+
+        if self.join_thread:
+          thread.join()
+          self.current_app.logger.info(f"Purging nginx caches finished after joining thread for {gemeente.gemeente_code}")
+
+    except Exception as e:
+        print(f"Exception occurred in CachePurger: {e}")
+        self.current_app.logger.info(f"Exception occurred in CachePurger: {e}")
+
+
+  def purge_target(self, app, gemeente, uuids):
+    app.logger.info(f"Purging nginx caches started in background for {gemeente.gemeente_code}")
+
+    try:
+        with app.app_context():
+            self.purge_for_url(app, url_for('index'))
+
+            self.purge_for_url(app, url_for('show_gemeente', gemeente=gemeente.gemeente_naam))
+            self.purge_for_url(app, url_for('embed_gemeente', gemeente=gemeente.gemeente_naam))
+
+            for uuid in uuids:
+              self.purge_for_url(app, url_for('show_stembureau', gemeente=gemeente.gemeente_naam, primary_key=uuid))
+              self.purge_for_url(app, url_for('embed_stembureau', gemeente=gemeente.gemeente_naam, primary_key=uuid))
+
+            self.purge_for_url(app, url_for('embed_alles'))
+
+            app.logger.info(f"Successfully purged nginx caches for {gemeente.gemeente_code}")
+    except Exception as e:
+        app.logger.info(f"Error purging nginx caches for {gemeente.gemeente_code}: {e}")
+
+
+  def purge_for_url(self, app, url):
+      cache_purge_key = f"{app.config['CACHE_PURGE_KEY']}"
+      return requests.get(url, headers={
+          cache_purge_key: 'true'
+      })

app/cli.py

@@ -13,17 +13,16 @@
 from app.procura import ProcuraManager
 
 from sqlalchemy import select
-from datetime import datetime, timedelta
+from datetime import datetime
 from dateutil import parser
-from flask import url_for
+from flask import url_for, current_app
 from pprint import pprint
 import click
 import copy
 import json
 import os
 import sys
 import uuid
-import pytz
 
 
 def create_cli_commands(app):
@@ -489,7 +488,8 @@ def publish_gemeente(gemeente_code):
         """
         Publishes the saved (draft) stembureaus of a gemeente
         """
-        publish_gemeente_records(gemeente_code)
+        with app.app_context():
+            publish_gemeente_records(gemeente_code, current_app, True)
 
 
     @CKAN.command()

app/routes.py

@@ -5,6 +5,7 @@
 from datetime import datetime
 from decimal import Decimal
 
+from app.cache_purger import CachePurger
 from flask import (
     render_template, request, redirect, url_for, flash, session,
     jsonify, current_app
@@ -27,7 +28,7 @@
 from app.validator import Validator
 from app.email import send_password_reset_email
 from app.models import Gemeente, User, Record, BAG, add_user, db
-from app.db_utils import db_exec_all, db_exec_first, db_exec_one, db_exec_one_optional
+from app.db_utils import db_exec_all, db_exec_first, db_exec_one_optional
 from app.utils import get_b64encoded_qr_image, get_gemeente, get_gemeente_by_id, get_gemeente_by_name, get_mysql_match_against_safe_string, remove_id
 from app.ckan import ckan
 from time import sleep
@@ -908,7 +909,6 @@ def gemeente_stemlokalen_dashboard():
                         ckan.elections[election]['draft_resource'],
                         records=records
                     )
-
                 flash(
                     'Het uploaden van stembureaus is gelukt! Controleer in het '
                     'overzicht hieronder of alles klopt en voer eventuele '
@@ -968,6 +968,8 @@ def gemeente_stemlokalen_overzicht():
                     remove_id(temp_gemeente_draft_records)
                     ckan.publish(election, gemeente.gemeente_code, temp_gemeente_draft_records)
 
+                    CachePurger(current_app).purge(gemeente, temp_gemeente_draft_records)
+
                 flash('De stembureaus zijn gepubliceerd.')
                 link_results = f'<a href="/s/{gemeente.gemeente_naam}" target="_blank">uw gemeentepagina</a>'
                 flash(Markup(f'De stembureaugegevens zijn nu openbaar beschikbaar op {link_results}. Kijk meteen of alles klopt.'))

app/stembureaumanager.py

@@ -120,7 +120,7 @@ def _save_draft_records(self, gemeente, gemeente_draft_records, elections, resul
             )
 
     def _publish_records(self, gemeente):
-        publish_gemeente_records(gemeente.gemeente_code)
+        publish_gemeente_records(gemeente.gemeente_code, current_app)
 
     def _send_error_email(self, gemeente, records, results, current_api):
         output = 'Er zijn fouten aangetroffen in de resultaten voor de gemeente %s (%s) via %s:\n\n' % (

app/utils.py

@@ -2,6 +2,7 @@
 import os
 from io import BytesIO
 
+from app.cache_purger import CachePurger
 from app.db_utils import db_exec_all, db_exec_one
 import fiona
 import shapely
@@ -57,7 +58,7 @@ def get_gemeente_by_id(id):
     return current_gemeente
 
 
-def publish_gemeente_records(gemeente_code):
+def publish_gemeente_records(gemeente_code, current_app, join_thread = False):
     """
     Publishes the saved (draft) stembureaus of a gemeente
     """
@@ -71,6 +72,8 @@ def publish_gemeente_records(gemeente_code):
         remove_id(temp_gemeente_draft_records)
         ckan.publish(election, current_gemeente.gemeente_code, temp_gemeente_draft_records)
 
+        CachePurger(current_app, join_thread).purge(current_gemeente, temp_gemeente_draft_records)
+
 
 def get_shapes(shape_file):
     shapes = []

config.py.example

@@ -7,6 +7,10 @@ from datetime import datetime
 # SECRET_KEY to a newly generated string using these python commands:
 # $ import os
 # $ os.urandom(32)
+# Same for CACHE_PURGE_KEY, which can only contain lowercase letters:
+# $ import secrets
+# $ import string
+# $ ''.join(secrets.choice(string.ascii_lowercase) for _ in range(32))
 
 basedir = os.path.abspath(os.path.dirname(__file__))
 locale.setlocale(locale.LC_NUMERIC, 'nl_NL.UTF-8')
@@ -22,6 +26,7 @@ class Config(object):
     USE_SESSION_FOR_NEXT = True
     SESSION_2FA_CONFIRMED_NAME = '2fa_confirmed'
     SESSION_2FA_LAST_ATTEMPT = '2fa_last_attempt'
+    CACHE_PURGE_KEY = False
 
     BABEL_DEFAULT_LOCALE = 'nl'
 

docker/docker-compose.yml.example

@@ -27,6 +27,13 @@ services:
     networks:
       - stm
     restart: always
+    # When you are using the production environment on your development laptop, you are using
+    # a dedicated local hostname such as `dev.waarismijnstemlokaal.nl`. To test the purging of
+    # nginx caches, pass the IP address (stm network) of the nginx container via `extra_hosts`
+    # so that cache purging actually works. On the production server `waarismijnstemlokaal.nl`
+    # will be resolved without additional measures.
+    #extra_hosts:
+    #  - "dev.waarismijnstemlokaal.nl:192.168.0.5"
   mysql:
     image: mysql:8.0
     # This root password will be overwritten with the password used in

docker/nginx/caching_params

@@ -0,0 +1,17 @@
+include uwsgi_params;
+uwsgi_pass stm_app_1:5000;
+uwsgi_read_timeout 1200;
+uwsgi_cache stm_cache;
+uwsgi_cache_key $uri;
+uwsgi_cache_valid any 1h;
+uwsgi_cache_bypass $bypass;
+# Remove the Vary header so that the same $uri for different clients always
+# ends up in the same cache file (without this, a `curl` to "/" produces
+# a different cache file than a browser accessing "/").
+# Remove the Cookie header so that no information can be leaked.
+uwsgi_hide_header Set-Cookie;
+uwsgi_hide_header Vary;
+uwsgi_ignore_headers Set-Cookie Vary;
+proxy_set_header Cookie "";
+proxy_set_header Vary "";
+add_header X-Cache-Status $upstream_cache_status;

docker/nginx/caching_setup.example

@@ -0,0 +1 @@
+set $bypass $http_<CACHE_PURGE_KEY>;

docker/nginx/conf.d/default.conf

@@ -3,6 +3,7 @@
 
 #uwsgi_cache_path /tmp/stm_cache levels=1:2 keys_zone=stm_cache:10m max_size=10g inactive=60m use_temp_path=off;
 
+
 # Redirect www to non-www
 server {
   server_name www.waarismijnstemlokaal.nl;
@@ -20,24 +21,14 @@ server {
 
   ## Cache only the homepage
   #location = / {
-  #  include uwsgi_params;
-  #  uwsgi_pass stm_app_1:5000;
-  #  uwsgi_read_timeout 1200;
-  #  uwsgi_cache stm_cache;
-  #  uwsgi_cache_key $uri;
-  #  uwsgi_cache_valid any 1h;
-  #  add_header X-Cache-Status $upstream_cache_status;
+  #  include caching_setup;
+  #  include caching_params;
   #}
 
   ## Cache also /s/ gemeente/stembureau and /e/ embed pages
   #location ~ ^/(s|e)/ {
-  #  include uwsgi_params;
-  #  uwsgi_pass stm_app_1:5000;
-  #  uwsgi_read_timeout 1200;
-  #  uwsgi_cache stm_cache;
-  #  uwsgi_cache_key $uri;
-  #  uwsgi_cache_valid any 1h;
-  #  add_header X-Cache-Status $upstream_cache_status;
+  #  include caching_setup;
+  #  include caching_params;
   #}
 
   location /static/dist/ {