diff --git a/.gitignore b/.gitignore index 08cb0994..42d45b90 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ app/data/gemeenten.json files/deels_vooringevuld/* app/data/nieuwe-gemeenten.json config.py +caching_setup backup.sh last_processed_line.csv update_bag.sh diff --git a/README.md b/README.md index 12fbeeb2..be909975 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,14 @@ Collecting and presenting stembureaus: [WaarIsMijnStemlokaal.nl](https://waarism - Fill in a password at `` - Copy `config.py.example` to `config.py` and edit it - Create a SECRET_KEY as per the instructions in the file + - Create a CACHE_PURGE_KEY as per the instructions in the file - Fill in your CKAN URL and CKAN API key - Fill in the same `` as used in `docker/docker-compose.yml` - Specify the name(s) of the election(s) and its corresponding CKAN draft and publish resource IDs - NOTE: Use the exact same `` values in the 'verkiezingen' field in 'app/data/gemeenten.json' - Specify email related information in order for the application to send emails +- Copy `docker/nginx/caching_setup.example` to `docker/nginx/caching_setup` and edit it + - Fill in the same `` as used in `config.py` - Copy `app/data/gemeenten.json.example` to `app/data/gemeenten.json` and edit it - Fill in the email addresses of the gemeenten - Add the name(s) of the election(s) for each gemeenten in which it participates. NOTE: make sure that these names are exactly the same as the name(s) of the election(s) in `app/config.py` diff --git a/app/cache_purger.py b/app/cache_purger.py new file mode 100644 index 00000000..6e190801 --- /dev/null +++ b/app/cache_purger.py @@ -0,0 +1,72 @@ +import threading +import requests + +from flask import url_for + +class CachePurger: + # Uses a background daemon to purge the nginx caches for + # - the home page / + # - the gemeente page + # - all stembureau pages for the gemeente + # Note that when logged in, `Cache-Control` is set to `private` in `routes.py` meaning pages + # you request that are not yet cached will not be cached. + # + # Development + # - When you are using the production environment on your development laptop, you are using + # a dedicated local hostname such as `dev.waarismijnstemlokaal.nl`. To test the purging of + # nginx caches, pass the IP address (stm network) of the nginx container via `extra_hosts` + # so that cache purging actually works. On the production server `waarismijnstemlokaal.nl` + # will be resolved without additional measures. See `docker-compose.yml.example`. + + + def __init__(self, current_app, join_thread = False): + self.current_app = current_app + self.join_thread = join_thread + + + def purge(self, gemeente, gemeente_records): + self.current_app.logger.info(f"Purging nginx caches started for {gemeente.gemeente_code}") + + try: + uuids = map(lambda record: record['UUID'], gemeente_records) + args = [self.current_app._get_current_object(), gemeente, uuids] + thread = threading.Thread(target=self.purge_target, args=args, daemon=True) + thread.start() + + self.current_app.logger.info(f"Purging nginx caches sent to background for {gemeente.gemeente_code}") + + if self.join_thread: + thread.join() + self.current_app.logger.info(f"Purging nginx caches finished after joining thread for {gemeente.gemeente_code}") + + except Exception as e: + print(f"Exception occurred in CachePurger: {e}") + self.current_app.logger.info(f"Exception occurred in CachePurger: {e}") + + + def purge_target(self, app, gemeente, uuids): + app.logger.info(f"Purging nginx caches started in background for {gemeente.gemeente_code}") + + try: + with app.app_context(): + self.purge_for_url(app, url_for('index')) + + self.purge_for_url(app, url_for('show_gemeente', gemeente=gemeente.gemeente_naam)) + self.purge_for_url(app, url_for('embed_gemeente', gemeente=gemeente.gemeente_naam)) + + for uuid in uuids: + self.purge_for_url(app, url_for('show_stembureau', gemeente=gemeente.gemeente_naam, primary_key=uuid)) + self.purge_for_url(app, url_for('embed_stembureau', gemeente=gemeente.gemeente_naam, primary_key=uuid)) + + self.purge_for_url(app, url_for('embed_alles')) + + app.logger.info(f"Successfully purged nginx caches for {gemeente.gemeente_code}") + except Exception as e: + app.logger.info(f"Error purging nginx caches for {gemeente.gemeente_code}: {e}") + + + def purge_for_url(self, app, url): + cache_purge_key = f"{app.config['CACHE_PURGE_KEY']}" + return requests.get(url, headers={ + cache_purge_key: 'true' + }) diff --git a/app/cli.py b/app/cli.py index 28a247dd..81580e20 100644 --- a/app/cli.py +++ b/app/cli.py @@ -13,9 +13,9 @@ from app.procura import ProcuraManager from sqlalchemy import select -from datetime import datetime, timedelta +from datetime import datetime from dateutil import parser -from flask import url_for +from flask import url_for, current_app from pprint import pprint import click import copy @@ -23,7 +23,6 @@ import os import sys import uuid -import pytz def create_cli_commands(app): @@ -489,7 +488,8 @@ def publish_gemeente(gemeente_code): """ Publishes the saved (draft) stembureaus of a gemeente """ - publish_gemeente_records(gemeente_code) + with app.app_context(): + publish_gemeente_records(gemeente_code, current_app, True) @CKAN.command() diff --git a/app/routes.py b/app/routes.py index 3b12ead6..03859e7a 100644 --- a/app/routes.py +++ b/app/routes.py @@ -5,6 +5,7 @@ from datetime import datetime from decimal import Decimal +from app.cache_purger import CachePurger from flask import ( render_template, request, redirect, url_for, flash, session, jsonify, current_app @@ -27,7 +28,7 @@ from app.validator import Validator from app.email import send_password_reset_email from app.models import Gemeente, User, Record, BAG, add_user, db -from app.db_utils import db_exec_all, db_exec_first, db_exec_one, db_exec_one_optional +from app.db_utils import db_exec_all, db_exec_first, db_exec_one_optional from app.utils import get_b64encoded_qr_image, get_gemeente, get_gemeente_by_id, get_gemeente_by_name, get_mysql_match_against_safe_string, remove_id from app.ckan import ckan from time import sleep @@ -908,7 +909,6 @@ def gemeente_stemlokalen_dashboard(): ckan.elections[election]['draft_resource'], records=records ) - flash( 'Het uploaden van stembureaus is gelukt! Controleer in het ' 'overzicht hieronder of alles klopt en voer eventuele ' @@ -968,6 +968,8 @@ def gemeente_stemlokalen_overzicht(): remove_id(temp_gemeente_draft_records) ckan.publish(election, gemeente.gemeente_code, temp_gemeente_draft_records) + CachePurger(current_app).purge(gemeente, temp_gemeente_draft_records) + flash('De stembureaus zijn gepubliceerd.') link_results = f'uw gemeentepagina' flash(Markup(f'De stembureaugegevens zijn nu openbaar beschikbaar op {link_results}. Kijk meteen of alles klopt.')) diff --git a/app/stembureaumanager.py b/app/stembureaumanager.py index 49e96a89..79b1f153 100644 --- a/app/stembureaumanager.py +++ b/app/stembureaumanager.py @@ -120,7 +120,7 @@ def _save_draft_records(self, gemeente, gemeente_draft_records, elections, resul ) def _publish_records(self, gemeente): - publish_gemeente_records(gemeente.gemeente_code) + publish_gemeente_records(gemeente.gemeente_code, current_app) def _send_error_email(self, gemeente, records, results, current_api): output = 'Er zijn fouten aangetroffen in de resultaten voor de gemeente %s (%s) via %s:\n\n' % ( diff --git a/app/utils.py b/app/utils.py index 898a120c..8fde3546 100644 --- a/app/utils.py +++ b/app/utils.py @@ -2,6 +2,7 @@ import os from io import BytesIO +from app.cache_purger import CachePurger from app.db_utils import db_exec_all, db_exec_one import fiona import shapely @@ -57,7 +58,7 @@ def get_gemeente_by_id(id): return current_gemeente -def publish_gemeente_records(gemeente_code): +def publish_gemeente_records(gemeente_code, current_app, join_thread = False): """ Publishes the saved (draft) stembureaus of a gemeente """ @@ -71,6 +72,8 @@ def publish_gemeente_records(gemeente_code): remove_id(temp_gemeente_draft_records) ckan.publish(election, current_gemeente.gemeente_code, temp_gemeente_draft_records) + CachePurger(current_app, join_thread).purge(current_gemeente, temp_gemeente_draft_records) + def get_shapes(shape_file): shapes = [] diff --git a/config.py.example b/config.py.example index b6c7cc0e..85c212e3 100644 --- a/config.py.example +++ b/config.py.example @@ -7,6 +7,10 @@ from datetime import datetime # SECRET_KEY to a newly generated string using these python commands: # $ import os # $ os.urandom(32) +# Same for CACHE_PURGE_KEY, which can only contain lowercase letters: +# $ import secrets +# $ import string +# $ ''.join(secrets.choice(string.ascii_lowercase) for _ in range(32)) basedir = os.path.abspath(os.path.dirname(__file__)) locale.setlocale(locale.LC_NUMERIC, 'nl_NL.UTF-8') @@ -22,6 +26,7 @@ class Config(object): USE_SESSION_FOR_NEXT = True SESSION_2FA_CONFIRMED_NAME = '2fa_confirmed' SESSION_2FA_LAST_ATTEMPT = '2fa_last_attempt' + CACHE_PURGE_KEY = False BABEL_DEFAULT_LOCALE = 'nl' diff --git a/docker/docker-compose.yml.example b/docker/docker-compose.yml.example index e0a6f233..2a1def59 100644 --- a/docker/docker-compose.yml.example +++ b/docker/docker-compose.yml.example @@ -27,6 +27,13 @@ services: networks: - stm restart: always + # When you are using the production environment on your development laptop, you are using + # a dedicated local hostname such as `dev.waarismijnstemlokaal.nl`. To test the purging of + # nginx caches, pass the IP address (stm network) of the nginx container via `extra_hosts` + # so that cache purging actually works. On the production server `waarismijnstemlokaal.nl` + # will be resolved without additional measures. + #extra_hosts: + # - "dev.waarismijnstemlokaal.nl:192.168.0.5" mysql: image: mysql:8.0 # This root password will be overwritten with the password used in diff --git a/docker/nginx/caching_params b/docker/nginx/caching_params new file mode 100644 index 00000000..3bff5544 --- /dev/null +++ b/docker/nginx/caching_params @@ -0,0 +1,17 @@ +include uwsgi_params; +uwsgi_pass stm_app_1:5000; +uwsgi_read_timeout 1200; +uwsgi_cache stm_cache; +uwsgi_cache_key $uri; +uwsgi_cache_valid any 1h; +uwsgi_cache_bypass $bypass; +# Remove the Vary header so that the same $uri for different clients always +# ends up in the same cache file (without this, a `curl` to "/" produces +# a different cache file than a browser accessing "/"). +# Remove the Cookie header so that no information can be leaked. +uwsgi_hide_header Set-Cookie; +uwsgi_hide_header Vary; +uwsgi_ignore_headers Set-Cookie Vary; +proxy_set_header Cookie ""; +proxy_set_header Vary ""; +add_header X-Cache-Status $upstream_cache_status; diff --git a/docker/nginx/caching_setup.example b/docker/nginx/caching_setup.example new file mode 100644 index 00000000..f0a1bef7 --- /dev/null +++ b/docker/nginx/caching_setup.example @@ -0,0 +1 @@ +set $bypass $http_; \ No newline at end of file diff --git a/docker/nginx/conf.d/default.conf b/docker/nginx/conf.d/default.conf index 5c6f332e..5d9c3517 100644 --- a/docker/nginx/conf.d/default.conf +++ b/docker/nginx/conf.d/default.conf @@ -3,6 +3,7 @@ #uwsgi_cache_path /tmp/stm_cache levels=1:2 keys_zone=stm_cache:10m max_size=10g inactive=60m use_temp_path=off; + # Redirect www to non-www server { server_name www.waarismijnstemlokaal.nl; @@ -20,24 +21,14 @@ server { ## Cache only the homepage #location = / { - # include uwsgi_params; - # uwsgi_pass stm_app_1:5000; - # uwsgi_read_timeout 1200; - # uwsgi_cache stm_cache; - # uwsgi_cache_key $uri; - # uwsgi_cache_valid any 1h; - # add_header X-Cache-Status $upstream_cache_status; + # include caching_setup; + # include caching_params; #} ## Cache also /s/ gemeente/stembureau and /e/ embed pages #location ~ ^/(s|e)/ { - # include uwsgi_params; - # uwsgi_pass stm_app_1:5000; - # uwsgi_read_timeout 1200; - # uwsgi_cache stm_cache; - # uwsgi_cache_key $uri; - # uwsgi_cache_valid any 1h; - # add_header X-Cache-Status $upstream_cache_status; + # include caching_setup; + # include caching_params; #} location /static/dist/ {