From 1feb98912af475c984b866d15771f515f133f62c Mon Sep 17 00:00:00 2001 From: Rob van Dijk <697696+robvandijk@users.noreply.github.com> Date: Thu, 22 Jan 2026 13:21:14 +0100 Subject: [PATCH 1/2] Adds emptying address fields when BAG id is 0000000000000000 --- README.md | 1 + app/cli.py | 30 ++++++++++++++--------------- app/models.py | 45 +++++++++++++++++++++++++++++--------------- app/routes.py | 35 +++++++++++++++++++--------------- tests/test_routes.py | 34 +++++++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 45 deletions(-) create mode 100644 tests/test_routes.py diff --git a/README.md b/README.md index 12fbeeb..7756344 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ Collecting and presenting stembureaus: [WaarIsMijnStemlokaal.nl](https://waarism - Retrieve the IP address of the nginx container `sudo docker inspect stm_nginx_1` and add it to your hosts file `/etc/hosts`: ` waarismijnstemlokaal.nl` - Useful commands - Run the tests: `sudo docker exec -it stm_app_1 nose2` + - Run 1 test: e.g. `sudo docker exec -it stm-app-1 nose2 tests.test_forms` - Remove and rebuild everything (this also removes the MySQL volume containing all gemeente, verkiezingen and BAG data (this is required if you want to load the .sql files from `docker/docker-entrypoint-initdb.d` again), but not the stembureaus data stored in CKAN) - Production: `sudo docker-compose down --rmi all && sudo docker volume rm stm_stm-mysql-volume && sudo docker-compose up -d` - Development: `sudo docker-compose -f docker-compose.yml -f docker-compose-dev.yml down --rmi all && sudo docker volume rm stm_stm-mysql-volume && sudo docker-compose -f docker-compose.yml -f docker-compose-dev.yml up -d` diff --git a/app/cli.py b/app/cli.py index 8a264c1..e900d7a 100644 --- a/app/cli.py +++ b/app/cli.py @@ -1,5 +1,5 @@ from app.changes_monitor import ChangesMonitor -from app.models import Gemeente, User, Gemeente_user, Election, BAG, add_user, db +from app.models import Gemeente, User, Gemeente_user, Election, BAG, add_user, db, get_bag_conversions from app.ckan import ckan from app.email import send_invite, send_update from app.parser import UploadFileParser @@ -138,15 +138,15 @@ def fix_bag_addresses(resource_type): bag_found += 1 r['BAG Nummeraanduiding ID'] = bag.nummeraanduiding - bag_conversions = { - 'verblijfsobjectgebruiksdoel': 'Gebruiksdoel van het gebouw', - 'openbareruimte': 'Straatnaam', - 'huisnummer': 'Huisnummer', - 'huisletter': 'Huisletter', - 'huisnummertoevoeging': 'Huisnummertoevoeging', - 'postcode': 'Postcode', - 'woonplaats': 'Plaats' - } + bag_conversions = get_bag_conversions([ + 'verblijfsobjectgebruiksdoel', + 'openbareruimte', + 'huisnummer', + 'huisletter', + 'huisnummertoevoeging', + 'postcode', + 'woonplaats' + ]) for bag_field, record_field in bag_conversions.items(): bag_field_value = getattr(bag, bag_field, None) @@ -659,11 +659,11 @@ def import_rug(rug_file_path, # Retrieve gebruiksdoel, postcode and nummeraanduiding # from BAG if bag_object: - bag_conversions = { - 'verblijfsobjectgebruiksdoel': 'Gebruiksdoel van het gebouw', - 'postcode': 'Postcode', - 'nummeraanduiding': 'BAG Nummeraanduiding ID' - } + bag_conversions = get_bag_conversions([ + 'verblijfsobjectgebruiksdoel', + 'postcode', + 'nummeraanduiding' + ]) for bag_field, record_field in bag_conversions.items(): bag_field_value = getattr(bag_object, bag_field, None) diff --git a/app/models.py b/app/models.py index 4336445..55554fa 100644 --- a/app/models.py +++ b/app/models.py @@ -284,6 +284,26 @@ def to_json(self): # A json-encodable dict return fields +def get_bag_conversions(fields): + all_conversions = { + 'gemeente': 'Gemeente', + 'verblijfsobjectgebruiksdoel': 'Gebruiksdoel van het gebouw', + 'nummeraanduiding': 'BAG Nummeraanduiding ID', + 'openbareruimte': 'Straatnaam', + 'huisnummer': 'Huisnummer', + 'huisletter': 'Huisletter', + 'huisnummertoevoeging': 'Huisnummertoevoeging', + 'postcode': 'Postcode', + 'woonplaats': 'Plaats', + 'lat': 'Latitude', + 'lon': 'Longitude', + 'x': 'X', + 'y': 'Y' + } + + selected = {k: v for k, v in all_conversions.items() if k in fields} + + return selected class Record(object): def __init__(self, *args, **kwargs): @@ -370,24 +390,19 @@ def expand(self): full_address = "0000000000000000 (geen adres beschikbaar in de BAG; kies deze optie voor Bonaire, Sint Eustatius en Saba)" self.record['adres_stembureau'] = full_address - for fld in [ + bag_conversions = get_bag_conversions([ 'gemeente', - 'straatnaam', + 'openbareruimte', 'huisnummer', 'huisletter', 'huisnummertoevoeging', 'postcode', - 'plaats', - # wijknaam - # cbs wijknummer - # buurtnaam - # cbs buurtnummer - # x - # y - ]: - # TODO: we need to know if these are the right fields for in CKAN. - fld_val = getattr(bag_record, fld, None) - if fld_val is not None: - self.record[fld] = fld_val.encode('utf-8').decode() + 'woonplaats' + ]) + + for bag_field, record_field in bag_conversions.items(): + bag_field_value = getattr(bag_record, bag_field, None) + if bag_field_value is not None: + self.record[record_field] = bag_field_value.encode('utf-8').decode() else: - self.record[fld] = None + self.record[record_field] = None diff --git a/app/routes.py b/app/routes.py index 9c9625f..75e27e3 100644 --- a/app/routes.py +++ b/app/routes.py @@ -26,7 +26,7 @@ from app.parser import UploadFileParser from app.validator import Validator from app.email import send_password_reset_email -from app.models import Gemeente, User, Record, BAG, add_user, db +from app.models import Gemeente, User, Record, BAG, add_user, db, get_bag_conversions from app.db_utils import db_exec_all, db_exec_first, db_exec_one, db_exec_one_optional from app.utils import get_b64encoded_qr_image, get_gemeente, get_gemeente_by_id, get_mysql_match_against_safe_string, remove_id from app.ckan import ckan @@ -1210,19 +1210,19 @@ def create_record(form, stemlokaal_id, gemeente, election): if bag_record is not None: - bag_conversions = { - 'verblijfsobjectgebruiksdoel': 'Gebruiksdoel van het gebouw', - 'openbareruimte': 'Straatnaam', - 'huisnummer': 'Huisnummer', - 'huisletter': 'Huisletter', - 'huisnummertoevoeging': 'Huisnummertoevoeging', - 'postcode': 'Postcode', - 'woonplaats': 'Plaats', - 'lat': 'Latitude', - 'lon': 'Longitude', - 'x': 'X', - 'y': 'Y' - } + bag_conversions = get_bag_conversions([ + 'verblijfsobjectgebruiksdoel', + 'openbareruimte', + 'huisnummer', + 'huisletter', + 'huisnummertoevoeging', + 'postcode', + 'woonplaats', + 'lat', + 'lon', + 'x', + 'y' + ]) for bag_field, record_field in bag_conversions.items(): bag_field_value = getattr(bag_record, bag_field, None) @@ -1237,7 +1237,6 @@ def create_record(form, stemlokaal_id, gemeente, election): ).decode() else: record[record_field] = None - ## We stopped adding the wijk and buurt data as the data ## supplied by CBS is not up to date enough as it is only ## released once a year and many months after changes @@ -1256,6 +1255,12 @@ def create_record(form, stemlokaal_id, gemeente, election): # record['Buurtnaam'] = bu_naam #if bu_code: # record['CBS buurtnummer'] = bu_code + else: + # If no bag is given make sure that any pre-saved address fields are emptied + record['Straatnaam'] = '' + record['Huisnummer'] = '' + record['Huisletter'] = '' + record['Huisnummertoevoeging'] = '' return record diff --git a/tests/test_routes.py b/tests/test_routes.py new file mode 100644 index 0000000..7dc63d9 --- /dev/null +++ b/tests/test_routes.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python + +import unittest +import uuid + +from app.routes import create_record +from app.utils import get_gemeente +from tests import app +from werkzeug.datastructures import MultiDict +from app.forms import EditForm +from app.models import Record + +from tests.record_to_test import record_to_test + + +class TestCreateRecord(unittest.TestCase): + def test_emptying_address_fields_for_zerosbag(self): + # When user fills in 0000000000000000 for BAG id, any pre-existing address fields should be emptied + with app.test_request_context('/'): + r = Record(**record_to_test(app.config["ELECTION_DATE"])) + r.record['bag_nummeraanduiding_id'] = '0000000000000000' + form = EditForm(MultiDict(r.record)) + + # Pre-conditions + self.assertNotEqual(r.record['Straatnaam'], '') + self.assertNotEqual(r.record['Straatnaam'], None) + + stemlokaal_id = uuid.uuid4().hex + gemeente = get_gemeente('GM0518') + election = f'{app.config["ELECTION_TYPE"]} {app.config["ELECTION_DATE"][0:4]}' + record = create_record(form, stemlokaal_id, gemeente, election) + + # Post-conditions + self.assertEqual(record['Straatnaam'], '') From f1c2c19ba7566fcb0a25889db63cbead0ac8daca Mon Sep 17 00:00:00 2001 From: Rob van Dijk <697696+robvandijk@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:52:36 +0100 Subject: [PATCH 2/2] Adds test for not emptying address fields --- README.md | 6 ++++-- tests/test_routes.py | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7756344..2e024b7 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,6 @@ Collecting and presenting stembureaus: [WaarIsMijnStemlokaal.nl](https://waarism - NOT NEEDED AS THIS DATA IS NOT UP TO DATE: Get buurt data: `sudo docker exec -it stm_app_1 /opt/stm/bin/get_address_data.sh` - Retrieve the IP address of the nginx container `sudo docker inspect stm_nginx_1` and add it to your hosts file `/etc/hosts`: ` waarismijnstemlokaal.nl` - Useful commands - - Run the tests: `sudo docker exec -it stm_app_1 nose2` - - Run 1 test: e.g. `sudo docker exec -it stm-app-1 nose2 tests.test_forms` - Remove and rebuild everything (this also removes the MySQL volume containing all gemeente, verkiezingen and BAG data (this is required if you want to load the .sql files from `docker/docker-entrypoint-initdb.d` again), but not the stembureaus data stored in CKAN) - Production: `sudo docker-compose down --rmi all && sudo docker volume rm stm_stm-mysql-volume && sudo docker-compose up -d` - Development: `sudo docker-compose -f docker-compose.yml -f docker-compose-dev.yml down --rmi all && sudo docker volume rm stm_stm-mysql-volume && sudo docker-compose -f docker-compose.yml -f docker-compose-dev.yml up -d` @@ -67,6 +65,10 @@ Development - Compile CSS/JS to `static/dist` directory (with map files): `sudo docker exec stm_nodejs_1 yarn build` - Automatically compile CSS/JS when a file changes (simply refresh the page in your browser after a change): `sudo docker exec stm_nodejs_1 yarn watch` +## Testing + - Run the tests: `sudo docker exec -it stm_app_1 nose2` + - Run 1 test: e.g. `sudo docker exec -it stm-app-1 nose2 tests.test_forms` + ## CLI To access the CLI of the app run `sudo docker exec -it stm_app_1 bash` and run `flask`, `flask ckan` and `flask mysql` to see the available commands. Here are some CLI commands: diff --git a/tests/test_routes.py b/tests/test_routes.py index 7dc63d9..89bf58f 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -22,8 +22,7 @@ def test_emptying_address_fields_for_zerosbag(self): form = EditForm(MultiDict(r.record)) # Pre-conditions - self.assertNotEqual(r.record['Straatnaam'], '') - self.assertNotEqual(r.record['Straatnaam'], None) + self.assertEqual(r.record['Straatnaam'], 'Spui') stemlokaal_id = uuid.uuid4().hex gemeente = get_gemeente('GM0518') @@ -32,3 +31,21 @@ def test_emptying_address_fields_for_zerosbag(self): # Post-conditions self.assertEqual(record['Straatnaam'], '') + + def test_keeping_address_fields(self): + # When user fills in a real BAG id, any pre-existing address fields should not be emptied + with app.test_request_context('/'): + r = Record(**record_to_test(app.config["ELECTION_DATE"])) + r.record['bag_nummeraanduiding_id'] = '0518200000747446' + form = EditForm(MultiDict(r.record)) + + # Pre-conditions + self.assertEqual(r.record['Straatnaam'], 'Spui') + + stemlokaal_id = uuid.uuid4().hex + gemeente = get_gemeente('GM0518') + election = f'{app.config["ELECTION_TYPE"]} {app.config["ELECTION_DATE"][0:4]}' + record = create_record(form, stemlokaal_id, gemeente, election) + + # Post-conditions + self.assertEqual(record['Straatnaam'], 'Spui')