Adds emptying address fields when BAG id is 0000000000000000

README.md

@@ -49,7 +49,6 @@ Collecting and presenting stembureaus: [WaarIsMijnStemlokaal.nl](https://waarism
    - NOT NEEDED AS THIS DATA IS NOT UP TO DATE: Get buurt data: `sudo docker exec -it stm_app_1 /opt/stm/bin/get_address_data.sh`
    - Retrieve the IP address of the nginx container `sudo docker inspect stm_nginx_1` and add it to your hosts file `/etc/hosts`: `<IP_address> waarismijnstemlokaal.nl`
 - Useful commands
-   - Run the tests: `sudo docker exec -it stm_app_1 nose2`
    - Remove and rebuild everything (this also removes the MySQL volume containing all gemeente, verkiezingen and BAG data (this is required if you want to load the .sql files from `docker/docker-entrypoint-initdb.d` again), but not the stembureaus data stored in CKAN)
       - Production: `sudo docker-compose down --rmi all && sudo docker volume rm stm_stm-mysql-volume && sudo docker-compose up -d`
       - Development: `sudo docker-compose -f docker-compose.yml -f docker-compose-dev.yml down --rmi all && sudo docker volume rm stm_stm-mysql-volume && sudo docker-compose -f docker-compose.yml -f docker-compose-dev.yml up -d`
@@ -66,6 +65,10 @@ Development
 - Compile CSS/JS to `static/dist` directory (with map files): `sudo docker exec stm_nodejs_1 yarn build`
 - Automatically compile CSS/JS when a file changes (simply refresh the page in your browser after a change): `sudo docker exec stm_nodejs_1 yarn watch`
 
+## Testing
+   - Run the tests: `sudo docker exec -it stm_app_1 nose2`
+   - Run 1 test: e.g. `sudo docker exec -it stm-app-1 nose2 tests.test_forms`
+
 ## CLI
 To access the CLI of the app run `sudo docker exec -it stm_app_1 bash` and run `flask`, `flask ckan` and `flask mysql` to see the available commands. Here are some CLI commands:
 

app/cli.py

@@ -1,5 +1,5 @@
 from app.changes_monitor import ChangesMonitor
-from app.models import Gemeente, User, Gemeente_user, Election, BAG, add_user, db
+from app.models import Gemeente, User, Gemeente_user, Election, BAG, add_user, db, get_bag_conversions
 from app.ckan import ckan
 from app.email import send_invite, send_update
 from app.parser import UploadFileParser
@@ -138,15 +138,15 @@ def fix_bag_addresses(resource_type):
                     bag_found += 1
                     r['BAG Nummeraanduiding ID'] = bag.nummeraanduiding
 
-                    bag_conversions = {
-                        'verblijfsobjectgebruiksdoel': 'Gebruiksdoel van het gebouw',
-                        'openbareruimte': 'Straatnaam',
-                        'huisnummer': 'Huisnummer',
-                        'huisletter': 'Huisletter',
-                        'huisnummertoevoeging': 'Huisnummertoevoeging',
-                        'postcode': 'Postcode',
-                        'woonplaats': 'Plaats'
-                    }
+                    bag_conversions = get_bag_conversions([
+                        'verblijfsobjectgebruiksdoel',
+                        'openbareruimte',
+                        'huisnummer',
+                        'huisletter',
+                        'huisnummertoevoeging',
+                        'postcode',
+                        'woonplaats'
+                    ])
 
                     for bag_field, record_field in bag_conversions.items():
                         bag_field_value = getattr(bag, bag_field, None)
@@ -659,11 +659,11 @@ def import_rug(rug_file_path,
                 # Retrieve gebruiksdoel, postcode and nummeraanduiding
                 # from BAG
                 if bag_object:
-                    bag_conversions = {
-                        'verblijfsobjectgebruiksdoel': 'Gebruiksdoel van het gebouw',
-                        'postcode': 'Postcode',
-                        'nummeraanduiding': 'BAG Nummeraanduiding ID'
-                    }
+                    bag_conversions = get_bag_conversions([
+                        'verblijfsobjectgebruiksdoel',
+                        'postcode',
+                        'nummeraanduiding'
+                    ])
 
                     for bag_field, record_field in bag_conversions.items():
                         bag_field_value = getattr(bag_object, bag_field, None)

app/models.py

@@ -284,6 +284,26 @@ def to_json(self):
         # A json-encodable dict
         return fields
 
+def get_bag_conversions(fields):
+    all_conversions = {
+        'gemeente': 'Gemeente',
+        'verblijfsobjectgebruiksdoel': 'Gebruiksdoel van het gebouw',
+        'nummeraanduiding': 'BAG Nummeraanduiding ID',
+        'openbareruimte': 'Straatnaam',
+        'huisnummer': 'Huisnummer',
+        'huisletter': 'Huisletter',
+        'huisnummertoevoeging': 'Huisnummertoevoeging',
+        'postcode': 'Postcode',
+        'woonplaats': 'Plaats',
+        'lat': 'Latitude',
+        'lon': 'Longitude',
+        'x': 'X',
+        'y': 'Y'
+    }
+
+    selected = {k: v for k, v in all_conversions.items() if k in fields}
+
+    return selected
 
 class Record(object):
     def __init__(self, *args, **kwargs):
@@ -370,24 +390,19 @@ def expand(self):
             full_address = "0000000000000000 (geen adres beschikbaar in de BAG; kies deze optie voor Bonaire, Sint Eustatius en Saba)"
             self.record['adres_stembureau'] = full_address
 
-        for fld in [
+        bag_conversions = get_bag_conversions([
             'gemeente',
-            'straatnaam',
+            'openbareruimte',
             'huisnummer',
             'huisletter',
             'huisnummertoevoeging',
             'postcode',
-            'plaats',
-            # wijknaam
-            # cbs wijknummer
-            # buurtnaam
-            # cbs buurtnummer
-            # x
-            # y
-        ]:
-            # TODO: we need to know if these are the right fields for in CKAN.
-            fld_val = getattr(bag_record, fld, None)
-            if fld_val is not None:
-                self.record[fld] = fld_val.encode('utf-8').decode()
+            'woonplaats'
+        ])
+
+        for bag_field, record_field in bag_conversions.items():
+            bag_field_value = getattr(bag_record, bag_field, None)
+            if bag_field_value is not None:
+                self.record[record_field] = bag_field_value.encode('utf-8').decode()
             else:
-                self.record[fld] = None
+                self.record[record_field] = None

app/routes.py

@@ -26,7 +26,7 @@
 from app.parser import UploadFileParser
 from app.validator import Validator
 from app.email import send_password_reset_email
-from app.models import Gemeente, User, Record, BAG, add_user, db
+from app.models import Gemeente, User, Record, BAG, add_user, db, get_bag_conversions
 from app.db_utils import db_exec_all, db_exec_first, db_exec_one, db_exec_one_optional
 from app.utils import get_b64encoded_qr_image, get_gemeente, get_gemeente_by_id, get_mysql_match_against_safe_string, remove_id
 from app.ckan import ckan
@@ -1210,19 +1210,19 @@ def create_record(form, stemlokaal_id, gemeente, election):
 
 
     if bag_record is not None:
-        bag_conversions = {
-            'verblijfsobjectgebruiksdoel': 'Gebruiksdoel van het gebouw',
-            'openbareruimte': 'Straatnaam',
-            'huisnummer': 'Huisnummer',
-            'huisletter': 'Huisletter',
-            'huisnummertoevoeging': 'Huisnummertoevoeging',
-            'postcode': 'Postcode',
-            'woonplaats': 'Plaats',
-            'lat': 'Latitude',
-            'lon': 'Longitude',
-            'x': 'X',
-            'y': 'Y'
-        }
+        bag_conversions = get_bag_conversions([
+            'verblijfsobjectgebruiksdoel',
+            'openbareruimte',
+            'huisnummer',
+            'huisletter',
+            'huisnummertoevoeging',
+            'postcode',
+            'woonplaats',
+            'lat',
+            'lon',
+            'x',
+            'y'
+        ])
 
         for bag_field, record_field in bag_conversions.items():
             bag_field_value = getattr(bag_record, bag_field, None)
@@ -1237,7 +1237,6 @@ def create_record(form, stemlokaal_id, gemeente, election):
                     ).decode()
             else:
                 record[record_field] = None
-
         ## We stopped adding the wijk and buurt data as the data
         ## supplied by CBS is not up to date enough as it is only
         ## released once a year and many months after changes
@@ -1256,6 +1255,12 @@ def create_record(form, stemlokaal_id, gemeente, election):
         #    record['Buurtnaam'] = bu_naam
         #if bu_code:
         #    record['CBS buurtnummer'] = bu_code
+    else:
+        # If no bag is given make sure that any pre-saved address fields are emptied
+        record['Straatnaam'] = ''
+        record['Huisnummer'] = ''
+        record['Huisletter'] = ''
+        record['Huisnummertoevoeging'] = ''
 
     return record
 

tests/test_routes.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+import unittest
+import uuid
+
+from app.routes import create_record
+from app.utils import get_gemeente
+from tests import app
+from werkzeug.datastructures import MultiDict
+from app.forms import EditForm
+from app.models import Record
+
+from tests.record_to_test import record_to_test
+
+
+class TestCreateRecord(unittest.TestCase):
+  def test_emptying_address_fields_for_zerosbag(self):
+    # When user fills in 0000000000000000 for BAG id, any pre-existing address fields should be emptied
+    with app.test_request_context('/'):
+      r = Record(**record_to_test(app.config["ELECTION_DATE"]))
+      r.record['bag_nummeraanduiding_id'] = '0000000000000000'
+      form = EditForm(MultiDict(r.record))
+
+      # Pre-conditions
+      self.assertEqual(r.record['Straatnaam'], 'Spui')
+
+      stemlokaal_id = uuid.uuid4().hex
+      gemeente = get_gemeente('GM0518')
+      election = f'{app.config["ELECTION_TYPE"]} {app.config["ELECTION_DATE"][0:4]}'
+      record = create_record(form, stemlokaal_id, gemeente, election)
+
+      # Post-conditions
+      self.assertEqual(record['Straatnaam'], '')
+
+  def test_keeping_address_fields(self):
+    # When user fills in a real BAG id, any pre-existing address fields should not be emptied
+    with app.test_request_context('/'):
+      r = Record(**record_to_test(app.config["ELECTION_DATE"]))
+      r.record['bag_nummeraanduiding_id'] = '0518200000747446'
+      form = EditForm(MultiDict(r.record))
+
+      # Pre-conditions
+      self.assertEqual(r.record['Straatnaam'], 'Spui')
+
+      stemlokaal_id = uuid.uuid4().hex
+      gemeente = get_gemeente('GM0518')
+      election = f'{app.config["ELECTION_TYPE"]} {app.config["ELECTION_DATE"][0:4]}'
+      record = create_record(form, stemlokaal_id, gemeente, election)
+
+      # Post-conditions
+      self.assertEqual(record['Straatnaam'], 'Spui')