From 7b4086ac1660118859761aa871c848b2c4ec0555 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 16 Apr 2025 16:43:41 +0200 Subject: [PATCH 01/60] Fix field type from TIME to DATETIME in BigQuery converter and schema esolves #728 --- datacontract/export/bigquery_converter.py | 2 +- tests/fixtures/bigquery/export/bq_table_schema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datacontract/export/bigquery_converter.py b/datacontract/export/bigquery_converter.py index e41a20429..71222d46c 100644 --- a/datacontract/export/bigquery_converter.py +++ b/datacontract/export/bigquery_converter.py @@ -103,7 +103,7 @@ def map_type_to_bigquery(field: Field) -> str: elif field_type.lower() == "date": return "DATE" elif field_type.lower() == "timestamp_ntz": - return "TIME" + return "DATETIME" elif field_type.lower() in ["number", "decimal", "numeric"]: return "NUMERIC" elif field_type.lower() == "double": diff --git a/tests/fixtures/bigquery/export/bq_table_schema.json b/tests/fixtures/bigquery/export/bq_table_schema.json index c0d9df5c1..e4a33567c 100644 --- a/tests/fixtures/bigquery/export/bq_table_schema.json +++ b/tests/fixtures/bigquery/export/bq_table_schema.json @@ -106,7 +106,7 @@ }, { "name": "timestamp_ntz_field", - "type": "TIME", + "type": "DATETIME", "mode": "NULLABLE", "description": "a simple timestamp_ntz field" }, From 9f62ad7fc7efa99a8cd83ccd222ad1778e3e763e Mon Sep 17 00:00:00 2001 From: Simon Harrer Date: Wed, 16 Apr 2025 17:03:05 +0200 Subject: [PATCH 02/60] Update dependency constraints for python-dotenv and boto3 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 20ca1b69d..2240a11d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,8 @@ dependencies = [ "soda-core-duckdb>=3.3.20,<3.5.0", # remove setuptools when https://github.com/sodadata/soda-core/issues/2091 is resolved "setuptools>=60", - "python-dotenv~=1.0.0", - "boto3>=1.34.41,<1.37.23", + "python-dotenv>=1.0.0,<2.0.0", + "boto3>=1.34.41,<2.0.0", "Jinja2>=3.1.5", "jinja_partials >= 0.2.1", ] From 3bc0b9e3e6d2e9c3cf45e6b3b12ae24f4fea1294 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Apr 2025 14:56:08 +0200 Subject: [PATCH 03/60] Update pre-commit requirement from <4.2.0,>=3.7.1 to >=3.7.1,<4.3.0 (#722) Updates the requirements on [pre-commit](https://github.com/pre-commit/pre-commit) to permit the latest version. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v3.7.1...v4.2.0) --- updated-dependencies: - dependency-name: pre-commit dependency-version: 4.2.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2240a11d2..54c86cdc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,7 +123,7 @@ dev = [ "kafka-python", "moto==5.1.1", "pandas>=2.1.0", - "pre-commit>=3.7.1,<4.2.0", + "pre-commit>=3.7.1,<4.3.0", "pytest", "pytest-xdist", "pymssql==2.3.2", From 88918078163e6eee883f4ed0d6d3f8ad7fedba8c Mon Sep 17 00:00:00 2001 From: ezhao-mck Date: Fri, 18 Apr 2025 09:04:38 -0400 Subject: [PATCH 04/60] Feature/odcsv3 #715 (#724) * Fix required in export and added item and fields format * Removed not from not field.required * Update odcs_v3_importer.py * Update odcs_v3_importer.py Changed server.storageAccount * Updated regex requirements * Updated regex to be more universal * Fixed formatting * Fixed more formatting * fix turn nullable to required in test * fix PR issues --------- Co-authored-by: Damien Maresma --- datacontract/export/odcs_v3_exporter.py | 2 +- datacontract/imports/odcs_v3_importer.py | 22 ++++++- .../odcs_v3/adventureworks.datacontract.yml | 64 +++++++++++++++++++ .../fixtures/odcs_v3/adventureworks.odcs.yaml | 50 +++++++++++++++ tests/test_export_odcs_v3.py | 6 +- 5 files changed, 139 insertions(+), 5 deletions(-) diff --git a/datacontract/export/odcs_v3_exporter.py b/datacontract/export/odcs_v3_exporter.py index 6a34af946..6520befe7 100644 --- a/datacontract/export/odcs_v3_exporter.py +++ b/datacontract/export/odcs_v3_exporter.py @@ -218,7 +218,7 @@ def to_property(field_name: str, field: Field) -> dict: if field.description is not None: property["description"] = field.description if field.required is not None: - property["nullable"] = not field.required + property["required"] = field.required if field.unique is not None: property["unique"] = field.unique if field.classification is not None: diff --git a/datacontract/imports/odcs_v3_importer.py b/datacontract/imports/odcs_v3_importer.py index 226d9d0f4..c64224327 100644 --- a/datacontract/imports/odcs_v3_importer.py +++ b/datacontract/imports/odcs_v3_importer.py @@ -1,5 +1,6 @@ import datetime import logging +import re from typing import Any, Dict, List from venv import logger @@ -134,7 +135,7 @@ def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None: server.outputPortId = odcs_server.get("outputPortId") server.driver = odcs_server.get("driver") server.roles = import_server_roles(odcs_server.get("roles")) - + server.storageAccount = re.search(r"(?:@|://)([^.]+)\.",odcs_server.get("location"),re.IGNORECASE) if server.type == "azure" else None servers[server_name] = server return servers @@ -288,8 +289,27 @@ def import_fields( else None, tags=odcs_property.get("tags") if odcs_property.get("tags") is not None else None, quality=odcs_property.get("quality") if odcs_property.get("quality") is not None else [], + fields=import_fields(odcs_property.get("properties"), custom_type_mappings, server_type) + if odcs_property.get("properties") is not None else {}, config=import_field_config(odcs_property, server_type), + format=odcs_property.get("format") if odcs_property.get("format") is not None else None, ) + #mapped_type is array + if field.type == "array" and odcs_property.get("items") is not None : + #nested array object + if odcs_property.get("items").get("logicalType") == "object": + field.items= Field(type="object", + fields=import_fields(odcs_property.get("items").get("properties"), custom_type_mappings, server_type)) + #array of simple type + elif odcs_property.get("items").get("logicalType") is not None: + field.items= Field(type = odcs_property.get("items").get("logicalType")) + + # enum from quality validValues as enum + if field.type == "string": + for q in field.quality: + if hasattr(q,"validValues"): + field.enum = q.validValues + result[property_name] = field else: logger.info( diff --git a/tests/fixtures/odcs_v3/adventureworks.datacontract.yml b/tests/fixtures/odcs_v3/adventureworks.datacontract.yml index fbe37277e..969d21a0b 100644 --- a/tests/fixtures/odcs_v3/adventureworks.datacontract.yml +++ b/tests/fixtures/odcs_v3/adventureworks.datacontract.yml @@ -4407,3 +4407,67 @@ models: criticalDataElement: false partitioned: false physicalType: timestamp + StoreHolidayHours: + title: StoreHolidayHours + type: array + required: false + primaryKey: false + unique: false + items: + type: object + fields: + Date: + title: Date + type: date + primaryKey: false + examples: + - '2024-08-13' + config: + physicalType: string + Close: + title: Close + type: date + primaryKey: false + examples: + - 02:00 PM + config: + physicalType: string + Open: + title: Open + type: date + primaryKey: false + examples: + - 10:00 AM + config: + physicalType: string + config: + physicalType: array + extendedData: + title: extendedData + type: object + required: true + primaryKey: false + unique: false + fields: + pharmacyUUID: + title: pharmacyUUID + type: string + required: true + primaryKey: false + unique: true + examples: + - ec43dd63-c258-4506-8965-88a9e0c130ad + config: + physicalType: string + config: + physicalType: object + ArrayComments: + title: ArrayComments + type: array + required: false + primaryKey: false + unique: false + items: + type: string + config: + physicalType: array \ No newline at end of file diff --git a/tests/fixtures/odcs_v3/adventureworks.odcs.yaml b/tests/fixtures/odcs_v3/adventureworks.odcs.yaml index c5e2a42dc..b1fed2cbc 100644 --- a/tests/fixtures/odcs_v3/adventureworks.odcs.yaml +++ b/tests/fixtures/odcs_v3/adventureworks.odcs.yaml @@ -5320,4 +5320,54 @@ schema: criticalDataElement: false primaryKey: false required: false + - name: StoreHolidayHours + businessName: StoreHolidayHours + logicalType: array + physicalType: array + required: false + unique: false + items: + logicalType: object + properties: + - name: Date + businessName: Date + logicalType: date + physicalType: string + examples: + - "2024-08-13" + - name: Close + businessName: Close + logicalType: date + physicalType: string + examples: + - "02:00 PM" + - name: Open + businessName: Open + logicalType: date + physicalType: string + examples: + - "10:00 AM" + - name: extendedData + businessName: extendedData + logicalType: object + physicalType: object + required: true + unique: false + properties: + - name : pharmacyUUID + businessName: pharmacyUUID + logicalType: string + physicalType: string + required: true + unique: true + examples: + - "ec43dd63-c258-4506-8965-88a9e0c130ad" + - name: ArrayComments + businessName: ArrayComments + logicalType: array + physicalType: array + required: false + unique: false + items: + logicalType: string contractCreatedTs: "2023-09-28T20:24:49.331+00:00" diff --git a/tests/test_export_odcs_v3.py b/tests/test_export_odcs_v3.py index ee922aee1..c82d2a93c 100644 --- a/tests/test_export_odcs_v3.py +++ b/tests/test_export_odcs_v3.py @@ -46,7 +46,7 @@ def test_to_odcs(): maxLength: 10 pattern: ^B[0-9]+$ physicalType: varchar - nullable: false + required: true unique: true tags: - "order_id" @@ -65,7 +65,7 @@ def test_to_odcs(): minimum: 0 maximum: 1000000 physicalType: bigint - nullable: false + required: true description: The order_total field quality: - type: sql @@ -77,7 +77,7 @@ def test_to_odcs(): - name: order_status logicalType: string physicalType: text - nullable: false + required: true quality: - type: sql description: Row Count From 4a133c1e376e18abc49932935fd116e9d611566b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Apr 2025 15:05:06 +0200 Subject: [PATCH 05/60] Update databricks-sql-connector requirement (#721) Updates the requirements on [databricks-sql-connector](https://github.com/databricks/databricks-sql-python) to permit the latest version. - [Release notes](https://github.com/databricks/databricks-sql-python/releases) - [Changelog](https://github.com/databricks/databricks-sql-python/blob/main/CHANGELOG.md) - [Commits](https://github.com/databricks/databricks-sql-python/compare/v3.7.0...v4.0.2) --- updated-dependencies: - dependency-name: databricks-sql-connector dependency-version: 4.0.2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 54c86cdc6..5633edaa2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ csv = [ databricks = [ "soda-core-spark-df>=3.3.20,<3.5.0", "soda-core-spark[databricks]>=3.3.20,<3.5.0", - "databricks-sql-connector>=3.7.0,<3.8.0", + "databricks-sql-connector>=3.7.0,<4.1.0", "databricks-sdk<0.50.0", ] From 02eb4b185af1454e36eb5bfa1f1944a0747604ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Apr 2025 15:05:24 +0200 Subject: [PATCH 06/60] Bump moto from 5.1.1 to 5.1.3 (#720) Bumps [moto](https://github.com/getmoto/moto) from 5.1.1 to 5.1.3. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/5.1.1...5.1.3) --- updated-dependencies: - dependency-name: moto dependency-version: 5.1.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5633edaa2..e8df6f920 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,7 +121,7 @@ dev = [ "datacontract-cli[all]", "httpx==0.28.1", "kafka-python", - "moto==5.1.1", + "moto==5.1.3", "pandas>=2.1.0", "pre-commit>=3.7.1,<4.3.0", "pytest", From e3a584adcba40c032259c5eed1a6dc891111325e Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 19 Apr 2025 16:22:09 +0200 Subject: [PATCH 07/60] Update dependency constraints for various packages to allow newer versions Hopefully resolves #729 --- pyproject.toml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e8df6f920..2f546356e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,17 +22,17 @@ dependencies = [ "fastjsonschema>=2.19.1,<2.22.0", "fastparquet>=2024.5.0,<2025.0.0", "numpy>=1.26.4,<2.0.0", # transitive dependency, needs to be <2.0.0 https://github.com/datacontract/datacontract-cli/issues/575 - "python-multipart==0.0.20", - "rich>=13.7,<13.10", + "python-multipart>=0.0.20,<1.0.0", + "rich>=13.7,<14.0", "sqlglot>=26.6.0,<27.0.0", "duckdb>=1.0.0,<2.0.0", - "soda-core-duckdb>=3.3.20,<3.5.0", + "soda-core-duckdb>=3.3.20,<3.6.0", # remove setuptools when https://github.com/sodadata/soda-core/issues/2091 is resolved "setuptools>=60", "python-dotenv>=1.0.0,<2.0.0", "boto3>=1.34.41,<2.0.0", - "Jinja2>=3.1.5", - "jinja_partials >= 0.2.1", + "Jinja2>=3.1.5,<4.0.0", + "jinja_partials>=0.2.1,<1.0.0", ] [project.optional-dependencies] @@ -51,8 +51,8 @@ csv = [ ] databricks = [ - "soda-core-spark-df>=3.3.20,<3.5.0", - "soda-core-spark[databricks]>=3.3.20,<3.5.0", + "soda-core-spark-df>=3.3.20,<3.6.0", + "soda-core-spark[databricks]>=3.3.20,<3.6.0", "databricks-sql-connector>=3.7.0,<4.1.0", "databricks-sdk<0.50.0", ] @@ -63,11 +63,11 @@ iceberg = [ kafka = [ "datacontract-cli[avro]", - "soda-core-spark-df>=3.3.20,<3.5.0" + "soda-core-spark-df>=3.3.20,<3.6.0" ] postgres = [ - "soda-core-postgres>=3.3.20,<3.5.0" + "soda-core-postgres>=3.3.20,<3.6.0" ] s3 = [ @@ -77,15 +77,15 @@ s3 = [ snowflake = [ "snowflake-connector-python[pandas]>=3.6,<3.15", - "soda-core-snowflake>=3.3.20,<3.5.0" + "soda-core-snowflake>=3.3.20,<3.6.0" ] sqlserver = [ - "soda-core-sqlserver>=3.3.20,<3.5.0" + "soda-core-sqlserver>=3.3.20,<3.6.0" ] trino = [ - "soda-core-trino>=3.3.20,<3.5.0" + "soda-core-trino>=3.3.20,<3.6.0" ] dbt = [ From 944d48bab776404449ddb2aa1003e43368dd90ba Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 19 Apr 2025 16:29:24 +0200 Subject: [PATCH 08/60] Update changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d35d9fe24..e2c472230 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Updated dependencies + ### Fixed -- Fix to handle logicalType format wrt avro mentioned in issue #687 +- Fix to handle logicalType format wrt avro mentioned in issue (#687) +- Fix field type from TIME to DATETIME in BigQuery converter and schema (#728) +- Fix encoding issues. (#712) +- ODCS: Fix required in export and added item and fields format (#724) ## [0.10.23] - 2025-03-03 From 48ec80bca5b615833f966c6f17fe658f16109bb4 Mon Sep 17 00:00:00 2001 From: Stefan McKinnon Edwards Date: Sat, 19 Apr 2025 16:34:28 +0200 Subject: [PATCH 09/60] feat: more descriptive csv importer (#692) * feat: more descriptive csv importer Replaced csv importer with a duckdb-based method. Output is more descriptive, including required and unique, when these differ from default value, up to 5 examples of each non-binary/non-boolean field, and min/max values on numeric fields. * Fix ruff format * fixed test * fix: supressing deprecation warning in lint/resolve.py --------- Co-authored-by: jochen --- CHANGELOG.md | 2 + datacontract/imports/csv_importer.py | 168 ++++++++++++------ datacontract/lint/resolve.py | 20 ++- .../csv/data/sample_data_5_column.csv | 11 ++ tests/test_import_csv.py | 36 +++- 5 files changed, 172 insertions(+), 65 deletions(-) create mode 100644 tests/fixtures/csv/data/sample_data_5_column.csv diff --git a/CHANGELOG.md b/CHANGELOG.md index e2c472230..ca9b3d230 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `datacontract import --format csv` produces more descriptive output. Replaced + using clevercsv with duckdb for loading and sniffing csv file. - Updated dependencies ### Fixed diff --git a/datacontract/imports/csv_importer.py b/datacontract/imports/csv_importer.py index 485c1130f..98ac275b8 100644 --- a/datacontract/imports/csv_importer.py +++ b/datacontract/imports/csv_importer.py @@ -1,89 +1,143 @@ import os +from typing import Any, Dict, List -import clevercsv +import duckdb from datacontract.imports.importer import Importer -from datacontract.model.data_contract_specification import DataContractSpecification, Example, Field, Model, Server +from datacontract.model.data_contract_specification import DataContractSpecification, Model, Server class CsvImporter(Importer): def import_source( self, data_contract_specification: DataContractSpecification, source: str, import_args: dict ) -> DataContractSpecification: - return import_csv(data_contract_specification, self.import_format, source) + return import_csv(data_contract_specification, source) -def import_csv(data_contract_specification: DataContractSpecification, format: str, source: str): - include_example = False - - # detect encoding and dialect - encoding = clevercsv.encoding.get_encoding(source) - with open(source, "r", newline="") as fp: - dialect = clevercsv.Sniffer().sniff(fp.read(10000)) - - # using auto detecting of the format and encoding - df = clevercsv.read_dataframe(source) - - if data_contract_specification.models is None: - data_contract_specification.models = {} - +def import_csv( + data_contract_specification: DataContractSpecification, source: str, include_examples: bool = False +) -> DataContractSpecification: # use the file name as table name table_name = os.path.splitext(os.path.basename(source))[0] + # use duckdb to auto detect format, columns, etc. + con = duckdb.connect(database=":memory:") + con.sql( + f"""CREATE VIEW "{table_name}" AS SELECT * FROM read_csv_auto('{source}', hive_partitioning=1, auto_type_candidates = ['BOOLEAN', 'INTEGER', 'BIGINT', 'DOUBLE', 'VARCHAR']);""" + ) + dialect = con.sql(f"SELECT * FROM sniff_csv('{source}', sample_size = 1000);").fetchnumpy() + tbl = con.table(table_name) + if data_contract_specification.servers is None: data_contract_specification.servers = {} + delimiter = None if dialect is None else dialect['Delimiter'][0] + + if dialect is not None: + dc_types = [map_type_from_duckdb(x["type"]) for x in dialect['Columns'][0]] + else: + dc_types = [map_type_from_duckdb(str(x)) for x in tbl.dtypes] + data_contract_specification.servers["production"] = Server( - type="local", path=source, format="csv", delimiter=dialect.delimiter + type="local", path=source, format="csv", delimiter=delimiter ) + rowcount = tbl.shape[0] + + tallies = dict() + for row in tbl.describe().fetchall(): + if row[0] not in ["count", "max", "min"]: + continue + for i in range(tbl.shape[1]): + tallies[(row[0], tbl.columns[i])] = row[i + 1] if row[0] != "count" else int(row[i + 1]) + + samples: Dict[str, List] = dict() + for i in range(tbl.shape[1]): + field_name = tbl.columns[i] + if tallies[("count", field_name)] > 0 and tbl.dtypes[i] not in ["BOOLEAN", "BLOB"]: + sql = f"""SELECT DISTINCT "{field_name}" FROM "{table_name}" WHERE "{field_name}" IS NOT NULL USING SAMPLE 5 ROWS;""" + samples[field_name] = [x[0] for x in con.sql(sql).fetchall()] + + formats: Dict[str, str] = dict() + for i in range(tbl.shape[1]): + field_name = tbl.columns[i] + if tallies[("count", field_name)] > 0 and tbl.dtypes[i] == "VARCHAR": + sql = f"""SELECT + count_if("{field_name}" IS NOT NULL) as count, + count_if(regexp_matches("{field_name}", '^[\\w-\\.]+@([\\w-]+\\.)+[\\w-]{{2,4}}$')) as email, + count_if(regexp_matches("{field_name}", '^[[a-z0-9]{{8}}-?[a-z0-9]{{4}}-?[a-z0-9]{{4}}-?[a-z0-9]{{4}}-?[a-z0-9]{{12}}]')) as uuid + FROM "{table_name}"; + """ + res = con.sql(sql).fetchone() + if res[1] == res[0]: + formats[field_name] = "email" + elif res[2] == res[0]: + formats[field_name] = "uuid" + fields = {} - for column, dtype in df.dtypes.items(): - field = Field() - field.type = map_type_from_pandas(dtype.name) - fields[column] = field + for i in range(tbl.shape[1]): + field_name = tbl.columns[i] + dc_type = dc_types[i] + + ## specifying "integer" rather than "bigint" looks nicer + if ( + dc_type == "bigint" + and tallies[("max", field_name)] <= 2147483647 + and tallies[("min", field_name)] >= -2147483648 + ): + dc_type = "integer" + + field: Dict[str, Any] = {"type": dc_type, "format": formats.get(field_name, None)} + + if tallies[("count", field_name)] == rowcount: + field["required"] = True + if dc_type not in ["boolean", "bytes"]: + distinct_values = tbl.count(f'DISTINCT "{field_name}"').fetchone()[0] # type: ignore + if distinct_values > 0 and distinct_values == tallies[("count", field_name)]: + field["unique"] = True + s = samples.get(field_name, None) + if s is not None: + field["examples"] = s + if dc_type in ["integer", "bigint", "float", "double"]: + field["minimum"] = tallies[("min", field_name)] + field["maximum"] = tallies[("max", field_name)] + + fields[field_name] = field + + model_examples = None + if include_examples: + model_examples = con.sql(f"""SELECT DISTINCT * FROM "{table_name}" USING SAMPLE 5 ROWS;""").fetchall() data_contract_specification.models[table_name] = Model( - type="table", - description=f"Csv file with encoding {encoding}", - fields=fields, + type="table", description="Generated model of " + source, fields=fields, examples=model_examples ) - # multiline data is not correctly handled by yaml dump - if include_example: - if data_contract_specification.examples is None: - data_contract_specification.examples = [] - - # read first 10 lines with the detected encoding - with open(source, "r", encoding=encoding) as csvfile: - lines = csvfile.readlines()[:10] - - data_contract_specification.examples.append(Example(type="csv", model=table_name, data="".join(lines))) - return data_contract_specification -def map_type_from_pandas(sql_type: str): +_duck_db_types = { + "BOOLEAN": "boolean", + "BLOB": "bytes", + "TINYINT": "integer", + "SMALLINT": "integer", + "INTEGER": "integer", + "BIGINT": "bigint", + "UTINYINT": "integer", + "USMALLINT": "integer", + "UINTEGER": "integer", + "UBIGINT": "bigint", + "FLOAT": "float", + "DOUBLE": "double", + "VARCHAR": "string", + "TIMESTAMP": "timestamp", + "DATE": "date", + # TODO: Add support for NULL +} + + +def map_type_from_duckdb(sql_type: None | str): if sql_type is None: return None - sql_type_normed = sql_type.lower().strip() - - if sql_type_normed == "object": - return "string" - elif sql_type_normed.startswith("str"): - return "string" - elif sql_type_normed.startswith("int"): - return "integer" - elif sql_type_normed.startswith("float"): - return "float" - elif sql_type_normed.startswith("bool"): - return "boolean" - elif sql_type_normed.startswith("timestamp"): - return "timestamp" - elif sql_type_normed == "datetime64": - return "date" - elif sql_type_normed == "timedelta[ns]": - return "timestamp_ntz" - else: - return "variant" + sql_type_normed = sql_type.upper().strip() + return _duck_db_types.get(sql_type_normed, "string") diff --git a/datacontract/lint/resolve.py b/datacontract/lint/resolve.py index 78eb5f170..ac5395f28 100644 --- a/datacontract/lint/resolve.py +++ b/datacontract/lint/resolve.py @@ -1,5 +1,6 @@ import logging import os +import warnings import fastjsonschema import yaml @@ -259,12 +260,27 @@ def _resolve_data_contract_from_str( if inline_definitions: inline_definitions_into_data_contract(spec) - if spec.quality and inline_quality: - _resolve_quality_ref(spec.quality) + ## Suppress DeprecationWarning when accessing spec.quality, + ## iif it is in fact *not* used. + with warnings.catch_warnings(record=True) as recorded_warnings: + spec_quality = spec.quality + for w in recorded_warnings: + if not issubclass(w.category, DeprecationWarning) or spec_quality is not None: + warnings.warn_explicit( + message=w.message, + category=w.category, + filename=w.filename, + lineno=w.lineno, + source=w.source, + ) + if spec_quality and inline_quality: + _resolve_quality_ref(spec_quality) return spec + + def _to_yaml(data_contract_str) -> dict: try: yaml_dict = yaml.safe_load(data_contract_str) diff --git a/tests/fixtures/csv/data/sample_data_5_column.csv b/tests/fixtures/csv/data/sample_data_5_column.csv new file mode 100644 index 000000000..7603ba5a7 --- /dev/null +++ b/tests/fixtures/csv/data/sample_data_5_column.csv @@ -0,0 +1,11 @@ +field_one,field_two,field_three,field_four,field_five,field_six +CX-263-DU,50,2023-06-16 13:12:56,,true,test1@gmail.com +IK-894-MN,47,2023-10-08 22:40:57,,true,test1@gmail.com +ER-399-JY,22,2023-05-16 01:08:22,,true,test1@gmail.com +MT-939-FH,47,2023-03-15 05:15:21,,false,test1@gmail.com +LV-849-MI,50,2023-09-08 20:08:43,,false,test1@gmail.com +VS-079-OH,22,2023-04-15 00:50:32,,false,test1@gmail.com +DN-297-XY,50,2023-11-08 12:55:42,,false,test1@gmail.com +ZE-172-FP,14,,,true,test1@gmail.com +ID-840-EG,89,2023-10-02 17:17:58,,true, +FK-230-KZ,64,2023-11-27 15:21:48,,true,test1@gmail.com diff --git a/tests/test_import_csv.py b/tests/test_import_csv.py index 2e8b75388..9ce8eda33 100644 --- a/tests/test_import_csv.py +++ b/tests/test_import_csv.py @@ -24,10 +24,20 @@ def test_cli(): assert result.exit_code == 0 -def test_import_sql(): - result = DataContract().import_from_source("csv", csv_file_path) +def test_import_csv(): + source = "fixtures/csv/data/sample_data_5_column.csv" + result = DataContract().import_from_source("csv", source) + model = result.models["sample_data_5_column"] + assert model is not None + assert len(model.fields["field_one"].examples) == 5 + assert len(model.fields["field_two"].examples) > 0 + assert len(model.fields["field_three"].examples) > 0 + assert model.fields["field_four"].examples is None + assert model.fields["field_five"].examples is None + for k in model.fields.keys(): + model.fields[k].examples = None - expected = """dataContractSpecification: 1.1.0 + expected = f"""dataContractSpecification: 1.1.0 id: my-data-contract-id info: title: My Data Contract @@ -36,19 +46,33 @@ def test_import_sql(): production: type: local format: csv - path: fixtures/csv/data/sample_data.csv + path: {source} delimiter: ',' models: - sample_data: - description: Csv file with encoding ascii + sample_data_5_column: + description: Generated model of fixtures/csv/data/sample_data_5_column.csv type: table fields: field_one: type: string + required: true + unique: true field_two: type: integer + required: true + minimum: 14 + maximum: 89 field_three: + type: timestamp + unique: true + field_four: type: string + field_five: + type: boolean + required: true + field_six: + type: string + format: email """ print("Result", result.to_yaml()) assert yaml.safe_load(result.to_yaml()) == yaml.safe_load(expected) From 4b3020db1156343eb2bfabcd171c28766d79c238 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 19 Apr 2025 16:42:34 +0200 Subject: [PATCH 10/60] Deprecated QualityLinter is now removed --- CHANGELOG.md | 4 ++ datacontract/data_contract.py | 2 - .../lint/linters/quality_schema_linter.py | 52 ------------------- datacontract/lint/resolve.py | 15 +++--- tests/test_api.py | 2 +- tests/test_lint.py | 2 +- tests/test_quality_schema_linter.py | 35 ------------- 7 files changed, 13 insertions(+), 99 deletions(-) delete mode 100644 datacontract/lint/linters/quality_schema_linter.py delete mode 100644 tests/test_quality_schema_linter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ca9b3d230..d5dbd4990 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix encoding issues. (#712) - ODCS: Fix required in export and added item and fields format (#724) +### Removed + +- Deprecated QualityLinter is now removed + ## [0.10.23] - 2025-03-03 ### Added diff --git a/datacontract/data_contract.py b/datacontract/data_contract.py index 1ebe81268..ad65781fd 100644 --- a/datacontract/data_contract.py +++ b/datacontract/data_contract.py @@ -24,7 +24,6 @@ from datacontract.lint.linters.field_pattern_linter import FieldPatternLinter from datacontract.lint.linters.field_reference_linter import FieldReferenceLinter from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter -from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter from datacontract.model.data_contract_specification import DataContractSpecification from datacontract.model.exceptions import DataContractException @@ -58,7 +57,6 @@ def __init__( self._inline_quality = inline_quality self._ssl_verification = ssl_verification self.all_linters = { - QualityUsesSchemaLinter(), FieldPatternLinter(), FieldReferenceLinter(), NoticePeriodLinter(), diff --git a/datacontract/lint/linters/quality_schema_linter.py b/datacontract/lint/linters/quality_schema_linter.py deleted file mode 100644 index a2c17de3c..000000000 --- a/datacontract/lint/linters/quality_schema_linter.py +++ /dev/null @@ -1,52 +0,0 @@ -import yaml - -from datacontract.model.data_contract_specification import DataContractSpecification, Model - -from ..lint import Linter, LinterResult - - -class QualityUsesSchemaLinter(Linter): - @property - def name(self) -> str: - return "Quality check(s) use model" - - @property - def id(self) -> str: - return "quality-schema" - - def lint_sodacl(self, check, models: dict[str, Model]) -> LinterResult: - result = LinterResult() - for sodacl_check in check.keys(): - table_name = sodacl_check[len("checks for ") :] - if table_name not in models: - result = result.with_error(f"Quality check on unknown model '{table_name}'") - return result - - def lint_montecarlo(self, check, models: dict[str, Model]) -> LinterResult: - return LinterResult().with_warning("Linting montecarlo checks is not currently implemented") - - def lint_great_expectations(self, check, models: dict[str, Model]) -> LinterResult: - return LinterResult().with_warning("Linting great expectations checks is not currently implemented") - - def lint_implementation(self, contract: DataContractSpecification) -> LinterResult: - result = LinterResult() - models = contract.models - check = contract.quality - if not check: - return LinterResult() - if not check.specification: - return LinterResult.cautious("Quality check without specification.") - if isinstance(check.specification, str): - check_specification = yaml.safe_load(check.specification) - else: - check_specification = check.specification - match check.type: - case "SodaCL": - result = result.combine(self.lint_sodacl(check_specification, models)) - case "montecarlo": - result = result.combine(self.lint_montecarlo(check_specification, models)) - case "great-expectations": - result = result.combine(self.lint_great_expectations(check_specification, models)) - case _: - result = result.with_warning("Can't lint quality check " f"with type '{check.type}'") - return result diff --git a/datacontract/lint/resolve.py b/datacontract/lint/resolve.py index ac5395f28..28d1f7d1d 100644 --- a/datacontract/lint/resolve.py +++ b/datacontract/lint/resolve.py @@ -17,6 +17,7 @@ ) from datacontract.model.exceptions import DataContractException from datacontract.model.odcs import is_open_data_contract_standard +from datacontract.model.run import ResultEnum def resolve_data_contract( @@ -38,7 +39,7 @@ def resolve_data_contract( else: raise DataContractException( type="lint", - result="failed", + result=ResultEnum.failed, name="Check that data contract YAML is valid", reason="Data contract needs to be provided", engine="datacontract", @@ -59,7 +60,7 @@ def resolve_data_contract_dict( else: raise DataContractException( type="lint", - result="failed", + result=ResultEnum.failed, name="Check that data contract YAML is valid", reason="Data contract needs to be provided", engine="datacontract", @@ -153,7 +154,7 @@ def _resolve_definition_ref(ref, spec) -> Definition: else: raise DataContractException( type="lint", - result="failed", + result=ResultEnum.failed, name="Check that data contract YAML is valid", reason=f"Cannot resolve reference {ref}", engine="datacontract", @@ -166,7 +167,7 @@ def _find_by_path_in_spec(definition_path: str, spec: DataContractSpecification) if definition_key not in spec.definitions: raise DataContractException( type="lint", - result="failed", + result=ResultEnum.failed, name="Check that data contract YAML is valid", reason=f"Cannot resolve definition {definition_key}", engine="datacontract", @@ -196,7 +197,7 @@ def _fetch_file(path) -> str: if not os.path.exists(path): raise DataContractException( type="export", - result="failed", + result=ResultEnum.failed, name="Check that data contract definition is valid", reason=f"Cannot resolve reference {path}", engine="datacontract", @@ -231,7 +232,7 @@ def _get_quality_ref_file(quality_spec: str | object) -> str | object: if not os.path.exists(ref): raise DataContractException( type="export", - result="failed", + result=ResultEnum.failed, name="Check that data contract quality is valid", reason=f"Cannot resolve reference {ref}", engine="datacontract", @@ -279,8 +280,6 @@ def _resolve_data_contract_from_str( return spec - - def _to_yaml(data_contract_str) -> dict: try: yaml_dict = yaml.safe_load(data_contract_str) diff --git a/tests/test_api.py b/tests/test_api.py index 68098f86e..7db685a0c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -16,7 +16,7 @@ def test_lint(): assert response.status_code == 200 print(response.json()) assert response.json()["result"] == "passed" - assert len(response.json()["checks"]) == 7 + assert len(response.json()["checks"]) == 6 assert all([check["result"] == "passed" for check in response.json()["checks"]]) diff --git a/tests/test_lint.py b/tests/test_lint.py index 5bb6552f2..5cc1fb514 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -27,7 +27,7 @@ def test_lint_invalid_data_contract(): def test_lint_cli_valid(): data_contract_file = "fixtures/lint/valid_datacontract.yaml" - expected_output = "🟢 data contract is valid. Run 7 checks." + expected_output = "🟢 data contract is valid. Run 6 checks." result = runner.invoke(app, ["lint", data_contract_file]) diff --git a/tests/test_quality_schema_linter.py b/tests/test_quality_schema_linter.py deleted file mode 100644 index 278f87fb8..000000000 --- a/tests/test_quality_schema_linter.py +++ /dev/null @@ -1,35 +0,0 @@ -import datacontract.lint.resolve as resolve -from datacontract.lint.linters.quality_schema_linter import QualityUsesSchemaLinter -from datacontract.model.run import Check - - -def construct_error_check(msg: str) -> Check: - return Check( - type="lint", - name="Linter 'Quality check(s) use model'", - result="warning", - engine="datacontract", - reason=msg, - ) - - -success_check = Check(type="lint", name="Linter 'Quality check(s) use model'", result="passed", engine="datacontract") - -data_contract_file = "fixtures/lint/datacontract_quality_schema.yaml" - - -def test_lint_correct_sodacl(): - base_contract_sodacl = resolve.resolve_data_contract_from_location(data_contract_file) - result = QualityUsesSchemaLinter().lint(base_contract_sodacl) - assert result == [success_check] - - -def test_lint_incorrect_sodacl(): - base_contract_sodacl = resolve.resolve_data_contract_from_location(data_contract_file) - incorrect_contract = base_contract_sodacl.model_copy(deep=True) - incorrect_contract.quality.specification = """ - checks for tests: - - freshness(column_1) < 1d - """ - result = QualityUsesSchemaLinter().lint(incorrect_contract) - assert result == [construct_error_check("Quality check on unknown model 'tests'")] From ce95fa5aca810121288533f5d94ea28ae3d336f6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 19 Apr 2025 16:44:17 +0200 Subject: [PATCH 11/60] chore: remove clevercsv dependency from pyproject.toml --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2f546356e..22f1c7da7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,6 @@ bigquery = [ ] csv = [ - "clevercsv >= 0.8.2", "pandas >= 2.0.0", ] From a44b0f550427dd8bb6a5df531b38da45c06dc370 Mon Sep 17 00:00:00 2001 From: Stefan McKinnon Edwards Date: Sat, 19 Apr 2025 17:08:31 +0200 Subject: [PATCH 12/60] deep nesting of json objects in duckdb (#681) * deep nesting of json objects in duckdb --- .../soda/connections/duckdb_connection.py | 58 ++++++++++++-- datacontract/export/csv_type_converter.py | 36 --------- datacontract/export/duckdb_type_converter.py | 57 ++++++++++++++ .../local-json/data/nested_types.json | 28 +++++++ tests/test_duckdb_json.py | 77 +++++++++++++++++++ 5 files changed, 214 insertions(+), 42 deletions(-) delete mode 100644 datacontract/export/csv_type_converter.py create mode 100644 datacontract/export/duckdb_type_converter.py create mode 100644 tests/fixtures/local-json/data/nested_types.json create mode 100644 tests/test_duckdb_json.py diff --git a/datacontract/engines/soda/connections/duckdb_connection.py b/datacontract/engines/soda/connections/duckdb_connection.py index f05fce2f6..f05f1c762 100644 --- a/datacontract/engines/soda/connections/duckdb_connection.py +++ b/datacontract/engines/soda/connections/duckdb_connection.py @@ -1,10 +1,10 @@ import os -from typing import Any +from typing import Any, Dict import duckdb -from datacontract.export.csv_type_converter import convert_to_duckdb_csv_type -from datacontract.model.data_contract_specification import DataContractSpecification, Server +from datacontract.export.duckdb_type_converter import convert_to_duckdb_csv_type, convert_to_duckdb_json_type +from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model, Server from datacontract.model.run import Run @@ -12,8 +12,8 @@ def get_duckdb_connection( data_contract: DataContractSpecification, server: Server, run: Run, - duckdb_connection: duckdb.DuckDBPyConnection = None, -): + duckdb_connection: duckdb.DuckDBPyConnection | None = None, +) -> duckdb.DuckDBPyConnection: if duckdb_connection is None: con = duckdb.connect(database=":memory:") else: @@ -43,9 +43,16 @@ def get_duckdb_connection( json_format = "newline_delimited" elif server.delimiter == "array": json_format = "array" - con.sql(f""" + columns = to_json_types(model) + if columns is None: + con.sql(f""" CREATE VIEW "{model_name}" AS SELECT * FROM read_json_auto('{model_path}', format='{json_format}', hive_partitioning=1); """) + else: + con.sql( + f"""CREATE VIEW "{model_name}" AS SELECT * FROM read_json_auto('{model_path}', format='{json_format}', columns={columns}, hive_partitioning=1);""" + ) + add_nested_views(con, model_name, model.fields) elif server.format == "parquet": con.sql(f""" CREATE VIEW "{model_name}" AS SELECT * FROM read_parquet('{model_path}', hive_partitioning=1); @@ -77,6 +84,45 @@ def to_csv_types(model) -> dict[Any, str | None] | None: return columns +def to_json_types(model: Model) -> dict[Any, str | None] | None: + if model is None: + return None + columns = {} + for field_name, field in model.fields.items(): + columns[field_name] = convert_to_duckdb_json_type(field) + return columns + + +def add_nested_views(con: duckdb.DuckDBPyConnection, model_name: str, fields: Dict[str, Field] | None): + model_name = model_name.strip('"') + if fields is None: + return + for field_name, field in fields.items(): + if field.type is None or field.type.lower() not in ["array", "object"]: + continue + field_type = field.type.lower() + if field_type == "array" and field.items is None: + continue + elif field_type == "object" and field.fields is None: + continue + + nested_model_name = f"{model_name}__{field_name}" + max_depth = 2 if field_type == "array" else 1 + + ## if parent field is not required, the nested objects may respolve + ## to a row of NULLs -- but if the objects themselves have required + ## fields, this will fail the check. + where = "" if field.required else f" WHERE {field_name} IS NOT NULL" + con.sql(f""" + CREATE VIEW IF NOT EXISTS "{nested_model_name}" AS + SELECT unnest({field_name}, max_depth := {max_depth}) as {field_name} FROM "{model_name}" {where} + """) + if field_type == "array": + add_nested_views(con, nested_model_name, field.items.fields) + elif field_type == "object": + add_nested_views(con, nested_model_name, field.fields) + + def setup_s3_connection(con, server): s3_region = os.getenv("DATACONTRACT_S3_REGION") s3_access_key_id = os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID") diff --git a/datacontract/export/csv_type_converter.py b/datacontract/export/csv_type_converter.py deleted file mode 100644 index 79dfe1668..000000000 --- a/datacontract/export/csv_type_converter.py +++ /dev/null @@ -1,36 +0,0 @@ -# https://duckdb.org/docs/data/csv/overview.html -# ['SQLNULL', 'BOOLEAN', 'BIGINT', 'DOUBLE', 'TIME', 'DATE', 'TIMESTAMP', 'VARCHAR'] -def convert_to_duckdb_csv_type(field) -> None | str: - type = field.type - if type is None: - return "VARCHAR" - if type.lower() in ["string", "varchar", "text"]: - return "VARCHAR" - if type.lower() in ["timestamp", "timestamp_tz"]: - return "TIMESTAMP" - if type.lower() in ["timestamp_ntz"]: - return "TIMESTAMP" - if type.lower() in ["date"]: - return "DATE" - if type.lower() in ["time"]: - return "TIME" - if type.lower() in ["number", "decimal", "numeric"]: - # precision and scale not supported by data contract - return "VARCHAR" - if type.lower() in ["float", "double"]: - return "DOUBLE" - if type.lower() in ["integer", "int", "long", "bigint"]: - return "BIGINT" - if type.lower() in ["boolean"]: - return "BOOLEAN" - if type.lower() in ["object", "record", "struct"]: - # not supported in CSV - return "VARCHAR" - if type.lower() in ["bytes"]: - # not supported in CSV - return "VARCHAR" - if type.lower() in ["array"]: - return "VARCHAR" - if type.lower() in ["null"]: - return "SQLNULL" - return "VARCHAR" diff --git a/datacontract/export/duckdb_type_converter.py b/datacontract/export/duckdb_type_converter.py new file mode 100644 index 000000000..cf57398e1 --- /dev/null +++ b/datacontract/export/duckdb_type_converter.py @@ -0,0 +1,57 @@ +from typing import Dict + +from datacontract.model.data_contract_specification import Field + + +# https://duckdb.org/docs/data/csv/overview.html +# ['SQLNULL', 'BOOLEAN', 'BIGINT', 'DOUBLE', 'TIME', 'DATE', 'TIMESTAMP', 'VARCHAR'] +def convert_to_duckdb_csv_type(field) -> None | str: + datacontract_type = field.type + if datacontract_type is None: + return "VARCHAR" + if datacontract_type.lower() in ["string", "varchar", "text"]: + return "VARCHAR" + if datacontract_type.lower() in ["timestamp", "timestamp_tz"]: + return "TIMESTAMP" + if datacontract_type.lower() in ["timestamp_ntz"]: + return "TIMESTAMP" + if datacontract_type.lower() in ["date"]: + return "DATE" + if datacontract_type.lower() in ["time"]: + return "TIME" + if datacontract_type.lower() in ["number", "decimal", "numeric"]: + # precision and scale not supported by data contract + return "VARCHAR" + if datacontract_type.lower() in ["float", "double"]: + return "DOUBLE" + if datacontract_type.lower() in ["integer", "int", "long", "bigint"]: + return "BIGINT" + if datacontract_type.lower() in ["boolean"]: + return "BOOLEAN" + if datacontract_type.lower() in ["object", "record", "struct"]: + # not supported in CSV + return "VARCHAR" + if datacontract_type.lower() in ["bytes"]: + # not supported in CSV + return "VARCHAR" + if datacontract_type.lower() in ["array"]: + return "VARCHAR" + if datacontract_type.lower() in ["null"]: + return "SQLNULL" + return "VARCHAR" + + +def convert_to_duckdb_json_type(field: Field) -> None | str: + datacontract_type = field.type + if datacontract_type is None: + return "VARCHAR" + if datacontract_type.lower() in ["array"]: + return convert_to_duckdb_json_type(field.items) + "[]" # type: ignore + if datacontract_type.lower() in ["object", "record", "struct"]: + return convert_to_duckdb_object(field.fields) + return convert_to_duckdb_csv_type(field) + + +def convert_to_duckdb_object(fields: Dict[str, Field]): + columns = [f'"{x[0]}" {convert_to_duckdb_json_type(x[1])}' for x in fields.items()] + return f"STRUCT({', '.join(columns)})" diff --git a/tests/fixtures/local-json/data/nested_types.json b/tests/fixtures/local-json/data/nested_types.json new file mode 100644 index 000000000..2354407ea --- /dev/null +++ b/tests/fixtures/local-json/data/nested_types.json @@ -0,0 +1,28 @@ +[ + { + "id": 1, + "tags": [ + { + "foo": "bar", + "arr": [ 1, 2, 3 ] + }, + { + "foo": "lap", + "arr": [ 4 ] + } + ], + "name": { + "first": "John", + "last": "Doe" + } + }, + { + "id": 2, + "tags": [ + { + "foo": "zap", + "arr": [ ] + } + ] + } +] \ No newline at end of file diff --git a/tests/test_duckdb_json.py b/tests/test_duckdb_json.py new file mode 100644 index 000000000..e7e34f70f --- /dev/null +++ b/tests/test_duckdb_json.py @@ -0,0 +1,77 @@ +from datacontract.engines.soda.connections.duckdb_connection import get_duckdb_connection +from datacontract.lint import resolve +from datacontract.model.run import Run + + +def test_nested_json(): + data_contract_str = """ +dataContractSpecification: 1.1.0 +id: "61111-0002" +info: + title: Sample data of nested types + version: 1.0.0 +servers: + sample: + type: local + path: ./fixtures/local-json/data/nested_types.json + format: json + delimiter: array +models: + sample_data: + type: object + fields: + id: + type: integer + required: true + tags: + type: array + required: true + items: + type: object + fields: + foo: + type: string + required: true + arr: + type: array + items: + type: integer + name: + type: object + required: false + fields: + first: + type: string + last: + type: string + """ + data_contract = resolve.resolve_data_contract(data_contract_str=data_contract_str) + run = Run.create_run() + con = get_duckdb_connection(data_contract, data_contract.servers["sample"], run) + tbl = con.table("sample_data") + assert tbl.columns == ["id", "tags", "name"] + assert [x[1].lower() for x in tbl.description] == ["number", "list", "dict"] + # test that duckdb correct unpacked the nested structures. + assert tbl.fetchone() == ( + 1, + [{"foo": "bar", "arr": [1, 2, 3]}, {"foo": "lap", "arr": [4]}], + {"first": "John", "last": "Doe"}, + ) + assert tbl.fetchone() == (2, [{"foo": "zap", "arr": []}], None) + assert tbl.fetchone() is None + ## check nested tables + tbl = con.table("sample_data__tags") + assert tbl.columns == ["foo", "arr"] + assert [x[1].lower() for x in tbl.description] == ["string", "list"] + assert tbl.fetchone() == ("bar", [1, 2, 3]) + assert tbl.fetchone() == ("lap", [4]) + assert tbl.fetchone() == ("zap", []) + assert tbl.fetchone() is None + tbl = con.table("sample_data__tags__arr") + assert tbl.columns == ["arr"] + assert [x[1].lower() for x in tbl.description] == ["number"] + assert tbl.fetchall() == [(1,), (2,), (3,), (4,)] + tbl = con.table("sample_data__name") + assert tbl.columns == ["first", "last"] + assert [x[1].lower() for x in tbl.description] == ["string", "string"] + assert tbl.fetchall() == [("John", "Doe")] From 388deb3f096d7c47107dfcd60bcc67394102e785 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 19 Apr 2025 17:10:48 +0200 Subject: [PATCH 13/60] chore: update changelog to include datacontract test with DuckDB --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5dbd4990..889c217ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `datacontract test` with DuckDB: Deep nesting of json objects in duckdb (#681) + ### Changed - `datacontract import --format csv` produces more descriptive output. Replaced From 91d8dfe2cd6adec79c8df571b00c15da5135889c Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 19 Apr 2025 17:21:41 +0200 Subject: [PATCH 14/60] chore: update version to 0.10.24 --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- release | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 889c217ec..fcda2ad78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## [0.10.24] - 2025-04-19 + ### Added - `datacontract test` with DuckDB: Deep nesting of json objects in duckdb (#681) diff --git a/pyproject.toml b/pyproject.toml index 22f1c7da7..0d64ec031 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datacontract-cli" -version = "0.10.23" +version = "0.10.24" description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library." readme = "README.md" authors = [ diff --git a/release b/release index 7c3ed60f1..ec2b24890 100755 --- a/release +++ b/release @@ -1,6 +1,12 @@ #!/bin/bash set -e +# Release steps: +# 1. Update release version in pyproject.toml +# 2. Update CHANGELOG.md header +# 3. Run ./release +# 4. Update release notes in Github + # pip install toml-cli VERSION=$(toml get --toml-path pyproject.toml project.version) TAG_VERSION=v$VERSION From 8af13e1a4baa6f215d582d7cba03e96c362e2ddb Mon Sep 17 00:00:00 2001 From: Emmanuel Ferdman Date: Sat, 19 Apr 2025 19:43:04 +0300 Subject: [PATCH 15/60] Resolve FastAPI deprecation warning for example fields (#730) Signed-off-by: Emmanuel Ferdman --- datacontract/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datacontract/api.py b/datacontract/api.py index 422016083..89fe711ab 100644 --- a/datacontract/api.py +++ b/datacontract/api.py @@ -162,7 +162,7 @@ async def test( server: Annotated[ str | None, Query( - example="production", + examples=["production"], description="The server name to test. Optional, if there is only one server.", ), ] = None, @@ -191,7 +191,7 @@ async def lint( schema: Annotated[ str | None, Query( - example="https://datacontract.com/datacontract.schema.json", + examples=["https://datacontract.com/datacontract.schema.json"], description="The schema to use for validation. This must be a URL.", ), ] = None, @@ -220,7 +220,7 @@ def export( server: Annotated[ str | None, Query( - example="production", + examples=["production"], description="The server name to export. Optional, if there is only one server.", ), ] = None, From 50ce8a537b3a9f790106f7eb9f119dfc08ba34d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Apr 2025 18:35:54 +0200 Subject: [PATCH 16/60] Update databricks-sdk requirement from <0.50.0 to <0.51.0 (#733) Updates the requirements on [databricks-sdk](https://github.com/databricks/databricks-sdk-py) to permit the latest version. - [Release notes](https://github.com/databricks/databricks-sdk-py/releases) - [Changelog](https://github.com/databricks/databricks-sdk-py/blob/main/CHANGELOG.md) - [Commits](https://github.com/databricks/databricks-sdk-py/compare/v0.0.1...v0.50.0) --- updated-dependencies: - dependency-name: databricks-sdk dependency-version: 0.50.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0d64ec031..5b112e03e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ databricks = [ "soda-core-spark-df>=3.3.20,<3.6.0", "soda-core-spark[databricks]>=3.3.20,<3.6.0", "databricks-sql-connector>=3.7.0,<4.1.0", - "databricks-sdk<0.50.0", + "databricks-sdk<0.51.0", ] iceberg = [ From d8943af787d87652608a46281149bcfbc338f967 Mon Sep 17 00:00:00 2001 From: gguglielmoni <92776454+gilles-guglielmoni@users.noreply.github.com> Date: Tue, 29 Apr 2025 08:43:30 +0200 Subject: [PATCH 17/60] In export to odcs, keep nested structures in the schema (currently flattened when exporting cds schema) (#740) Co-authored-by: Gilles Guglielmoni --- datacontract/export/odcs_v3_exporter.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/datacontract/export/odcs_v3_exporter.py b/datacontract/export/odcs_v3_exporter.py index 6520befe7..f7f23132d 100644 --- a/datacontract/export/odcs_v3_exporter.py +++ b/datacontract/export/odcs_v3_exporter.py @@ -210,6 +210,16 @@ def to_physical_type(type: str) -> str | None: def to_property(field_name: str, field: Field) -> dict: property = {"name": field_name} + if field.fields: + properties = [] + for field_name_, field_ in field.fields.items(): + property_ = to_property(field_name_, field_) + properties.append(property_) + property["properties"]=properties + if field.items: + items = to_property(field_name, field.items) + del items["name"] + property["items"]=items if field.title is not None: property["businessName"] = field.title if field.type is not None: From a6ce1f64a989402d6b5abf517578283938d96e07 Mon Sep 17 00:00:00 2001 From: Simon Auger <44462523+sauger92@users.noreply.github.com> Date: Mon, 5 May 2025 09:31:49 +0200 Subject: [PATCH 18/60] Include custom quality rules in great expectations export (#738) * feat: support quality definition at field and model level in great expectation export * fix: test_export_great_expectation failing test --------- Co-authored-by: Simon Auger --- .../export/great_expectations_converter.py | 51 ++++++++- .../datacontract_quality_column.yaml | 35 ++++++ .../datacontract_quality_yaml.yaml | 26 +++++ tests/test_export_great_expectations.py | 103 ++++++++++++++++++ 4 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/great-expectations/datacontract_quality_column.yaml create mode 100644 tests/fixtures/great-expectations/datacontract_quality_yaml.yaml diff --git a/datacontract/export/great_expectations_converter.py b/datacontract/export/great_expectations_converter.py index a8202ea9b..d49129f7d 100644 --- a/datacontract/export/great_expectations_converter.py +++ b/datacontract/export/great_expectations_converter.py @@ -19,6 +19,7 @@ from datacontract.export.sql_type_converter import convert_to_sql_type from datacontract.model.data_contract_specification import ( DataContractSpecification, + DeprecatedQuality, Field, Quality, ) @@ -91,8 +92,14 @@ def to_great_expectations( model_key=model_key, contract_version=data_contract_spec.info.version ) model_value = data_contract_spec.models.get(model_key) - quality_checks = get_quality_checks(data_contract_spec.quality) + + # Support for Deprecated Quality + quality_checks = get_deprecated_quality_checks(data_contract_spec.quality) + + expectations.extend(get_quality_checks(model_value.quality)) + expectations.extend(model_to_expectations(model_value.fields, engine, sql_server_type)) + expectations.extend(checks_to_expectations(quality_checks, model_key)) model_expectation_suite = to_suite(expectations, expectation_suite_name) @@ -135,6 +142,7 @@ def model_to_expectations(fields: Dict[str, Field], engine: str | None, sql_serv add_column_order_exp(fields, expectations) for field_name, field in fields.items(): add_field_expectations(field_name, field, expectations, engine, sql_server_type) + expectations.extend(get_quality_checks(field.quality, field_name)) return expectations @@ -173,6 +181,8 @@ def add_field_expectations( expectations.append(to_column_length_exp(field_name, field.minLength, field.maxLength)) if field.minimum is not None or field.maximum is not None: expectations.append(to_column_min_max_exp(field_name, field.minimum, field.maximum)) + if field.enum is not None and len(field.enum) != 0: + expectations.append(to_column_enum_exp(field_name, field.enum)) return expectations @@ -266,7 +276,24 @@ def to_column_min_max_exp(field_name, minimum, maximum) -> Dict[str, Any]: } -def get_quality_checks(quality: Quality) -> Dict[str, Any]: +def to_column_enum_exp(field_name, enum_list: List[str]) -> Dict[str, Any]: + """Creates a expect_column_values_to_be_in_set expectation. + + Args: + field_name (str): The name of the field. + enum_list (Set[str]): enum list of value. + + Returns: + Dict[str, Any]: Column value in set expectation. + """ + return { + "expectation_type": "expect_column_values_to_be_in_set", + "kwargs": {"column": field_name, "value_set": enum_list}, + "meta": {}, + } + + +def get_deprecated_quality_checks(quality: DeprecatedQuality) -> Dict[str, Any]: """Retrieves quality checks defined in a data contract. Args: @@ -288,6 +315,26 @@ def get_quality_checks(quality: Quality) -> Dict[str, Any]: return quality_specification +def get_quality_checks(qualities: List[Quality], field_name: str | None = None) -> List[Dict[str, Any]]: + """Retrieves quality checks defined in a data contract. + + Args: + qualities (List[Quality]): List of quality object from the model specification. + field_name (str | None): field name if the quality list is attached to a specific field + + Returns: + Dict[str, Any]: Dictionary of quality checks. + """ + quality_specification = [] + for quality in qualities: + if quality is not None and quality.engine is not None and quality.engine.lower() == "great-expectations": + ge_expectation = quality.implementation + if field_name is not None: + ge_expectation["column"] = field_name + quality_specification.append(ge_expectation) + return quality_specification + + def checks_to_expectations(quality_checks: Dict[str, Any], model_key: str) -> List[Dict[str, Any]]: """Converts quality checks to a list of expectations. diff --git a/tests/fixtures/great-expectations/datacontract_quality_column.yaml b/tests/fixtures/great-expectations/datacontract_quality_column.yaml new file mode 100644 index 000000000..dcd59ff07 --- /dev/null +++ b/tests/fixtures/great-expectations/datacontract_quality_column.yaml @@ -0,0 +1,35 @@ +dataContractSpecification: 1.1.0 +id: my-data-contract-id +info: + title: Orders Unit Test + version: 1.1.1 + owner: checkout + description: The orders data contract + contact: + email: team-orders@example.com + url: https://wiki.example.com/teams/checkout +models: + orders: + description: test + fields: + id: + description: Unique identifier for each alert. + type: string + required: true + primaryKey: true + unique: true + type: + description: The type of alert that has fired. + type: string + required: true + enum: [ "A", "B", "C", "D", "E" ] + quality: + - type: custom + engine: great-expectations + description: "Accepted Values for type" + implementation: + expectation_type: expect_column_value_lengths_to_equal + kwargs: + value: 1 + meta: + notes: "Ensures that column length is 1." \ No newline at end of file diff --git a/tests/fixtures/great-expectations/datacontract_quality_yaml.yaml b/tests/fixtures/great-expectations/datacontract_quality_yaml.yaml new file mode 100644 index 000000000..a122019c5 --- /dev/null +++ b/tests/fixtures/great-expectations/datacontract_quality_yaml.yaml @@ -0,0 +1,26 @@ +dataContractSpecification: 0.9.1 +id: my-data-contract-id + +info: + title: Orders Unit Test + version: 1.0.0 + owner: checkout + description: The orders data contract + contact: + email: team-orders@example.com + url: https://wiki.example.com/teams/checkout +models: + orders: + description: test + fields: + order_id: + type: string + required: true + quality: + - type: custom + engine: great-expectations + implementation: + expectation_type: expect_table_row_count_to_be_between + kwargs: + min_value: 10 + meta: {} diff --git a/tests/test_export_great_expectations.py b/tests/test_export_great_expectations.py index 223f4e6ef..d66573832 100644 --- a/tests/test_export_great_expectations.py +++ b/tests/test_export_great_expectations.py @@ -88,6 +88,22 @@ def data_contract_great_expectations_quality_file() -> DataContractSpecification ) +@pytest.fixture +def data_contract_great_expectations_quality_yaml() -> DataContractSpecification: + return resolve.resolve_data_contract_from_location( + "./fixtures/great-expectations/datacontract_quality_yaml.yaml", + inline_quality=True, + ) + + +@pytest.fixture +def data_contract_great_expectations_quality_column() -> DataContractSpecification: + return resolve.resolve_data_contract_from_location( + "./fixtures/great-expectations/datacontract_quality_column.yaml", + inline_quality=True, + ) + + @pytest.fixture def expected_json_suite() -> Dict[str, Any]: return { @@ -119,6 +135,66 @@ def expected_json_suite() -> Dict[str, Any]: } +@pytest.fixture +def expected_json_suite_table_quality() -> Dict[str, Any]: + return { + "data_asset_type": "null", + "expectation_suite_name": "orders.1.0.0", + "expectations": [ + {"expectation_type": "expect_table_row_count_to_be_between", "kwargs": {"min_value": 10}, "meta": {}}, + { + "expectation_type": "expect_table_columns_to_match_ordered_list", + "kwargs": {"column_list": ["order_id"]}, + "meta": {}, + }, + { + "expectation_type": "expect_column_values_to_be_of_type", + "kwargs": {"column": "order_id", "type_": "string"}, + "meta": {}, + }, + ], + "meta": {}, + } + + +@pytest.fixture +def expected_json_suite_with_enum() -> Dict[str, Any]: + return { + "data_asset_type": "null", + "expectation_suite_name": "orders.1.1.1", + "expectations": [ + { + "expectation_type": "expect_table_columns_to_match_ordered_list", + "kwargs": {"column_list": ["id", "type"]}, + "meta": {}, + }, + { + "expectation_type": "expect_column_values_to_be_of_type", + "kwargs": {"column": "id", "type_": "string"}, + "meta": {}, + }, + {"expectation_type": "expect_column_values_to_be_unique", "kwargs": {"column": "id"}, "meta": {}}, + { + "expectation_type": "expect_column_values_to_be_of_type", + "kwargs": {"column": "type", "type_": "string"}, + "meta": {}, + }, + { + "expectation_type": "expect_column_values_to_be_in_set", + "kwargs": {"column": "type", "value_set": ["A", "B", "C", "D", "E"]}, + "meta": {}, + }, + { + "expectation_type": "expect_column_value_lengths_to_equal", + "kwargs": {"value": 1}, + "meta": {"notes": "Ensures that column length is 1."}, + "column": "type", + }, + ], + "meta": {}, + } + + @pytest.fixture def expected_spark_engine() -> Dict[str, Any]: return { @@ -290,6 +366,11 @@ def test_to_great_expectation(data_contract_basic: DataContractSpecification): "kwargs": {"column": "order_status", "type_": "text"}, "meta": {}, }, + { + "expectation_type": "expect_column_values_to_be_in_set", + "kwargs": {"column": "order_status", "value_set": ["pending", "shipped", "delivered"]}, + "meta": {}, + }, ], "meta": {}, } @@ -602,3 +683,25 @@ def test_to_great_expectation_missing_quality_json_file(): assert False except DataContractException as dataContractException: assert dataContractException.reason == "Cannot resolve reference ./fixtures/great-expectations/missing.json" + + +def test_to_great_expectation_quality_yaml( + data_contract_great_expectations_quality_yaml: DataContractSpecification, + expected_json_suite_table_quality: Dict[str, Any], +): + """ + Test with Quality definition in a model quality list + """ + result = to_great_expectations(data_contract_great_expectations_quality_yaml, "orders") + assert result == json.dumps(expected_json_suite_table_quality, indent=2) + + +def test_to_great_expectation_quality_column( + data_contract_great_expectations_quality_column: DataContractSpecification, + expected_json_suite_with_enum: Dict[str, Any], +): + """ + Test with quality definition in a field quality list + """ + result = to_great_expectations(data_contract_great_expectations_quality_column, "orders") + assert result == json.dumps(expected_json_suite_with_enum, indent=2) From 869a578df10ff1eb2d45881b947d9d16dcaaac75 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 5 May 2025 20:14:52 +0200 Subject: [PATCH 19/60] fix: update help text for source option in cli.py --- datacontract/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datacontract/cli.py b/datacontract/cli.py index 7af97463e..c4f6ad29d 100644 --- a/datacontract/cli.py +++ b/datacontract/cli.py @@ -244,7 +244,7 @@ def import_( ] = None, source: Annotated[ Optional[str], - typer.Option(help="The path to the file or Glue Database that should be imported."), + typer.Option(help="The path to the file that should be imported."), ] = None, dialect: Annotated[ Optional[str], From 33165b863a6665c707e359f9193e56a9f2736d82 Mon Sep 17 00:00:00 2001 From: "Dr. Simon Harrer" Date: Tue, 6 May 2025 10:37:22 +0200 Subject: [PATCH 20/60] Use pip package for dcs (#705) * Use pip package for dcs --- .../model/data_contract_specification.py | 327 ------------------ .../data_contract_specification/__init__.py | 1 + pyproject.toml | 4 + 3 files changed, 5 insertions(+), 327 deletions(-) delete mode 100644 datacontract/model/data_contract_specification.py create mode 100644 datacontract/model/data_contract_specification/__init__.py diff --git a/datacontract/model/data_contract_specification.py b/datacontract/model/data_contract_specification.py deleted file mode 100644 index 0a3c1cd01..000000000 --- a/datacontract/model/data_contract_specification.py +++ /dev/null @@ -1,327 +0,0 @@ -import os -from typing import Any, Dict, List - -import pydantic as pyd -import yaml - -DATACONTRACT_TYPES = [ - "string", - "text", - "varchar", - "number", - "decimal", - "numeric", - "int", - "integer", - "long", - "bigint", - "float", - "double", - "boolean", - "timestamp", - "timestamp_tz", - "timestamp_ntz", - "date", - "array", - "bytes", - "object", - "record", - "struct", - "null", -] - - -class Contact(pyd.BaseModel): - name: str | None = None - url: str | None = None - email: str | None = None - - model_config = pyd.ConfigDict( - extra="allow", - ) - - -class ServerRole(pyd.BaseModel): - name: str | None = None - description: str | None = None - model_config = pyd.ConfigDict( - extra="allow", - ) - - -class Server(pyd.BaseModel): - type: str | None = None - description: str | None = None - environment: str | None = None - format: str | None = None - project: str | None = None - dataset: str | None = None - path: str | None = None - delimiter: str | None = None - endpointUrl: str | None = None - location: str | None = None - account: str | None = None - database: str | None = None - schema_: str | None = pyd.Field(default=None, alias="schema") - host: str | None = None - port: int | None = None - catalog: str | None = None - topic: str | None = None - http_path: str | None = None # Use ENV variable - token: str | None = None # Use ENV variable - dataProductId: str | None = None - outputPortId: str | None = None - driver: str | None = None - storageAccount: str | None = None - roles: List[ServerRole] = None - - model_config = pyd.ConfigDict( - extra="allow", - ) - - -class Terms(pyd.BaseModel): - usage: str | None = None - limitations: str | None = None - billing: str | None = None - noticePeriod: str | None = None - description: str | None = None - - model_config = pyd.ConfigDict( - extra="allow", - ) - - -class Definition(pyd.BaseModel): - domain: str | None = None - name: str | None = None - title: str | None = None - description: str | None = None - type: str | None = None - enum: List[str] = [] - format: str | None = None - minLength: int | None = None - maxLength: int | None = None - pattern: str | None = None - minimum: int | None = None - exclusiveMinimum: int | None = None - maximum: int | None = None - exclusiveMaximum: int | None = None - pii: bool | None = None - classification: str | None = None - fields: Dict[str, "Field"] = {} - items: "Field" = None - tags: List[str] = [] - links: Dict[str, str] = {} - example: str | None = None - examples: List[Any] | None = None - - model_config = pyd.ConfigDict( - extra="allow", - ) - - -class Quality(pyd.BaseModel): - type: str | None = None - description: str | None = None - query: str | None = None - dialect: str | None = None - mustBe: int | None = None - mustNotBe: int | None = None - mustBeGreaterThan: int | None = None - mustBeGreaterThanOrEqualTo: int | None = None - mustBeLessThan: int | None = None - mustBeLessThanOrEqualTo: int | None = None - mustBeBetween: List[int] = None - mustNotBeBetween: List[int] = None - engine: str | None = None - implementation: str | Dict[str, Any] | None = None - - model_config = pyd.ConfigDict( - extra="allow", - ) - - -class Field(pyd.BaseModel): - ref: str = pyd.Field(default=None, alias="$ref") - title: str | None = None - type: str | None = None - format: str | None = None - required: bool | None = None - primary: bool = pyd.Field( - default=None, - deprecated="Removed in Data Contract Specification v1.1.0. Use primaryKey instead.", - ) - primaryKey: bool | None = None - unique: bool | None = None - references: str | None = None - description: str | None = None - pii: bool | None = None - classification: str | None = None - pattern: str | None = None - minLength: int | None = None - maxLength: int | None = None - minimum: int | None = None - exclusiveMinimum: int | None = None - maximum: int | None = None - exclusiveMaximum: int | None = None - enum: List[str] | None = [] - tags: List[str] | None = [] - links: Dict[str, str] = {} - fields: Dict[str, "Field"] = {} - items: "Field" = None - keys: "Field" = None - values: "Field" = None - precision: int | None = None - scale: int | None = None - example: Any | None = pyd.Field( - default=None, - deprecated="Removed in Data Contract Specification v1.1.0. Use examples instead.", - ) - examples: List[Any] | None = None - quality: List[Quality] | None = [] - config: Dict[str, Any] | None = None - - model_config = pyd.ConfigDict( - extra="allow", - ) - - -class Model(pyd.BaseModel): - description: str | None = None - type: str | None = None - namespace: str | None = None - title: str | None = None - fields: Dict[str, Field] = {} - quality: List[Quality] | None = [] - primaryKey: List[str] | None = [] - examples: List[Any] | None = None - config: Dict[str, Any] = None - tags: List[str] | None = None - - model_config = pyd.ConfigDict( - extra="allow", - ) - - -class Info(pyd.BaseModel): - title: str | None = None - version: str | None = None - status: str | None = None - description: str | None = None - owner: str | None = None - contact: Contact | None = None - - model_config = pyd.ConfigDict( - extra="allow", - ) - - -class Example(pyd.BaseModel): - type: str | None = None - description: str | None = None - model: str | None = None - data: str | object = None - - -# Deprecated Quality class -class DeprecatedQuality(pyd.BaseModel): - type: str | None = None - specification: str | object = None - - -class Availability(pyd.BaseModel): - description: str | None = None - percentage: str | None = None - - -class Retention(pyd.BaseModel): - description: str | None = None - period: str | None = None - unlimited: bool | None = None - timestampField: str | None = None - - -class Latency(pyd.BaseModel): - description: str | None = None - threshold: str | None = None - sourceTimestampField: str | None = None - processedTimestampField: str | None = None - - -class Freshness(pyd.BaseModel): - description: str | None = None - threshold: str | None = None - timestampField: str | None = None - - -class Frequency(pyd.BaseModel): - description: str | None = None - type: str | None = None - interval: str | None = None - cron: str | None = None - - -class Support(pyd.BaseModel): - description: str | None = None - time: str | None = None - responseTime: str | None = None - - -class Backup(pyd.BaseModel): - description: str | None = None - interval: str | None = None - cron: str | None = None - recoveryTime: str | None = None - recoveryPoint: str | None = None - - -class ServiceLevel(pyd.BaseModel): - availability: Availability | None = None - retention: Retention | None = None - latency: Latency | None = None - freshness: Freshness | None = None - frequency: Frequency | None = None - support: Support | None = None - backup: Backup | None = None - - -class DataContractSpecification(pyd.BaseModel): - dataContractSpecification: str | None = None - id: str | None = None - info: Info | None = None - servers: Dict[str, Server] = {} - terms: Terms | None = None - models: Dict[str, Model] = {} - definitions: Dict[str, Definition] = {} - examples: List[Example] = pyd.Field( - default_factory=list, - deprecated="Removed in Data Contract Specification " "v1.1.0. Use models.examples instead.", - ) - quality: DeprecatedQuality | None = pyd.Field( - default=None, - deprecated="Removed in Data Contract Specification v1.1.0. Use " "model-level and field-level quality instead.", - ) - servicelevels: ServiceLevel | None = None - links: Dict[str, str] = {} - tags: List[str] = [] - - @classmethod - def from_file(cls, file): - if not os.path.exists(file): - raise FileNotFoundError(f"The file '{file}' does not exist.") - with open(file, "r") as file: - file_content = file.read() - return DataContractSpecification.from_string(file_content) - - @classmethod - def from_string(cls, data_contract_str): - data = yaml.safe_load(data_contract_str) - return DataContractSpecification(**data) - - def to_yaml(self): - return yaml.safe_dump( - self.model_dump(mode="json", exclude_defaults=True, exclude_none=True, by_alias=True), - sort_keys=False, - allow_unicode=True, - ) diff --git a/datacontract/model/data_contract_specification/__init__.py b/datacontract/model/data_contract_specification/__init__.py new file mode 100644 index 000000000..2141d2da0 --- /dev/null +++ b/datacontract/model/data_contract_specification/__init__.py @@ -0,0 +1 @@ +from datacontract_specification.model import * diff --git a/pyproject.toml b/pyproject.toml index 5b112e03e..c11407d07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "boto3>=1.34.41,<2.0.0", "Jinja2>=3.1.5,<4.0.0", "jinja_partials>=0.2.1,<1.0.0", + "datacontract-specification >= 1.1.0,<2.0.0", ] [project.optional-dependencies] @@ -155,3 +156,6 @@ line-length = 120 extend-select = [ "I", # re-order imports in alphabetic order ] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401", "F403"] From 6dc0a7d4af5497901b08c4831f3defc760006b14 Mon Sep 17 00:00:00 2001 From: "Dr. Simon Harrer" Date: Wed, 7 May 2025 08:00:26 +0200 Subject: [PATCH 21/60] Use Open Data Contract Standard pip module (#746) - Extracted the DataContractSpecification and the OpenDataContractSpecification in separate pip modules and use them in the CLI. - `datacontract import --format excel`: Import from Excel template https://github.com/datacontract/open-data-contract-standard-excel-template (#742) - Resolves #745 --- CHANGELOG.md | 5 + README.md | 3 +- datacontract/export/odcs_v3_exporter.py | 293 +++--- datacontract/export/spark_converter.py | 2 +- datacontract/imports/avro_importer.py | 46 +- datacontract/imports/csv_importer.py | 4 +- datacontract/imports/excel_importer.py | 850 ++++++++++++++++++ datacontract/imports/importer.py | 6 +- datacontract/imports/importer_factory.py | 5 + datacontract/imports/odcs_v3_importer.py | 347 ++++--- datacontract/imports/protobuf_importer.py | 2 - .../lint/linters/description_linter.py | 4 +- .../lint/linters/field_reference_linter.py | 3 +- .../lint/linters/notice_period_linter.py | 4 +- .../lint/linters/valid_constraints_linter.py | 6 +- pyproject.toml | 12 +- tests/fixtures/excel/shipments-odcs.xlsx | Bin 0 -> 112519 bytes tests/fixtures/excel/shipments-odcs.yaml | 176 ++++ .../odcs_v3/adventureworks.datacontract.yml | 20 +- tests/test_export_odcs_v3.py | 14 +- tests/test_import_excel.py | 40 + 21 files changed, 1508 insertions(+), 334 deletions(-) create mode 100644 datacontract/imports/excel_importer.py create mode 100644 tests/fixtures/excel/shipments-odcs.xlsx create mode 100644 tests/fixtures/excel/shipments-odcs.yaml create mode 100644 tests/test_import_excel.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fcda2ad78..528f25945 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added +- Extracted the DataContractSpecification and the OpenDataContractSpecification in separate pip modules and use them in the CLI. +- `datacontract import --format excel`: Import from Excel + template https://github.com/datacontract/open-data-contract-standard-excel-template (#742) + ## [0.10.24] - 2025-04-19 ### Added diff --git a/README.md b/README.md index f2e24fc9d..4f14a93cc 100644 --- a/README.md +++ b/README.md @@ -1906,7 +1906,8 @@ pytest ```bash # make sure uv is installed uv python pin 3.11 -uv sync --all-extras +uv pip install -e '.[dev]' +uv run ruff check uv run pytest ``` diff --git a/datacontract/export/odcs_v3_exporter.py b/datacontract/export/odcs_v3_exporter.py index f7f23132d..479f6e0ac 100644 --- a/datacontract/export/odcs_v3_exporter.py +++ b/datacontract/export/odcs_v3_exporter.py @@ -1,6 +1,17 @@ from typing import Dict -import yaml +from open_data_contract_standard.model import ( + CustomProperty, + DataQuality, + Description, + OpenDataContractStandard, + Role, + SchemaObject, + SchemaProperty, + Server, + ServiceLevelAgreementProperty, + Support, +) from datacontract.export.exporter import Exporter from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model @@ -12,155 +23,148 @@ def export(self, data_contract, model, server, sql_server_type, export_args) -> def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str: - odcs = { - "apiVersion": "v3.0.1", - "kind": "DataContract", - "id": data_contract_spec.id, - "name": data_contract_spec.info.title, - "version": data_contract_spec.info.version, - "status": to_status(data_contract_spec.info.status), - } + result = OpenDataContractStandard( + apiVersion="v3.0.1", + kind="DataContract", + id=data_contract_spec.id, + name=data_contract_spec.info.title, + version=data_contract_spec.info.version, + status=to_status(data_contract_spec.info.status), + ) if data_contract_spec.terms is not None: - odcs["description"] = { - "purpose": data_contract_spec.terms.description.strip() + result.description = Description( + purpose=data_contract_spec.terms.description.strip() if data_contract_spec.terms.description is not None else None, - "usage": data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None, - "limitations": data_contract_spec.terms.limitations.strip() + usage=data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None, + limitations=data_contract_spec.terms.limitations.strip() if data_contract_spec.terms.limitations is not None else None, - } + ) - odcs["schema"] = [] + result.schema_ = [] for model_key, model_value in data_contract_spec.models.items(): odcs_schema = to_odcs_schema(model_key, model_value) - odcs["schema"].append(odcs_schema) + result.schema_.append(odcs_schema) if data_contract_spec.servicelevels is not None: slas = [] if data_contract_spec.servicelevels.availability is not None: slas.append( - { - "property": "generalAvailability", - "value": data_contract_spec.servicelevels.availability.description, - } + ServiceLevelAgreementProperty( + property="generalAvailability", value=data_contract_spec.servicelevels.availability.description + ) ) if data_contract_spec.servicelevels.retention is not None: - slas.append({"property": "retention", "value": data_contract_spec.servicelevels.retention.period}) + slas.append( + ServiceLevelAgreementProperty( + property="retention", value=data_contract_spec.servicelevels.retention.period + ) + ) if len(slas) > 0: - odcs["slaProperties"] = slas + result.slaProperties = slas if data_contract_spec.info.contact is not None: support = [] if data_contract_spec.info.contact.email is not None: - support.append( - { - "channel": "email", - "url": "mailto:" + data_contract_spec.info.contact.email, - } - ) + support.append(Support(channel="email", url="mailto:" + data_contract_spec.info.contact.email)) if data_contract_spec.info.contact.url is not None: - support.append( - { - "channel": "other", - "url": data_contract_spec.info.contact.url, - } - ) + support.append(Support(channel="other", url=data_contract_spec.info.contact.url)) if len(support) > 0: - odcs["support"] = support + result.support = support if data_contract_spec.servers is not None and len(data_contract_spec.servers) > 0: servers = [] for server_key, server_value in data_contract_spec.servers.items(): - server_dict = {} - server_dict["server"] = server_key - if server_value.type is not None: - server_dict["type"] = server_value.type + server = Server(server=server_key, type=server_value.type or "") + + # Set all the attributes that are not None if server_value.environment is not None: - server_dict["environment"] = server_value.environment + server.environment = server_value.environment if server_value.account is not None: - server_dict["account"] = server_value.account + server.account = server_value.account if server_value.database is not None: - server_dict["database"] = server_value.database + server.database = server_value.database if server_value.schema_ is not None: - server_dict["schema"] = server_value.schema_ + server.schema_ = server_value.schema_ if server_value.format is not None: - server_dict["format"] = server_value.format + server.format = server_value.format if server_value.project is not None: - server_dict["project"] = server_value.project + server.project = server_value.project if server_value.dataset is not None: - server_dict["dataset"] = server_value.dataset + server.dataset = server_value.dataset if server_value.path is not None: - server_dict["path"] = server_value.path + server.path = server_value.path if server_value.delimiter is not None: - server_dict["delimiter"] = server_value.delimiter + server.delimiter = server_value.delimiter if server_value.endpointUrl is not None: - server_dict["endpointUrl"] = server_value.endpointUrl + server.endpointUrl = server_value.endpointUrl if server_value.location is not None: - server_dict["location"] = server_value.location + server.location = server_value.location if server_value.host is not None: - server_dict["host"] = server_value.host + server.host = server_value.host if server_value.port is not None: - server_dict["port"] = server_value.port + server.port = server_value.port if server_value.catalog is not None: - server_dict["catalog"] = server_value.catalog + server.catalog = server_value.catalog if server_value.topic is not None: - server_dict["topic"] = server_value.topic + server.topic = server_value.topic if server_value.http_path is not None: - server_dict["http_path"] = server_value.http_path + server.http_path = server_value.http_path if server_value.token is not None: - server_dict["token"] = server_value.token + server.token = server_value.token if server_value.driver is not None: - server_dict["driver"] = server_value.driver + server.driver = server_value.driver + if server_value.roles is not None: - server_dict["roles"] = [ - {"name": role.name, "description": role.description} for role in server_value.roles - ] - servers.append(server_dict) + server.roles = [Role(role=role.name, description=role.description) for role in server_value.roles] + + servers.append(server) if len(servers) > 0: - odcs["servers"] = servers + result.servers = servers - odcs["customProperties"] = [] + custom_properties = [] if data_contract_spec.info.owner is not None: - odcs["customProperties"].append({"property": "owner", "value": data_contract_spec.info.owner}) + custom_properties.append(CustomProperty(property="owner", value=data_contract_spec.info.owner)) if data_contract_spec.info.model_extra is not None: for key, value in data_contract_spec.info.model_extra.items(): - odcs["customProperties"].append({"property": key, "value": value}) - if len(odcs["customProperties"]) == 0: - del odcs["customProperties"] + custom_properties.append(CustomProperty(property=key, value=value)) + + if len(custom_properties) > 0: + result.customProperties = custom_properties - return yaml.safe_dump(odcs, indent=2, sort_keys=False, allow_unicode=True) + return result.to_yaml() -def to_odcs_schema(model_key, model_value: Model) -> dict: - odcs_table = { - "name": model_key, - "physicalName": model_key, - "logicalType": "object", - "physicalType": model_value.type, - } +def to_odcs_schema(model_key, model_value: Model) -> SchemaObject: + schema_obj = SchemaObject( + name=model_key, physicalName=model_key, logicalType="object", physicalType=model_value.type + ) + if model_value.description is not None: - odcs_table["description"] = model_value.description + schema_obj.description = model_value.description + properties = to_properties(model_value.fields) if properties: - odcs_table["properties"] = properties + schema_obj.properties = properties model_quality = to_odcs_quality_list(model_value.quality) if len(model_quality) > 0: - odcs_table["quality"] = model_quality + schema_obj.quality = model_quality - odcs_table["customProperties"] = [] + custom_properties = [] if model_value.model_extra is not None: for key, value in model_value.model_extra.items(): - odcs_table["customProperties"].append({"property": key, "value": value}) - if len(odcs_table["customProperties"]) == 0: - del odcs_table["customProperties"] + custom_properties.append(CustomProperty(property=key, value=value)) - return odcs_table + if len(custom_properties) > 0: + schema_obj.customProperties = custom_properties + + return schema_obj def to_properties(fields: Dict[str, Field]) -> list: @@ -204,86 +208,95 @@ def to_logical_type(type: str) -> str | None: def to_physical_type(type: str) -> str | None: - # TODO: to we need to do a server mapping here? return type -def to_property(field_name: str, field: Field) -> dict: - property = {"name": field_name} +def to_property(field_name: str, field: Field) -> SchemaProperty: + property = SchemaProperty(name=field_name) + if field.fields: properties = [] for field_name_, field_ in field.fields.items(): property_ = to_property(field_name_, field_) properties.append(property_) - property["properties"]=properties + property.properties = properties + if field.items: items = to_property(field_name, field.items) - del items["name"] - property["items"]=items + items.name = None # Clear the name for items + property.items = items + if field.title is not None: - property["businessName"] = field.title + property.businessName = field.title + if field.type is not None: - property["logicalType"] = to_logical_type(field.type) - property["physicalType"] = to_physical_type(field.type) + property.logicalType = to_logical_type(field.type) + property.physicalType = to_physical_type(field.type) + if field.description is not None: - property["description"] = field.description + property.description = field.description + if field.required is not None: - property["required"] = field.required + property.required = field.required + if field.unique is not None: - property["unique"] = field.unique + property.unique = field.unique + if field.classification is not None: - property["classification"] = field.classification + property.classification = field.classification + if field.examples is not None: - property["examples"] = field.examples.copy() + property.examples = field.examples.copy() + if field.example is not None: - property["examples"] = [field.example] + property.examples = [field.example] + if field.primaryKey is not None and field.primaryKey: - property["primaryKey"] = field.primaryKey - property["primaryKeyPosition"] = 1 + property.primaryKey = field.primaryKey + property.primaryKeyPosition = 1 + if field.primary is not None and field.primary: - property["primaryKey"] = field.primary - property["primaryKeyPosition"] = 1 + property.primaryKey = field.primary + property.primaryKeyPosition = 1 - property["customProperties"] = [] + custom_properties = [] if field.model_extra is not None: for key, value in field.model_extra.items(): - property["customProperties"].append({"property": key, "value": value}) + custom_properties.append(CustomProperty(property=key, value=value)) + if field.pii is not None: - property["customProperties"].append({"property": "pii", "value": field.pii}) - if property.get("customProperties") is not None and len(property["customProperties"]) == 0: - del property["customProperties"] + custom_properties.append(CustomProperty(property="pii", value=field.pii)) - property["tags"] = [] - if field.tags is not None: - property["tags"].extend(field.tags) - if not property["tags"]: - del property["tags"] + if len(custom_properties) > 0: + property.customProperties = custom_properties - property["logicalTypeOptions"] = {} + if field.tags is not None and len(field.tags) > 0: + property.tags = field.tags + + logical_type_options = {} if field.minLength is not None: - property["logicalTypeOptions"]["minLength"] = field.minLength + logical_type_options["minLength"] = field.minLength if field.maxLength is not None: - property["logicalTypeOptions"]["maxLength"] = field.maxLength + logical_type_options["maxLength"] = field.maxLength if field.pattern is not None: - property["logicalTypeOptions"]["pattern"] = field.pattern + logical_type_options["pattern"] = field.pattern if field.minimum is not None: - property["logicalTypeOptions"]["minimum"] = field.minimum + logical_type_options["minimum"] = field.minimum if field.maximum is not None: - property["logicalTypeOptions"]["maximum"] = field.maximum + logical_type_options["maximum"] = field.maximum if field.exclusiveMinimum is not None: - property["logicalTypeOptions"]["exclusiveMinimum"] = field.exclusiveMinimum + logical_type_options["exclusiveMinimum"] = field.exclusiveMinimum if field.exclusiveMaximum is not None: - property["logicalTypeOptions"]["exclusiveMaximum"] = field.exclusiveMaximum - if property["logicalTypeOptions"] == {}: - del property["logicalTypeOptions"] + logical_type_options["exclusiveMaximum"] = field.exclusiveMaximum + + if logical_type_options: + property.logicalTypeOptions = logical_type_options if field.quality is not None: quality_list = field.quality quality_property = to_odcs_quality_list(quality_list) if len(quality_property) > 0: - property["quality"] = quality_property - - # todo enum + property.quality = quality_property return property @@ -296,33 +309,35 @@ def to_odcs_quality_list(quality_list): def to_odcs_quality(quality): - quality_dict = {"type": quality.type} + quality_obj = DataQuality(type=quality.type) + if quality.description is not None: - quality_dict["description"] = quality.description + quality_obj.description = quality.description if quality.query is not None: - quality_dict["query"] = quality.query + quality_obj.query = quality.query # dialect is not supported in v3.0.0 if quality.mustBe is not None: - quality_dict["mustBe"] = quality.mustBe + quality_obj.mustBe = quality.mustBe if quality.mustNotBe is not None: - quality_dict["mustNotBe"] = quality.mustNotBe + quality_obj.mustNotBe = quality.mustNotBe if quality.mustBeGreaterThan is not None: - quality_dict["mustBeGreaterThan"] = quality.mustBeGreaterThan + quality_obj.mustBeGreaterThan = quality.mustBeGreaterThan if quality.mustBeGreaterThanOrEqualTo is not None: - quality_dict["mustBeGreaterThanOrEqualTo"] = quality.mustBeGreaterThanOrEqualTo + quality_obj.mustBeGreaterOrEqualTo = quality.mustBeGreaterThanOrEqualTo if quality.mustBeLessThan is not None: - quality_dict["mustBeLessThan"] = quality.mustBeLessThan + quality_obj.mustBeLessThan = quality.mustBeLessThan if quality.mustBeLessThanOrEqualTo is not None: - quality_dict["mustBeLessThanOrEqualTo"] = quality.mustBeLessThanOrEqualTo + quality_obj.mustBeLessOrEqualTo = quality.mustBeLessThanOrEqualTo if quality.mustBeBetween is not None: - quality_dict["mustBeBetween"] = quality.mustBeBetween + quality_obj.mustBeBetween = quality.mustBeBetween if quality.mustNotBeBetween is not None: - quality_dict["mustNotBeBetween"] = quality.mustNotBeBetween + quality_obj.mustNotBeBetween = quality.mustNotBeBetween if quality.engine is not None: - quality_dict["engine"] = quality.engine + quality_obj.engine = quality.engine if quality.implementation is not None: - quality_dict["implementation"] = quality.implementation - return quality_dict + quality_obj.implementation = quality.implementation + + return quality_obj def to_status(status): diff --git a/datacontract/export/spark_converter.py b/datacontract/export/spark_converter.py index be2b6cae5..0c7d86bcb 100644 --- a/datacontract/export/spark_converter.py +++ b/datacontract/export/spark_converter.py @@ -175,7 +175,7 @@ def indent(text: str, level: int) -> str: Returns: str: The indented text. """ - return "\n".join([f'{" " * level}{line}' for line in text.split("\n")]) + return "\n".join([f"{' ' * level}{line}" for line in text.split("\n")]) def repr_column(column: types.StructField) -> str: """ diff --git a/datacontract/imports/avro_importer.py b/datacontract/imports/avro_importer.py index 5309a6ce2..e7df6048d 100644 --- a/datacontract/imports/avro_importer.py +++ b/datacontract/imports/avro_importer.py @@ -55,7 +55,7 @@ def import_avro(data_contract_specification: DataContractSpecification, source: engine="datacontract", original_exception=e, ) - # type record is being used for both the table and the object types in data contract + # type record is being used for both the table and the object types in data contract # -> CONSTRAINT: one table per .avsc input, all nested records are interpreted as objects fields = import_record_fields(avro_schema.fields) @@ -92,19 +92,19 @@ def handle_config_avro_custom_properties(field: avro.schema.Field, imported_fiel LOGICAL_TYPE_MAPPING = { - "decimal": "decimal", - "date": "date", - "time-millis": "time", - "time-micros": "time", - "timestamp-millis": "timestamp_tz", - "timestamp-micros": "timestamp_tz", - "local-timestamp-micros": "timestamp_ntz", - "local-timestamp-millis": "timestamp_ntz", - "duration": "string", - "uuid": "string", - } - - + "decimal": "decimal", + "date": "date", + "time-millis": "time", + "time-micros": "time", + "timestamp-millis": "timestamp_tz", + "timestamp-micros": "timestamp_tz", + "local-timestamp-micros": "timestamp_ntz", + "local-timestamp-millis": "timestamp_ntz", + "duration": "string", + "uuid": "string", +} + + def import_record_fields(record_fields: List[avro.schema.Field]) -> Dict[str, Field]: """ Import Avro record fields and convert them to data contract fields. @@ -150,15 +150,15 @@ def import_record_fields(record_fields: List[avro.schema.Field]) -> Dict[str, Fi if not imported_field.config: imported_field.config = {} imported_field.config["avroType"] = "enum" - else: - logical_type = field.type.get_prop("logicalType") - if logical_type in LOGICAL_TYPE_MAPPING: - imported_field.type = LOGICAL_TYPE_MAPPING[logical_type] - if logical_type == "decimal": - imported_field.precision = field.type.precision - imported_field.scale = field.type.scale - else: - imported_field.type = map_type_from_avro(field.type.type) + else: + logical_type = field.type.get_prop("logicalType") + if logical_type in LOGICAL_TYPE_MAPPING: + imported_field.type = LOGICAL_TYPE_MAPPING[logical_type] + if logical_type == "decimal": + imported_field.precision = field.type.precision + imported_field.scale = field.type.scale + else: + imported_field.type = map_type_from_avro(field.type.type) imported_fields[field.name] = imported_field return imported_fields diff --git a/datacontract/imports/csv_importer.py b/datacontract/imports/csv_importer.py index 98ac275b8..f58d2fc34 100644 --- a/datacontract/imports/csv_importer.py +++ b/datacontract/imports/csv_importer.py @@ -31,10 +31,10 @@ def import_csv( if data_contract_specification.servers is None: data_contract_specification.servers = {} - delimiter = None if dialect is None else dialect['Delimiter'][0] + delimiter = None if dialect is None else dialect["Delimiter"][0] if dialect is not None: - dc_types = [map_type_from_duckdb(x["type"]) for x in dialect['Columns'][0]] + dc_types = [map_type_from_duckdb(x["type"]) for x in dialect["Columns"][0]] else: dc_types = [map_type_from_duckdb(str(x)) for x in tbl.dtypes] diff --git a/datacontract/imports/excel_importer.py b/datacontract/imports/excel_importer.py new file mode 100644 index 000000000..bf8089a0d --- /dev/null +++ b/datacontract/imports/excel_importer.py @@ -0,0 +1,850 @@ +import logging +import os +from typing import Any, Dict, List, Optional + +import openpyxl +from open_data_contract_standard.model import ( + AuthoritativeDefinition, + CustomProperty, + DataQuality, + OpenDataContractStandard, + Role, + SchemaObject, + SchemaProperty, + Server, + ServiceLevelAgreementProperty, + Support, + Team, +) +from openpyxl.cell.cell import Cell +from openpyxl.workbook.workbook import Workbook +from openpyxl.worksheet.worksheet import Worksheet + +from datacontract.imports.importer import Importer +from datacontract.model.data_contract_specification import ( + DataContractSpecification, +) +from datacontract.model.exceptions import DataContractException + +logger = logging.getLogger(__name__) + + +class ExcelImporter(Importer): + def import_source( + self, data_contract_specification: DataContractSpecification, source: str, import_args: dict + ) -> OpenDataContractStandard: + return import_excel_as_odcs(source) + + +def import_excel_as_odcs(excel_file_path: str) -> OpenDataContractStandard: + """ + Import an Excel file and convert it to an OpenDataContractStandard object + + Args: + excel_file_path: Path to the Excel file + + Returns: + OpenDataContractStandard object + """ + if not os.path.exists(excel_file_path): + raise FileNotFoundError(f"Excel file not found: {excel_file_path}") + + try: + workbook = openpyxl.load_workbook(excel_file_path, data_only=True) + except Exception as e: + raise DataContractException( + type="schema", + name="Parse excel contract", + reason=f"Failed to open Excel file: {excel_file_path}", + engine="datacontract", + original_exception=e, + ) + + try: + # Get description values + purpose = get_cell_value_by_name(workbook, "description.purpose") + limitations = get_cell_value_by_name(workbook, "description.limitations") + usage = get_cell_value_by_name(workbook, "description.usage") + + # Build description dict + description = None + if purpose or limitations or usage: + description = {"purpose": purpose, "limitations": limitations, "usage": usage} + + # Get tags as a list + tags_str = get_cell_value_by_name(workbook, "tags") + tags = None + if tags_str: + tags = [tag.strip() for tag in tags_str.split(",") if tag.strip()] + + # Import other components + schemas = import_schemas(workbook) + support = import_support(workbook) + team = import_team(workbook) + roles = import_roles(workbook) + sla_properties = import_sla_properties(workbook) + servers = import_servers(workbook) + price = import_price(workbook) + custom_properties = import_custom_properties(workbook) + + # Create the ODCS object with proper object creation + odcs = OpenDataContractStandard( + apiVersion=get_cell_value_by_name(workbook, "apiVersion"), + kind=get_cell_value_by_name(workbook, "kind"), + id=get_cell_value_by_name(workbook, "id"), + name=get_cell_value_by_name(workbook, "name"), + version=get_cell_value_by_name(workbook, "version"), + status=get_cell_value_by_name(workbook, "status"), + domain=get_cell_value_by_name(workbook, "domain"), + dataProduct=get_cell_value_by_name(workbook, "dataProduct"), + tenant=get_cell_value_by_name(workbook, "tenant"), + description=description, + tags=tags, + schema=schemas, + support=support, + price=price, + team=team, + roles=roles, + slaDefaultElement=get_cell_value_by_name(workbook, "slaDefaultElement"), + slaProperties=sla_properties, + servers=servers, + customProperties=custom_properties, + ) + + return odcs + except Exception as e: + logger.error(f"Error importing Excel file: {str(e)}") + raise DataContractException( + type="schema", + name="Parse excel contract", + reason=f"Failed to parse Excel file: {excel_file_path}", + engine="datacontract", + original_exception=e, + ) + finally: + workbook.close() + + +def import_schemas(workbook) -> Optional[List[SchemaObject]]: + """Extract schema information from sheets starting with 'Schema '""" + schemas = [] + + for sheet_name in workbook.sheetnames: + if sheet_name.startswith("Schema ") and sheet_name != "Schema ": + sheet = workbook[sheet_name] + schema_name = get_cell_value_by_name_in_sheet(sheet, "schema.name") + + if not schema_name: + continue + + schema = SchemaObject( + name=schema_name, + logicalType="object", + physicalType=get_cell_value_by_name_in_sheet(sheet, "schema.physicalType"), + physicalName=get_cell_value_by_name_in_sheet(sheet, "schema.physicalName"), + description=get_cell_value_by_name_in_sheet(sheet, "schema.description"), + businessName=get_cell_value_by_name_in_sheet(sheet, "schema.businessName"), + dataGranularityDescription=get_cell_value_by_name_in_sheet(sheet, "schema.dataGranularityDescription"), + authoritativeDefinitions=None, + properties=import_properties(sheet), + quality=None, + customProperties=None, + tags=None, + ) + + # Get tags + tags_str = get_cell_value_by_name_in_sheet(sheet, "schema.tags") + if tags_str: + schema.tags = [tag.strip() for tag in tags_str.split(",") if tag.strip()] + + schemas.append(schema) + + return schemas if schemas else None + + +def import_properties(sheet) -> Optional[List[SchemaProperty]]: + """Extract properties from the schema sheet""" + try: + # Find the properties table + properties_range = get_range_by_name_in_sheet(sheet, "schema.properties") + if not properties_range: + return None + + # Get header row to map column names to indices + header_row = list(sheet.rows)[properties_range[0] - 1] # Convert to 0-based indexing + headers = {} + for i, cell in enumerate(header_row): + if cell.value: + headers[cell.value.lower()] = i + + # Process property rows + property_lookup = {} # Dictionary to keep track of properties by name for nesting + + # First, create all properties + for row_idx in range(properties_range[0], properties_range[1]): + if len(list(sheet.rows)) < row_idx + 1: + break + row = list(sheet.rows)[row_idx] + + # Skip empty rows or header row + property_name = get_cell_value(row, headers.get("property")) + if not property_name or row_idx == properties_range[0] - 1: + continue + + # Create property object + property_obj = SchemaProperty( + name=property_name, + logicalType=get_cell_value(row, headers.get("logical type")), + logicalTypeOptions=import_logical_type_options(row, headers), + physicalType=get_cell_value(row, headers.get("physical type")), + physicalName=get_cell_value(row, headers.get("physical name")), + description=get_cell_value(row, headers.get("description")), + businessName=get_cell_value(row, headers.get("business name")), + required=parse_boolean(get_cell_value(row, headers.get("required"))), + unique=parse_boolean(get_cell_value(row, headers.get("unique"))), + primaryKey=parse_boolean(get_cell_value(row, headers.get("primary key"))), + primaryKeyPosition=parse_integer(get_cell_value(row, headers.get("primary key position"))), + partitioned=parse_boolean(get_cell_value(row, headers.get("partitioned"))), + partitionKeyPosition=parse_integer(get_cell_value(row, headers.get("partition key position"))), + criticalDataElement=parse_boolean(get_cell_value(row, headers.get("critical data element status"))), + classification=get_cell_value(row, headers.get("classification")), + transformLogic=get_cell_value(row, headers.get("transform logic")), + transformDescription=get_cell_value(row, headers.get("transform description")), + encryptedName=get_cell_value(row, headers.get("encrypted name")), + properties=None, + items=None, + tags=get_property_tags(headers, row), + ) + + # Authoritative definitions + authoritative_definition_url = get_cell_value(row, headers.get("authoritative definition url")) + authoritative_definition_type = get_cell_value(row, headers.get("authoritative definition type")) + if authoritative_definition_url and authoritative_definition_type: + property_obj.authoritativeDefinitions = [ + AuthoritativeDefinition( + url=authoritative_definition_url, + type=authoritative_definition_type, + ) + ] + + # Quality + quality_type = get_cell_value(row, headers.get("quality type")) + quality_description = get_cell_value(row, headers.get("quality description")) + if quality_type and quality_description: + property_obj.quality = [ + DataQuality( + type=quality_type, + description=quality_description, + ) + ] + + # Transform sources + transform_sources = get_cell_value(row, headers.get("transform sources")) + if transform_sources: + property_obj.transformSourceObjects = [ + src.strip() for src in transform_sources.split(",") if src.strip() + ] + + # Examples + examples = get_cell_value(row, headers.get("example(s)")) + if examples: + property_obj.examples = [ex.strip() for ex in examples.split(",") if ex.strip()] + + # Add to lookup dictionary + property_lookup[property_name] = property_obj + + # Now organize nested properties + root_properties = [] + for name, prop in property_lookup.items(): + if "." in name: + # This is a nested property + parent_name = name.rsplit(".", 1)[0] + child_name = name.rsplit(".", 1)[1] + + if parent_name in property_lookup: + parent_prop = property_lookup[parent_name] + # Update the property name to be just the child part + prop.name = child_name + + # If parent is an array, set as items + if parent_prop.logicalType == "array": + parent_prop.items = prop + else: + # Otherwise add to properties list + if parent_prop.properties is None: + parent_prop.properties = [] + parent_prop.properties.append(prop) + else: + # This is a root property + root_properties.append(prop) + + return root_properties if root_properties else None + except Exception as e: + logger.warning(f"Error importing properties: {str(e)}") + return None + + +def import_logical_type_options(row, headers): + """Import logical type options from property row""" + + required_props = get_cell_value(row, headers.get("required properties")) + + required_props_list = None + if required_props: + required_props_list = [prop.strip() for prop in required_props.split(",") if prop.strip()] + + logical_type_options_dict = { + "minLength": parse_integer(get_cell_value(row, headers.get("minimum length"))), + "maxLength": parse_integer(get_cell_value(row, headers.get("maximum length"))), + "pattern": get_cell_value(row, headers.get("pattern")), + "format": get_cell_value(row, headers.get("format")), + "exclusiveMaximum": parse_boolean(get_cell_value(row, headers.get("exclusive maximum"))), + "exclusiveMinimum": parse_boolean(get_cell_value(row, headers.get("exclusive minimum"))), + "minimum": get_cell_value(row, headers.get("minimum")), + "maximum": get_cell_value(row, headers.get("maximum")), + "multipleOf": get_cell_value(row, headers.get("multiple of")), + "minItems": parse_integer(get_cell_value(row, headers.get("minimum items"))), + "maxItems": parse_integer(get_cell_value(row, headers.get("maximum items"))), + "uniqueItems": parse_boolean(get_cell_value(row, headers.get("unique items"))), + "maxProperties": parse_integer(get_cell_value(row, headers.get("maximum properties"))), + "minProperties": parse_integer(get_cell_value(row, headers.get("minimum properties"))), + "required": required_props_list, + } + + for dict_key in list(logical_type_options_dict.keys()): + if logical_type_options_dict[dict_key] is None: + del logical_type_options_dict[dict_key] + + if len(logical_type_options_dict) == 0: + return None + return logical_type_options_dict + + +def get_property_tags(headers, row): + tags_value = get_cell_value(row, headers.get("tags")) + if tags_value: + return [tag.strip() for tag in tags_value.split(",") if tag.strip()] + return None + + +def parse_boolean(value): + """Parse a string value to boolean""" + if value is None: + return None + value = value.lower().strip() + return value == "true" or value == "yes" or value == "1" + + +def parse_integer(value): + """Parse a string value to integer""" + if value is None: + return None + try: + return int(value) + except (ValueError, TypeError): + return None + + +def get_range_by_name_in_workbook(workbook: Workbook, name: str) -> tuple | None: + """Find the range (start_row, end_row) of a named range in a workbook""" + try: + for named_range in workbook.defined_names: + if named_range == name: + destinations = workbook.defined_names[named_range].destinations + for sheet_title, range_address in destinations: + if ":" in range_address: + # Convert Excel range to row numbers + start_ref, end_ref = range_address.split(":") + start_row = int("".join(filter(str.isdigit, start_ref))) + end_row = int("".join(filter(str.isdigit, end_ref))) + return start_row, end_row + else: + # Single cell + row = int("".join(filter(str.isdigit, range_address))) + return row, row + except Exception as e: + logger.warning(f"Error finding range by name {name}: {str(e)}") + return None + + +def get_range_by_name_in_sheet(sheet: Worksheet, name: str) -> tuple | None: + """Find the range (start_row, end_row) of a named range in a sheet""" + try: + for named_range in sheet.defined_names: + if named_range == name: + destinations = sheet.defined_names[named_range].destinations + for sheet_title, range_address in destinations: + if sheet_title == sheet.title: + # For named ranges that refer to entire rows or multiple rows + if ":" in range_address: + # Convert Excel range to row numbers + start_ref, end_ref = range_address.split(":") + start_row = int("".join(filter(str.isdigit, start_ref))) + end_row = int("".join(filter(str.isdigit, end_ref))) + return (start_row, end_row) + else: + # Single cell + row = int("".join(filter(str.isdigit, range_address))) + return (row, row) + except Exception as e: + logger.warning(f"Error finding range by name {name}: {str(e)}") + return None + + +def get_cell_by_name_in_workbook(workbook: Workbook, name: str) -> Cell | None: + """Find a cell by name within a workbook""" + try: + for named_range in workbook.defined_names: + if named_range == name: + destinations = workbook.defined_names[named_range].destinations + for sheet_title, coordinate in destinations: + sheet = workbook[sheet_title] + if sheet_title == sheet.title: + return sheet[coordinate] + except Exception as e: + logger.warning(f"Error finding cell by name {name}: {str(e)}") + return None + + +def get_cell_value_by_name(workbook: Workbook, name: str) -> str | None: + """Get the value of a named cell""" + try: + cell = get_cell_by_name_in_workbook(workbook, name) + if cell.value is not None: + return str(cell.value) + except Exception as e: + logger.warning(f"Error getting cell value by name {name}: {str(e)}") + return None + + +def get_cell_value_by_name_in_sheet(sheet: Worksheet, name: str) -> str | None: + """Get the value of a named cell within a specific sheet""" + try: + for named_range in sheet.defined_names: + if named_range == name: + destinations = sheet.defined_names[named_range].destinations + for sheet_title, coordinate in destinations: + if sheet_title == sheet.title: + cell = sheet[coordinate] + if cell.value is not None: + return str(cell.value) + except Exception as e: + logger.warning(f"Error getting cell value by name {name} in sheet {sheet.title}: {str(e)}") + return None + + +def get_cell_value(row, col_idx): + """Safely get cell value from a row by column index""" + if col_idx is None: + return None + try: + cell = row[col_idx] + return str(cell.value) if cell.value is not None else None + except (IndexError, AttributeError): + return None + + +def get_cell_value_by_position(sheet, row_idx, col_idx): + """Get cell value by row and column indices (0-based)""" + try: + cell = sheet.cell(row=row_idx + 1, column=col_idx + 1) # Convert to 1-based indices + return str(cell.value) if cell.value is not None else None + except Exception as e: + logger.warning(f"Error getting cell value by position ({row_idx}, {col_idx}): {str(e)}") + return None + + +def import_support(workbook: Workbook) -> Optional[List[Support]]: + """Extract support information from the Support sheet""" + try: + support_sheet = workbook["Support"] + if not support_sheet: + return None + + support_range = get_range_by_name_in_workbook(workbook, "support") + if not support_range: + return None + + header_row = list(support_sheet.rows)[support_range[0] - 1] + headers = {} + for i, cell in enumerate(header_row): + if cell.value: + headers[cell.value.lower()] = i + + support_channels = [] + for row_idx in range(support_range[0], support_range[1]): + if len(list(support_sheet.rows)) < row_idx + 1: + break + row = list(support_sheet.rows)[row_idx] + + channel = get_cell_value(row, headers.get("channel")) + if not channel or row_idx == support_range[0] - 1: + continue + + support_channel = Support( + channel=channel, + url=get_cell_value(row, headers.get("channel url")), + description=get_cell_value(row, headers.get("description")), + tool=get_cell_value(row, headers.get("tool")), + scope=get_cell_value(row, headers.get("scope")), + invitationUrl=get_cell_value(row, headers.get("invitation url")), + ) + + support_channels.append(support_channel) + except Exception as e: + logger.warning(f"Error importing support: {str(e)}") + return None + + return support_channels if support_channels else None + + +def import_team(workbook: Workbook) -> Optional[List[Team]]: + """Extract team information from the Team sheet""" + try: + team_sheet = workbook["Team"] + if not team_sheet: + return None + + team_range = get_range_by_name_in_workbook(workbook, "team") + if not team_range: + return None + + header_row = list(team_sheet.rows)[team_range[0] - 1] + headers = {} + for i, cell in enumerate(header_row): + if cell.value: + headers[cell.value.lower()] = i + + team_members = [] + for row_idx in range(team_range[0], team_range[1]): + if len(list(team_sheet.rows)) < row_idx + 1: + break + row = list(team_sheet.rows)[row_idx] + + username = get_cell_value(row, headers.get("username")) + name = get_cell_value(row, headers.get("name")) + role = get_cell_value(row, headers.get("role")) + + if (not (username or name or role)) or row_idx == team_range[0] - 1: + continue + + team_member = Team( + username=username, + name=name, + description=get_cell_value(row, headers.get("description")), + role=role, + dateIn=get_cell_value(row, headers.get("date in")), + dateOut=get_cell_value(row, headers.get("date out")), + replacedByUsername=get_cell_value(row, headers.get("replaced by username")), + ) + + team_members.append(team_member) + except Exception as e: + logger.warning(f"Error importing team: {str(e)}") + return None + + return team_members if team_members else None + + +def import_roles(workbook: Workbook) -> Optional[List[Role]]: + """Extract roles information from the Roles sheet""" + try: + roles_sheet = workbook["Roles"] + if not roles_sheet: + return None + + roles_range = get_range_by_name_in_sheet(roles_sheet, "roles") + if not roles_range: + return None + + header_row = list(roles_sheet.rows)[roles_range[0] - 1] + headers = {} + for i, cell in enumerate(header_row): + if cell.value: + headers[cell.value.lower()] = i + + roles_list = [] + for row_idx in range(roles_range[0], roles_range[1]): + row = list(roles_sheet.rows)[row_idx] + + role_name = get_cell_value(row, headers.get("role")) + if not role_name or row_idx == roles_range[0] - 1: + continue + + role = Role( + role=role_name, + description=get_cell_value(row, headers.get("description")), + access=get_cell_value(row, headers.get("access")), + firstLevelApprovers=get_cell_value(row, headers.get("1st level approvers")), + secondLevelApprovers=get_cell_value(row, headers.get("2nd level approvers")), + customProperties=None, + ) + + roles_list.append(role) + except Exception as e: + logger.warning(f"Error importing roles: {str(e)}") + return None + + return roles_list if roles_list else None + + +def import_sla_properties(workbook: Workbook) -> Optional[List[ServiceLevelAgreementProperty]]: + """Extract SLA properties from the SLA sheet""" + try: + sla_sheet = workbook["SLA"] + if not sla_sheet: + return None + + sla_range = get_range_by_name_in_sheet(sla_sheet, "slaProperties") + if not sla_range: + return None + + header_row = list(sla_sheet.rows)[sla_range[0] - 1] + headers = {} + for i, cell in enumerate(header_row): + if cell.value: + headers[cell.value.lower()] = i + + sla_properties = [] + for row_idx in range(sla_range[0], sla_range[1]): + if len(list(sla_sheet.rows)) < row_idx + 1: + break + row = list(sla_sheet.rows)[row_idx] + + property_name = get_cell_value(row, headers.get("property")) + if not property_name or row_idx == sla_range[0] - 1: + continue + + sla_property = ServiceLevelAgreementProperty( + property=property_name, + value=get_cell_value(row, headers.get("value")), + valueExt=get_cell_value(row, headers.get("extended value")), + unit=get_cell_value(row, headers.get("unit")), + element=get_cell_value(row, headers.get("element")), + driver=get_cell_value(row, headers.get("driver")), + ) + + sla_properties.append(sla_property) + except Exception as e: + logger.warning(f"Error importing SLA properties: {str(e)}") + return None + + return sla_properties if sla_properties else None + + +def import_servers(workbook) -> Optional[List[Server]]: + """Extract server information from the Servers sheet""" + try: + sheet = workbook["Servers"] + if not sheet: + return None + + # Find the server cells + server_cell = get_cell_by_name_in_workbook(workbook, "servers.server") + if not server_cell: + return None + + # Get servers (horizontally arranged in the sheet) + servers = [] + col_idx = server_cell.column - 1 # 0-based index + row_idx = server_cell.row - 1 # 0-based index + + index = 0 + while True: + server_name = get_cell_value_by_position(sheet, row_idx, col_idx + index) + if not server_name: + break + + server = Server( + server=server_name, + description=get_server_cell_value(workbook, sheet, "servers.description", index), + environment=get_server_cell_value(workbook, sheet, "servers.environment", index), + type=get_server_cell_value(workbook, sheet, "servers.type", index), + ) + + # Get type-specific fields + server_type = server.type + if server_type: + if server_type == "azure": + server.location = get_server_cell_value(workbook, sheet, "servers.azure.location", index) + server.format = get_server_cell_value(workbook, sheet, "servers.azure.format", index) + server.delimiter = get_server_cell_value(workbook, sheet, "servers.azure.delimiter", index) + elif server_type == "bigquery": + server.project = get_server_cell_value(workbook, sheet, "servers.bigquery.project", index) + server.dataset = get_server_cell_value(workbook, sheet, "servers.bigquery.dataset", index) + elif server_type == "databricks": + server.catalog = get_server_cell_value(workbook, sheet, "servers.databricks.catalog", index) + server.host = get_server_cell_value(workbook, sheet, "servers.databricks.host", index) + server.schema = get_server_cell_value(workbook, sheet, "servers.databricks.schema", index) + elif server_type == "glue": + server.account = get_server_cell_value(workbook, sheet, "servers.glue.account", index) + server.database = get_server_cell_value(workbook, sheet, "servers.glue.database", index) + server.format = get_server_cell_value(workbook, sheet, "servers.glue.format", index) + server.location = get_server_cell_value(workbook, sheet, "servers.glue.location", index) + elif server_type == "kafka": + server.format = get_server_cell_value(workbook, sheet, "servers.kafka.format", index) + server.host = get_server_cell_value(workbook, sheet, "servers.kafka.host", index) + server.topic = get_server_cell_value(workbook, sheet, "servers.kafka.topic", index) + elif server_type == "postgres": + server.database = get_server_cell_value(workbook, sheet, "servers.postgres.database", index) + server.host = get_server_cell_value(workbook, sheet, "servers.postgres.host", index) + server.port = get_server_cell_value(workbook, sheet, "servers.postgres.port", index) + server.schema = get_server_cell_value(workbook, sheet, "servers.postgres.schema", index) + elif server_type == "s3": + server.delimiter = get_server_cell_value(workbook, sheet, "servers.s3.delimiter", index) + server.endpointUrl = get_server_cell_value(workbook, sheet, "servers.s3.endpointUrl", index) + server.format = get_server_cell_value(workbook, sheet, "servers.s3.format", index) + server.location = get_server_cell_value(workbook, sheet, "servers.s3.location", index) + elif server_type == "snowflake": + server.account = get_server_cell_value(workbook, sheet, "servers.snowflake.account", index) + server.database = get_server_cell_value(workbook, sheet, "servers.snowflake.database", index) + server.host = get_server_cell_value(workbook, sheet, "servers.snowflake.host", index) + server.port = get_server_cell_value(workbook, sheet, "servers.snowflake.port", index) + server.schema = get_server_cell_value(workbook, sheet, "servers.snowflake.schema", index) + server.warehouse = get_server_cell_value(workbook, sheet, "servers.snowflake.warehouse", index) + elif server_type == "sqlserver": + server.database = get_server_cell_value(workbook, sheet, "servers.sqlserver.database", index) + server.host = get_server_cell_value(workbook, sheet, "servers.sqlserver.host", index) + server.port = get_server_cell_value(workbook, sheet, "servers.sqlserver.port", index) + server.schema = get_server_cell_value(workbook, sheet, "servers.sqlserver.schema", index) + else: + # Custom server type - grab all possible fields + server.account = get_server_cell_value(workbook, sheet, "servers.custom.account", index) + server.catalog = get_server_cell_value(workbook, sheet, "servers.custom.catalog", index) + server.database = get_server_cell_value(workbook, sheet, "servers.custom.database", index) + server.dataset = get_server_cell_value(workbook, sheet, "servers.custom.dataset", index) + server.delimiter = get_server_cell_value(workbook, sheet, "servers.custom.delimiter", index) + server.endpointUrl = get_server_cell_value(workbook, sheet, "servers.custom.endpointUrl", index) + server.format = get_server_cell_value(workbook, sheet, "servers.custom.format", index) + server.host = get_server_cell_value(workbook, sheet, "servers.custom.host", index) + server.location = get_server_cell_value(workbook, sheet, "servers.custom.location", index) + server.path = get_server_cell_value(workbook, sheet, "servers.custom.path", index) + server.port = get_server_cell_value(workbook, sheet, "servers.custom.port", index) + server.project = get_server_cell_value(workbook, sheet, "servers.custom.project", index) + server.schema = get_server_cell_value(workbook, sheet, "servers.custom.schema", index) + server.stagingDir = get_server_cell_value(workbook, sheet, "servers.custom.stagingDir", index) + server.table = get_server_cell_value(workbook, sheet, "servers.custom.table", index) + server.view = get_server_cell_value(workbook, sheet, "servers.custom.view", index) + server.warehouse = get_server_cell_value(workbook, sheet, "servers.custom.warehouse", index) + server.region = get_server_cell_value(workbook, sheet, "servers.custom.region", index) + server.regionName = get_server_cell_value(workbook, sheet, "servers.custom.regionName", index) + server.serviceName = get_server_cell_value(workbook, sheet, "servers.custom.serviceName", index) + + servers.append(server) + index += 1 + except Exception as e: + logger.warning(f"Error importing servers: {str(e)}") + return None + + return servers if servers else None + + +def get_server_cell_value(workbook: Workbook, sheet: Worksheet, name: str, col_offset: int): + """Get cell value for server properties (arranged horizontally)""" + try: + cell = get_cell_by_name_in_workbook(workbook, name) + if not cell: + return None + + row = cell.row - 1 # 0-based + col = cell.column - 1 + col_offset # 0-based + return get_cell_value_by_position(sheet, row, col) + except Exception as e: + logger.warning(f"Error getting server cell value for {name}: {str(e)}") + return None + + +def import_price(workbook) -> Optional[Dict[str, Any]]: + """Extract price information""" + try: + price_amount = get_cell_value_by_name(workbook, "price.priceAmount") + price_currency = get_cell_value_by_name(workbook, "price.priceCurrency") + price_unit = get_cell_value_by_name(workbook, "price.priceUnit") + + if not (price_amount or price_currency or price_unit): + return None + + # Create a dictionary for price since the class doesn't seem to be directly available + return { + "priceAmount": price_amount, + "priceCurrency": price_currency, + "priceUnit": price_unit, + } + except Exception as e: + logger.warning(f"Error importing price: {str(e)}") + return None + + +def import_custom_properties(workbook: Workbook) -> List[CustomProperty]: + """Extract custom properties""" + custom_properties = [] + + owner = get_cell_value_by_name(workbook, "owner") + + # Add owner as a custom property + if owner: + custom_properties.append( + CustomProperty( + property="owner", + value=owner, + ) + ) + + try: + # Get other custom properties + custom_properties_sheet = workbook["Custom Properties"] + if custom_properties_sheet: + custom_properties_range = get_range_by_name_in_workbook(workbook, "CustomProperties") + if custom_properties_range: + # Skip header row + for row_idx in range(custom_properties_range[0], custom_properties_range[1]): + if row_idx == custom_properties_range[0] - 1: + continue + + property_name = get_cell_value_by_position(custom_properties_sheet, row_idx, 0) + if not property_name or property_name == "owner": + continue + + property_value = get_cell_value_by_position(custom_properties_sheet, row_idx, 1) + parsed_value = parse_property_value(property_value) + + custom_properties.append( + CustomProperty( + property=property_name, + value=parsed_value, + ) + ) + except Exception as e: + logger.warning(f"Error importing custom properties: {str(e)}") + + return custom_properties + + +def parse_property_value(value: str) -> Any: + """Parse a property value into the appropriate type based on Excel values""" + if value is None: + return None + + # Try to convert to boolean (simple case) + if isinstance(value, str): + value_lower = value.lower().strip() + if value_lower == "true": + return True + if value_lower == "false": + return False + + # Try numeric conversions + try: + # Check if it's an integer + if isinstance(value, str) and value.isdigit(): + return int(value) + + # Try float conversion + float_val = float(value) + # If it's a whole number, return as int + if float_val.is_integer(): + return int(float_val) + return float_val + except (ValueError, TypeError, AttributeError): + # If conversion fails, return original string + return value diff --git a/datacontract/imports/importer.py b/datacontract/imports/importer.py index c3056a03d..9ff7ea020 100644 --- a/datacontract/imports/importer.py +++ b/datacontract/imports/importer.py @@ -1,7 +1,8 @@ from abc import ABC, abstractmethod from enum import Enum -from datacontract.model.data_contract_specification import DataContractSpecification +from datacontract_specification.model import DataContractSpecification +from open_data_contract_standard.model import OpenDataContractStandard class Importer(ABC): @@ -14,7 +15,7 @@ def import_source( data_contract_specification: DataContractSpecification, source: str, import_args: dict, - ) -> DataContractSpecification: + ) -> DataContractSpecification | OpenDataContractStandard: pass @@ -33,6 +34,7 @@ class ImportFormat(str, Enum): parquet = "parquet" csv = "csv" protobuf = "protobuf" + excel = "excel" @classmethod def get_supported_formats(cls): diff --git a/datacontract/imports/importer_factory.py b/datacontract/imports/importer_factory.py index 8df55721d..c7c107440 100644 --- a/datacontract/imports/importer_factory.py +++ b/datacontract/imports/importer_factory.py @@ -114,3 +114,8 @@ def load_module_class(module_path, class_name): module_path="datacontract.imports.protobuf_importer", class_name="ProtoBufImporter", ) +importer_factory.register_lazy_importer( + name=ImportFormat.excel, + module_path="datacontract.imports.excel_importer", + class_name="ExcelImporter", +) diff --git a/datacontract/imports/odcs_v3_importer.py b/datacontract/imports/odcs_v3_importer.py index c64224327..a1042bc43 100644 --- a/datacontract/imports/odcs_v3_importer.py +++ b/datacontract/imports/odcs_v3_importer.py @@ -4,7 +4,8 @@ from typing import Any, Dict, List from venv import logger -import yaml +from datacontract_specification.model import Quality +from open_data_contract_standard.model import CustomProperty, OpenDataContractStandard, SchemaProperty from datacontract.imports.importer import Importer from datacontract.lint.resources import read_resource @@ -15,7 +16,6 @@ Field, Info, Model, - Quality, Retention, Server, ServerRole, @@ -41,7 +41,7 @@ def import_odcs_v3_from_str( data_contract_specification: DataContractSpecification, source_str: str ) -> DataContractSpecification: try: - odcs_contract = yaml.safe_load(source_str) + odcs = OpenDataContractStandard.from_string(source_str) except Exception as e: raise DataContractException( type="schema", @@ -51,41 +51,44 @@ def import_odcs_v3_from_str( original_exception=e, ) - data_contract_specification.id = odcs_contract["id"] - data_contract_specification.info = import_info(odcs_contract) - data_contract_specification.servers = import_servers(odcs_contract) - data_contract_specification.terms = import_terms(odcs_contract) - data_contract_specification.servicelevels = import_servicelevels(odcs_contract) - data_contract_specification.models = import_models(odcs_contract) - data_contract_specification.tags = import_tags(odcs_contract) + return import_from_odcs_model(data_contract_specification, odcs) + +def import_from_odcs_model(data_contract_specification, odcs): + data_contract_specification.id = odcs.id + data_contract_specification.info = import_info(odcs) + data_contract_specification.servers = import_servers(odcs) + data_contract_specification.terms = import_terms(odcs) + data_contract_specification.servicelevels = import_servicelevels(odcs) + data_contract_specification.models = import_models(odcs) + data_contract_specification.tags = import_tags(odcs) return data_contract_specification -def import_info(odcs_contract: Dict[str, Any]) -> Info: +def import_info(odcs: Any) -> Info: info = Info() - info.title = odcs_contract.get("name") if odcs_contract.get("name") is not None else "" + info.title = odcs.name if odcs.name is not None else "" - if odcs_contract.get("version") is not None: - info.version = odcs_contract.get("version") + if odcs.version is not None: + info.version = odcs.version # odcs.description.purpose => datacontract.description - if odcs_contract.get("description") is not None and odcs_contract.get("description").get("purpose") is not None: - info.description = odcs_contract.get("description").get("purpose") + if odcs.description is not None and odcs.description.purpose is not None: + info.description = odcs.description.purpose # odcs.domain => datacontract.owner - owner = get_owner(odcs_contract.get("customProperties")) + owner = get_owner(odcs.customProperties) if owner is not None: info.owner = owner # add dataProduct as custom property - if odcs_contract.get("dataProduct") is not None: - info.dataProduct = odcs_contract.get("dataProduct") + if odcs.dataProduct is not None: + info.dataProduct = odcs.dataProduct # add tenant as custom property - if odcs_contract.get("tenant") is not None: - info.tenant = odcs_contract.get("tenant") + if odcs.tenant is not None: + info.tenant = odcs.tenant return info @@ -96,96 +99,92 @@ def import_server_roles(roles: List[Dict]) -> List[ServerRole] | None: result = [] for role in roles: server_role = ServerRole() - server_role.name = role.get("role") - server_role.description = role.get("description") + server_role.name = role.role + server_role.description = role.description result.append(server_role) -def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None: - if odcs_contract.get("servers") is None: +def import_servers(odcs: OpenDataContractStandard) -> Dict[str, Server] | None: + if odcs.servers is None: return None servers = {} - for odcs_server in odcs_contract.get("servers"): - server_name = odcs_server.get("server") + for odcs_server in odcs.servers: + server_name = odcs_server.server if server_name is None: logger.warning("Server name is missing, skipping server") continue server = Server() - server.type = odcs_server.get("type") - server.description = odcs_server.get("description") - server.environment = odcs_server.get("environment") - server.format = odcs_server.get("format") - server.project = odcs_server.get("project") - server.dataset = odcs_server.get("dataset") - server.path = odcs_server.get("path") - server.delimiter = odcs_server.get("delimiter") - server.endpointUrl = odcs_server.get("endpointUrl") - server.location = odcs_server.get("location") - server.account = odcs_server.get("account") - server.database = odcs_server.get("database") - server.schema_ = odcs_server.get("schema") - server.host = odcs_server.get("host") - server.port = odcs_server.get("port") - server.catalog = odcs_server.get("catalog") - server.topic = odcs_server.get("topic") - server.http_path = odcs_server.get("http_path") - server.token = odcs_server.get("token") - server.dataProductId = odcs_server.get("dataProductId") - server.outputPortId = odcs_server.get("outputPortId") - server.driver = odcs_server.get("driver") - server.roles = import_server_roles(odcs_server.get("roles")) - server.storageAccount = re.search(r"(?:@|://)([^.]+)\.",odcs_server.get("location"),re.IGNORECASE) if server.type == "azure" else None + server.type = odcs_server.type + server.description = odcs_server.description + server.environment = odcs_server.environment + server.format = odcs_server.format + server.project = odcs_server.project + server.dataset = odcs_server.dataset + server.path = odcs_server.path + server.delimiter = odcs_server.delimiter + server.endpointUrl = odcs_server.endpointUrl + server.location = odcs_server.location + server.account = odcs_server.account + server.database = odcs_server.database + server.schema_ = odcs_server.schema_ + server.host = odcs_server.host + server.port = odcs_server.port + server.catalog = odcs_server.catalog + server.topic = getattr(odcs_server, "topic", None) + server.http_path = getattr(odcs_server, "http_path", None) + server.token = getattr(odcs_server, "token", None) + server.driver = getattr(odcs_server, "driver", None) + server.roles = import_server_roles(odcs_server.roles) + server.storageAccount = ( + re.search(r"(?:@|://)([^.]+)\.", odcs_server.location, re.IGNORECASE) if server.type == "azure" else None + ) servers[server_name] = server return servers -def import_terms(odcs_contract: Dict[str, Any]) -> Terms | None: - if odcs_contract.get("description") is None: +def import_terms(odcs: Any) -> Terms | None: + if odcs.description is None: return None - if ( - odcs_contract.get("description").get("usage") is not None - or odcs_contract.get("description").get("limitations") is not None - or odcs_contract.get("price") is not None - ): + if odcs.description.usage is not None or odcs.description.limitations is not None or odcs.price is not None: terms = Terms() - if odcs_contract.get("description").get("usage") is not None: - terms.usage = odcs_contract.get("description").get("usage") - if odcs_contract.get("description").get("limitations") is not None: - terms.limitations = odcs_contract.get("description").get("limitations") - if odcs_contract.get("price") is not None: - terms.billing = f"{odcs_contract.get('price').get('priceAmount')} {odcs_contract.get('price').get('priceCurrency')} / {odcs_contract.get('price').get('priceUnit')}" + if odcs.description.usage is not None: + terms.usage = odcs.description.usage + if odcs.description.limitations is not None: + terms.limitations = odcs.description.limitations + if odcs.price is not None: + terms.billing = f"{odcs.price.priceAmount} {odcs.price.priceCurrency} / {odcs.price.priceUnit}" return terms else: return None -def import_servicelevels(odcs_contract: Dict[str, Any]) -> ServiceLevel: +def import_servicelevels(odcs: Any) -> ServiceLevel: # find the two properties we can map (based on the examples) - sla_properties = odcs_contract.get("slaProperties") if odcs_contract.get("slaProperties") is not None else [] - availability = next((p for p in sla_properties if p["property"] == "generalAvailability"), None) - retention = next((p for p in sla_properties if p["property"] == "retention"), None) + sla_properties = odcs.slaProperties if odcs.slaProperties is not None else [] + availability = next((p for p in sla_properties if p.property == "generalAvailability"), None) + retention = next((p for p in sla_properties if p.property == "retention"), None) if availability is not None or retention is not None: servicelevel = ServiceLevel() if availability is not None: - value = availability.get("value") + value = availability.value if isinstance(value, datetime.datetime): value = value.isoformat() servicelevel.availability = Availability(description=value) if retention is not None: - servicelevel.retention = Retention(period=f"{retention.get('value')}{retention.get('unit')}") + servicelevel.retention = Retention(period=f"{retention.value}{retention.unit}") return servicelevel else: return None -def get_server_type(odcs_contract: Dict[str, Any]) -> str | None: - servers = import_servers(odcs_contract) +def get_server_type(odcs: OpenDataContractStandard) -> str | None: + servers = import_servers(odcs) if servers is None or len(servers) == 0: return None # get first server from map @@ -193,49 +192,106 @@ def get_server_type(odcs_contract: Dict[str, Any]) -> str | None: return server.type -def import_models(odcs_contract: Dict[str, Any]) -> Dict[str, Model]: - custom_type_mappings = get_custom_type_mappings(odcs_contract.get("customProperties")) +def import_models(odcs: Any) -> Dict[str, Model]: + custom_type_mappings = get_custom_type_mappings(odcs.customProperties) - odcs_schemas = odcs_contract.get("schema") if odcs_contract.get("schema") is not None else [] + odcs_schemas = odcs.schema_ if odcs.schema_ is not None else [] result = {} for odcs_schema in odcs_schemas: - schema_name = odcs_schema.get("name") - schema_physical_name = odcs_schema.get("physicalName") - schema_description = odcs_schema.get("description") if odcs_schema.get("description") is not None else "" + schema_name = odcs_schema.name + schema_physical_name = odcs_schema.physicalName + schema_description = odcs_schema.description if odcs_schema.description is not None else "" model_name = schema_physical_name if schema_physical_name is not None else schema_name - model = Model(description=" ".join(schema_description.splitlines()), type="table") - model.fields = import_fields( - odcs_schema.get("properties"), custom_type_mappings, server_type=get_server_type(odcs_contract) - ) - if odcs_schema.get("quality") is not None: - # convert dict to pydantic model - - model.quality = [Quality.model_validate(q) for q in odcs_schema.get("quality")] + model = Model(description=" ".join(schema_description.splitlines()) if schema_description else "", type="table") + model.fields = import_fields(odcs_schema.properties, custom_type_mappings, server_type=get_server_type(odcs)) + if odcs_schema.quality is not None: + model.quality = convert_quality_list(odcs_schema.quality) model.title = schema_name - if odcs_schema.get("dataGranularityDescription") is not None: - model.config = {"dataGranularityDescription": odcs_schema.get("dataGranularityDescription")} + if odcs_schema.dataGranularityDescription is not None: + model.config = {"dataGranularityDescription": odcs_schema.dataGranularityDescription} result[model_name] = model return result -def import_field_config(odcs_property: Dict[str, Any], server_type=None) -> Dict[str, Any]: +def convert_quality_list(odcs_quality_list): + """Convert a list of ODCS DataQuality objects to datacontract Quality objects""" + quality_list = [] + + if odcs_quality_list is not None: + for odcs_quality in odcs_quality_list: + quality = Quality(type=odcs_quality.type) + + if odcs_quality.description is not None: + quality.description = odcs_quality.description + if odcs_quality.query is not None: + quality.query = odcs_quality.query + if odcs_quality.mustBe is not None: + quality.mustBe = odcs_quality.mustBe + if odcs_quality.mustNotBe is not None: + quality.mustNotBe = odcs_quality.mustNotBe + if odcs_quality.mustBeGreaterThan is not None: + quality.mustBeGreaterThan = odcs_quality.mustBeGreaterThan + if odcs_quality.mustBeGreaterOrEqualTo is not None: + quality.mustBeGreaterThanOrEqualTo = odcs_quality.mustBeGreaterOrEqualTo + if odcs_quality.mustBeLessThan is not None: + quality.mustBeLessThan = odcs_quality.mustBeLessThan + if odcs_quality.mustBeLessOrEqualTo is not None: + quality.mustBeLessThanOrEqualTo = odcs_quality.mustBeLessOrEqualTo + if odcs_quality.mustBeBetween is not None: + quality.mustBeBetween = odcs_quality.mustBeBetween + if odcs_quality.mustNotBeBetween is not None: + quality.mustNotBeBetween = odcs_quality.mustNotBeBetween + if odcs_quality.engine is not None: + quality.engine = odcs_quality.engine + if odcs_quality.implementation is not None: + quality.implementation = odcs_quality.implementation + if odcs_quality.businessImpact is not None: + quality.model_extra["businessImpact"] = odcs_quality.businessImpact + if odcs_quality.dimension is not None: + quality.model_extra["dimension"] = odcs_quality.dimension + if odcs_quality.rule is not None: + quality.model_extra["rule"] = odcs_quality.rule + if odcs_quality.schedule is not None: + quality.model_extra["schedule"] = odcs_quality.schedule + if odcs_quality.scheduler is not None: + quality.model_extra["scheduler"] = odcs_quality.scheduler + if odcs_quality.severity is not None: + quality.model_extra["severity"] = odcs_quality.severity + if odcs_quality.method is not None: + quality.model_extra["method"] = odcs_quality.method + if odcs_quality.customProperties is not None: + quality.model_extra["customProperties"] = [] + for item in odcs_quality.customProperties: + quality.model_extra["customProperties"].append( + { + "property": item.property, + "value": item.value, + } + ) + + quality_list.append(quality) + + return quality_list + + +def import_field_config(odcs_property: SchemaProperty, server_type=None) -> Dict[str, Any]: config = {} - if odcs_property.get("criticalDataElement") is not None: - config["criticalDataElement"] = odcs_property.get("criticalDataElement") - if odcs_property.get("encryptedName") is not None: - config["encryptedName"] = odcs_property.get("encryptedName") - if odcs_property.get("partitionKeyPosition") is not None: - config["partitionKeyPosition"] = odcs_property.get("partitionKeyPosition") - if odcs_property.get("partitioned") is not None: - config["partitioned"] = odcs_property.get("partitioned") - - if odcs_property.get("customProperties") is not None and isinstance(odcs_property.get("customProperties"), list): - for item in odcs_property.get("customProperties"): - config[item["property"]] = item["value"] - - physical_type = odcs_property.get("physicalType") + if odcs_property.criticalDataElement is not None: + config["criticalDataElement"] = odcs_property.criticalDataElement + if odcs_property.encryptedName is not None: + config["encryptedName"] = odcs_property.encryptedName + if odcs_property.partitionKeyPosition is not None: + config["partitionKeyPosition"] = odcs_property.partitionKeyPosition + if odcs_property.partitioned is not None: + config["partitioned"] = odcs_property.partitioned + + if odcs_property.customProperties is not None: + for item in odcs_property.customProperties: + config[item.property] = item.value + + physical_type = odcs_property.physicalType if physical_type is not None: if server_type == "postgres" or server_type == "postgresql": config["postgresType"] = physical_type @@ -255,13 +311,13 @@ def import_field_config(odcs_property: Dict[str, Any], server_type=None) -> Dict return config -def has_composite_primary_key(odcs_properties) -> bool: - primary_keys = [prop for prop in odcs_properties if prop.get("primaryKey") is not None and prop.get("primaryKey")] +def has_composite_primary_key(odcs_properties: List[SchemaProperty]) -> bool: + primary_keys = [prop for prop in odcs_properties if prop.primaryKey is not None and prop.primaryKey] return len(primary_keys) > 1 def import_fields( - odcs_properties: Dict[str, Any], custom_type_mappings: Dict[str, str], server_type + odcs_properties: List[SchemaProperty], custom_type_mappings: Dict[str, str], server_type ) -> Dict[str, Field]: logger = logging.getLogger(__name__) result = {} @@ -270,50 +326,51 @@ def import_fields( return result for odcs_property in odcs_properties: - mapped_type = map_type(odcs_property.get("logicalType"), custom_type_mappings) + mapped_type = map_type(odcs_property.logicalType, custom_type_mappings) if mapped_type is not None: - property_name = odcs_property["name"] - description = odcs_property.get("description") if odcs_property.get("description") is not None else None + property_name = odcs_property.name + description = odcs_property.description if odcs_property.description is not None else None field = Field( description=" ".join(description.splitlines()) if description is not None else None, type=mapped_type, - title=odcs_property.get("businessName"), - required=odcs_property.get("required") if odcs_property.get("required") is not None else None, - primaryKey=odcs_property.get("primaryKey") - if not has_composite_primary_key(odcs_properties) and odcs_property.get("primaryKey") is not None + title=odcs_property.businessName, + required=odcs_property.required if odcs_property.required is not None else None, + primaryKey=odcs_property.primaryKey + if not has_composite_primary_key(odcs_properties) and odcs_property.primaryKey is not None else False, - unique=odcs_property.get("unique"), - examples=odcs_property.get("examples") if odcs_property.get("examples") is not None else None, - classification=odcs_property.get("classification") - if odcs_property.get("classification") is not None - else None, - tags=odcs_property.get("tags") if odcs_property.get("tags") is not None else None, - quality=odcs_property.get("quality") if odcs_property.get("quality") is not None else [], - fields=import_fields(odcs_property.get("properties"), custom_type_mappings, server_type) - if odcs_property.get("properties") is not None else {}, + unique=odcs_property.unique if odcs_property.unique else None, + examples=odcs_property.examples if odcs_property.examples is not None else None, + classification=odcs_property.classification if odcs_property.classification is not None else None, + tags=odcs_property.tags if odcs_property.tags is not None else None, + quality=convert_quality_list(odcs_property.quality), + fields=import_fields(odcs_property.properties, custom_type_mappings, server_type) + if odcs_property.properties is not None + else {}, config=import_field_config(odcs_property, server_type), - format=odcs_property.get("format") if odcs_property.get("format") is not None else None, + format=getattr(odcs_property, "format", None), ) - #mapped_type is array - if field.type == "array" and odcs_property.get("items") is not None : - #nested array object - if odcs_property.get("items").get("logicalType") == "object": - field.items= Field(type="object", - fields=import_fields(odcs_property.get("items").get("properties"), custom_type_mappings, server_type)) - #array of simple type - elif odcs_property.get("items").get("logicalType") is not None: - field.items= Field(type = odcs_property.get("items").get("logicalType")) - + # mapped_type is array + if field.type == "array" and odcs_property.items is not None: + # nested array object + if odcs_property.items.logicalType == "object": + field.items = Field( + type="object", + fields=import_fields(odcs_property.items.properties, custom_type_mappings, server_type), + ) + # array of simple type + elif odcs_property.items.logicalType is not None: + field.items = Field(type=odcs_property.items.logicalType) + # enum from quality validValues as enum if field.type == "string": for q in field.quality: - if hasattr(q,"validValues"): + if hasattr(q, "validValues"): field.enum = q.validValues result[property_name] = field else: logger.info( - f"Can't map {odcs_property.get('column')} to the Datacontract Mapping types, as there is no equivalent or special mapping. Consider introducing a customProperty 'dc_mapping_{odcs_property.get('logicalName')}' that defines your expected type as the 'value'" + f"Can't map {odcs_property.name} to the Datacontract Mapping types, as there is no equivalent or special mapping. Consider introducing a customProperty 'dc_mapping_{odcs_property.logicalType}' that defines your expected type as the 'value'" ) return result @@ -331,28 +388,28 @@ def map_type(odcs_type: str, custom_mappings: Dict[str, str]) -> str | None: return None -def get_custom_type_mappings(odcs_custom_properties: List[Any]) -> Dict[str, str]: +def get_custom_type_mappings(odcs_custom_properties: List[CustomProperty]) -> Dict[str, str]: result = {} if odcs_custom_properties is not None: for prop in odcs_custom_properties: - if prop["property"].startswith("dc_mapping_"): - odcs_type_name = prop["property"].substring(11) - datacontract_type = prop["value"] + if prop.property.startswith("dc_mapping_"): + odcs_type_name = prop.property[11:] # Changed substring to slice + datacontract_type = prop.value result[odcs_type_name] = datacontract_type return result -def get_owner(odcs_custom_properties: List[Any]) -> str | None: +def get_owner(odcs_custom_properties: List[CustomProperty]) -> str | None: if odcs_custom_properties is not None: for prop in odcs_custom_properties: - if prop["property"] == "owner": - return prop["value"] + if prop.property == "owner": + return prop.value return None -def import_tags(odcs_contract) -> List[str] | None: - if odcs_contract.get("tags") is None: +def import_tags(odcs: OpenDataContractStandard) -> List[str] | None: + if odcs.tags is None: return None - return odcs_contract.get("tags") + return odcs.tags diff --git a/datacontract/imports/protobuf_importer.py b/datacontract/imports/protobuf_importer.py index 1152f6cee..cea238898 100644 --- a/datacontract/imports/protobuf_importer.py +++ b/datacontract/imports/protobuf_importer.py @@ -238,7 +238,6 @@ def import_protobuf( os.remove(descriptor_file) - class ProtoBufImporter(Importer): def __init__(self, name): # 'name' is passed by the importer factory. @@ -263,4 +262,3 @@ def import_source( """ # Wrap the source in a list because import_protobuf expects a list of sources. return import_protobuf(data_contract_specification, [source], import_args) - diff --git a/datacontract/lint/linters/description_linter.py b/datacontract/lint/linters/description_linter.py index 615dd3edc..2f88615b8 100644 --- a/datacontract/lint/linters/description_linter.py +++ b/datacontract/lint/linters/description_linter.py @@ -23,9 +23,7 @@ def lint_implementation(self, contract: DataContractSpecification) -> LinterResu result = result.with_error(f"Model '{model_name}' has empty description.") for field_name, field in model.fields.items(): if not field.description: - result = result.with_error( - f"Field '{field_name}' in model '{model_name}'" f" has empty description." - ) + result = result.with_error(f"Field '{field_name}' in model '{model_name}' has empty description.") for definition_name, definition in contract.definitions.items(): if not definition.description: result = result.with_error(f"Definition '{definition_name}' has empty description.") diff --git a/datacontract/lint/linters/field_reference_linter.py b/datacontract/lint/linters/field_reference_linter.py index 97213cd72..0a9a52435 100644 --- a/datacontract/lint/linters/field_reference_linter.py +++ b/datacontract/lint/linters/field_reference_linter.py @@ -34,8 +34,7 @@ def lint_implementation(self, contract: DataContractSpecification) -> LinterResu if ref_model not in contract.models: result = result.with_error( - f"Field '{field_name}' in model '{model_name}'" - f" references non-existing model '{ref_model}'." + f"Field '{field_name}' in model '{model_name}' references non-existing model '{ref_model}'." ) else: ref_model_obj = contract.models[ref_model] diff --git a/datacontract/lint/linters/notice_period_linter.py b/datacontract/lint/linters/notice_period_linter.py index bb09703e1..7051bc4f6 100644 --- a/datacontract/lint/linters/notice_period_linter.py +++ b/datacontract/lint/linters/notice_period_linter.py @@ -41,10 +41,10 @@ def lint_implementation(self, contract: DataContractSpecification) -> LinterResu if not period: return LinterResult.cautious("No notice period defined.") if not period.startswith("P"): - return LinterResult.erroneous(f"Notice period '{period}' is not a valid" "ISO8601 duration.") + return LinterResult.erroneous(f"Notice period '{period}' is not a valid ISO8601 duration.") if period == "P": return LinterResult.erroneous( - "Notice period 'P' is not a valid" "ISO8601 duration, requires at least one" "duration to be specified." + "Notice period 'P' is not a valid ISO8601 duration, requires at least one duration to be specified." ) if ( not self.simple.fullmatch(period) diff --git a/datacontract/lint/linters/valid_constraints_linter.py b/datacontract/lint/linters/valid_constraints_linter.py index c1f764787..54afea84d 100644 --- a/datacontract/lint/linters/valid_constraints_linter.py +++ b/datacontract/lint/linters/valid_constraints_linter.py @@ -40,7 +40,7 @@ def check_minimum_maximum(self, field: Field, field_name: str, model_name: str) ): case (True, True, _, _) if min > max: return LinterResult.erroneous( - f"Minimum {min} is greater than maximum {max} on " f"field '{field_name}' in model '{model_name}'." + f"Minimum {min} is greater than maximum {max} on field '{field_name}' in model '{model_name}'." ) case (_, _, True, True) if xmin >= xmax: return LinterResult.erroneous( @@ -68,11 +68,11 @@ def check_string_constraints(self, field: Field, field_name: str, model_name: st result = LinterResult() if field.minLength and field.maxLength and field.minLength > field.maxLength: result = result.with_error( - f"Minimum length is greater that maximum length on" f" field '{field_name}' in model '{model_name}'." + f"Minimum length is greater that maximum length on field '{field_name}' in model '{model_name}'." ) if field.pattern and field.format: result = result.with_error( - f"Both a pattern and a format are defined for field" f" '{field_name}' in model '{model_name}'." + f"Both a pattern and a format are defined for field '{field_name}' in model '{model_name}'." ) return result diff --git a/pyproject.toml b/pyproject.toml index c11407d07..fa9c6dd62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ name = "datacontract-cli" version = "0.10.24" description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library." +license = "MIT" readme = "README.md" authors = [ { name = "Jochen Christ", email = "jochen.christ@innoq.com" }, @@ -10,7 +11,6 @@ authors = [ ] classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ] requires-python = ">=3.10" @@ -33,7 +33,8 @@ dependencies = [ "boto3>=1.34.41,<2.0.0", "Jinja2>=3.1.5,<4.0.0", "jinja_partials>=0.2.1,<1.0.0", - "datacontract-specification >= 1.1.0,<2.0.0", + "datacontract-specification>=1.1.1,<2.0.0", + "open-data-contract-standard>=3.0.4,<4.0.0", ] [project.optional-dependencies] @@ -50,6 +51,11 @@ csv = [ "pandas >= 2.0.0", ] +excel = [ + "openpyxl>=3.1.5,<4.0.0", +] + + databricks = [ "soda-core-spark-df>=3.3.20,<3.6.0", "soda-core-spark[databricks]>=3.3.20,<3.6.0", @@ -114,7 +120,7 @@ protobuf = [ ] all = [ - "datacontract-cli[kafka,bigquery,csv,snowflake,postgres,databricks,sqlserver,s3,trino,dbt,dbml,iceberg,parquet,rdf,api,protobuf]" + "datacontract-cli[kafka,bigquery,csv,excel,snowflake,postgres,databricks,sqlserver,s3,trino,dbt,dbml,iceberg,parquet,rdf,api,protobuf]" ] dev = [ diff --git a/tests/fixtures/excel/shipments-odcs.xlsx b/tests/fixtures/excel/shipments-odcs.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..e3a94351854a8169c75cf77af4ebecd043d31a63 GIT binary patch literal 112519 zcmeEt1AAplw{C3PwrzLJj_ssl8y(y3uw&a!I=1bOZFPKC_uk+6_C4o2zu?~WJXv$4 zFl$uJG2ZcxQ8hoyfP$d`K>$Gk0Ra&Mxr;7Ww*UhHwZZ@ap#njHXn(S^bvCti)>rYc zH+9lwaJR80$^!$T$^ilawEzE~|A$v#JY~jql?g56lJXp%WJ|vP$6}0pkY3HXqZp1C z(3AiY>`;@p4vby5@h8Pdme1fyPNv6g;y{Y14vW@zEe} z;?j#1Z2np;Bdyx}URp5P0bzB1jq3j~1iW-E%jTO$bKH5cN4~gu?k^xb4HJt_z-C=A&*@$pTOm_b$D6#t9a)_e`v2 zh^y5@2@L*2ok*Y)H#ju%DugSg7r0QE9ctpMw~76iE=-;rP5Mq*S87{DtkW_NCIy?}*C;9?(_2UB+==1+& z@QtcWq<4TCS%5vm0t{Z?(bU?Bk>U6E|2Opihpqb`Q?E#nQ|M!Y3%Qhh4IO@5*oZ?C zk#Q52Y$sOn@snCdY>dt)!`_L(Tr}?H*b3R%wZ)GHk4m1QxZeuIrqL=4Q$tASswkI z(0%2@S0e$Dge98*-}gU_Wd5xTMFLR618|Z!Kv2N$){KAIiJP6Hm64sD)o;J}FPi}d zI7dL+|F=IK$y1g9of*6g?EG}~)ghtDf;auHhWW_ll;8)KG+z30&&~VZxfzHI29+LS z4`!eV-PYB7c!Q>-$Cf==pW+qBx;D)XUV!Ky-uRMMsI+JSrZ={=>x zVFKt)AFY?xhaXnubrsxvwC^(kZ0cImjD?I;KvDFg&fpZJ#xTeB^|18Jxl4vS3khsg zsUbTP=CmC0qV)}tUldbu3p85|n!bNs<@qX&aX)ewjsN0cVjrgahH2v)$QrH;H#OPM zkUGyQd!j<2moT_o2sxJoM~p9DOoBTVVVxPV{C4TGF&GbPcAoa-JYdBuWQ8|cO9f{V z=bcQ0zW5qnxf{*-VfRoFpgsRn4Ff9?)w&^pfRssqfDi#K0BZP81uW3Iw9DtfdKa+x z_=F(VrU2~^^ps1XJj>QB-Ko6raM7ediJ93?YcR=S(`NDUq6eBwQDbPkfKT1y^ZM?- zIW%}Nz&J#8#&0|b9gP{nj)%>U_rgYW?PKWevy4zA9t(>GNmanChgF@5+fC>De!LT0 zS1ci&?Foc*8|;8Yl9oa7h{z@4Es2WPmxynQjAB3bK%syo4lHOump9!YK{^$M6dsLd zIO)uEWGms_k?VuAt+7W07b@xZ^eTknBNFM2-vU)z%!d7oO&Vw?K1j6VOO80>&Ee8Z zD$5;Kz?#I^IX?F5?ayxZfXJ^n&;$!15n)qQ#k|a5=B%7;2i@!}^yMf4#{3YTk})G! ztnu7*l}4M0TKmZwo4Bd$&s*jgpJ{hT@d6$L&223*Z0HzW$@=vjPaZxec#Fi3tC-)3 z^be>CfgxZFg4rI!R(bc#nq(y)eb*u#BEUYVE4ZZ(#2tR4K}#Kw{3(M2o1~7I;C}1b<|1MLn85!c4dUPjk(g#3@B2tg@dAovEmvn7 z$9oe+I8DlyNU%IR9<0c0mzdEUobBJCGR}(r!eRr?TR)|jxzOh_4lZMeh9nv1SlXre zBbp#%EPmO=K#FrgU=8g~4~7nAL7*;RO#Z>6_(k@~5G-N1%2HxYi6_Pie~v6ymTFKF zDZ-KHu;UF>Jbu-^g3WabcT16|dlg5tGU&mwXuV>-=|^eZdBGwEE4LlZxv0OT z7RydwMcGcf^vjCJ%UBXD&V9}i>Nr@bXD$TVED7vQ(ksxVWE5y=h!FUG6%m-Uq$2mA zlH6VUyXjUZQ!;{Fr&Bpve#)g)NSE)D{{4oB??!W_nh>=yCHjkaK${#YJyZ=9P|8xf z_PGFf>Pr+^ooEPgc-ZhaR2Mq0dE3Z<v z6qvk{i8#@^D#7|`jWU`$c@*Py%O9wc=KRj*Au=oFj~Pud68c8tIF6>x53)IR*G%nI zBC-S(tZ!ydq32+o`gldaI8$R2Vnxg$F>BP=ZA!~~ z(aEf#i{UlQtdW;rk?oe*daqi1&Y=iZmnYiG%G+X)GCM-74)?KLlX4ePFXE-&tax5@ z-~5=*mRmbaH-htjV6w5Qo1OKWPzU*RXEu7p^kW5&q;XHcm&i;FejvE8OXo8h(^G~M zdcH1OIo7Ngce$$KD)@QOjpW0Y^0;1T-3}Lot|I?z$x?4V3gyU_K^Yg$g3`}lD+%Ud z-h%3y12_s>Vr)GUOKLl-IXTYLvtUFm|>!@E4Ndi4x>~_$BuCi&jg>= zd@m!S`@K|n@em;<8#=#sy}qPCD!ccEX&^c=q0&w)WQHqk`FU?b2#0KG(M$(orX;0F z;4ypp0va9{4*RrRB}Q>~StA?g>Vp2EraQe-&(i5mV=D`##5UR|KqhV#jA>E99(wjO zoADdj?R9sB68_g&(7|qcRJ=DjP-##(yf+UhAe@zxqg%420_k%%EWskib|ijwSif?{ znGP^Z6<~GW-0KjP-ST{eXU-%X!z=_@+D=P@iZy>qa`{`GBUj1CN2V$FdWIH!5fzdJ zkndf7kouZTLd!>1Sxvn@YVbN%mgkJowFAHZ*l&hN78_o3vutSQIkebQL`k#>M?Xay z{Xq28wy~!P-%Hr{M%=P2)^pg@zx0|!y=MbLKs8Vp={DtmBA$w?NMo<7cY^gsyU5%xV72hoO3HXYFQ8_g%~ZO^N$HbIWFrTYBcD8)vgGLpY2b*RI=~qvQ4N zdOQEiT1I)Ooqls$v^wEZMzYV#!p6qQV`%x7{HN4t;NV14$(D{UX`Bkah;^?3kN!V> zJ)|r{OQkB7k6mm&>j~{p0_k{>J7|#^> zq_woHYAJ@f2WD{IwMp&kV~mst$v(xQB%R(B=g0V~tbR5H;R=c6%=HOf-t zVg9A7?rq9ZNpHk6?zxwg--$ZJ%tVU58xEK4Bvq1aKrg1Viq?i;O;=lO9@ENAo7o4H zoLAv0Qu7E^c=Sz_hLT83pSjFQi!H+MeT5?t$;uC)4Z_v$yE{dU2P>Ag)WpUd-qgG` zelGpy8H7+U^qSoZ7GJ#leGOvs^Y#lF`1KDMJEm884VE9)ttw5srZMkDKD9rf2lGjK zuczXH30s-c(Qg}S69Gyiw zO%E+*xt^n(dH6~Q8eAwS1qu6?z5s=hT3^D+YbMt~vK!RM#RHR5;K~ACn}QbUHor-z z<#Xied{`3sjXgQ25|vn>4NTFntuQQoRH-X4d(82#Ss^CYl+P@GDbpYJn6z{%4~6c~CjsC6p2Op;EWNbVtm2e2q?^n43{Um!bk?@pZefr${ok!V>9KBm4u#$y&A)<2{9 zks*N?BFz}RH1Zwo-W4ppb&w4sK?R0ktl6zN`pPVC>N8S%5+gw%(>LjlNk`6wOdlSS z3x^0{i8d_z&}UFQqt}8<5rOp*Nd>~+0-bHO(!b=>8s2&X|JSWn2=UJl|E@3cx$XXt?D`YC{(1H~ zG5m-DOjto5(sTH=-k+pVg_d{6z5r@s?t z5Qq0$G1S%)dQ_$iG9>&X?|;uqHAx?czXQ;32K9eMKlVT9m!ugB$Uz2g(cTM+`b>4f zi3TBC@iM9|+^=vU)YceVX4sUyKk88;#km`bgCs2f;yv8-GPv_bTVGKwOtXeT^8|J@ z|70?f?&9zBuF(d%XC9b=;-$>cq z;GZQ;;}6eQ)>z#fu~WxS)7fG3jinDH3R!Ql$8m(Kh=@%yYFRHe=iq_MAXEPFCYgO+HH@Q6=kG<4 z)?Ln+$8;`chA6FbBTTN@A=^Ze6b?+fd~ct26-Nwun`k6v%7hY`KG1@P31e4*-i+>Y z>@Kp@sR!Hw&MSJ+l-31(YoYfJRWnwTlW@IW!Pmiv3aE?w4~BZ|SLU$yukNKldOqf) zDAg7(UF@EkgZQZ}-03F_V@zlnzqmZ&LKM=Vzt)hKRo^ZM&5yQb#}-fcX)YhrxH%VN%eLxR!N}?^hY6$C_3qf??!(Y8*RCfSu5)znZ4>WTug{-`uI;@QTH3TZ+b`m_ zZW6Yfe9Uf`j~{K@Pg1tpZ9OD?ORsb~7ZzYu*IF$nALO3TtW?#j(pl;b^~(>exA;cN z&UNnjCYi+6rA-K2`Y}nY@N-2>tXOnLt(~(zkFKA}(wR2xTe;{vR5~`<&7OKgt~OfO zwV0%QAO%g#N(4Z~2j0vf^QbP+pAt&zl}L*4+NrJ&Iz!EoFy%{Lz)a zL9-U&H)!e?9>?U+i$O7=&5tZ}O8bY5#ukF1?zU^oVmY_HT!geYps48tsmY_?k+${B zuff?le1BJyG>2zh%DYU#iAE$ng}Y_qW)4+Ilr0;*vYL{?U?%YFVW#S6DbS8}>g~X% zhYH8${w%V!Jf-|$$D;r_GhELU?I_q#u@P;6YG|S|SGC!DCHE<+%7A`d-LkNxKr_rh z1#}rbr4U<|vq@?774$zb##~Xa;#>e`vLgSln91=MV_b{*H)G@`grPD?KFGO{Q@wXT z$3m?zkhL*OY&dCmgYy?1m2)NU^~3kQof@AyJMPCl4VE4d)h61yjuX=+kzTxjx!Zji zl#efDvz1v7OTjpjMfXha_T}V-X;o)0Tm=y+hf%TG*sm&=l;c*5IWmXkO^(B}a!=+} zcS+c(6YR(3yz1!RPi-_O+PgsxDWMTRl8yr{t|TBz;Lr&4QLUGcW|JrAaiDt)oumkG z^INV9Id(!6!=QY+twKk!!!3E)lStK|gjNUYwwIl&-ZI^ZlFSt?#3yPGJb1Zb<#b|B zlYG1v47w&&pT+zF;VuSlm$)~CO54)g%h{U%{biN>7L*qDj`JBS*br~fHHEyib6He| z7d{}l^Ew~#c<^?qXZ0oYpgy4|NKQ4SK-C|^rwhS*Z_*6F8Cg74IHwi~iGOhhOjNvL z>F{;>av)eUln*v)6F(gMVlBBQiqC^59~8t0xf1!-;NQ(i!t{Pg>|!$8ro6d!sMuHZ z_=IU;Q=2C<^J-LBMNCaaZ4rzZ-NjHOj{~811bN-?q>3k~*i_CApp2S>HXa^&ief(w za^|3;AV_c|q1+-_Z(9Rou1jNwi1o2}d{8{Xg=xQXZ1@Q$1kMJ;CNSgx1Gn>o?0_Si z$=}d`!8J<=;i;O)bE*D{gEM!X^5bF&$Y9{__e=1C=#NM4tV1(@o1P|+dfuRIhDPY?}TcE<)YuN{B6_bz|gsaEjK-QMj`Y3AtP2_!1<+*dV9|X zXVo*C(PL_9y-IeIytC=lgTJfk-aT8-;_9oXX)&T~<%P0S6ZCOCp)zKlva-0F4-dOp zMva;>=G@Et+PU?MWM)ZJc_Gm?@9Si>87-w3&Tx#g`2eb`;9OMHL z%s4UwjOSj(@wPu-4Wy!hCmrlR|Eu`F>I$;&#=2uO+4=eYbd`;LUGLD@!{Ctni=El* zKCH}bAianceaHQa9j+yvbuTjtZ4AN+hXClC4J$o&DP9xhesCVL@c@7f;F%oc?JWViaDRF|bL&Nh3 zHJMe7(03p+my!|R`Gpmi2_AqZKpU=5YCy6@XpY=f`_19fj%PQ}dR}u?oKY~U>>I8# zrlE<;ob6`MA6WZ|x6WyqT#=+ZGE@R#k6R(bx@>Em?6~qD17TSw8CU{vAfU~82~EynEXr__sBP1?k6Iy$f9VbtopSI-#7q0D*jwHLGA6blLCNi6s#x)HT##} zcfC^i@x@L%SPEOuq=PzvxGVDOb~c(s1$`?v-~f?u6-nipMmQi6E`Az9tPfP--CzPl z!p?1?(ac}BU(z`Mk?_cOml`D*KqRctc<)k;RT?6fci%@O`Bm6GbTUYUzcjjzrrYKae1Xx{RvVIj$mLwkV+~QE5AR^W1u8 zp})k+pnK^hAKAq*I;8B3>oNqa!|%GEA53vMtB=5huIzXu(`A3;K2_{Y8=j6TRxr$f zV4gCGynAzi9buY5e!abTjaTw6?tg%xjN3VWiIg+&N^30LkZi4A(-)F6@PEg5;4gS6 zcjOSQFSPc(9cA+avK_Fjvzu^XxvisuBf+DX+X3*s@n22MHRRx@9K2#KfM}Pv;qwBl zjV_IL4bbms_g45|*Z~Xi&{J`5n+-wfceL9ntbL@1OD!&iIgJfm4hk{|OES`VCkJJ| z;OFO{67My$cSVeS9SmG2A@1%ZjmZ3S9Ri;zChAwTnm{yUx;VCEn}O{MJ9RBJ*6Kg8 zYe~^|jR^^Cx*Us7h<-L^4I;rmy}cuJ8)sNuOH_Pyv#TI%xt&*($OjCHKzu$drFDd> zTc?_nXKQlk`g|#SVr{QuZM=J`+UfoL=_dzc^GI;#+Uxzx?f%oDVB6%kQ>(%4u}S

mqGYwBrqnKh49P zyM$%rQsPTbOy|D!?9=OXVv_T=QeNj6+PUQ?dvvgl#SFyC`T%3(EQ%e>nRqE$>fS9Y z-_QDK_|5bmnbnnJ99t(5} z`u99M4DV=CnDUgNKH9fE9=qH1@ zaW!2fs<+EVSbtka{)rCOD*--LyZokIT4$bB4*rVn zkG3VXC_`0z26x-BORd${#Yd^knF1)eQAFfs0mCBZB-(|A)`%Uf&#-16iwZT^mp$1! zP_E8yvJa7z)Dyho*cL7UuH=cLjTJ=V_VoRU@id2=dJhtfr3|HyCU0Dnmjm!pyuL_S zPac^M{Js0G`&gYqQCCgHBP*Lv*-3klel4LUDK_Ibw*L5m{~lg{uFB}Sg9ZYs zW&dBPBG><Whd9G%oG|uSSynh77K2dsbDD7oFmeH3OAe|f~>OH zhR(FG9@Ul&_bAIsM)W)elP_xt(i-pV7g z319X==QXuY7AiUWY21cn7$b_sAid5{5!dbIQ*;N zY0)QyRrW}-?cNuYBCOWPZXM$)0061fsZkYIAd#EUGUBOn-$va!=%%-rbdHzER%i4nmA*tgN>cYPrEMS4996j zwv!F!jS7l#cbl{IWBgYUhdnv)X_MQ{O^Zpo$ef4@V}p2!^$!ql{GRhENPY=~w-&(x z89vF_Y>he&2`l(A5w}%IW-1;hc`&YXXr}Mbig!dHGz+9c4}R_{19Le$B&02CJxGE9 zDu^Jx<#1ccq2`i>S#U|qG6d3J*i2X@^VkTum-Yz;gqR)lC;p;?e>5s7-O6@d4K1+Q zXdJ}ylv~~a_lWO80B%Xhr%>3s;QXQ-lAQ!`C}K)*7nI!LLUL~gU$En=*VnIt{M~|r z{I9Ps*%{ff?Uru054%q<0kVeBy5-^W9@0Ie=ITw3F%RFIB3Gm34bevF7+FgjT2go?CeA#B?W^-s1 zCtQi%s!@%gVGNEf%`r%|rja~=QcLbcS9Gzo@8qaF$;I*u75GJzlP033#K0u9TI$aY z1jMrvZHTdVV$L2#(P$LU)GWCw1H;^YM_-YjFQAZpBgzmheA>bm6%1$kao*7MS*CWW z25(C~4eNe^!5(1VUR0=}oD?qu797qqY2s*;2A3CUonj1@ZI7iLlirKdm!1%#>1{}G-xJ$!9&2gBr+z%K-U4az#bR}|lDw&{t=CPQ{re#afL;*!FW zF|jGSkbcmemWcc&lmF7+`E?%a3Ybo$m8D7C2DP z6E(E3L@~K9;)VKCYy=koLo+xQv1!r3^UF{St&tB|bXRIBYb#_*tu4-Cn6%Fk?VvaQ zm&{rKncZ*zGVlDBS>uAmfj4}=3w%a3=F$Qw-jvjg^!H3myv%q>h}PKQkhQ-ni1x); zz@Ro}T0~E3fr`>`HjUmIT5~6`$~Rd4SFwqcBVbjvEmDh^6o=Agj|9?Z+GXPuc9^)? zPA}3|$NFu|Sb2tBzYF)%TJnn@#Os`f?|bxSPl&VDV>&0vkKP>Tg&e&DdD& zSr;%Vj-@r6)7uEoJFf4ChS6)UrvqdgOKUl|5rH*?^I? z|0*)_$hL@)&ghM!!|{G57Wls^zRS_ItPT4Snx^nyf?EFA40Z zZ0!Ob&R~1FKlY$#;q5V9y3|wK#l?Ga+35?Pa?Jd6brNf+EX+ClER1rz-~9eioCNo` zSlD8t%HeSL=PLWM;lmQHX2m1j{5+XXD?Mk$Gf6e)TGZWs!wcQSoTEC@>rEBI6^(ep z!v~au!;0J6Gv~WC3d^?+SS3N}9X9!ND(K#WEWCH>UOvtWD6`eL4>V_&#T7z%xo-rw z6jrZXHY|c8c|WEYzbzuxcgXY_FN19yZ@BPExzuNVw-n)on5Uznn!Z#czO(MqR#Cv> z!d(>8imvgMuf|(odH{~1+M_a5aW_UdtBK(sU(fR*JukV$OI@a4#;tJHqOOq5`GR9R z#Xr0+#4RZX8B-%=^Ch^wi<=3BW6f$Uuvd}MnkDDcbmEE->{BY2F7mo*^a(D_w1P0F zdak0ZqV_Owgu40iWtcGhNWJj!OQAqDDdw2W^{E5qmC za8Rqi4SIAe2RMs1*2lRgCLM^as~{qls&x{jKU4lkrs(x3Fq#HPH(CKsKK?J;(A
YY%kb`<5uk5FH5-vkxL1)sq=pin%y)?Em9N_ zz^PrTdqg!d9pdl5)M9U`FB+iFJaBC$Br&`L>KAtz%@#)^r5`=Cv7_5^j(d#%qLY&! z9N3fgJrsMbT6|VoV^?<9IBo>(*5#R%g;YnU;Plnb$XvX_Th$PFi|U~(&fKSV88@X1 zm6kyxl|lPLRd-vzT(iq__?Sg&znsp9&MP8iOcMjWzL;G1P^wK2v-{dek5*$l4!y3f z^eQ0*5u76ONkecenXkq7`;3C|UF1*0QuBCYv{5JHafa7laD5cHw5=!Y1Y@`LVz?K3 zqt1dV=h{RDxbt<}FO$B2JhPIGHe%(^m{Z-wVu@ogG@>8OunkQ3{h;avs>_m}1z))~ z5m84XQZY1@n1nI~gN3liYnOTL?mU*eXH9-A&G@~n7&xTYkrg(RGTkZ@iFtn0B}(si z`|$5Yiw=MN_oiY+;tWh65n6~oxwl{1!6FPQhtVZ5!%-2=@#@0^FNH=pW}u7W$o8T4 zmVUp>8e36{IE(;7wMhk%aJbK~USDZ}eHGIBf^tS~JG?RF1fIpVFt+q&kd*4Cpi92$ zTpts^iKM+Lr|wx9H@p-4oD>@MP~uOG9&kRy#+2&CLv9{^f$x-z-H&8 zrGdICl|Cm?SI_I0Y6eFgF{-g$dg6v}sk-Q4GeYBHO-6?;Sl+{({byxDbg=z8CbVi` zM($6zU4K;l&zDZ@1mK)1QjosbS!;}y(TGDFxQh=%VPUaItTg=^N!?QXVnN~E#wnPN zq&)9!?b!IJbfVAFr2L!_p@wOo0N3q7EiM&W9MDUDZE>X1jcdltqXCRP2NKgc-glYe zm)(1+%#Kpn<>FH_;Ipx632pSN#Pd zUAS5DXx^z;R}noqMwU3%)ugKRRqyHBk}NNB!~RpxRd& z#nlJ(*HJX0CF{eB3s|3x&7+C$|6?*uxF!Hv60k>b3xtv;ddVasF$ES7C^^Ed>4|3WAzizsa;H_&?TH=T;7 zy4a+Ngr(*nH6hv9;h&{sAw|!3B4?mAVKIX~!V}iZ1Wkqxe*&H66O=ou#MGr?P3d9m z6?VYh_;X7oj~z4JB=#rS!kX?WE4j8%g&6%=X&{M09pGFEmE010=1Z9IXh(jC0+_yI z{PE&tQh|XMw(2T!T;S&I+w~j~JQc3wFkO;8n6n`8Bls)rrS7b@ag=VY(bFFurT?4I87=T~xhC zkRnrtEATVXyM_jbk=W#vAs2i zuzG!4hsIFv+449=M7VHm&9{-Es^4y_SGASx7dAgJ?8n_KKSx%MmsBoLR6k zKR8~avb`>>@&TV{$tsN_gFcxYq|!)TsFP=3#dr*w?m-yV+nVsJoMXeeTXb7Ts$+q@ zW8VpY6WC;OZaeI-V-L3!F=S9GvJC3VqriPzngY?uMm+%*@MM#s!uD1!-s z-Y_7rj`=!%WEUK?SYuBg;!{3z$5c1MOo=U6^^0w#;smd|x8bUTi$or~Ai2*&g-c3P zRUW552yA9V;q@q5SA665xsI99dZtP(U-Oj;&W=w}@rpLhUa-5ITQWpO?mw#MD-tq+kaDW~6_uGYk;K$kA)W($Y&o}dL;5pNru*Ky> z>cTwYMepJy>63%UMsqf?mxa>?ujwR3vPjHSdqka`#Wc44kAp#(pv@D=`Mi;+fC( zuNcYpRz6$@Pn9cVY*hquMeFlc+0JXwePqznmfTAenjQ;#IO`*1W z87W!~UI7{Li)?wP*6rv+R2MK)W7w%{dzP(>_!N*mRzQ(j3d@9+UFYieSvvo-C9%~k z=}sl5Yw~9+SQgH;bJtGIVWV)IxHm2BAJytOjg_j9c0w5X`ahOdwX@{=dT`?gI5Dm| zX(z-d25>+3nsaCgS%p4$g5P7C46npsb|GBdO9N-;6XATBDb$4XC`uX7inw%|0ri7m zsW1qj}a~8K9Z8VsC0(Ru{v`%9Hq7xi8bMfQJ(_{5_K{3D| zx6iM$U)u^pu>|;nuQRib92Y<780@^C&dq#%-k6_vk2x6$eYc0kSl*Wi1)lZ~r{08y zb6Q0iyS#7K>M0q!p6?zn3A}!4gCLGx3CM+uHV-4O9#S8Df!mdqhuHDveTFWYM92iP zK|HRK#`{?b{Y5}0qh+w2Hpl&!-_p<{z9u3Hv)X*$lXwEp58X-44A^R5TXQ{f*vBEw zGc$%k({+KtP=QVSOzx`l7^jl-xg2NfEvYcl1A@M^UgnW7k!O5DRtW<5T>flZ4WE>+#`=p{TUKi)E_CM)#*6PcxSiwUkWxK@E>RCsuXzY) zGoK|N97abWy2Dj>P;U%KdReMy(cO6{qVjD;3+~Qa>4mlu=Ijhg)b*M#>!eOdg7ZiT zCo)oRSsXKBd7$N;s>j)6Mwb!yphVE}`K!FIbT7Vy`9KK%v@UJGRN85Ixf8Ju9gLB@ z1)8xN%qv-yf0MI#Xfqn8QrUGIaQEb~>6zJ+3C}_$cgMca*KlvX=2{~~I*o*!YHxd~ zx`6n2YHO$aHl<~A2#%4y=b>-iw&!e-$X5`BUjj|aS$LdCj)$R$h#?1RSJbjHokV%n zNOgh0fU$fg98exxB8`BO`6*VHom8I=cD7cO`y10*nklPy0&2qqmsIWqN1~zb6kHR5 zWQD7T$+xA^`t7KwG&EyYUwQ0G3lKBYCfEJ*RMkoqnTdlkHTJwTrNa^qWo+-35*)h8 z3TR!ZfKBjPFJ!Z>%}%%T^y&A(ZITpRB*C!_n~?%#IFxe6ra0mRgvZ>M zXoSfv?CdMeKV2~>lUEi}YBD@8Z93TSZ4dQkJ-kTpHPz(hQ*!gwrYou!LD6j@dS<|54U>h$d0n_*r zVo&;Ei9hbC2(v8d3rfAXNR0K)LH#49%4)=3Qbg6&)nZ^>5lcu>`bh)XAx>1n{KudX zVc?5PNq_)yvOTXMO@NPIIh&2m?I``E*laRm4cugvaSCM%RWpxRahmluMXVW1DkW_t zGNSYCsBIE&Hx0!;qhzup*2k2}%ptT6W`)=)DDnCpOq6q169|J!Z!XL}G`trGL#yn$ zpSl03hC944_~XbyH&N(w_XIcK7(2V{SIg`{$}P zDpERW+O?2-aW!34{_tO$AgTO6b1oE6fkL>kyssaU1VYr>I=+`?#mXJrHbJ)W*3hSH z_zvn7sL?zZ=I0#qzsM@E(?V8rFHd>QuSPWHTwA`4OV}WM6OjDyzm1KETOp*p^ItuD z`K*onx@fRQ3#RyWgo8j5Q&O7%a++&J3|BTi$wnjM;-PnqswJL*X9HV*c$V|ndCKBU z)9Ikj$?^JrrLZmhfQt z0jzV~lVs>i=C{{8$*#@$0eH*xf1b5QlLUQK25d2Z1-w@U|Cd{Ga`vz`b^7g)*0s)K z^SH1+d`8|7gD&B6ewH9}?sAc3oY``uweEG1VAwQ$uA0|eAA7reZ19`EhIeu9xQ%RR zn7Tc^-Aw5z=A&4pHNq|mFJ5y?smdiQP|h6ZdyB=~Iy%uB3^c((N+(j#&y)RNdwpvb z9{dKzKZU%56DP@{knQySL5>ynkP4xWH$4w*Q#5G+IFuqIO>^`!(?owkYAXHMNveY% zYG=-LY#)W_j^CR`fw<%@+fVlZ=mQxknWdeY508}37_v}86E>7kjq4XRDJk_4;b$8^ zQFr5JqQqYjj?&p8?87*=AI8692eFgqc;k?-u~#mxt~$ArkPn$0p~K!E_X&%lezzdj zC0n2K0#XUf`5?^|MAh#Af&Z2t5_tCQ0SD=**2ReniaJ+zH-{%K^GqrL3!RhzG=LH= z(ohk}xQ?z(JMWWaJ07NST;~xRcnrvdJri<1jo!wm4gw|ldU%}jr-Va+3djROozt1u z9z!mS9gYg#;-yoUEU%$9w{Wg z?T|)_*=m&7ebh^1o1$O03}oF>h>6XweIYac!OW9b2AY`pnIl+Mb3lC!#Ea@V1T=sI zq3FkzHN!i%T(IHiN#ZOxNM*cna}*?G+;Z#_S;^WsIsU9EVqCSJUzOa@804YiyW|+q zg){bkI>feYT^j4Ywl?q$ipSTI)cQ)>!BSsK2$m{eC~{u2H-5536B=${n%NtkpJ`ZP z>l5pQhQ*m7tA9kDq{7dz$2OQCzc7EmmEw$YDpsh)#^sL){BR8p9g4+aJw|_;^ zrH@sEbIq*-HV7}Pw4y0nu%?$(u0sRwqS_e47%$aE1e;30LI({2Ejf%V;wXmu2naVK zCAx_mzwl_#maJS0043p;;K^Cmb#!A8*F+}fTf_lhP<^o1$S2i|)Dh`3W9aaE_klRv z^y}fSNv_HhHHQ4v6h;~Lbo1-e^;o&PMKK0WNj(QMkS{Dn6YQphr(fCb;IEja>;2aU zS&0=?n5n2IJL)x(-X!j3=;LmTr2I=bF|O*jBdifl?Ny0>BZr$&2dX+o>u%?BO4wP< zq%l5*h?r>E4pNl!As;zfeorxJ;D?^@Yxuj$8};eUC1wnr%M*)N$%Y7P;gHsNrdc-x zUoU5GSoTH~DYqVA26c^II?PJ1p>w%{{%D>p(0ItejvF_{seduDxjC`kmA9(&vX0%T?eNh$t+KOS zY&#vJQQkII>&Q)dPv|Lxezk%P;eWdW|qDu>U8TIE6Ou`>hr@#xQjfyPQR|WEfj5K!MWpy zQLP_A!S{=kwNmBcGMB3jKAC~6GH0c|YOOchYH}35P3P~;K^!cxrdoR^ zs#dwqr|o@1u?P(uAjVE_W$dg)-mUfd`DvpC7Z`=3*QQD;X>7mh(=U$dlvoy73uRL- z&K^7!+*s1~=#kM~Z1ox+9-iqH4@ZWy;C06?67^26geXHH53ce8Mn>rnT*tS25E@$_ z2u-zUyRVhLRwPOf_`Jl=DkdjEyqIJc#P^YKCPzhm$fbiWVxW{Fhfv=hKOZSs_^_OrAEiu>N+RmQZ+1~M_Fb4|~ zTBBqbA~~SUUin1~>`6s=RqfBWzc8aP9>ddSN(xX4L6!5Im-KrwaG0i4Y)wWd&vg@6 zCA09DMMH8~dL5R=_Y{Q%kU;W*c6QSPuch}{%#b=u%Il`jk`xlh^_=4C)2ffXu~IB; zBBF{Y%=(^cV70oVN{D8W;&JcnONZjxC@C!z>->2*(52a^**2Gs^R1$br-<#*?o0kNL- zl6@|ySqcvP7WTtL+)$g;h!q$@Qk6~%VQ!~<-rLGB6;x7RZlamq7xM72wXEsqf$1_R zyVXaoAMCynS2>E!>Q2c?gknNZWcN4Kfr)I2_|ZnAFO%|B0at4xA^nexF4Uy`z({Rs z3t5)<2=Bi(o_B8eK7Qc~L~1kUrF%oW5l4rGo8wHmaB8SueRjLDX>o@zpp3VTjBS)f ztv81?5W-S5)j~s6v?A{)3{n52lXEa5d!QV|6xvzK@(44uAcSntDPyGCicLptaE?=6 z>AaM^e}%(}EAoEL2eO0(PD25dDpJDLGYQo;*6AdVZS9K0q+(^wtjlI*l(kpfvtpqg zWo5iUnb5j%wc1cV*YE=BQ~O#Xsm#q#M)asX7|5uG)@ip;Az5qO)j<+8Xs8HszPLs| zc%Hckjc3~4zhzs(7*xgNdzfWnNG{^`#$ha@(-@w%={0e)6 zfR$8sI=;j|D*3~bOn;mPFLO);i!v4BJ4$&flBe|>b|}MrSY`K%wr>5&rOKfdR5+WK zNvW9?jylcg^uvXiVY0H}39^!}&k#z4MnP&(bg4-P1j7+t##g+qP}nHm9v=+qP}n zwmr?ezvnsUyzg4~AGkj}d#&1;wN_1X}- zL0g~9e*TSMNW4EWE-UrKz82UoBZ)By-m*i<$_)fr^uZd}4Hb%0&keseaQ(Q_P(^ol zbCLdbiNgTQpwZZ_Xza7LJP>i#R2dntaSoc?G3Cjjb}f7(D~j&&QOfSu9RFhk1z5uK zg+K>vkzXWn(vtFA+EKU_l9b>lTpip$W>6n?ECEsBSO9-AZ57bN5raQor z1wM|YSYR<+ssx(!U*IFVmWaIPgXF|7Rx!=)kxP$la@AjCWo=1!0$6XS=fG9tvFbc~ zDd?`rDUon?Mt4j?3c)?szkwRG(D)jgV3v30L~pqXRIO7yf&-0&+f!g!OVM7T-fmqO zKjjPZcc0)o=5*Cy$a)d(!GD2dkZGus=G&yyN$QeOJGNFKq99T%S?}+dE}6J{?$hOX z-{wgttPPefI7o1rtmVEF#X7rXpSSV3PO3WSHijYJ$P*d_N$1zOjyybRW2(A9%XEq;PsdhUFB-3s_J8T1?jktWcG?Hzi;fzt6TujtZMW=EWSOh#N zR9;Tgg^NSb0k#HABxx3fB34PDG$uYv@m7ojYU-6s5DamN^h5N9utk{7!zR%^=?rG6 zt_=O?fl>4m+RUFVeu>(uNx6LBi=NYO2j#E2PpNc*jhI_vEind7%S6<5JVgL^VRhT@ zuj}k}zxziGF&;mL>qobPuGqnyKc8DAa_{e-X762i4~1kL!hHx}0S{sU`KWHE$pysMuAIGac73Vzywr>F@i~nQNeLLon=cq8Wu(d;*uHMo@3wRr zgcc7I`)mlEl=?nkqvFqOte_XnYMRF;Xt&q><*!tYU>g}wtu+qB9s!-aPoUV=%GXbQ ze27A)`x)dr6L@siaPqDN%l`f-Du((-hABU>RPV9}3(_rT@k6N7q@3oZW}h`OvN?Rn z5;m-w6L`N7?d#Do0Y^c4tJQ5d6FlyzgtzWVDi6{ak>CgHvaGccHs^&8OfVDnCx&vyjSkSy8P`ea+9wz}ZtYP_Tv`Hsme3|ArxX3SOI*OBI2bCJ(>GuwDn+5t@I!WR7km1Cjtx7GaZ*`RbO6jNw7!^wx6j#2n}7JJkj$DOk-6p6n%ixT zP}nyBlK;O^5hlotni2qB;siiO{_XlVb~JGNMO0x)4l4A@R-BRudkXIb_sR&xfzAH|_gF5=UEpe_axLX$6|2c=Z$l zmaE!U!yi|Dnhe_pkYc{IlCg3K?N&4^%|E1Ivi21hlg7Opb1Ct@;>>YvJsane$B;MX znzx<{O*eB*h+Bw6W&`GBc~Q-_PG!_PAXic>B_cRMAn+?J57rD2*3dH*Ett$orV3T} zE0vx_s=8cONmGrbre9|`{){@IFH|IbxgV?~$?k$@JJr#6oLCD94L5$?brZgLm+uYS zWn;^v6QN`BT>I}}$A+~NhW;Sf3(h|J6Q5|zeNQGbBeG{&;j@0>ortP{CPp{9UP+_#`LUuC4O)?%`Yc9o4+6BsaIb^G zv$535H(P~4@w*m@GYt>6m81QFzb4+>JTuQ+GYvP3!o%{V#?|nW-R!WgpbG~*XhHP{ z`MD{==q}_H9>wjT-9$@zM38rf*56o6)4k=#u1%%_>S!6d$saQlM0^xDQ_<{rNU!xp z@-QQQWMs9xejVwff7=9KXgS+N6 z2KwjtwCXmXD%1a zfCAV_QA`l1dK%{pa1v~yE@3Jo1M~|}hX4WtMgjs14M+tBh@b$*0s?T+ztVz90%QNT zt_V!=?`r^j7G?ni@XY_~8cjg?Pu2gQzyFp$a)JN99&qXlJ zC?M^{HJpHexZwUdfG3;tf`EYdfh2?ll-+?Z|3G@99DRQ0n3+29BN0IEE5IwGFq16h z=R=IGYoaFDgVi;wURhK$HOT+VBkU`b4M8j!f3Q7ytH8mC4~73Ip0)*Te3| zpRm(y3gy-e_KN&PYJCReCBg8KmL~6y4;O5|9aIxpNzwmyf3oiZQ)26#l9cPO%HbwT_i26PTx!&a+?uH`nIl!}D>UwNp2NDxNjd!|QL_{}2!AERuWuBb|IbX;At5SdcQ>0HL^B_rdE#g0{6JQHEN_hEmfKm_r z1Th<<`NIT)|43gvQUJ94Xst&uGt(0JVNgsE zxhK@<6^A2~wG4?d=zrb`)^dPnfUPzs`Q2|vA?N|4&?mskx{y*#sM)Gvw7(tjBpu>9AxzJ5^{bR6C>Uyy9{qu(Lgu@STfKvzjS%bx)>mnqW&3x zAkM#Ac(v^-V4!yS{4Qg-fn5!piSVZ(I=Ki0*i^Uqf}2x+%}>(7p3vY9BZ9 zHw0iM$ao?1U)!r8bUpT84idb2r1^n*5j<>1r^!HQ?Zc8QGtCdPG^BkT8=f*k;QyMb z1W4)!@X*)BfRi$9+$b9Up6Dw68GbEh5M4b->GI%LkO^BKYUXFSll2Wv_K#;#Julp< zdIV$<$xV1*M~HV|~n!{p@3%sUxNfkUzSVY{`j$ zo0@>QRY69JMvg3x`}@V2@2?l|P?DCfqnDXn*K`WzK{}bFR=k*WPDQhe;1ll80GKeW z12P`9q8{%~qSar_c$o7ciM<05xxk-k4p$}W<;O(>va@l)$n4oV^!ob4v&zvB*WzHZWLL+J8x>opxJQMX)d-t5n=KynqlC-zSXfD#f-P&5Y_${BWDP3pVaFbiZ;<6F zZ+VROBiE&@`-sR#o1Y(lfo6U_RyEC$8hy_wBp)X%H5`#*{mk7zHt+}grt1(&CGg6L zVzh#HwY$riRjr&KVm>_LX(E7qeD)c>3$I}e$sADef`3e{h+@T(5NOiVo;1U(q${t4;Au{~M3LQh|pH}AqgvMnzI1(MM`kUQmhFf>zz(pP5U+PNI;j&iu zmpdGhL*jokstM!aMG_;x@*?^@4SmN243vl!;lB4Zy+Weh_<<-~lh0Az^NpYd!?67U zP*O}BTbVoF1CWV}mdxypmD0x2DOr6RkDMRxyjO|VI4`3CiE&aAPcr?xZqYgW=micA zk3oy<5xS00&$QJwC0blfW?xI4mr=TkvxHHQmDK9=zyt4+2uC!-(0?h>D&+h*5d}4< z2DH6u+1aVmfPRPPDr`l|c*ZTY0MszL9QWSqzm0-0Y|FOoo`utq4z*%M?jaoiVT7xGh= zqEd&)my>4gx22$$zMI%Yfm#*x<&$h*Sz#AIj@PpNBE#Egp|B7Ll-!M_1CO3Jf&jr*`B}DmhAS9j(E=pGGUcm1`kO7d$i#2`IQoS{0LmOLCGFkmF9i#?$v7K zJ9fTdUhK`rs{1Mlmlx7h$tD}wPerXPQErUD-y)e7l|Iy0&%!vkrZHR^ZLv1S>l?B= zuHqiAt8b>IW0lD}jlgx`>|*iayV`h}Q%zG60j9=hbexk-f;rE3+) zN7%l8jN0Sy?)rk3B)R<0s@<|kd)3Ji&VNnV7A-WXjj$ANv;Y-`fIXCpDwo1%krCMk zttenB(F=_r-02IPdW)V|ZC;Yh+PYq*R>zpj=n3v<_tj}}8$xqKSJgdx?s9kfVs8&? zIFxv*EQEtGdllk>pDD88+wy}2$bA7^x@ky~65Ql&v>&%4>bbGXY2rXJ2RA1neIC2Y(!R6|SJ2|AsqLMtUgMMK~}cs&d~ zKfd8|AiO}dHXQe#1SZ88c@x_F07Kg)Lam3;*C2^z@|HyY-0`X!d4&NiySYha+8 zFpgdTZd!;vZnVO98cW!|m$N9S^MY2>mR>E;p_(MnF-ph`LA!2E_3<#HmTm{jUom%_ z=lo1=PoVe;{jSx%)6euXIqUeL4*wv)1GYW5jB1J!btUQ##%mnr?PWxD^RySQrA}Qz zDoHk{vWondjrL01UOV1ySs;0RtviRnOn+j2)RvT%A6Y%o;W=5DJnhUt;h7GGq~}{l z+MesW0r_bd6$FILyMn<~<%qR528-S3qhQ`Gw*JA21Md8%?ZG&sz9u&Ee#;-7b#7_i zGPd}gIgQ6W_A{Y{YcuV>C^#P^L4Ci)H@P04 z!>>b?K}2pp9SqPIoUr#QDokg_qc zF_Mxqy3?_!k$2Y>3K4GD8rk&XORI83%~B2x=I3enoj7#)%qz=rYY?Gr`d^oA&&I_Cyg@;|u&4JvX*x2#j7XQ|-jczf$ta|& zibX5%e_tA7{iB64k~91 zUE4BL>7NC(FIKoF9P6LyK>}HPpvTXKRwH-P`aJQFPyl0tQB_}D|w{g@`Sf5KA>+*rQ2lk#gUHqK{60l zP*qYtb(_!HGyoX;yn_ z&^SWTR46^$y_xt>pg*cJKuzEn9tr{D1XS@3lXmG}Vq@HTSA#l`^uvM%o5NeIn{kGUH9wu!u zgqf}}87I-$CiB1ZzziQmrP@O<4I+D1I@3EH$sS}mgduCjL(0S58Zo1g{<~2gC?cgd zswZqEO_qy9A&5c#n0#=O-M@6GsfkV{8&5k0cVd%@u=kOl!GS+YZD{-axWS=xoj{pr z?2+m}JUGL;qQ;#j4wC$JkNN9hrZ+T6cwL~~hF^at+W0E9s_zMY%6KNsM*latbS{KJ z*3#rs;m8$u*Rk8=w*UKqsq zIVQ-y-m0Cew}Jihw=o(Ao6|6kwF;YDv2?-_4G2aDOGC54*h9pxrtWu$XYX%5XRUW6 zlOJbIbih6dcngDq-3JT3g*5Okt1F5pTB2}yeVmnP_A_;5lIQQ?aBvJ1f%P8IKzN-^ zM4K7}UJ)UilV=TW`I9dgvcIZ9LDJ;w^P0Q&xgqmmB1);%)hii`0#(iEgJ+b|=|7Hz z2wz_!EhSex%XiVxQre?o`(;&>@JY>P3Ap1-2c8n52!B#N4Y`^S^)4Gj-~N=Dm>Uf~ zN7XP;NGZ)%J*H~~-5Qh{#gtZtWtXB0-HG1Ut{lD%d^c{%nO|OAGY8_q{GL>t_g*939-`pw+ zwX>Y6*oE+j;R!(1=rAIIN#$A%0(Xk-KkjBx}V~#9-HC{+NFNLRa#>%*9-a zyuG7#IZ4(sxF0JKxuj^E?;B^@8ym?C*uoVX*qVf^fREan5ZU+I(4wk zABZgB0~cAMJ2qb*q{IcEC*k0s%*sJ0wpo_S;vNcEdi~u=%4}iAtQ*n%Rx;frHIXQ| z4gr-{6M`L0H>p_Qpnw@ zW|~RizQc3IvbmGZzL50E)OKJVsuEEf7*c1J0O2Z**6|*3Ju0Wziao_WYayVOzfylX zA4V1eTdbR66yWHCiGmeW6Y@)o6v&ER_9e&Y^q#!r#2nkMw`%zLm2_~Ic<0AwU6 zwoQ7|g5so1T48)Xp{LRJ=(R2Vnz}Gc@wX5Lm>6M_Rlc~WL^%H@=d(g%?HYd%_x_z3s9;g}p%8&4}a5#coIdjFuLVT{?#o%;t zI0L@ZCt7%B%ag~%Xjx?72+-H9Us-y>AJX3qY zD)gHf1El;jnSYZ>v$dtU`?w@k4nVCLMET;Pa!H^iLLd#OQz(8gW(0sBRj#^=c1V)^ zpd`q>1pRbHaN=JD#&s}Ov&jbjPqzQOTX`VEYoD!#0|}|BuoDs-;B_ry>lDe)h(Ht& zcfD5`zc=?3ik|cVw%>c{xe?!LW(+i5W;f=|F+VIgFzGKl`#wVX92Y*l+{p?=T#_f zZr_sn4uB-18%>g^zZ3#uipqJ*>M~q|WASvBS6QXI67r`e1o;gSUh@xDCCqtoba03u zTKWk9p6+KgvQYP!amheZkb14>z|l@X3$K$3TwJCM16?#U5fn{f1FC7)6pnbWc-i;> z>1~uS*UX4GB|xnKxX65b|NJK_{g5j_N5;sLL>Ur8mj-2ZUiAbqAC#5C&bUKP&$xK5 zUJZ0Gjhg_Fh9n1cAhcp@sd@KXCFOKZV3#zcoCCR>So9PHvIz1z;a(UJ zJtm>j1(O5-Ek7CBUi%?RPU#WFVSuM&CDAbz$%DLS+aG&QV|^(Yz=*2qA@vz0%%WBs z)SEZE4=Dp(NzS6+|1=N+PtR1|)8#fXdMU$l1oQZx@=-mYXNBGci9I<+!XX9-Be#}0 zO6%t;6ANBFFr?VWDW{WA6N1vGAL8dQcUx-c7)liWPSfyK5QLJFjNs=61x@8tpu2}GSQ|BJ3~Me=LVK`-ak8A8D}m8NVNj}^&kN2RSqKbzV?}@TFOc)cH0#3 z&WnH=BlbFVtEW)6j1q>`wqzVG>TF`8ahk$|U`V+ybO(GOP0b0?ycl6-kVOzlBn^S? z!a!9u__QTkhR9bpD&hlp1PRjHXDJ+8iDcZVlfn_gt4uJ2DH7qiKLl2NXQKRDLnZ&{Z`8D?+a0Cq88MGA|1aR_`6eT|d#U6z zdxJxS=h9vuXjLdz)3|}hM2OXBpdQs`0UO+^D1ru=X6*zsMBqG)_SsfB#1ORP=tTrz zCkT68<5+z&Nz$-du`bevfRlxIWDBM=gHMxg7i=u3ttEF_Ly%9C9<*lc>m(h zEkT>NcXJ8TPoqMX`s^J|2A9YAY4HzlZ`#%Elz_Q)`$;SAtw zp~%B(!I9o9v_S4VVgTDuy#px>LTsX>Lq!M(mUn|qVmmWZTzQ+zA4KRw;iL!&1k=bD z!zn~MP#j1K)j)n3F?sUr^#vUwxK=!l41dr=BKAZ7&tT1<^c%1JM_+Bk#@H17)_KPn9101}+-Qh3ECaQ+u%n1~lGf zvxD4)o_xM0Bzf2Y6CFM;k7}L+7pk9nlGq-!?fZwcZyPeaKBOS^kgtt}Rs;{Jj7PTH zN_VQds$&R+h=>P2V7G>BuMo zZuFD-d=8J+a*qansuJ$>Dq~U3p%#$6=*xSp6%3x7ybS(8thP9en-Ce-p zsgg}wcab+r>}v*?n`&f+N92z5yFyOWx!w<8h!5212Kdnh3+1KS`uNEd_a{p2mc|SW zx#sWggI4OX!o>b0bHn`zS7pc7BbW+fS1f_N!q2bEYmWspvSdeZm{@AOJNZM5a%9LH zu5>p;3{N|UWNV41+Q~5M8DARRQ(}e!FC!tl9V$*Of@2KCA{AP^Ra@pzX3{ZPNlo#M zs%q4_P`TeKRb48!@tj#L|C&cxGgD`6HJS_-MXQ$9icdE_(iTO^Z(8;ixp2^~jHQb~fHhf#*{vP?4 zQ;1C+9A}U;qnJl?rIq=Zft9n!A5j0)Hhl|3Ko1XTbusX-Y=Ox;ef(4L1XXW*6u(dh57 zC1W1V*Qx7~Gp`%aU4-lP<&T+z>W$a;>TRXgZ}Ulb%NC-tU+T_uwpm)6n!HNmq=Rsy zKKU_W_;irdYp%>4Bv+-CkNAB7ubAGw$i`=bG3}8qb@S`RP&|R=r$55{I)A;=?cuaL zMVBH_)p8G^2-&}lGILrDvtM4>j1t}mQU6I01~PJNqQiQI185HOL=wh5hPw#1C8J89 z((Z{velr$(vr}oWcR3Jc)I6Pht9`y<=$F<~s9f@xf!ISKee=n1%sP={hX{DR>MHWO zkmRAyEYa8XFCia&s3^E6yt7fiR*7R5>++;Ei|k5fd~G@TkGw+Fg@lWWnACGzu!S7l=Tl1(9? z{Dp{$e9htNpHbceF<(Ls+_w#LH#=Vf9%vq|MXp>GwF~krw4lb{D_Ap^M{dAJtL|PO>O{lQPeEi+fDsB*RKyVE3VU& zQ{x9pHFp5*WSh_0>JA4CR`p`PAyIq1&Jqn!7)|>7ERStJIANTa41yF9DMBw#n@C_< zsR#)bbKW6@dLRqIuh$@nrJ{q&Q<(GTa~^hV^+bHVC)W&KLE z;ieA`I|HnCj<5qNA5UU}v~KRn7DP$Dj?!Xhu!QiYdT&$usi2-WvBN_}=F=~$Lu6iSeKk#~Jj)tFaD?hkBGHX-ZA{F2 z0uS(rh8`h(8UYeoc>U3O!=fhATs%w{t!#Vlo58f>!5p*gw$Aw93iuru@G`=Xc5&oA zpZAV>j+XPvOqWQS;q5U~K-eYHmtD4HLgAzZ(@_h?C1y9h{qt_jH`Z^G&Ro&@T}zV| zJH70iBlMl70HB)>iAeXirr{z^#v$`bIBrsYq%T{Y0aY=B=kJFMx3DC4D_ z7uW4L4yfQBfZ4FbPF>kwSGNkaPO!)Vf`^b9NDzZje0OG%^`4#TbM9nqPLtbkJrrk)AY37fut# zrC6Y2JrCM}w>9wh6z|&Ytb!>lDJ$k~>4IG;d$8eXF&kwzR#r&~+;)75I(_jWYpEwn zb*6lt8Lw8e=IRO3*To-<`vl*ky0=Rw+&$crS1?@kW+>%&)&rbXhA-H{c{vF=EuT7Q zgZp+?Ak%veckz)>!+jv;1lgA|d$W0BLIX)HQCfr7LhkcByJU=%wcLTwRjcibAS?61 zM%mL!8qr!Mb(ceAspu^p@D;x?#3UzSFuNB@Yo2V%&zKrsr~}>d`+6xnj#yOkMjM}s zVvA~CVG6BN~GAp>pJumB&1_aqYxM?cgt zkkJk)GKMNyPI|@;nKJltrjBR=#7T72nzzusdG7H|PfLD5c=$0o?j9b*$6XjTIWd*Y zOpj6XGI(+fD-j_TN7$ddC>(#a_r*{-=*vbceP{oeik&d-9Cd$hn8e3O=x4<0k`#cj zaW#0V4CzYdGaFrOKoRlL${C#7@ZiE3$h0xC8A!;h1>+LIeAhn-d8%GthkKVWGJTy5 zN(_irIO9>XJ&umhHmLCAlM*Z%5#-)p{W9L#;C({GRDd=f7 z|HGHr_O|?IP8@%X%&gFuv@4wTu-|-sB)%AS!e^_g&9|)EHFxisM$@O_9-UU$F21`B z$mbdP)ZFgsb^h_GW!VcX{De5TT>Qn@86gHQbSNBO*zxBRHowYJ$K@r=IqUh>a6v=K z5Q`MfCw~g;E;#8e)|_n z7g%B~^#ogi%TKY|2ud=Jpf@lEeuMEY4-HZQDzZFh3YwQ(wNBD!s<|c(MMcHL8b_Nm zPf84qU_fq*aMU$FM}3ARU0Uej$B>dcAbJyO%^Ii!_R`>kKPZDO!+F~&8Yj5Pg2UlFXa&QcZ_{6=R6w2frYsO%}rnOd27(;^Bmh zs~5>tJn||5Gl3{^7b-+C82e1C;J7us?`Z#XTWRBl0>^iA`Pk1M zH-=3wY6)vlYSqO!6|sst{S#%buT@tt=Je|KNrn32TMFJt;m4(7`*sS^sEXk0i;hYb zM95vE(5P|_d47<)C3kQU$`{%m7P~XM+3cf}Ob&yacxES^yH?x;;UKm{J-5{*NyPg+ zlQ(+00U}l`C)1%OA@WldUAr^%a!pou(chtg<&1%@{WM2P!G^+5G--!FLg5Z0q3t4Z zrm2OHeLW^aF6%jPt9KPym6T@BskF-v|G>^e+0OTu-6{zc-7_VN z2~R&QA#ajNL9(N>9>U!{{tBz=}BXE)x;tS$^=f|@Jd>hDU%og3qcrK3GwfbZS$4>hvUk`6xUBVd}+k3GW>bg=o}W$9;c z3L!tI&~Z&2Y<{HHX8ws9c6Tl?Xu4YVzDtstiK5RC@i-5l!ah@(%97SZ4!*weABS9Ua4L{5Ae;8iH^^!;Wc7! zymQ_RIkmzUw{~q_x6c_>)A)K$YGfq|gpc>@TPW)g57hrWLq^y|;k*Ux#B_wXcs3CV}#nCKLs(##OpwJ^1X;~O?z8xnr3W+iiNcl!%3{o?_9#<%NwcAHQTEzEe zP6PKg>>EfNAwJW&cd~37SX`X^4yox@HyFPiB#(!J9B|iciGQ$Pd3XewiFb`SB&ydJ z{{HPLu8c?TL8#DPHGg3dh9ET^T@TfG56q`QY`fF)_crA&p9I##gFo`Zec+})hb1l!R?hJvK8x3_O&5N zHzh+*vprNJ2F*702*q&?3J4C+rL`g4Em@P!3J?NDX92Wp$`0^!5UG6IQeJSZCNqK1YVn z|2n0$8Wp-@x)6&MAExHJ1YuTkD6kb<{+J4%Y)j`RiUJHjp#snsM6W=1k4|&uHPQ*? z8Qd6A@Ze0l#M12{$6`4hUV_QEcDGc9-f(TyEfrPg2h|@$q;J-XT8UCxZ;%C@ zeBA2bdK=&f@sRYc-=XYp*J_{jE0+%lPor4kaOaIIbO+}8e4e3)d^+eK6&Y^bkN7sh zZq7xLuizqTn@P|LZUv8E!sH;tQ%)sok!Co9TRk?<5TNSv?5C)S8}hJ_ylJqM>GESg9*cos za~AtnyUuC{7^nxWh6lEu=@8REbDP4}4YA&d`{B+QO(!R5jV@(Jwg1)!du`8l5ip}Q z4|L&-{d@$?LO;{49NGJU8I?a=J_>v`2i=rs{ABa(;$P;Fd8!eHkO(^ynVAg>W*&&G zRk}#{<1CJUdTp6FR>Wgh!jAy8ra#*r;tqkEPm;Qhq~RNmPP=o%i7nv^dKGEvF!5_B zEeiyZY5{K@;7&j4>!V-TV@{XV+yATfB9nGTB|&$B64LbSxGEba``ZMjJ@*Kn;kQZF zkz?naPNmUPq0s9|XNZNl^cLHIdie8lN;MSm3pAGhljuDuC%mbj4GSR0CJiif`#p2P zU)*H5+6DmPQxuy#$YS&@p%K`HYQ?j3<Mw{uo)F!NSC&i99J4 zwzjsvicA-!BA{-bGm;Gr zZireB#wWWxMO6SGOUYl$QI6zOxT)GqtG25+PB-Ju0*t0|Gf7y zbiEUx&31;kd%kIRFfxZ(7nXXE0;ghq?&(8lA4?g^rgPwU{8`)GZQ)dj2KoVvuaqQa zICb*)$Sae`ek^yBfz%vFY_Di;#b^M?Yh{XlIH|V> z)edHmD3Hg-*H(XpJcT1+VL71+6t;zN zXdzbwqTi$c@dr-qiDb#ap0~pNb62&t3M+{8@@MXpg_BUng} z)JNdp6fz8Jl?i2;!7kcEcf~W<;tJ?nSK1+zA z=VJ8a>SE!SszAF5+LKH;0);7fz`!;wp*^|M5mi|`qsmGDqw6(h16E|hlLj2U*$u*= zW$xNg7`$<-D`mqL9E@ulH1S?0-h%IUT2i9tUYbH3hU!Y~rCm7QhOHxHzQ$F4*TrA; zaU=%72kl<+5Q#3!t_{0px1AtRT@?ng`|`+l2S3S|tIml~o4WFzG&-?;5YIYeVq2J* zT|3dDcmUyAbo_K?lMT>Da|Q>Eo^D9l*p9s6M40PW0d3nP{zz9e>0 zIBS-@qrjo{+JP~+ictSm0f?DqbVij6CymRTx6F8sth|RY`%eJ!+Qh&Z`Eq@(L z(-ox%y-eNkPHZH;>|tM>cqLmJWfe*sOihimkKpKx`Cdd*AGiIpXX`?x*e5(RhMuUV z+4sW89x6spPc-Q{0em1N#m2VMy*O2A7>+UvM{t-EYa{a@yU?N%1SxYu3y#p}gdDAf zW1VY5@Ma=qa$sLzjnQOmff+4aMAhUgP$JwrI)XH=V%|}Ku;D$ZV%&Y>p%0`V5Sni3 ze%rH~VS`S<8_i>OUP>smi!%J(m!Mrxi?iTxC*_zF+z_qNfp~+xuQq%?J_Z){SirjM zScPNpUPN$z6Fg?f80B3^O{m*usnd%nBpS=i+fAJ90D+JFFhC;Q<#;e)GP0TP`}U?) z3Jw&KeL{y(?Vd*BF zw-VgTAqwWvM0EJ#{OV3{#U< zm-qkuQXLH=Hh`rg*#wBGi>>qGsXz4bI&pljQ3^}N9GDa6dMw|^35%!NOC(-Ve>9ur z^_(GGyfa@{%P(2iQa2BAft@INfguz$r#G~e=$ldt%npGoatZkx zd0E!S1edoj<#?6}&t@YQ@*KO|V;B(ixYB0W%jw&)Q$+3xCmt8jhsdoC&7u zTniP<+EnducJYaYZWQ>>-UElY8Ty|)Y9(MlV$~gg4WGySYBR$T6poOMj0_(@av(gc zYAbRE^w1KVitIw~kJ zLBmog9Ai96Cxxbi|BeuIT@}z7#&~hFl9^G>8@Y z3VAT?Lx7C|+A7s{cz@N@9Emrt=hEXXXe)+$L#I-$6F^pU97kV_8fQk{Pp65ZE=xf2 zghJ@L#^$`QPzpu7lwhiEMef7f`HQ7E*BihVFFGej&<}w<2ypCVyi}pmTapoBh&tSp zaQB+qLStWnPR6VZ{FU*)>IJL=rJpV_6U+J{qu|lUaldl(H$qM`5Z=$B6eM<1!>J@4 z9?Q1U6oWZ8xKcXmjOSAR4^8LbUPsry?IsO&yki?pW4m#hG*)9fZS2O_u^QX9?Hx9@ zZQK0q=RLmfADCli&6>HfuJgM8v7?gXU=U+5Vtpy-B{NV-8r)1(&X6K;yV7efF+AbM zLmHTM?(jVQCYhJhMHkcfMU^`@AdA!IHC~p`Ay_u+)5o($3fov*{|r4ufP(q_|hncC`RB0d~GY3$sbcbC3Au=i{uR_KWnox+WG zwr5%ha1Bv}QTI2Koph4AdPq6K4+f&>rN+_GXld7c4jctcqW|LNJa_MXrQJpU^+4S1 zU-mG|rP__Zisy*$_4TWe-{1Ik>b4JBxV8;yUcQibm~%XM{=WKV3Q*~GIx7gUVunV_ z<}4+k&grfgAzvOMSz&#>-T!>;7O^z!f0m)u3>Z(VXI*pW=16~gYPgf>R9mQ(PF-Ce z=4DxXU?h65@+e^!S^Ks@NXSfl6CoBtShkjcU1mSG>Hv$*V8|M|7QR4I?{6DYt>z1> zZW~;lJh(4#gqFWu@gaG&yfyo}`5^LpEX^m@@&mxd3X2YECP`&GJ5or*uvweZPwgBJ zM;$eHo&2I(eC15&;n{+Y9GH9$NoVb0Q3O}t!1Hz*vf#HbhF|fnSc9OkX6MycBSt1q zEjV8d?~UPg&#&eBPq)V##}S|WGpMH^uhoX>==a@HZfOgiRU~XF8+F+7X+weO#!8?M-9;y{b6&{a-qp0sd{~IRd4U@ zU6%Mx8Ti}~?sIcLzi?pvOQ)7k&m;GrzGihb;B4>EV!D+(((T{LTEF|6`v7?#;7Zq~ zJ^9Ma++FM)U3&^NXU?*z@g8(;!NYBosHZ7o{`e?|g3X@K;qU)88tF_dmooXU-bK0k zG|*ytAQtW<&Wk(oHdHn|pT5vcEc)hGE|hPrGfGm3?+R#95Ci=c4Dr=hOwi9{ z4(OAtpm(tSHH(1z#yFnsU)v9*6efT^vDTPf^6uo(` zw^fzNrU2#_={55G9zHG`uUhA)?s4nBB!?QO{C_Wi2yr?YUsx;&C0ud7i-aAwD8`JZ z0k6Vuap`?i{HTW7gM-JPB~FBp z)Kl^JhI}ez3{kcIFnB0(V;K|_<5BwrKVz>(5G~cC~3+_12%=vtp9;hv%kfduuNl zl8L^$p@%{aNk)2dFV5@`Go~!;a`-{5`sP%nM=1xsdR1OtlWi>arak@$5_5H?Q5#4L zoD3ca?sq>YVbayo=jd)olEZ&$Tq0eGd9;D=IvXx9LWNn*AnoApfocqNdLG)Ly z_$Nd)G77q ze+vp;hOcs6Bhp6jmF

rK9`28PiQi0-yM$P2O78+TD268#MO6kCw}~`&P+cl_?G_(UC!7Ud)kEr zLsTCXsJL(Fzih$1&BJna{$j(RU2Y2Aho9(69UMFQwg+UB@K}pXC~+k$`|-oIwZ>{H zz67Ea!eq5OZiW12SW#T4pDOV8l$(>SgLGz0f4w^5^G)iW(gp+X2^;|Yd~Jx+OL)ead3^CXe) z(C?u;>A=NTF=@M#1n0LN zcOkC-$9m*mSdnaH^JE^ZB1DoB)^yk~Rp_e9hkUQxeEs-e&yHvKoF`IOljVbgCqRAR z>p;&^f3Keb8CBaS4by$pCANy#h3Q4Ae)7J03`t&4vU~wQ$YMLfvzgvQA>!|0COnJG z_}8IF-3Md&*UBpKocto@V0O1k%(EnGb~7A^c*#l0)gG-HQk>k??o@Qu!hgnn>)Wm$ zM9DhHg}&3oT6@|XHJ5g^%nGx1z=+iPEhAZX+qC*0)VSSaUm_QVABS&Ht3J`_q&MOgIf(_rJUpvuC%Hb5CCEvKXPnJkF`n-9Rjp|@S#(OsOUXTtBqS-e9 zeGTTewxg;&-g7JVUx>uSm`z`^_0)=-@lUuMEBB9mjNhCPOz{yT&;hqc37-zQGqwM| zziuOWpBIPz{!d{<{abiIizh(;qCtn`Td^fX>8L&DF;32)qkaEXt#WOW5V?s+tl-`L z&wFm&nsVm)aE)zYdgpnzs}ON+nVdTZ&liv5KW=Wa4h(8NRL;03UR9_kZ1ePTTR}wN z63&7w2;JFRb7$kX)y!O2L`|w6aJDXAC#<_lSNa-W`Ez#Hm27Boak?3$y5!R+<)V(N z%JF!Pu|3;Ex>)hebajGa`s!&$Mx9d@SWkz`rl;hdYc|>WxqRD;eXqS0z*(K6^dfde z*ksH7oyCDk;_*DnbN<~NI2X4MM3h@J>bcV>?hWN^zkx9;6T%2*VlI@3?&B@ZZCBZ2 z{)o{XsbUssEH#)K!TWfmTK@&NG&e536sQLp#3)!fcmH+-Y;@rg3J;rl`09KHAK5eY z^~Z4;{WWgpX`$#lL^P7REu*!ym*Vie*rO=*E;^eRJbJm~cW_xrU86F8$Al%3%Am=0 z_XYkpWWFF^dDuS?cQG*d9H55H@Qj&A=7AQqN$pZI zBJXl&ZSJVDxCWKOkNwzz8;Ps#-Kl^OL8E5E-K(|~a#4|h+i zI8vw_b4_RpHj>ionFfdh+0nS<3i_ges;b^GCp~>%9Y^_UrijdTL*={j!3;H; zjx@ZOo`cCt9NsaPrTZXG`=9o5DT2Eq`;ucLbRyiCuCJSpr$?Tq|Iq{{-T&~NEE7}X zm6*j63)n#SH*zR47u=+^(} z4sjWP702k7I!B&`c6?@u#qyW2Vf87Ig4kivWu^<@Zcnp%V0POZujRDft+9;$)!2!b zi#NyM=R546!SSwi#1j4vjK^3#I_3{O`f{e1R4d{?nloL|renkpxs!y!cC@bzt=~rU zPR`Q(jESNgzL0cB9sdxSJzQFTrdG$Ny~trWgwqYetu6^hoy6^h*$go-MN$h`%U{O~ zQ(GAC+mGE$KLffgvI)ryG7=^3Vjj?jU1Q&#r3c=+=Rc@Mz~A0FF0(w6;Mdq*HArxp z`W)LQd%I9>76(M*B)?aSTB(tG$#N95uKB&JN{VS3KhJaIDz&GSihu+%zeJQoIi@VGc_}D_z&?RKM{)N)%g? z>AzdQY5KRlJqsy)8uQCfF67>U#pjZ>^oABL*AwJ7JWCFk)Cx|+YjL=kZt2`7Y9TYh zXeoRUKsUM5Wggcqs0%k5d6+Qe({h8!D?h&crW;RKT$CLgrFj8+5Wu1rz4EBBS&WxlDM0@Era$WO4WZg*oOm++*!wQnm zbm7-Vwg8#G8|91sNrH~Q{3q@Dx^g}hP?U*iKH1pDh)2h4&Nhff!Xm=MB-%V-H%^0R zv`87($z=$Cks7KOzxGXjN`b<_2O3}>%wZk#hSCrmI7{uCO;UP&V5nGip({Q*^kY8d zRJ-tddq_{Sj8>``Jr(2rzyElo?`-CnumfY=OhtK1-jJMmLOUZNPGZv=PP5$c2i zUG^w0LdAn;K6myNcah7G5&`LV&` z1eS|{_>1&9j&P2~4NymZe2)|)io}jA}G)>>;z-5R$WN8*CFV?{cc3 zSiB&mOk7R+#F^PpV}R`G>7~$<3yfGuRN8j>NYkBNsGG>hzGhzjITu@t*?bpz=-c>K z>LD(AH7?YxMu$eT**u?zrm4^GA##D*DDk3!%3j#HKf7P~L}5lLeTPQalf|zI<2Wv{ z^sh-Lbpws zx)h^KoOMtmBTOx@;|6|)l*F;zV zK%*0U#qdaK6fA`%1gKU2*c+`c?*6Z5LI#-MweS`+E!4bknk{c5mN$&ly#B$2ei;2p zbk>k@w3843S1eg2q~Vngp=!{0p1*pPM-24OSzPkH{E#jR6;_1-j{WMtQ`Km9AAc@p z!;|8m_`OZ$^yw8g9N=Me^+#N!iLe{X8Hd5L?(1<#5oO3)La z7|y=lVrMe^vw{ynFXRI)dc5eu3x`-x5P_?9QJHwDjr@5O4#S^~eMazaw~4+;4U_Og zMCCRmww;vs5%GCnAc(OCM(Cx7nDYolI}|)+dSS0P838o0DCP{!+-PiLhK8;5e=^L0 zQ)LC~!zd<76=G2)P#!VZYatX|aDKCTvrJ!^c?YK_YEiY5X>iyS3L}m*V5nbmk!zfb zF%d!&%rabxTJ0NTiZkgv>LQsE2mx!wU~Y#(aptAN)RH>U9fD-L0Brzwdy4oH z)Eu0BCObf&<=Ox|^BzRYeZkQ~BJ`N{jseAvunGGTY!=naHRHlR47qfW)>M+F@WRy{ zt4U72_$NQpsjo0y?G|_CbnNyKze*fZ7~Kk)f(`hzo4CIy<40~@0XUW z4YU%!eA%wS;0gSQnrD)hgfSxb&1_B53OKRwXw}G;h)E;@(~{lEPOoCMLX9YTF5mFH z(Lk>u9M*;Q5-`&tk9(KA*kg9y+XK#IX+%{Oh4fw>nmb3lHI+SxBL@Uk_Z4v58S_XUsGMIAWc!;@>*o6@ zms0M~u)Q4O^RzL>WMeZQp>O;r<;d)$rp$9S_>8okiPL`k zgHg(zu6_o2dCid>V%zAnh(pJBVb9reXB@o_OSSL?X&Z?@5IYX3KOeQ^^;nT2o|WT; z3Id2Xm(-9Jn0lJ-?4`@UE~yZoCBj! zbr*`Go9oTf4X?~3SB&4xb-)#EQq_eE|ZEBs{duJF$(m9pG`z=)Qi| z-%8KghT%|KqOGqsMKx3xdz5Q`Lpy}-Omn0Y+ zxLpTC+rw3xaGx|xy4yyFx=GPu(vVWA?8WB82OL-tQ>1Atb_GV zL*fVecG@AO3wq@xX-aW`IDe$CL6rf~ebcQoLxx!^;b{RNXIG1%GUd&lA1QjFXh;S} zZu?Z&KwmvbV-cMCU2_kaWH1aKj9*S+UfqNwe7FP8=m_VGbcypOZJWq*mMnN`U@-4% z`(?<+ub|C4!lq|~N=m4tfe=cTPfLG|BPPA4?ku_#t+z<#I z$m|*@+M@B@_frQqnf3SRs}Z#m4mr|;I&ywP z5B)stYE|Rqi7V)e6t(XdR<6**vPk-Q3VhSSSc_{K+$)l(IX@c8m@^R$dZO=|=;V|| zj4CL|7IEZ^RcK&Gn9{DFiADy>_Nvo&%gc8Xb0A~NN{i)Z@~Fu$^jE5_zMY?}$s5nL zb-dQMj~*@Dd5@>&El%xvBX=eHoqurs=7_~<19pdZe~2>dW~s8jOt{qi8*zb{5E-5I z{;_~Ou>dw&g@>G~Sgss}??5R6+B@%9TdKqVez5s8u^JNp2<|xM@!L6+q4_E?HxnfR zIUEdlK69L}Ee;2lqP)_sp9k!P3vVvQj&-LV+Nl@@X+LjZobwR20pGt(HU9OELjW`)?oN@zQv*bFFnXX%G0*% zLsqOZ=)~1@wkDf;hqrC!Kj=Z}r?TLg_vq9{ARw^evSl%I;#Sw?c?XnavA6nq%RdES z3GoGPBaYFMd&iAkUTn4fPrSJx{?bP#EWw7-AuUdInh{` znfnU@9M=Ss&bPMQXp_P$f(*_#M1jvAVZr2UX>XH3H}9I(V*R0SRP1L?Hun)tia(5H zRSaqj#QVAD=b7=58XDacs~D1f(9MrY?da33SP_-Yq5ZtRG3&%18-JGrDnNL#Y$js_ zpfP&Rgna)l==W6ME90o>I-!i7^lvz*8u)L0b=oeR88H^-EX=>3pVH|SV%YJlzumw% z<6bl=urTH@qfIG?ab=vZZU^obTFj(nR2I9B3u(DIl3*_9S?AE-a@6YLD3$1d}K3Fub8m)yUYRh*!DeRs_Rh!CMJqNcv`p^@(5b(i=pS?qZGpY54-l&7)sokb4V3ys`03Sz=Z zZwOOeKp#Yjg(8_=Pq-WQPk6gLUjg7=^o9B8Qjn(G%O&;w8N5=$C<)QE_4 zI0fF{Ow<3KVa#kI(x$<)w+w!p`nss$0GFWz!P9A)GjBDAG-^Kl#w+beWJ~0$c`Mb> zirR2&OOuQYkA6VJdmXk(VAL_`cgoDJWZfTG8UTaEK-1r%-y6fVYn=^E-mWJMkWUs^ zM&96_Vt3+-6+|ICn^%1t_etFKelM$4m;%KfSKOy#b5nNPPP<=nN9Jb7f`rrAzyrzY^wlerD-~_-#cdfW;cmq`BLtj zuDYP{lqF7(7RR|KoA70hdK%8rDSWMO<% ze+Fm^$BPxkH6632do{JERk~n^x9a@qv^`RA0o9d*ER)o%U>;XKM%xc}ZUOULPxU+v zI|%B*SR=#2J$N{J6qqay@gx(HI5rlKWX(QHMo$Z}(u&Y5_%bmp$H;lp4`uc6~A6~(q9cp&o*m}j$tQuy+X1<@6 z*K`*TJPHNZXM|mbjXp#CdX}>>p4qnk>l^V2zTIY#g*m|wq{Q4z;+A_I^o@ujjnS32 zv@~wyBS_o*C;gsD=s8i!D*7@6Ggu}8vV)}nshleQowAIJR#T2`-0B}Qhxf=|&SutN ztPejXNUDgTTQD(v-kvw6LFR#HkNd(VA={VBc>os4mulBs4Rn)qs`z|&cW-gLfJuQQu%KDUbvFx#^9liD1qN)l+t?h5p?2k%ykJrQQ6_v zl+hpr!KwvrU5(V814m)pnNhSVKg`Pd`kehdZI9q2{gNvV80UMb zV|Lg0%Z0NA#&5=zVJUm2GEbs4oOiVl9i=N}npp_Wk4AzNmxq(mI-1po4yq(Hnbjcc zwFD`r5CE4^`ees&V+ESr^0lHQg>|T!tNeX^+0VlFugkabOl(jLq)ybWtXbjd;A3`g zeVhA9>*g=fjaqd-pR3tNn17!nwc13VVZT2el%=|($t&xXl9t=9jwn0rQfFmI*MMGXBe?&rVpzx;rS~R@%)aE1A~GQx$5qJkgD8> zagp@qEQ=&Ou*#eQt`*TxQG23Xo{CN9K*3n1hm|cuq(dMq_Enf-M55KYk>K0U5LW2t zN3h?oYohW&=Wi52pCf+McrY=TlP#)fnR{eIgr7|-Ys=^Nd&sE*qF2-%p@(8F4?DMO zJbvSLhr4L5UDb2VFi*Ua|M9I~e>?&2PWLmDaB!xtxz99@kFAIPJ84Fj?T|LyesPnH ziuY;YQJ^5|(e)1}tRbzr9eT~NCgbOqwEm@K7yx^ap)SP6;NsXz9T1&o@;~!kEY1Qf zL;VhP$H9nQue9ax4@^UE$*ol7!81Y+_wQGev%9+u#b{oxu+LHyz{-sTr9uC(*uEc_ zacP{!)T=$BmeAI;fTmp^)+e{q{h+@US>^?G7}?+V^uG(cr%Jsr*;HqN(Rt@W z3wl;8zF6sFpgOPdKi#FBq-zBIFKMmQ@6C4BVo*1N-Oicv3l3r=9pw!u^7?sK;*+GI zqaEfSptv%iHDGJ$W?#jK!qHOLCr4nQiE!U zOMAG-*@#PnLZ6o9NPOdTmQGW3UkAoKb)9xAU-(==;u9pvyAa_YeX+Bw?`Ad1g2L#yASgAh! z@tJ~QBG>k!Z-yOFX)Er^7Q<}TRMSSh3Q?j1_-fkIfXlDlrkCFNKL4y2$9qg>mTrdf zMo)spPGs6B79oLUh?xs$bZ&-#u$RySSq^~l$I(6qjtZ0tmBwmZV%8afhQ_PenM6Y*8+qtlVoUKQ%Y<_Z-e_8`)vB4Te>*OXbd%R{&S3Yt-;;if1pp_>-~Fk^+~1< zOxCk_u;}Q-n3}oJOVRGWbKKaJUhQpgYc+%)k(p2v-+Z#(-RQg%vBxE}(0OQgkDrxm ze>X61X@}^_=_J3=LnjwWm7?!>Wm$YVAtd;dRvp~Z0^KV<(D!95b+tK3(R|xkr>x3o ze*Gryx7phY!PrZ~|Jo$r*!RFs64N3b)t0s@y_I>87RF1D@$CWuz3!pc3$FlB+Q{rXelCsg2z>}E=;getm|ZiSh1=u#Ed8si zW;vJrF^u*MgA1XHXCc^*_Xd5_* z!iwE?H!-lWY+W(Q{HWUaFFw*}vBaX*mQpfK(s1hdZ`NI3*H1?uIxjpnWcua}tM3eO z9D`IbOYpaDGODA}L=DdsUXSC5T}sf37d{muNtx(>;QEfix=&H@t&v@xRpKWO@D^R< zBk%w4Pntab)3#<%zE=-KIg@s)og0R-RFaqwL>i2LoNhllFr-lNJrIKe5BJ{XcV#6K zRDlugJU7;_qHoSbl?0VR^*<5AD{MRGUqcgeS%2{|EyYzsC7(W^zANA=`DNH!7Y3$xrBK_*Df>0WzKL;fvtZQmU31fBPNX z$*U=@mRRa}!O%Eut9op{zRLzURJ~6lJvG7Vbw1#$ucOsXHa1qR2y)G=EF@HZ%Md#^;1xi;%q8&@ z9-QyUHO)>_t09dLeVzC#f=9jIVQqxgdx_wa{f28ZQ51@yw!$@WrtMVww?N`Dp2Sp} z+KPt9+ZZXv$-8E*|9j4{tgU}L3fdA!*+^-br$6jgc`LE;t#73hvAirJZeo5us_-fG z+K4z`M1i=t?lOpB+28O_^xrh0I>)`h&%w-P`cs;<4CHsEFO;KTb*@KTUXO>Y^EvK4 zM0}7^w>J!7-028Wex?7JJvVUFrV%k`RhP8b4az!!Rzp#Rko<3)rYB_|ql5;_vw#fY zktL_n71)y?T)MI*RY&(;9{wk!7><#T^RcNT*$AWxioPz_aPF6lY+t zI`i3Jy9qfCQ4;sh#Sb6$$|cv(b_9#3XJSZm{$vVIwI3WQD84-{bj7w9dqUr88e%I- zirGh<2uGfsNG#f%hs^veIfTan*v%ZGu$rGSrPo~{MxFTS_`8Bi<<(?MCBH8|C_N%q zW@j*tN0S)poY2MGb|Aa-b7sXs$%Izf$qv=i5Ltv7hbhTVUEbBnK||-W&;JDOX3`_J zC1*S2^9&`e8G}DfTw&`mdQ5Xuri&p=(4e3t^(E2NM5UfJ7WT_6UjNpcB_^JRD!~$D z`r#v4Du*)cyvHZbs4PYat@(zi@Hg2>U3{%(5#*taMEOf5(n-V@84NH~U0apCEJ5e` z&|@DBnr`m1Rpss+WP%&1^$M)_=zjp#zT z`3gM(9gb(`vMRq_B99IlwKrY8zMO;z_7Aq<0C4Ho(diQN&97bs+Z9N}H8_O00xQ;G^=^{qZ|q(8tid zT$7c(J$=tUmS(f<;|1%x(w|V)7#?li^gS%(@=Ey2LP`u6gYFX8;7E<7M^2pVSKis- z_j-GUz_JcNOoW)gaxo`tf$WA8P5ue`H324CW*;RR%&abQbvD)K3TN%|pPZGqo=i7I z%gYII;d*IrKTTrA*QCLHq8X}+mbss^d%=EM5l2}8kv#`ob;E1$I=RuPqGTiLZcw8A zBdXC-^}oJ~w%{OfpwZpoFcSaT;D=U1lfR(KSP*6Wo{|2y^>dShzu|I80RfM8arI;` z%6q>p-as(GHB{KW3Kmq{rS?tUs@q0Qd!NT?L;- zRxiEYVPBlqR}|`*bm^@SSBlL;!e{ut2+*D8_q>5keR~j8*nhFCPdRNs3Qg>+6^A*? zN?d)Rqv~U=7$QjL#ju(T*&}f9>@OIjr#{3YD!Qqg1rK~?5@?(p-a6xp&)H1kYZ4%H zQ!Ir698U~A4u0w%mMHd2%@hL31@}FbY;T_i(M63Uci>qtf`plnp%G;kw@A>3o!!p> z^(LpQPF0fL#F-y4FF)1$%0tV{7#BZu56}P!???{>dlGk`GT=#d!*bT!`=)5=!7`@e zd>0o;PD=KwpuEralw)iftcihoCRlH{AOkhVQEV&@_$#uAB>Q>kaaPRpXl0BA&tTNk z&$FZmEc&`JP#AXB5h9)BCoH*LYx}6rswYGyOcdnX$qc8bTWx?&;=8eO1rt}}f^KFm ztH=Jk1N{(XEM0&$hT0lh-xa%T{u2$(_G!>UwGQoT%5!0D0-2vA1+k0hrxG;kiD4)H1{OWW-cke=5Q#JNX8)g%w3melhqx~$mbY5RD zu$mJGiVfh=t*#4|p1Jfv>FZ>HHn#1KKoUU=&=-OyyjpVq$c?l&wfSbIoLKcJ5PC&Q zj;s@c+dk+{5i9Oao(<|de9qXj?U%N-e18>9`LXv%?ELcDOLfY-i>XIKUsd^n>*nB@ zV!LJfM|5G~$VvfVV02js^fc5^(#szDp&*swih_B2#IlLIO>%M1jT7mFchlHmG(le9 zWY1A7v7wgK6P8)=&E3QtXAP8I7hFNdxc%kiO6*T#gZ0FXc3P5;^-tcrMEbIgdQ57~ zP8dzUq^$%&Dcfi933ouZaM{4|_P-iwwVy0P+8EPXtK1zyR;*90M$^NZKH&-G>Th)~ zI9U|`h@Qe_&yV>sPKco*q6tLxg8yxHT{a=N6!5!WNH9ixCo9r76pY)$HPkuOR+d9g zn&(OsvaNr$DO7+n<4iW~Lys(`xC5LUCnu_*0g|ab3l)y-a>vkLR{mu63|=6V8=S(R zR;SMn@7f9Iq}w~jbHFhhISw37VJTx>Pc96(*=EC!F;e4^0wGt&rRkz8yYhaSs@NJ9 zE_hqdxSG_z1+an|JP0c~rZTeWp)8lYfN$yH-C|h*X;b*>WMqB44Sv^Y##7GZZBp`M zLe{)Am`Z0Dpbh1Y=~d6=_&aJ3qpgjf8@)^M@$V_Wcv0o~U7cVRkq-<6tc1t!jPP#5 z30-~`Tf)_oQNsyP9zjs4M-DV;?yZyxHG7tA3xr1@;_85kwcjxIxuIXs{U))YFQuN? zHU=ZVHJ6pS@C6r<9eae%MT9S0I%qdZo_g=}4kWp1Nx=)2U&Qz5DQy3cMgL;#QKsbP zl4Y|g)-PHbN83T}>v11>Ai^ zb0FC3r81k?m|Np6q}w4fH%A*f%~2M4QyzO+i)jX1$L4y@k0De1f6OED`>?zl3!IWdhT*2Jcmn&Q|Tkk zeVKi>3`z>Ex>Np!AF{gHuceihQ!R_=WC523$a|}!M&;|y_IU7j>zhheeMn1aZZ2St zsjy&+w9R&u^#L_JhsTlSEw9_n)Fgy+i5Hmk-om}d)fgfF?TPfe-UiuvZkynwfWhc- z9vCPIrE{^Af;JdM?@US)7<6@xjd4{=vUoD;Qd zABC{6RgdF|M7frCP0iH|0ixvhPfPDKzo;rc-X5*{8PRYq@`ZIb3FEQrbO~m?XUkvx zkiqhK5Ra4B&ooy)PJD9u2vDMAnE=l=-8u2(DGWgv+1odM2KCy8> z&b*v)%4-m`brdLuiLu*6ZeB`KixGK<-VK9`sb7OJ6^C2ndqz!nGWRWLDJ77SCoa=Z z-OlNuBBCW1w4_<{9VH@W^j)>YX60cJ_`n_hkMhdP;WiRT29qoC$uI{^v-#(m<@Y+) zW=QyF6roZD^H+qk79aZt_fA^D3qrf{wlhRJ_q-rF09XFZCNz3n`4O;;asGS;Nza1CmZv z9nz-A@X9JXR6}*+11h4qzrjjXNB{b4s_H)K>k)(u{wzH9k3lb5bK)O-dTuo66FKk; z%xqF6MPGf;z;!`Qb@?!J2X5J%4?a?4<})||VaHdWr^L+cr@~pij>>~P>7k51{IjOk zvr1Mo`K12~cDPC(;`+GS>>1Mr7WX(D3qh)2L6`ddzFmrG-lh*~d;Zc8(!{_E%i1Rw z);KI_7Vi1hd{(3_VQ3SeH+L~Fem`annoYy!!DLkm7%M+taD3Ln3q47w#NzSmG%1<+ zDZ#D}5P z5o_XJHzX1#e=9-;GEa#xsy!H;<^3<3Mb+B|t807FxV~XX()!o~8$M_w(z~(>uc7-< zgL?xxpP$?S2l^6OP?eOniyG%W0oGCB0q}(V8g;kERz{d(0OOf&=W-mMs@ZywSmM#M zUGOH#U4B#_h~Dkx=Z}1<3rf@vs=0Ge*_@fX(&_ND8ffvFi5DD1(BzA9WW@@6GrXCEm2H)Sy8#)ts$lw9qQk5M}dk#faB|Ls+AVb!_kkvVbu zv2dx<@pQ*>sxQSJpV2~M|CEoZd)Ip0{1MXo5NA;n&3G|Z>LuMgdzCo2xn0u(IL!*0 zAQOJBp1bs~LV;gO(-6;3yq$KCS}IY2r`5aIV%w2K(BzfIcpO<&x}-&eW5)9nP(d=} zzf+1l=UR8!#qpUda*16zuAQbcjsU9N8_r=7|F>0aVJ>xi6c8SbwUm&JSpr7Ze_=+5 zU{}71SoV_iZk*f=R5ERLF@CB0_?O&pa3Ys>*pCK<_v=^Be%&UgfQ;FrK6E-Tr9aC! z!~OGsOB)zyv(Mmr;qf;PukQtLr!4`8WB;fqyb^r#i*Z5B@Rm62wNq?TO-)^-41T%v zL+N9~-gm(@EBy`rJF@~-Xk7Ll=J-5E=W@%=`4QAixu8n0R3>QbvFGe1@@_Jf2^|{b zs|U%AuUddPGsmB0W?$*$2EpaAp!#=9W^!4b>`-6%3=oA)O8u{SF}|1;h3{*7k`O7G zM44sR&Rg*$yr{5*FoT4NA)R3kDRd>bze&}D8XMeQTTTcDIS>IFkkvrXAi#)Y4D@vS z^>JG?$_2-NoABsg@aJYgGS`E{n!u$43jnu7BA0;*jzOkIBt)_6~h_qpY+2A6QX9r z_XYU~T@$)UmB9xT?UkGtR-|8_WdJ+?sLa^Rm_LmHSZuo3>VzN!y^9GJ=zoRL&t&E#YobHt3dUMRCZo6Gr@`^@@cbFpA~_03=H`Fw8c zTuHzpHq@zB?m2;Yusoo7o=lZH9)e>MI)P(`fY2aeaX`q!%I7nouwqN@$izLyX1syT z&ucndcN`Jmympd5PmV(Bjp#vQ%>Sz;gi zC+U%eKGzbMp|QJ=1V-ltMe*?fp`4+)rq$E;gy&?H{x!gnhjqrfPR46TCAVTj;W^a( zWKL3hE4#)?_fsYc-y)t}o1CUod3bx|EqR-HR|88s`3%@xCROe9Y-)g_n_MTdDuK}F z!k;EU2)o8vq-;&Mg?Eu|pj$=bH;h;A55OWU)aihOpRP$mYkDOI zpoV~@kcg%XjqaZ$Km!2^6t=+iEYzk~Cj0Y`qoiB229$Ail^TKBT^vsvYFW+!evi*i zX4r&w-?+f!dlmX^C=Iy*5)2F}_ut5-G4ZM4<7Bee1v(*wLkG8h)NcT&I&@*48_eipHOWtyIX4wddBnSQHb-!uCr6I>zXpu_)2A>w<@6PZp@IQG zk$no-MUsG-ummtHWy1oUo6xF8Kf;4&gc;u|*5NZ}w7*oOn`#jE|GzfgBW|XnxfU@} zY&38Z$*)myr6$dl=}n5+-6nsB4lUX@&uBSV^iUj>urrqlNv`?J92?IMl?{@j-6? zbgUTC#WR3*%+S@}1d-{s1e`B)8EIAbCccG_oMOu^!&w~J770q}rqq=iykA7dwb1+* zt|IA;2&`SriH60Olp%F9-i}@K02$FhVc`zBr&@%YlB=jM@DLiRIcGk+aT7l+7032u z+o7tc^jbQc>Rc1`Lesq}l6XeA6)6)8pM$zzIzZGQhn--(bsvX5=+ldbDxLtWMut8R z#_08yNt~J%7BTs3sU3Ek(|P)JvF#B)5hDyl4nySo$v<{ZAJ#?gnYOuHDV*=;Q^A5` z=m)lf!@}OZQLOtj8hkXV=j$eW>958)TCK~Iis&wDnVBSyxGenS^ca8WEn@?x8JRln z$-~w&V7GM!8;haaUXnGTP*cVw4*DV|i6|}K5=C+x9>~`i=*#-(rki~VW?l+Qv`A2H za@J1d_7SA{72PXpKel^okp7sEC-FQT2up1VuRn7#`fOmGZG{7VR=J$0!u^>m2GBr+ z3N6zU<>R2(BnSYetk!o#4y1uv(V^kV$u#hS`YTxAeEffJ1ynvbh;n+%k11&PQT?lC zW9SGmC>gn2()~*}tzW`p{7(KIQEfv*X>DLdURZDI7v&SsTC^J(2FBbxrDx%*fG#>8 zUZx%o2f4`lX}k#KHvp6hI?&2IIglB$1>KdxiNbGih>j8#l?6QIC#N_5d2QOiDQ+1} zLlTWNSAs~5S%q_Ws_GcMc%E?)J(U-q@yrJ)T;AR1nw9gA&Vo(|M=pZ$aAPp9 zelAQ;YKnI20m!~^e=Ad%`gb3@%s1T%CKGQ?MixC;b)*WFLl+Kld#bd1oLyc^&}u(x zr?c(RP4{?5F`kTpExIFATTdEkZ{@4HHG8vgkt-3bKHS)v@3@=|Q*X5qF-L&%ASe6t z$m8+Dir7*Yoo#%KVcmc3a8MiGB^Cs%#)S^cT@8M%^B{Acr8lIP?_g$U<-OwNR2bLO zN@He>xcu@|v|^jyd>6wNP!QwD7Jeb|Cv^q*-KF27$3P&o{Re=|svXa8M>KA6CADJ5 z=keED4@{Qpmv6gPURkl1|3}j~hR4}`T|Caj6WeZV+h}YxYMcg*jfrg=jh)7}8>g|= zG&bMq^S|DY^JONv=gisrth0V=(~t)|2TiS9?N)?}dbZ)9r{YYq#KZdI7(G)q-~JH3 z+(5Y35^(4#t2>;wm)=X5ul2~(g90qBKNCzv9b-blkzcx2_e}m8(uEtd=hqz}1nJ^S znSGVhGvlA5f*?2MfH6k?Hl(&|O$c>>G8@_LILVTzjtM`t6ZAne9I}_kF}sWD>fHxr zg5K>jUa^)cwZSuvs_R>dnjnaPeTjJ}`8y>O6Dn1djL1%;+eG_7O95a;Nlug}LC5A- z7&9C!NexpK7+}nDjv6Ly%LWIYj+C35$$D(K63COpRyObGe3;Z`5JIu}kcyS;;ZnFt zjejR~iq@19`)lpHA$NWfC}0SEJJB+FNUEvWBbEn%UO`|gw=YL0-dh2nvqO!hzDGSgK{FPd7XM{+HvY3)*!ZGj8Oy z#3ys&z7-Wr1@eHSnetSOQJVhNUNWqa6>~=LB-vg&nEOxSZN)$9&(eTY6+@I0*Ej~* z<8wSaVuFNMCb$>;rmsMXWbX=Xs?K1|wPn1i>YDmpQu#8^GL^xi(XVzqnWVF{Fko4` zQQf6rkDVfR=I_`Qc28KEM$N=<$^tb^p*Vz5J~1Ovu2#$Ut7j{It%E7zzP497KQl|2 z`MA|nP=Ik%P<6FPm*a-vms81lefHnoNO9)k?-5j2wyMp6iT52+k<95a5Z8J7e_pqW zIkYg@^>GBQ+&jh6|1f-$TxreAfeSIGhS`k_0RHK>_x{1v#P~v?)~zr+EwsLua3PL$ z!@rN)qcXY#HK+6k>Fh!II{fgPAo|mqv59rhCa7}3P={dGJa2O8WWxod8ho-vLm}R%H)~pMbAO6Ix zrc7~4M84p$cMPJ+L1H=ZaO8u3sF>mXCixo`U|gu8g-h2lirSzjC2dIv`ggPP^~ta@ zz}}V*WL0ROs2@iL!o7&9YyDYMZmP+Wc?s_BX=DxBVM+RKysQ<{n4%$&2SS(_mJRuX zyMRCI(!HW+4(^%w6XecZKqbl{GB2Dok1BW(iJQ^kxgqGlGsEN={;_@xJtfY#1FAVq zvSPt`bPgywm6!W()ILg?RM?b(K13u{ZL%=@B-?7otQIbNqM~(m{ez^LPJ2T=D5=QtS0*<73Yk{2CTBO1;;Y2_8mJZWK6;%izr zzi7>#a1thRn}|kc`xH~u53nMMVDVW)Z>yfhkrtvo2?2gz`gt5zco$ZD%)#QGu+m`) z%okk1Q3U9+Pxg`eJ_HGd&-SxXcP9@nFi?>j#-pKRE0NVz));kF7Z$@Q zV9|EW3ofZ5qNu8b6@wBoK=3XYB)=kBb07bXOW>vI1U~j$Fp71d7STX5;0e4}EbmaB z^P**1)#e`6v>q+;8z`dpl-u5>-D^Zu8k!Vv_F2ZW;SDj_os<^B_7vrPHsbn&uMlNT zyd!C!Zwf!C`azs5pD^#P78;)nu(hQx!>q@X!vY?C_FjPVuyk7HELIF>x}GhI6EF)Sbp+&J{RlOE2&G2 zK10s=6Y>yemCjkEETY!@wZxUL@rVdHT>pH;j2%R0PR66{Q>HiIY8e!C_8ikOPN}DI zXLtU(pqO`yOaeDFEUKij7Zd<&HkoG5RpYQ#H+XDILmS4Hx&?725lnUN;->yXn=v2=dd{N!YyQ@!CtY@%Ec`B_sDbKPH^;U)Ob z7=6xmVUIMo2?M!`SYJuVLWIJcYLLajLjn7YgD=(aKxqS4msCU{7gx&0+PLqmY=k6n zKOl&*L9@guo;PuUzc_&G9azJxf6pw@R8 z+yG4-FgsK!w^)8CW~V+K^mlzgFLsqcr+cE5Ku7URb#uL2<_HiGgJZxCn>(m(LS`{U zV9}p+t{@ej#|1M!5u?&toIm*elR_4<@bYdwKjg5!J2S}E5_pgUJYC-#*8b9U z7@AL;!diEJ`A|=DC z)9|k7%B#d2JXo3|b#ssm+c`DZ{Sy1O+X~moG1;uV>x&fV~cl2n>H}b9Fdu>zxe38%O}i z=6mY!&(C%P4jImfcMzQN1S=J9x%~b^7e!}>r-$IDV4aHdmpFF4k2XQT!?5YCtm_?M zewV=yqd$^^!qU=iHp(X(M!z<3KiMEo=l?rC3nN7=xHd4*S)hMQ=1;AVsvr;&FW8~! z<~|C^_I%xK+wdTxBjhNA zuO?1)m9R)*=Ph-h8jR|HO&(p?J-z(O}(R)!WiG|Qnb23 zT5RrFw+=$^Uc2a*ONkc7!i*#gFD>*Y@?)EaxBk3o@=hTo0Sx%f`|}zo2}H}71cYhL zo(BNXe))iRNXx#ziU@zhm-}J+x4OXwjL&*wyPdGgp49gcr6$S0h$kEouPk_pGKIl8 z0;D72q0gCMe7SjfoY|wcKqIeMNh=+x_Bsf0wpIpB!_5#@8tYeZQ5kBQY0o<5qROFs*-W+FuicyMyC zG?VsT=+S2v2v?gJgk__Amx1EU4FA6_5#6iEfd*gDe>Ns9tt#5BMIqHz^)Rc%GRY*M zDa&X8T6ZDAwx@S4l7^;RkqZ5R{9!r|lwc=lOj9A}>Mg{A*7H!&w)TflZyqQ?>xkwI zBU2M;)F;*;aQa}7pjI;E@OD^qSQPEz`h5-d0cA#^1Q}1`W6x-b5JS7s0>2hanS7`l zjG+ne&|GN?Mk>rYEW(Q*0s?tZ3zWx$FilX$TP23LSG#@ZDlj1e^~Ioc00F>A9_j8w=qK)a|Shv$otj=Ffnr5BVf2Gc@M7xu#w6!;-#*@x%ocQi&j1Gv^< z&shP!L$$jjo}q9)WlAa8c9!iIM zJQ>FN)GioAq4`0_L$$PvlkK=v*4IUtH5Q3u=h`l9Fb=%Kn%wHBY1i*nOJCfuTz-Tn zg%b8+!;u?2(EaZHVbvSfu7eC|r_)VXZsqVP^17XBg`XNvPkT%4sbpx7%E?R2_;;(P zSq2qLzruG>me0iA_`}9U6v+yuRqgg6W+^UdrZ(NrhIN1ZgU4bSHmkJ=jXFusYj4uF zthw6GDB>8t>AlLLmzrv?eN~a2rb7ps5_;tW$~}nDK&~CoF&B1bT-B;-1&2=3`Roqw zY$Vf_DwAI3*}7RXO5rH>Vqa}`Y+HPdyIHBkssP3lV>^t2ZwN_?K|0o|{$Qs>OIVow zYL{#b#cTm2q&6O?t$X*$<0M~)nYcz&)bM0-r~|-Q8;pC1LQmvX`OQJl z10${my|=K+e;Ias*xff}S~@5|0J(qCwDs!gf)Wz6%8La_o(Us91-?q6dUvBE^72-n z-?c|wT+*@;GE_-)377STRQ);j6T={>ZoCWA)?zEThLdj=l-HI6*Kk!1J)aZ{G;$E$ zq@O3B;?tO$Txa_|VyL>3{5;%H;k$L>a0_VdC!3sRY?m{g8<-uC5Xz2*ut<*aNH?2} z^dM|}@uCJTP6@na5$JqwOuGA7_-zI4x3}c;uO9M!w|eY@C`}qg@POyNRXGOBHzqB; zg&t@~gkfE0ugvV-E2`4*MA?J^XSLCRE(6uh%N5TyDnSETqbkvXuQUu8Nqp)vJGFze z(jN7MN!q)NkPP0P9{4mI%xdVmvQm-=`Ye$e*?*!=;h~MY(Rx88BD}Pi?@JO8jg{4| z@9fQq04CmO6AuFupsDIuO8V3r7hHB(r2?4bBAoHRXCq-HbeyppMl#P*dE^cWX=!Gy z&$%DrJeSrPWlmJTQPvN~tsNMj1Bl?nVJR7TuG#GU`VMzM-i_dkky31C+<|hK=!}fq z+?U=|4yWsnXPvQ<)!n1=!Jc}E93zKh2n6n?u8KZMwEuEuLkBOCOCV0agwD>c6V2kC zI7Er#HlK>S$#2M_D|Me?k$14CPgsTb!`MuHh_+R_Q6t8KW%84p?B6^s*rH3p7%Il4 z&vxHzwP|K|6E4sc!ggpAzi*>Yiy_LO3VO6X3dMk~j$Zr}o}~e|(GV0u9rq2Xlr3ul1;=n) zcCs=ofU>oXG3%2L$k-nba|?56J9d86C0(B5(aqG53P?6}j=GEtfp_?BLHh?k04NT|>7k^kaouzEDV#!e`0d7GMTEdhf z`lmmA6;(GY)_&iTm~j>4wzu7_CeEJ|AIr&mL6Z~}orObo8Y<|my1g0?ufpH2s=n)( z3kSu>-QN+rgvRSasxeQCqYl$w!q5zhCp}|88IlKV>3**h62vR9cr`Tq@qLzj8!?!b z$M=*qH9r45O#zg!11W=SZtV@~RM1yDbr6WUAjXv>0+Ug|ZA&<;ku(rPl1mpig zMEBy)*S2JNL41`kByI^!^AIjF+oQ*odka2 zuEYv6pmX4HkgVDGzUgi2-K3(>Pt{m#I(;y~icTMZ5I%J60)$Ex->%W~|LvQaI11U{ zI!cyp6a}vz3q2gIK?ZYA32}ktwr|Fep^|Hl2Aat3q$}a~f6vKQNcBW08z$3=`od@FKpL&>f8&H@fm6WWybnpCS_VU}p&}12Yhga2iUeKO9XYupLJ;B7 zRDU{T4CfZG0Y13>48{h`vXjZA6DIQv+l^j@U?C!99>OE3Ws-Dij%B#^5vI{qePA#g zbAZFj!Hg9=G*gvbxf71%x2+=!u{C$iU>@)o8?sM+6z>e1`ZJ1w8Tr9;q@Fl z=5RKD8w6zMHjhb1Ep4*O4F^hxk+Y)7{%KI#4_5Np;44!-JasNOCnUqKd%h7R*?#js zAo}upC-`#3`9fa}LV zQaDo(JtaX%h`Xq-Ic>FH!K`yhMPeWlSsGVkAL9Hrn9}&6texpZmioWrvOkQ%GaQUU zYy`A|sTklm5`w1v2zn@zfPh0Z)VDzPgF$fO;qR0kh>2G0qY}OQ{op!?Z zNRKvB)fD}pE5X#B$|cs$Wn-PV$Ihi?d(y%Sz#S9~ZO4ErrGzl6iwRj7ZO=I%&18-a zhINb)<89q)^7A!E!%6)m1^_}41Hh5QJNr)^gm23B;KssD{NVTcQ+Woy51}){UV{YO zHC&$b^fniFuk83ewjoHK&0SR!mM|J80uTM4aIU?j+&ST=m&>akj|)v3+~H#I&dsCz60c5QJrLI+YO3 zlXNFq@UGV|qMd52jk3a}>ORJKwA_OczZphbeGHfK$JH|}W5>lXAmY87k1;hfDBukkpnXDzhZT2ZVZm)~snp^VQHuZP zVkJT^APzsY3HT)>guT43w0Mk_gbg@-s?Nzz``s&ss=kD-JOvw?{!|{Ig8`{HinGX* zs;R5S3y*m4DHzCtb`9jeq@hWy6kqFdztq6*o5u)CIsm1F<<%XOIKN_nYaWZS6Udx2ncx#uJ5I<>xun*8g$@m zJF)T@K@e(Jf*jKudccTdG^3oYF&?vqd(*Exi8Cy)n+d@fw(z_SgoxRqXvB5Xyh6Q@ z5(JtP<0+*owhN@y-q0Q7ih^>LN(lcA1<9Q3Z3B%hU`Jjh0dm=ZrUtk73dtq{gXDWM zHbAH=apPKw`oZ6AiCtf_o(V%vKh^fDhLlC`g-=DJ(B0qQ~m)(4Xe-nO_i ze{{Q%WaKArueDPeWc>WG9RCrcFEs1juO zmD&nE-2*&__Y9df)EoTH2yNE_hUOT%IjQ%=BC{c>%#w)U*?gOnY!?FN$frG93rdDh&6RA#KxW(VTI@#-tQ(>(a~-+@qcngiz62+_%iSl%N2pE7;hU*1OQ+;5h2 z_Eo>5r#L!3M>nJC^h5Itb~#!pSsdB|kUwxf?9?&nQrYsmK@gwd*E`5nHbrFzNgOh`qJ2S3;fUmM44&zlC705(r~Aea;4 zvmRl7>6|jK!G{oLo4BeH$u7%`Fp!w0>;&!FV~-vr=}j;~=(2kx3u9bWT(mwG%!;V} z6|4>n+ELRkPx|2-b<;MF2I9_&)M1WoHQg>Y&aiTG85#}-UtaEwx+={kanDk~FvbJ` z$#GB(r&hcsmR_4U{rCY#{K0kQ<-Qz71Uj=y@L+$#aS+6j+1J{Ku77%s=*&WhO@Ydz z)@$dVqB(^+d#0&A!opKC(Y@?wr0*CUT@HOOUI~!<5me;H)-ac9-L>D2cJESg=63os zo-etP^g(9A!^p3)jgOdL#Cz)jfGaiL1t05ynlvbj6P*m654?;Uo=d_D-#oL0sy%}Tfi?Pb_e`#V9fsX$LJD!4 z1~wwONCw>8d{utd|3@DDO0dx=UND^#2i+L|vA^gly0%P;5zEbDdlSYB{s^ctN4^&} zUg$RglvPHCV2EeXB4TF=HGw@23CJJ0zk4zL|NW(oZqbYiD+k4w7>zF3ou~IPWqMBh zo&F!^T5#mxIIHo$8%8{j{i58$*!nX}!-?4fC>B!t2#|lksSLGXSa(5JTl7Kpa6F#P zIR}xNMzhs2T~1xRmk6fLaR2OCRJWXOz2~5v(_vFGVuHtsd0NV+w^12~x=Jy()64Cy z`{L@=+dyxJoIXO03+FH!L%Dk}3Rs5)C0?p|{~_tnuRY4< zT=I_7dLavgOCDf;1=i5m=LMZ!^8L8o{cJ+x@&+^T80pml5YTF(9Ni91cwxO(6d4#&RuMiS#NvL~%K?fedWxz_Lcn#9 ze$H&|U+ZUgFf&=2E>R&wig2Tex>q4Yqo(mJt^@4mF?wDpKb>>D{z4#;qC)(gM2AI< z-tkIh@}yQ6FG;Zu95XPXk9@}c02RKcEMQ2Cl4Hxolgl}#4u(9Z?bXT#Sqh*p3#J>G zbYZn39f}F)zM=l^d5g}pAwF^>Zo=61l_q5%-L25`f8x+e{bk?q7cH;#v(qb?Hko@E zvUm%M;E63NRQ3np4iP`P{|_0&rrEiGvrmW_Z&Iq*jo@Erc*pRRx2$;W!FXx0 zAq9L`(tChyj!_k{cCy%dx=N&YwiQ^;(%gPfe~yfn^sJ6<;!>-+_gv<`^)3}2b_#va zl0L8@So1%YqB01k{7`PkV466T6f?_seSuxdI50>ftS9S_TdG6YQa!%!bV#*CW7R@L zSOaK(){zCLmkbXr&g^HxC$oZOTE=xR@r~(IY4Gj;-(z&gYB4|=6(~jo79va%z=yw|bWa3mWBr(sU+%1t z_seX}0heddg%Jjc76U4yaa9U4)mTz(wcYi*HM#n;nvFomX0i0Y41#C}bPF&J{DA*V zyvAE2bVUoi1P^ikHlmO=7 z#*9(U6tc5Hf-4Ej)X=`ypT>-f987x@6#!aS$zU3;-A(oaLkkrL!M@mSV;GQ?Y2Wrq za|Z(tsn6`DWj9I(bp_ve*qVOQqdeMPEl$CBkDTI@rG38m9pan`I?RI9r*)p!2UY?L z6BBujpU~n%9d)HIRQoP|R_Ly>(agYb87I*&L4gNkDe?~Is(p{Z;`_nx$Gz)ZE@(wpkKSKyHo zKRAKxs~_RAH(-%`bc#`W^i$|M;ASL&iVVgs1|mB6j|tXmUUiBJ1MXv~cK>(wu|qFB z#%>R4BAgyus|t@HV`<4vQ5a`mDH#AwSLDlE+4%|?n#+y$yJ1l`*f!s8D{)jE9Pmd7 zbgGiC_By|Wj{){(IFdri%6VByN*5H|%|uIcjc5zzMfaVnhC0=$fd*h#a(3<5;W|df zn6>$x1%r2AFT~&BBVy)6?d5r05b%TH#ms(VngdO(R*Qf3m~atdvoxRC%LlbV1e(`U z2~Fq(j>NWGyhQrJ6oD-m#ee&0D+5L)w)^X&N^azkg6cEt6+W*?BRjGmO?x7L(2<=|H z+f}K<)=_#%PX%uex@&RL{PyooV(S_-WtdO49zg35tsp>`CAzqpaJ9GNC69ZP+s`cjT9fBkf!~ib zkVT)z$46Fe={a;b^6OTQ_|{oaz=7q?6%2dFMblm@V^3U|;0}iJb_6c@FMP;8;Wzd= z+jLEBy2D$B+~Kd?WsNJ}-eQl(1sMsRsBeD^d4B6ckabw26ea104!x}W`8h_%z!~nW zZx5CC^M6kGO@QMPE!S_tcbki0?cJ$%9zR`t_1N7+OTN;Wv94yW%qxqJYo6L!T!wg! z-*gYnE#6VlK^*P&+o?!FzEt7`9M&CtEy%nlCvY%y+BH$C!}sbBz0j9y%*?;fvHJNs zfMcH6w%l961s&M_&;~p9*-=D;cH&uyV$hlmU8fQ-vqTXoO!j zt$!BkcJJP>1xuIV_2BT3iZ8(H4?V_!lL_28{@K&FIbrPD7y@R4$yQhMj3W@-_v-VE zIaz{_Tj+v>;_M4^9ah?`s6B2ok8_3^Fg&()+6=UuTjS$!4az)|WlR?D5?Jxz*<+yy z!NSIsbU~HlvA08~F6pR))#1Ka!}g4c_<@ZX5OWfxFJGyC1%P;sDoSg1%Kt{FpBhy@^OY==k@^@26H#nmISHb~6v z@IS~-h6cRVk-^+bf`5}Fzb41B-*F%a79}v}S^Ds*?9iGw8a$8Y;uLX9wILA&f8%FR z@@WT@3&wIdaf{DhzJgEjVAnFRvi@c0y-Us8x_a#Z4Ejb~B5SqrIjH0~M#kf5_cXpT zZ3Iw@gN?1md|+bs%tW?<9jP`x<5dyu^JE98ziSlpRnFH;H^ozYaQdmkblY-IS`a=; zkGXfE#ock#Aw{dr9KGFw9y{6_y`*kt0Awd!R z5&5KGy5c^dq+Q!u-eP=&RRi^;KzE&`59s3@XO&sws3n^9_qms|ksxLA0RKi5G2io3KeW5Xv|cJOleh4c?W z6vLplQgM&o*idMEG|;m3cXIO0>3KbPWaUhWN?xp``;uBxSvT&hX;_z2CLa$XkKM_ z@@?dQPx>oSSc#&LARE0&Rz{_T>Zw@Yz$rPHg+HHJnG@;?KWm*BSXsE0%sVpeE4DSx zj&sTmDpnZ;^a}X-`R3b64H9#5PL9#FCo9Ztak;pv;MUU;A?N<8t8}i!Og?3OpNkcd zGJSOsl#t%Z;75F#7}DrCr!r zzxdGN=PI=#Ro6n}gV3WBGN`RCf9clN{X+T26HAg+>}5rtY5OF~SoHKtJYsOfxQsyC z8^Y#Q-=!H^Cwu(HA8OOV?pgXvVjU8Z_2=yXjWFc9;4!n%%{wMloF$Zz^ATd7Dp$~N zo^dishFo{LhXMEA^r0{H&=;Y^mv-OmkXm@KYP+K%K_h#m&b(;s07|^=gp8pdo_PL3 zL8Z@JCy%oX)XjmBJppurAQz(W-tgqYfqh9g%KMHt1d`htrDqW!?ub85VL5jl3SZar zGS%ei_ip9u;w}W1nflm~dU#m$Pez)!bS$EnD@6BvI2I{X#i+7;SxDu;(0MF8m9XD3 z&v?|>i~)T%+lu93KdxP+5Nb0z52w-0isd&9Lm|MA1=XXb_R$rDrxaD!c4(3Q1`A|P zUt4*6q7+!s8&^Jo9gB5zw&?%dYfpn8+7)mRKLMD@u@oh#PTphCYAGSS0H$-6LlzaL zgh*;_Ogro2VIuvOa(3Q4{%$VUj-?tbju+_5Nq;`5B<>S>Wf1%sMysVc>L8?c<312` zkt*l*A6^KkpZwnmTZF={rsTJglq$)}fZNpf)l(BDW63p3aNa5HCZvX7 zeO>y;O{2RNSZt87{*xh35`ECL;5H@sLN@};Z{`8k=c2OehW1qx2`Q=`5sYIqf5az; zCAU96H*9SAy;^9stwq{s{PTmHUl~9uZ)0EXlbA{2+fybP=TTzZJ-GGd<^J?qf?FH? z97GTa{x(#5qtvj{`EIskW=qmaO%-^HUz<#h3P0}W;+bV4djDk$cz4sVN z+15o8R1RV;4MSh->}`6TA09CC!UX)6~okbX76CJ^L+@RQ<=rp{ zBclnEUd4rRRm14yZ3PePVd+>c0;N;hetADb^`TE3u6q36Hpkf0Sa303An{KuVnKkw zmmZb(pDt=s_7|D-2v@Nz;1VDm@zCOW`M9R)rv8Vgsh`6GE%LO&D8{krt=#tvuwRUe zW@&WKi#*V|QjQM$9bI(C{G${(ro+ln@vOp$RFPF1-qvxX;;+B);yv0_eCaE39V$KOzcu%Y{AZPfz6o z%h@74b+jt1+_;>u;xBy)-WrO@WsQA+@Ob0b@+HSV4rLHO`&6re$% zwgri!yD!0wk0i3;A(%Q@Q=YzI$A!B&v3LIbv)7-Hju!-h2+S)_Va46mtoF@Wh-27o zx#EUPi_P*r1=IPiiBv*z!}-MLH0OQ$CnIudb6UwSZ3-K3zfjeZqGu_=JoasRy(4%B zoUHCSEobOwbqJI6&@eQ6db5%t-_7l1-^sMeUw-Q0GXxyChab&9rnAD;H(+5b+W}44 z>u9vz-7ew;Cm(DG-`JQYBjBJH)q_kiig)cN z|4rT-Y6=9fat(gA;T4_=9tE!@6E#7wSNU(QIa~g#!^2W{hCd6EaK!RyG;qrvHFZ>{ z2R!^t&oGJ0(BYZeGSNrnvE_$&H34koJKoG=AWt7DGLG8?Q(q+cG zJIoef0zSfD%X@g{^|xB!jYQCm5JEni*hm^6z~%9mRT^J}rd(Q8`MuhAk~G1OiP9EI z=NFXkVwbq0$$tyiE<9HxR+NAUm9lW#vL3NU&o|NDCcd?uom)omtt*--`d6j~CDOoX z=M5Dfq_tRvTZ=x1A*NAk`|bAVL&B{kRk$-1*=ewqI7&}F~z)!6)cW5 z5c7bwg(l5e{nd0V6hl>>6#?J`JF*LxzWC)9hI{Yqgbc)^>XVTv5X2xaa?tFW0(DVk z$bYuL2Dy(&)SPzCWts3YhCOXZ!HevoTf*>)1aUkae1%0i|FPcBYvuqUzz#5j(oF87 zGr+aj-FN5;`V-8SP&92p39NBo?9q1=CJt&{@;1LGlN|VBYEUE=jyQ?v%|u|NWL2r; z24lHQ&?Q8CKbBH~Bu~1J;SWT@FsC(^;dhDYb*G1DAx#i=`CG_wXnnon{yAfMm=L@{ znTh(Kt-at@kqHM!Dvm%D!gegZOnmC!xzxdXS!1FjiEx2{V*SQR2)_Kc`K0?D!h6kE za^>D>fyn2#Y+o8m8@-c@*t7T`mIHSp+U!ppD7*_*1f!GBTvEMyi3ywu6yTLwEbBV^ z&DQvdIh)(J@&~hIkvbbie(3u{(*-_Ck)vyLaal(K08rDGUJq!g(j>N!jaDRae87pPU;N)E5YMd@Ug;6AxonWQD@cB1E^j9h%LCg_&e;cNL<2yFW3 z6T#U`8$4v1N;)-OkrdX-_SrEIrFt`0$}NThPJbxvrug>Nk(`&I5R1JJ`T>|XPm>rggj2iWr-R*YOMbS^;Q(cun! zcSS`O>B8__GiK#3lWMNloLv*F>fFG1uX6j(hXAD6kQz>P=P_}QhBq-+uLwCmwxOlN zI=F8ij`Y-{14m|j5Fx3P%*`!x`@+1?5k1nh1TJzCl7DyIeB;aZ{?gwFNK)Yk1S$al z_inZ-87jA{XCyq|j~Al4GXucYh9;mU<)A+(u}LH0Ssi5lEs)KlRf~W2_Rr-fcC9$! z+R6_s8lyb0-IqhSgZAMRqPM|!h{IS^Ff5JKe#-@d*}@NmyNh3S-6J1`%=Z>~-O5GF;mD6tB7qG0+BM$ZaNd1fYdv2ru|Dr6 zYJ`$1C;m1Qdv&5kydppr|J;#ke#f^(>plObG=$zW^PgmkSMUW)g(?^$3&)JnzoAA$ ztXOk~te|61IGk4eCguCg_2XCqu3$*U`G3AZl3oe-Voj+bB>X3GBB}PF$ozhz%zr&f z<#fG50u zrTaKMc$4S*_2O1}s^qMT3AvuMmZC!EFrFzO{Ie8vo*IcP9u@oXFSn<&{{4T{WO?VVTe>RhL&`w%*ri$Rd42?Kr7Sm`tgQd3(U>ZY5e?heU=ct{w*!R(^&;BUn zGiHYY4V~U>Y#g)EE5tg^Gyz(4ah9^toKHwIi2GRg zF%dx5ScE`Uq?}S$2cs&o4_ZH4+!3mPMSi4vLXXc1w^+72WiKzKrQVfSbHA{1m(_6! zzk)$Q#UF(eugagd|;o`P0{aq04cDUaF^zN#u7=&G#VM)PD_z z7{u>=NkijVJ@iBM1k(9$m3i3)CU0Zzr zQtm+D+%(=iFL|T(#y;(F+YI5k^n;f_BPms>t)YW+9gR@}~y06S=OJ z1mIGdwprXAu87Mh54oV{U=UszPYNxALhK|Jz%d0!9;$Gm+3KX}OUmI zqrW(VIM!5#5g$j}15aJ!@3}?$=iuktZa5J6m$(zx@8Pavq!NY@ZU8E3t^mOrD6szn0MnBMJ0g|e&IN%JUvXLqHoODymgDdf2c#+UF>6$7{Jxi5Z}D#e<^PDCT) zVC&1oWFuZaS~R8pNS!#@J=0m|<+df0KF+(x8>Uzk1I>Xsj&|>N*Mgw~muL?}q_~Ad zk+cJbPSL|_=ME^ywC!cHHzCgIUvYoj%+icG1Z(*{Z1sVOR9Cm^!U4hoQK-xhlPjo0mH{R zDTU9vF)yv@uRsVI*UU4qeCaD>Sr%DBG7 zrLpDjF|z1^V@=#i7P0Q_t={o*I7OoPUlFiJGGs^ls!oI3XjOK2eq&lqb;fn+lSb6yk8d0J5g53BJ6U;_|7sJ;J5#1woOXOHgd zp_qKS`N%0zmL|(52#uDDs}qy)FH5abebDC$iqp!ZgH;Qiqw)W>n)fUf8W0bn31-Do zJHumN;?k(1$R6$N`Jv)u{y@;ojswZnj3}@t_>~d2a1$T=PY6&J4!FA>B`mzoj~gSK zjzIA5hC&>LRZ`o+MK&>=qS9$q1bo50+=}(-UvwJi4~bS0mQE5ozWVf&K_J*?tj#Ml zGGQ-!3Z`cvqy?qiP80mARmiEvI3E4wMH$n9Hrx1K37ALq=G!)%j9-+q;K zg-@P+vi_smFihdPU?Xl{-KTI#;SjBXVuMQz1t=iFHjH!7Z~57>HZD;K^FH7OZ#z{7t6}VGMC1vIv)fvyd(Ll zADC&Qsc>^>y*Vb1bW!tcx2gVwe~sY3w8+JeD*#nzxj5brvX#HE1NaY-GlGLj%(awkHe=#vpk6_Jbz z=v7Gvbrt?e!)9#tHxZM|M5W?NQgLS&6W$S@f+;h5065YdJx8Mc=@whoCeVqoM;2cf z5C;Xtxm8E}Y&44V!8W5qRsMub|8Uv$? zmdQ4HAYf&0mK)18Qivfg1M#UpD^!r29VAZGb)LVz32b}>Cl-v$S5U0lVO{Eux0ua z%*Q*1WwT+gQc@VOM(Ud<8o4vP$gppt;<8w>U{iBpO#lYl0C4;4nc5Rci9K2v-NGC= z?1;(8Z?@?~7j1B5$jc5VzpzLIW1u(1z&kUHBc3q?D4&?Z-D92?)M? zdL69W?S`N^|FpOcp3{^Hy~T30WRVz^ZN2KZ@~sE-0Fxi(I0cJBpkI}GO!~LJ_GT}s zX9{C*u&ZB$xbXzE*!$*tP>?d0{9odHD{8()rZC=J5I|-$@*$19QndZ4bm7JU>%t4* z5k%oIvBeBrE+x#VG68u>RMbn%Q+XoPN`;702p?FrXtv&EyD-vKcJYK2)&;<8K39~X z;R>0fcfqRTQ(}!1g$5UAOW|7~XBfwsS8bDVG^py1m(K=#H|cU*jY?1VTa%QlkKahO zYLqpw?&%l@lqtdJuD28fAzCpFYUXcS0UIC}loL{%6jO$j03(Su{~W^=k7it=loVC{ zt+;5oOlFUDE>}@Ou^~(11{li6774e1ScvB$W}~=_>0!D9qu9!v=FYxDYui`L=8Dn- zG79>J?hpr{i=FgotqhWrO zTt%l5U+}V_(P`Y?*#ko*(gz)0{q*CuV2BQimFoy@cw1x)FB*zdfal<~bOF-366c6X z<;!p=NwwWfZSS_NL`Ey9m1Q2Y*#w>?wq`hWT15oft9^o&S47K_k60P?gm{L2YKm6M z1i2zTe5}&43+7^HKi{}WbWdso`fF2Ab_pB#h8ZVe%Peg$Yb3hzNuC!^bYC`IyNY@i zWdLyB+K73x=2NIR7NAH9Z z%+0jZ{<|wc$bqf_7fAtC(>daQ@mS1a5C^EwlnXa=4V`KdH(DV7o4rx$*HP*^f>IjX zxoxWm{Y8l~X5NZ_@h*D46t*+)l1tMY)eEm`RL>NWg8Fy6f1DL7GISRwv+C(o%QnC8 zH<9&b@boc!Xenr)&U1l9I=7gKhv|Rp{&ub(K^y7jg<(ERjhJY4;1HEjJ7dpnd8{)r zoYf}S4{~+hwgxd*LlKb>xhfIfzkp+?LGf-gxLrQ9VgiH48T2#Bssa*CdJ0Tz7?#lOe)2%l&jIPbx}c{qf`e2@MEs@?CE-k9_5}2br$faYO<_KlN6dpu zQRc$?3CkY&!WKaD4*^4N*zz_prBz1=O=nv$*Rf4oJX>_a(uTZ80i2=NUv_>LdL$F) zPaCrwYW9po-5$_lLi$H2QWeP#+%qGdLuFKiT{payd z=dW{YJ*1$l^BSYf*{@(z748XdtF}?)WR`z4qz$F;5Iq+;oUw^4F|ob-{=Duq87Q#M zMSp&`)XFuYOy~hG6O%9j83#N8%oo!ye2=!eyIPRMR7sc5+djS}Ps}q!?^kFVs{_g3 zk!sj>C11gu?)haUfGon;jpL{0d24MSsq@Jf6&Dm6<|WE73y!p*AM*My%ie23Q&r8b z4rX(f6y$G?o}zSYe2m5Bhni}|z@NcM|Hs~2cE#01-J*ox?(PJ)Zme;FySqbx0KtO0 z1q%|K;O;H~65QS0J-E9D?&f*VIiKzixOa@hfYGcj*|X+cYfDw_ssbVTO&{g`Ms&;k zue3CjD^bvYWT|MKeH`p?B1@qs>QDZK%5+_@n^sZ&c7~QEZz|$Jk+N>N0dubQCpy=;Zlv1n3^6G;1ZHB0J#)M{f%qx!hJgoqW|)Nv*=Dd+8B6Ks z`2W2Yz-OWcBt@2;Z?nyJTm9aR6_dCqWr1STxy)jdWkZW?U02l^YvG2(s)2!+7|hzl z^lEM!9VaA}vAfzA#76>s3Jvny0%*q1kHO zXUH+!3{?1=dzpP^(s{U`1D?R$%!D@m_`J!ZvQH`;ny+zPr(hJa;K*k9Xk;DF;-}%U zCTRN}U?eajV%DB6m${2Pk`qqjuli%WCXkZ&%%ShvY)B)tn#h9U3F9;Uaa_cRABB2+!xrj zPle#(TyVkXIoeJf0sP=i*bA0#zQ>=wW+u?m#D#(+KMYbF!*!HC!%kyr63&jA@1;vp z{c8G$71`0kP-gK$w-ye((EUsG`vf7#x1m4nx83tOXiMI({K?hL1X!w)y(v;TPKt!= z$It@Pf+`%aH0lfHkLTD11D?X3Q1u^WRJLc=uIrw#i~6?*xA8^gZU0cq_W3<_Y|Az) z(rs})jwwU`_)<0xsJ5GWZQKeo3{pIk<2?zN&=7v0mA%dSx(`3{@wHjYO7LCT{)<&% zc)ZqXn+js^CC0|`=Ry+jrD8&gx?!T0E)I3ZztyAEp`;EkqIrSBefF{Sx&Pil&r{K}(r+AL>Ao#SoyAACmO z`N@1|^ZEOXLy=-bxi>dPB(Pd0oRzcTA0=Q%ruZE0~f?tLw+G#vt^rG_yX?+P4a21v*(a`-5!Fq>rt_yh%YaxfJ)WxInawk3IXB;=tjv;nkS#F)CCDqFX zc>>@D9aTGYIOWR7jqjwoYN&N!z4>;WO3M2zDr>olTWXAo&Tm$!kl&W zOnH&gv4dmuZSuQo5SaL%J7)UDN4|I3`*$d`!RqBg-%R?O`6*}9JXL^Gx(xo?8p?<` zL8*2p&O?|+7r7n=hd99|`#`MF_QBi=wVc6p(K0kx47E1BPNn+@byey$T)(CF$NB40PT$ zrs2uK8a;bB^DAxLtVnOSP_WcbKIh}BtqnIg=YMRO-hi-t(shm=Ofzt?pBco=Ca9c5jJGQ)Suhxsj5IsGNA04U1ubFgK$^_qItItEL- z>unA6%6fX%^%1X{)TGncT3P9C^1L&b%utbz~QoyvY4}cwFicW>oZP6}YTW=$pm0(J762JIu_Tj@(#F z1?Wkxv^}VSNa5e2mwaxl;94g_Q^Gix@A>VDmSKyn(N~q7qS@e532vnj z%DPTdr7#v3V;r>ONe;J*p?B*}H|#%EZiw{=;RR~H@sFTE9|TCyxW{tYtu%sY$|6v4 zhZ>^n;%3WULdMP^Uyi;|%g{u zgvmY*reFnb6kEGWOztrSZlP@-MSr+WI_971FQ1?yqf5sp@1Ley?QLGhzu8ui|1){Y zJVWmO!-Rt-sG=J;udc#=u}Mk8rGv?@58+CCrWO@XSb4=#TT_;}=P$70@ z(MJ#5B!sYkOTZJWYFOfSH?rwfLX=}53t35Ez7~Ce%NMs$Le_{b$7Naf52tTyv;I06 zu5u zfvjVa=!ENu_pdcPs2QJ<<7v#l&8`cmFW*oFkU zRliT+k(xP_HgX=H8AhPS=ggDij8Q3h+tsgG_rd}bu`-F9B|q{^2Z8iksLc7KV-t~g zwk|}yE^gNvZsnZ4;G$WCe!DHipqoPo>hCknmj^`gnf} zujNB+X`=N15j1{Ci)7zVvElcgUNYwx?qoenZfMYT+Ji-isIz|3vOi48ZvD|OZelbQ z%146d!+cVbVmT92i4a>DCt(USdBUk27TV&tgiq6Yahqs^ToipR>0_5^+2!p<+nvQ9Gu0pNiW_zN?W!y+VVKZLpGAW5Ukw~#b zPSWC%_f!96Cn+KG`eZVaktfC)Iiw_#RQ7@kqwd6X>=^?=b{IURl{Pzk;u?X0A*s z1SCz{wTXYrBXmVHCdD4PXECdlw3~_AzV2ALekkL>(=(sp|6`;?qe)X%73gl{8q=30 zldum;Eb%!4IsZKSrs9*MNWULM@&j#aof6yU@^9ruSz%jHEX5eLJT!hoX0AvwOwpK% zz)ja#xptEf8}H3rn<{;J)aLXs=U!xK)??!y^M}x87DU}5ktxtih{+FyEO(T6yXuwQ zRRzz2TvDwrxm!)hg@6w+h*ar(eWmW4K|acBbQ<>ho1-kP5BCXW@MdO;CLw7?f{kQr zyS}WaXJ)nH?Y-?Y>00OOe!|1WE%=h3&)k<0S#tA0ESxU#@!AW^PeuGj5$l3L!tCzi zS4r_k2XRminFrT?wz;=xW8RR)VPn1IQ2_BzMzQ5dnK}KGe`MWpsrbVYv{6p^dAs7K zn(GTVKT?yPJcKA4#A{s2mpTzf+et|JV%{Urr(9Eg;P_3wO(v=~pa8#Lb%892^6l47 zWaL^7W(GZ5DOuQEw|)kD^H7)M5WWkl+%@URrB~SL)mjmZC5Epo)na=91YuHJQ6BGS z(Uu9Cr+WAxd_}HYKpF|EGVGXxcI2$Xcn&1Z!TxYTWBO=Pu5DcH#zc(p{S)!xcVFch zY4sKK=r|@qc*N0KDD&isK|F?{?twp{6Z`w_5FMYBzV5)d5h{-6Ppt&)zRs)H_$~~Z zYhKRahGS2t_f3=r?7-`mj^ZQde*$EWGQ%ByT|A&dsKG*PO{O>RZ81)>R@j0^$@5I- zJh2&XmP?yT*xc|u^+j$>?lBjP8s~JRzm0=3mDo{Cc~QDuvQTJ>8gk^bk)xcRFm0%s znPjk~<+oXg*UGeYTkD)Q^@=0*KCPNA6v2qf5C?sxPjEob;dBfap4H!5_b6I7CT#9M zNI1~YDYROv()j%~ntzf|NVRMvxobw{pM?44cy=7aVi|kk7hklVFz^+lwLw(hJ)ytP zDggA8qaPPaaU>zvZP^j9O!M(t^KVnl`Rj)qWsmu4b?@IH*+PV}zAn-SL1#K%P6bq1 z;_Ur`T-ydStM9|M+;7NQv?_l`bqWjtTbg9yU#oupa6oBF&B$>|nb7Dc%J}C7#m@m0 znObhYw+Z{Bp(LuHiQp%$0uS=ZNUs;HR#3T#vw0=N)fb;11PAsPi| zjJUF-c!qvV*Hynv~a3< zT4=xz(vqb)fptpDTAxabjmi&=Of>l*te-@Mv9FIh=elcaL0Ygs|6twVEBQ;is?5f` zcNRZah|zg)n#JN+jRUg%`!(FIaglzUv&UpnyHY)op&HIa#$F|i+@}zd=b~IyDsr-kzL(Qm?JGI2(nrt+t_#fSit@x0@La|@eDJj*mSV)n=btD+ce#isXH zHFk~;OD@-@A+ltLQuQ1KPR6LBk%-zi`jx4y3OU5}Fg9p3c!k^F0`z-)=p_pZ@=k6)(USb=c#KBh(kMJ&7uS*64UnVCQq z)&av@<7@L`<<4(VzS$OG@=e^W4>2+E`m#xpsJxnG8k-fJQG0JMKmEzo^1r^~Uu~US z2nCN-V-(Mq9$DX+qYKLFWl?{$L+B=CJ!r=Stg3ejZ`p{9aOC{xOn5~}w?7z2PH%h& z{^8=~Yzyu($KJrp5#`e2 zFf+0vul{bL-KxyJZ|;D49KtCa(W>n-nj)W{*?z{LvVR7hul4kiC}=oj^xI~_!=}cQ zMulNXWwV1^xpS+mZ1nP3e1yw1zezl@*yi3)cv&4mCCLAu1qRZEW<@bqy?U6{6h-7a zEGf-J&qgFRR3@1m7Y9T3D$$7+-)iJ`3@qpcRuVx$1<&N?b+~oq%bsG#6s(qju6pS;c=x{N?6M+F?qKIxaEI{K67rsdxgQ zorm{W9%wzbhKvmt)>B;9z3MUl8cHzTUp~uSJGHMgaGt-M5$ob)(?sqETc}ysx?zZ( z&VJ&Y3^EGl7ymsHm9@RnjURv~->Wlxh{ThPNcuXpyW~na%U+ehgzIU%>Ho9Bg#vyWe#*;#LT1I5!zhs1GZEoEfHKwbM z=8fBRgmT5M8iGy{AqpkJY)}G0oCw1GHLq$^v_G|~bL)0@SI8QP!Ri9}a;oFriL!>J zrkjZx6u26%=klGZ+219{4X4yl>u|+NZ!~dv6Cs5reRA6BS3jq(7c6LzV?|VrwSl<-Z-hMfP$`L!Cs}C``&=a{EJQ}FZv6GQGl{n)L7jA}SSoeC$ zI{fZCN?+G7J8b7Nl|q&&$RgIdarINNH7*vmyIm;79dSE(%OqKFU50?J8Un1Rg10Kt z`QAb>?_&SVU#%~G!@Pkb6N?>YdmuEa}^4jR#!LR4b9{Nr;t#2Iu#JOx}DP#+x8+PY)PDJXgw%q6zl%k!8E)$jIW zwlxSTt(r2CW*FW6vAJgNyS|jL!Vw&<&NuzKApCuqUA-;AV0U#Z@)+%%ooAtrk|{!8 zduAhMQ|K9Pky-o!x05P4iTlhqA3V4ZFRmxx9kp|C;Pf#Yy-i$0y)1Ns{?sQDJzN3T z00SXdR2MHveOAHXhYDN~DF~lNT1mAopc1tUl1jD?D;ZrNWy*yAaEzz2=P5n?-Syti z!C;9%&J*FJ8ZBMqJGplo=|{upWpWDwQ`}9hmdZVmN0|fn9aF`D&@~aL?;%TdZ83=` z=tAF_-q(ghEtSc^C!|;H1hXZ2XlZsHs1rD2NVz^9-seRQHAZ1C%hM|e>h%SQ^(m+6 zz|%EhT>~eelWfAEIT)&F5;Oa}pBT8qdrqbW18fYWVQV#K-0KOLS2@}{GPEem;g(j< z@q==DBs2}`^Rs;!@P{AB1%}NPr@%k_fW;{c7SygEf1>@!W2x{9v=HJqijmC_vcDfk zcNqrW_I^+@9z7FY9g?B-@#>l3s8S=sjvr+E?jRpqFzyx3$YBXbiARRSA;GN1(!iq$ zSKPtzDcFk_#mQpMexd$rD>ms~7_w+GBil+w}EJjAowr zShJ$cw%eZ0maN+OQxI@(OmkK0{CCkLbv;1kk6dWEJG*i6>(S1xL{(pNmd$-F=5Iel z!@rY*edziEo(Q%Y9up{{?n_&2W7QI@tFh+Cv-!YEAkQC&!ZHKlb}yXE{&EVdCAYh# zHuRszhjtq29&-8=PUTF_@m+YmuEmRHW|mxn);Y0aYK=Swdl2~?Bb~+1zST?@i~XE8 zGrr$bCFT3?_IP3S7hJlkt0HJk%x^lUj8aUvA<~4jwpK+_J*3>W$el)p;yb$P%50Mi z`>;G`!4_Jm&077w3_*wy7xkf8d7K`?HpXqJ6k9=q&D!L3w%SQoh{`snE$wutSn(_c zr|e_;IBs7``K0q+g>Nx+zk|}KrKaj?&Krsj3ISx~u{sjDFQ}wxs`^18GdH=>fjpfP zHYcE0N98}G{soD@GvLlHXt?8H89n1C<$;Uy5BNz6?{PTtGx5aB>UPbMF{S*8LIdQ? z^@%^(eK@g{ppRr9(dV6tw8aY^uqh9#)xP~JYpBH|E*XwEE*kMh;`mfKT^9TpV(5{u zYYu%5ouO%{TG305!nRp8>G0e!yyH@;7e!g=lGJY)fvXX1fSE$FE}q6d+F4z=~|=dm6epg4~3o z5D4qTT^}hjYqwGEvA$2YwT#Sxz%^{3~T;mPMMjRQK21f z__%-8x9Rf9o(uF+xTjis|LS5)-)=4(FoDW9Wa)tjj(vPRjM?ELtd6SGSN2$rZMt5X z;dwo1@R>w<4!KDuj8ybgcUjrhY|lf2Tl+RT$FwpuczInJdCF{o{mVUo1ED6r=QmuA zRS*?6DtJ)pI=MM&KUnX)QGzod?^FxvT zsTau)r-iyY*L_djo`|w@r6#6^_|@_q+nF(wgYQsic1G>YU?^F%rzz@mCg0GRzUrn$ z3Sh5-g7f|ra&xk-gy;M{8f`u4-;e^fnOi)q!wbV-~?~eUxWckBCFqutx~=s&u>_hA6sU|eyl&W4?e%V zJn$wIuvtaNq4Kd_d15tyr61Fbu~>#A_^RHX*5$;Kz4sBjVd6Ud}t&Yn!{TP8<7^ zu$lfeiSp$Fi_YLvmSm4!CS1;F6LPwUr7c<{xjN%*(66vH5oa2uxyb?B#YXqN5K?42 z+m)J%EYPU+Qz4vn zHmb+SU)9<10-nh>1z4D(ST@hGUZ6rE_wcfM8UJK}>Mq2TRAC{DGNzDIrcYMC>d#?p z58NP}W5O=mQ4g&kq$^lVaA;{U@8$MMN3Ml9|5AHhjl)Y=)v@kv;i>>%zY(wFDGUAl z{+P)#NtQn)E^61k`=_;lUzG0imqWZxD3nF_HGd1kqX^W5eG)&fis8~=mnOzQvfX?_ zSQpQGwy@$MTr6K(VtS^`dk1*YMXP|jb*2nQsEyId8@1@!*u9$Gy<1k0r>R5uae)|_ z`F(uH3j_J2;6KQz?qq%*Omh4!aW}^NLCg*V;lf%k&ZLv-gkfQ8a&cZw2u{R5KG`@| z;{goP3?|8Wg$KWsf#oU>xG^zH4IED$7vBmHW#pA=winLT)2`{(b;_J!$oZk5vB&Mo z-#~Ix!QJN{uGT1(B)(56zFcs3SMj20ZXl!UwuHXyv?6x?sV+U=?`7mRF@OE* z-~Z#CKl<}5E(2dH6m1~R(vL;>%}A+X4`YVb(wKVsIHCZWDWQ0&U`L_6G1PH zU-Q8w8OB!Bg)xiMnoZ1lI5o=k3Zdmgq5m|~;4b`laG2i}OCDA?UE-X6n%Q(;IHana z8y}h87yl^JzAHgHPeF^JYK}JIo44Xe1L5qy&tlFvpGFRe)6VtAhhS;<eIIP6Rb+AxP`Y$n9Da?~+b~PR92~tgb$L!NwQBN0v}} zl%9uq6wUv1m#I2+t{JmU%1IC1z$ck7&|ecy{%Gxd#19|J{sA)pc;>;wC>d!IH*WD( zO4%HqE?M6#3|7*?E`o5|J#4+S{t0>%MCo}_n0{_#pWV0I9`=E7vEFy?uF4rr7tGfV z@q$DPniB_`5*+B^;hdWw z?9=#O>^Du)7Sn@Auu`tUWUXlP@-wISK$U} z=eVgo8XNV%1{CAVu!XJ%w8o5J)FP{q$wW51um9e)Hq6OY5a1fPL>Iz;NH6<5_ zj)lkms`^NGK3)<-3X^i~DlFXO8(wC+^sS@Bn(+M*We>>meT#l< zek>|O?z#porM!O8b>cvJnnHDTki?>C_^_I#-K6@|6Ue4{ z6Ner+*MPSVVMhS9d;V859y2NLo*6-)n}s!Let|x_i(W~A#4Cz9mUiF--#E7Y@g956 zs!xNg{@Hxk)CH3LNeuxb{WJXTNID){R92S)L)K(A+iuVB*_RkG%te`viIuqYlBRHO zt62r|Pi|dV`*C{`yvY&E$n7QYT2bj&cfl4TLL+i~x>qD(qGO?oB&c82z(PW~a|$)9 zg{b8sf5_9-m6~9aORg;^9{(l>3lFQmG+nh?>}&8A6@P2*CLk%$b1nRvLaH)P) z-SZ7GZbfDogn1;`mdYtyo%<5UB(dR50te!>zW2rdhqxun1FuoGj5R?QeM#P z&&odLjiMmcaqd%VAEDWkGA@H%GnI3qT_h(?iWI&>T1E9@v9iNL-x&qYwNp`wl^w6_ zHbWJyX8a#()79bQx3--%QW`SRUBaTTGQLdRQ=7k|QNkv~KydY_ZXO{4>Xp&>J>LS( zyb1->dzN9oGKOLzts>E|O%>3ijFd>y+f>}5&q-q`etRb-ZxEZe!Hi*Pgu8~lkZn|% zBwG=hbN|gs(r}`2xSQc~RYb6esdE3Xh{=?@vO_BrJ`+Mq^c+|e3-Pr`CxO>xL;<#f z0I4rX0;=uTaf%ntH6dn~Qkjp)D_LISuqUv40KKKI3h&+`F7@v7$dXaD z(?b$!mY1RuwLMGM5MSukFMjCrczZ`W%0DIl&WN0J+~&ghp8b7Vm+sYI+zGD{%hIKi zP}6oAZIO2qXLrd8k!swRK& zs<5y)RsZPcHn{voBNehKA)rBY`LjB~x+d;nogkzg+mDzz5c4dPIs&1lk@$cQye3&ra=lI5ONkSdwR@y2bGpgAJFyU8$2CEc_(Ec96hskpE}|slCX1 z&PltLCw7K*%vKS&hcmz_ottgg9Jx$$8APrnsS5}+f$>H63x!V+D;HcdM*{0-#yRerT@ zR8fN&7%D|)LqVhDg&lAv#|HJV1bY4u)W?t7Bi22Y8h?N_xTFt-Q5E@!RB!iMY{VD@ z|B+U*&|L>+qsx!9wZZgPbjVV$;WQt~^c^E}uh!(WA8y~4U&E=#&S=^E>T>MiWhIHj zlbK1-OWdJ%3l;g=u7~z!h|K+1<>gdrzsW?_HZzx6Y{wozf=pCn94u45?=|FCuY2BREG)KO0!(9 z$!mW28Fy(rjdLb#)yBdf`8#Xac5%0%sNSaMB(HkH9MSt{fX=8LqsvY5W62$#$9F|w zSQ|_dm;Sm1LpI&>0fQ^A8vYcqHZVPGw?Q!k8?Tazy zV{bHK$EJ+?NIB?LOi=!XUJqfo-{k2QtyBA~O;)V^->6U(?R0tXKFWYP%nmETp-6$B z4bQb8YEJ~=k{J&%iyKyYl#a5KY9E__Mm_4L+LM4LLQLMRCr-tCQB3ODq24nL%rUjSbUu*a?WFa; zz00!ntG8{$5l_`uzm%y^(uFwT-}>rd_k?!XrfjkdNQ`eG5CqbR&J9oy1>%V@j7X)| zs+;fz_hqYj$|rsIkL&Y8A0#h3aUa+_N;~VM?`#tHxFkc&PlIYOZ>L-J_^jv4iNHTw ztlo8j+!{~di$ed-oX^V_4kz7@-uHWY67Iwv?QCz~Ow6a0Bx_Vs>&7n<6;_gY>%9a1 zB@z_DG5&hcvj4o)SJg+f_I5&`P`ebb`4Y6(4w!`FK2E`un_SH)A7RVT?c>x~*uC6c z)LRr$_ zbc61Smcj0TPHhsm_}Pc8PRm%)_;bFmdHgHZgap+1zQMnUKKz5Y`9IU?VOr;S|3yJK(LQoCe2t~3y2yq)#op@GVS}qpH!Nv}cQl$&cFA2HSf{T*MoE zdq{-Y!U7d&W88gyCSE{ICsyh2@8&O|^0&>6I#G+eQ!}_M15SP=#eZAZ3P2)r`y$;4c8k#U8-?R4P{U>BIdc^#n zP;0pmbsQy9MWQ^k&=Z93rGLUP6BINH8=mwi2!Z@rL2tqSp zHqXM}!wZx#7Jg?vK2%Zh`VeYB5`r>XmlkpXjDNsXw9?>r44!)gm%e{SKa=&d~LG-BQhVsYkV*4wfv+d25l))w>Luf zw@GmocQ$tzkarD9*$Unwi-&eN;lg_~>b@4~jzE}%;tKkduc?ZoeKSf=U|+Vog9I6! zcU0Lk{4ArfNAK0QBJmR8OrW3hjtbk+qsO`tbI&y=5a=%q%NBhLRzD@-YLWg(JTm&7 z76r&)3zI*fGNmn$Z?>Yscjc|vpbwiDb5g8WED~c*ipV&d7(X|sr zZQ{ZZ*=Y|4jfs5V1r0(9#lyR6G_Vk%wNIXFQfi3&R&*|r;mE)whiDu@SaVWgrK8I? zju!!E0?{zNu~VS=gZUYfUi;eMakSr5((a}@mQJ#*n?LLp_tQ$6f*K~jsq#1mk?v@u z8Mw0173z7TN^TDG?Wk(HVs@aR*%pQynz>CHx~dGwTLG$c8W0|mp|~9@P%-A2QNA-T z)yX?7T;0mIlnS5r`b0YDqBzBlif9(@Uw=P15DPgNTTit!X*d(am)uNiGFLNn#ZVr4 zen}1?WHXntde%T93X+-8(pa5|mZy157FU-XWQVp)FO0TJKD=Mq{MN-r5S(Js2dF;O z4!Y_qplY8~Qu2{=DV8M)F3CPpN^x%&YB#9!?$27X2%THO9+ecVEKlc0p0rqy?kX1g z;2&~8Q$WR4LexOdMD%Ljjo@b_?E)T)8@-~WDkZ^y-tQ1-t8wVmQqx#@8@;4+GRug` z^((#1)+fhMJ`8lFQ6gFg?U0gNrqwe}QPio_`&A@o80k_$wrSSJCD|o!5Z|>t%e!Cr z&Mv*2vczXLQtzy7Y$KJT509hj^m`dZ;C~{|GR{qXN9DgB^d-81{BEeyoFVPXf-sm$ zN+^t_EeaJw7w^Ui`Uynj#9H{d;zi(l7Q2W)T#mxwu;hvmqgjyVf_&dT868yBhNP8b z6=D6CN3{dZSzUKkb?^I~sTj5MK$4i#=M-|_6r9~7MT+UtC)bSsOlL=e3i+AwvMv>L z=_x=NM3OH;B5@xK#humXXGA^=+M#SOP9OE0qHtDjW|A6dec#{l+99G!dqMDy=VNNV zy0Be|p4Z)ni)hP`S&pyH3otZ0DDF8*L)EM!Li;cYT>`=mYkj-0R*STsHwk~CAH`CW zcV}kcib#PJKmpOpB?ngbxUEAQzua>-#U6^PRARSr+ z`Wj+Woc8GB)w^K_YrC~fMBL-Qy!mdUGLq6O-%YPdD_S>?^Ud|KzKTM%N5iMO9pB&L zsU*d=YLWBQ-^Rpc2Fv9ciLLJri9;F zP(SCk-PlRX!p{acW1kqUy{x)2s}V+a;m*ZIe~y$<-j{QIaJ%1z}J>2v8}K zP}@&%m^mLbUcxG#vl7lVeAVV-{|{jK;^t}Z9b3faC=2V1F0 z0c_GX)aTCec$jqtW4K{U-iBCW$$`-jHg-Y`P-fUmX6z-o5XR{U-#T=TW1Rw( zFNs<}=oc+r>Jz_Us;f&WYqbonRhPT&1d8)}sM?GiF|**U`$A;h`wXKbsj&NnxSW3d zFpEt63|vHS-X}4IjFUyMh*u-&MEzB+_3dz=VVwMq8791?UB0eEN7?fuNdoeS|AUPY zg>YWRTz|Y3F3*Ijp;x3Ra%Gn#DbO>uJ5I%Z5!(@8`MxSQ!VLv(b#P4gvw#d zJ*mE=wwHfNegXA#(*D(;9dZ<-38c+?&rV8q$QFx$=u89EzG1rC)-ZUmrdn7=rr5?- zmF^Rp6WN}NI_eOYX{D+JMe2;*_}o@c<2aq~!r@jRB(<9z0fj1R@WFOKN9Y*8f9UJj zym6N|E-na#?X27813EAP>9*_g$*V`bo#^+Mr4Zw8X>Cta>A@gm(1Dk%Y%>2c69MjCo+@KKB~_kQbVb@ld30T2X%r;_tm(L7g|)Z_HK?~0<(ep zj4-~Sv~)PXvK@e(g*FxaZV)qDB$=eBt}j(fGc8R^SNkV*?he5%6F%Gvg9Efcm{fsA zEQ$pj^;h)BZSyX5^oB~^=5*8_%}5MPm1Lg>Cgj~UC7Xd~V<~J?<#IYdcxeyDtW(2YL zg{WVOx@*^NH6rTM(5e`_+pI}h9HtSc;g!o%HL>ND^O!0MkI;JKkevs#G5Z@OhTwjG zeJyJ3epb1eo;JF(1{dE20G>@Y(4Dk?o0aT%kNG~B9j+_wdx(SIwk220`$^^73&))E zsc>1Emte|Jsl-U3rQ6BOqFR&pR%{%v>c2%=%c2Gm_Y`{LMwu}B6e@*b<6-?SydIkg zts3pFPNj%`W%D3LfZnDHjH2%dMLyIf+jTST0OxALAFeCv^`5)!s+ss4(oDcpk5zBR z%Sg3C*A|>>Dq%q+-3%l_%li^fraQXVnVKzzldm9B3 zhC<|bb<5EVG}ir`@*YjYEWV>yz$xWdK2-)zH=jX)x4YsxS)1kO z4b{XhcjjRDDhzZPp~yk+5z~?WfBFP*Uk@4TS_!_YPl>N^KiBWl)l)%yT^cH26y zOZEa1DKBrW_sjn~@Zu-tr4e5rJR;ti;Lgc7vIGolj+;?=?WPC;c&GbQd1L*R5Y$74 zBjji|>oQ6}r%KmhQ=J-WVYW}L-xQF4!s2)J(_{hHR9y8;SEd%7=?T*vxH(1g@6me* ztl-gowo=Ve?IU%R>4&%A_nXD9rwORTqCYb5X~66{pc73G#b;iQsGCv(M#9P5mv>sd z#E-7$iOAP7cwDPbS&4m@86@eQ>EOVMsG5VTKQ$#l%+J}Xu{x$ipIiQF5=}ovKoPtD zPqmR4JURJ=D8sRS-({eaHfYk4FlmwxEMSb0?K&o2xYCIBoP_2r)X`q5y?fFTLvVyU znUlf;&)ENRK+7KE4Ym9>AaIMaPyVEEvWfcza9-6OjQ<_^Hf1GW=MmU-G z;-?Rp@L9lYVA*sI8rP>05&N--8}y_kbQHu5hmMocpqe6iuUNoVULzt!#%}%wy*`ae zf4B^Nn^2*Mk@y?>Bk(i|#h4=T;oT}sIY+$2K}UpW*ZsRa!oxyhJ+hW>SJjr*Lm_>^ zdZzuxmToXVQQQA9g_wAwv$aJ`lpIVNJlnKJTa>)RM)O^rtNx4#c^;qBZ7^ab3c5`J z!y?9fq=Uxo#SS$scyO_lr`@0-K6Q>D=fd#Ew-D5Pc{LWBe%2uBQGr2V7TDP|jM$@0-!K9-k|3H-Dc!%{}!dSVgCUoy+ zpcRQN!+yK1J{Xe?9|k1QLfYL8vHzm)PTsRYU~gDLQT?eW9n<}DJFJo$YDj`t0abM5 zkop#vfSg;I<&)3bocRP5gXG2{#0iRP=Nn~StaIvsq4St2A=6EtMxrIZ#IpF4SxN!k zx$sTi&)ZUNHh3-z^YL1I2-<0Sp`x3})x(E7+B{q&lDX0A(b_D`f3a*{UZ21&)0t4h z3qdW?6ySH{!slVgea#;}LZ{Gu^`5+c;g!5kb0;@I>pm|YvHoj-dfvB&O#S$qhun2; zcr?P*U2FPy-e$2{M~Zm!ycd+m+UdmKBTSyt7;G1A6o9wyx}4FmMK0262(OHnB4 zOfZKYlpBhQa1~=6-D4rS?_Lx*r@Z%eSED%#>50Q+EzP%o>;BjX*}S_b`}# zfU;MOQ+NMO7V}2y^;wqxg^I`(A=ntQn3SQIhGE{WR#F*j90Ez2Bp>O!n?w!QDVB_p zhipjNKzq-_E3ly3;72zU@(B+qp$14;toZ8NDO!@?VIHEcDWFvO!qu!T}04l6*(OQtp*z<2mw zZQwyx{H4~T|5GOFk$-klJacI|%aBU2QLj*J-nptEKCn61>VY*WB^|fO7}0=7L#tp@ z>EppcRdf6x&~2Oy^!xRAyC=a&Qv|}meZRVLl=Jf8{m?T=@PQw*Z!;-Jkx(EkQ%DG% z+Vcri{)fzP+!K@Qwb5ZOcXon5RU~1A?*~lvCl6(MpaY$4!9=@0@+!g9s}1ai^N+j{ zH{>s|?2uMI2qF>et-|@c9Job`Xbb=x$_KcR6-IM8QN6>O5Y1-SH`M0)xKr900)pEv z_=3to{v)T^mq3%BZ?=!>@)zxi=~$LGKI%Rz7TZ5qJfKBc&$g6vE3UrZJuq$UcAg(oUc6OK4};oOUMz|C4Mutc8@iU+ zPu*ye`p(jdD75QJ^#e*Cfm2KPMh#P+{IeLw&-dGt{u=mkvcAvw@sD4tP{~@klf7)U z&%d>QChl_WW?KAy|Kl^~qe}kj^G0HWEjdz6WnhntKaBMM{r$fT{Qr&tRMBl#C@83O z8F3Ny{||zvAcTG;Z)%-#Qdv4Xf8}Fib8~ZJb>m>QceG$*2Z2CrK>7cbh0oo_%Fg=#EhCthmzV9o zR4NK9GeIg5Eh-9oS98bDHui2nek(IGb2};_0WXSZs3C({APkVc_?_mtY9uq4hk@Y6$0UgQ1EcEa)P~cusa85ojAH2Y`i}6#@pZaf4X7-kJt{4Pj@e;C?InMga)K3gP9Z;O1uK<^`~V zL7c1{Um&c0E5|CIU!&Q2v7{d&I8!oKt((h{|Uea z)CX_~K!X5H1q1%^rojKQz<~TfrT@nSoCI*CH}yCHp20vv z0iZxCH|ULat~ag!$I~Ey$N>F3T>r`UpDX-tRo~!^9(EmJ!1E3QG zAbo=X2Atq69>Bmq?EiTj2j_n_pc4=0{|W%0f$Sgv)PDiN#r+l#Z*oBZ7$5)@$6Gpt z6%6{%137pAqTb|qlZzL^3gQ90=#B0-^UY`8#NuJ)lNH1ZdBYAI_btkR zY#cyb{g3!u6kNbj0b$;><^sOv0+ir-Lk~nZ7eEydSL}e-0XCo{ATMBp*#AohsNv%M zf7*NVc&fthZCIufN~S_ZrljILXRUo`dV0R^_xE|<_pkTu^C{hD?|tw4Uh7)dy4Knocs>OK4oyp^E8ui= zWrQ9C4xC0rEp7(limC)-Kq5&2j0K)kf>pvE6%euLXexLL0ZhWpk-#%lvKSe5LpUD8 zm>81KL5#45G1!6z425H$G#oRS55~w6KuHvY6~YVnjs})R+{5H_upICinTAnJLIxxF zPe~ErgJU{G8=ylOgCEhVNP#^9!WA$M0ES8k515t=cg5orybdUe3}FZ!r&4fLN5fm{fV9ywCV0lGY9t<(&0gNOHf)ipIz9@r#kff)E$n6+>WN-(} zp-hH*fMsAzR)oC)T9k=U0LD;+0Hnd^RDd{`DJw&2fK);!!)gIUFufRr3ZwvHOrrt%D6k3z8mtB)P7y2)V?`3u z{I4;@95Gf>0(`)605P#!VoanKKqb&Tzz0wYrlx|YAWhK7V5KS466l8AO`)9l9QOgK zi3--k;G)rq>y$8dVO~0>K*9>Z(-4wW1(*-+L!rU^0B1!q1Roryg6k2^G$pVuVMQ<( z1HW)StOt<@>3~dwFo63hlfY|$SfCV`AIwXq0wN)J6)3Qt$*9DjM7uyXqyfY*gM(aw zpog%cfwvSD@p^`t5U?l+=yXOlgJ6X*$O)RxpYp;?7cQqOK)M2O(NS-J8K??KRghJI zi_i>o$X5jT=l}wOV<5j^o>9axgdtKMj3NA##gqUnR6xblm_i3m004Fh$EQDDm?SQ4-@Fbfb8u{0QC3IG!-0WU$ufguGc04$?O z!Hf&CKmqs+97m^P-oqdUoPZycfoT{Ci@?845E>sN5}<0s-($uN@e5*3M0`0Fkl%!qaQqr(hA0m;}D-TNCv;gs7v5DT}~0` zdXnA2VnCZj^aHRUq5L*RSp*g%auciv=BGdk{58fr4IY@f9@0IDk;@qKq09nV1O%!C zIS2_9VG3hVkT^#6CoZQE{0uq-tOo8@Mjl4KMqhw`NhHWQ$o9y`gpW|ag0GZGa2)s; zi~#gXrGW9l$e=GFA!BX;mogGD_=-rtNb>+Eh8lpBglPfr0l`H07J>}5AsJB*oCZ-& zs0%ur2ggCn!ExYs3XsgN*m>m-;j1^J;!U};aVKu0AFu#M=QNZ)3#;{rt$mjx44>0?r#3F1?!z2d> zuoyt2qE%pVm}cNOcpTCWY@Q0&1H_aurWu+_4BqSqDakM`tb!1}0AN_O0;W(PE^sC> zD~$x3L1Kmf5Uav51;(h_ah1dvaHNR9z(@w*L+AlAttYGnff&g9HOeLM={5 z%?MGXgp~*u`VgF8LC|pE7H|U{)D28eB0-iQN(0bv#NE+rSZILwz%&cq#7Y7Pjv=3s zt^p-5CGLhuuQ&!ZhvJ1q0a2s?Ih6#vgs~!I8)B>o=z%ens0A?R!tS90q0q6Y1vDZ* z0+njD(RyBSU9dJJf%9>8MPM0NJ_-2% zVU6MkVu6M^6|Il;AAm!VjMo#nnNTfIgu)oyL;-UXss_V?NHc^vKq7#7h!zDLC+>&9 z2O$5`u$})MzHyCd3FqX5Y)#YMBoX6i&1aG&7n}i zH7a5~fV>GojN}Bm9&Cxo1Rf!X4|E<71r;VE$^!5d9L0D!Cr z-UQ?;*b6lp+)J5g!ypNu$OiaQfj&WXKoG-mh${3K*auT4XmBMg35Y5aODeb-1p2Qr z!W-@iGZ3C<)USXyV(rKkK!8MsrjwA$u@ZsQ3OfOx1Il1WD82~P!g;U?V0Lf^)`^e} ziQXiPVRN7eVU>VR3b=zP6d`p$$dbT&1Qx(+3?_uja9ZRF1T%&*6ku53F_@LuoiZ>o z1{Ii@y)?IKrRPzw}P-YN!CwK%+hdm9V z{wJVMr6b`XaA}Z{5&XY^Y^IB=K|TUOp+r?S=G8KV|wIbHS2vEj#P?^Bz5cdiQY>Zc^*okC~@r$bjAj6o3y+-g1G$&y1FfRrS z!ABU^057PfCIbh?h(+`iV3k-O0)sJzX^8cL_=fpl-Pq+IYzaw*s9hlq5Sk4!jr@iI z3EPATe%=od042;MUjhFCz0wqbor(2hCjqk(Xh6&Ym@EMyP&yO)2cH2XrjBFP2J;X} z6xkh%O1uy96!?%~ZP*84e4r%4`U*OKIO9JO5+*L}s9~=Qy0)N}80}NcACM>kJ;0RU zIg~LdW^oLSbr>`Hri>0BplGtMiiuJQEP`EIC|uyHvK(H`=-J{|$d|;Y6!1Tzw+oR& zq*u%s2!IB3 zL+C-=43X@J<8-1jgKOzSu z8GsZakI70|0VEj%oM3-NEN)@F@PGv7XNY**4@3KaTM?x2IR))U6uodi%wmXr_?##t zh;^bJ82(~t7(5P+f!v3F!P1;zC)gK&mr7_M#`79z0um1fa9P-}Li=NJjoOds%@Z06 z#t`X{reGc*6D;`wogk5kc@+`VFjgW;c?=Fb4nc>p40a+~1SsJ9N&(!%-`=6SR zVJ0{a{RDP_v`E5e0OlbRFlIbQ!kC9gcj#6O_t7CR6D(8>!W3T zr;H7lsc3-QOd|3Rp)H_Kgrx_f|JRtJi0GV*9yks)E}(I0jO#qAVBVCTn^TngfB3H zGAdqVdD_qJAgm5zQ5iG_q7Yl8m?g-tYsT^*_NK=0F%f%HV_Xi=YQp6(3QZy@GNuFL z(#crL5J?O|8*&B;AcE_`H-v9tc7|_t{_^#IBwa$yKvWYwK)RwDJWf!8k^)=B&>~@U zTcINeNd_qxTDHI`lah$>$d7pM0;qy`83CM1gQP=fB9N=t?!;~#baaW*jd;i~)z^ls z0S~L7SqC=*f>wZnfzh3Vo-ahd5?0obdr^S?HYOf!5uItENH`8`1$}UE2bMk17*)dO z}x=Sno1PhU{$m}+!jPPaU7T!mI0W;CK=J=h1ei$178`oQPBCz zR{zniKYJ_4!pc-ah9K|&#{|q6eIdXRbV9&!D5#+zLKq-r6Im5t2keLMui#|(Y=_`j zLU04V5O$C`@I4pA1w75fF+62PoG{vMa6WVx7%)_T+XAW~BO#1Y%n)O!l<|Cq==phn z7%Rek0QOL2;`}HA;C@7*hcE_*W0w(w7NZF6kMGTZwu5>AkE7Tj3O@xx+#xW*KX5(- zHSQN1M^Hc!#!&{9@cWqh%dr1wEJKAug3c(~20EWKHQ)o90*)sm7IhE2f$sV|ZKyvw}ZSltc4a5uUx{7;Hn}1{#P^fc|N#|7cgFE^HhzC=38g zP#Z8RL3!{Me27fR_#ByYa!mLRF*`M$RD5t9oDE!r@3AltihPQ48>tJRL&!~(y@;Sm z*$w9tc^nW&#=AjY!uB^qX5(D|c?7Xe_MM5yU|f%gWKbg92j*k+r-AQ@)*})zA|H{8 zi2q}^I)9D-qooLXB~}LSC8!Zxj0F!NkcmVI37cUr>;gif1RyY0hE*@hE^JTXO0e(% zAE8)fn2!JoymrC}d3>LX5h9F41h057_=BipkP-+$(n%QdlXVw^Q5ckhJzfIW=m1YkCaGVHXG&H!60bJ-M<9hI=9#|hf*T>fT)O;X(ln^#x70kzk zanQghD3mFTTr@dS`j=Vp6~@*0;v_#4JJSr}#mU1a%0~CunV1R%n3(1LVR6*D54@0oD$;2cuWio}e!dtBzT2)gUIkX19w z{GwTA8Ecwsl&BFLeD??CDJ|zT9r{8?~Lf406Sf4o(ioDTCq#?k|MS{ z>x^$N6p@)JDN>T=hgjmgV6I$c&v6c!;LWRh1b!TN6**BF6Dn1+ zKKsLPb05iV|M=SJoK9bLOghvZpX>3nN$q&N-BvzPV~eDu9_e25Nzumf1#>eCLL}&v z2SqNM7M{Ja)6%nsEGakrk}OYfhw4o4?G+b4H{2Q1ir#kq=&GBpx5Qr5ZIL>#(Y}FW znwtpg9WwJsrHXz}tL<}+X_YQKJoRDM-aY!fpFd^u#B*lV41>e3{Wb-Ey?&65wpVUj zz(n(G2mhd;*W`_rsS7rzg{mY8b&Az9iFmDOULzXf6H{I;Y+rs{?cCeqcNOiDu3d>q z!2w)F)_(c4Bj!HF-$g&&ZeORYA93VpPVSZ#AMfx)kc}L@yYJjW#VqQF{5hh4enUqdk_5R5%xQeULJsZb+p<>{ zsC`(=`)$zy9%^ypboZs1y^#|NOI?#flkIsPxiF8m2}dbfc{~scWM4>)d^AlY?YuJQ zuCUwwg%Q>#r9U?=cyf7BTGzV|yPX9q?w#Db-1&H>@-u0LSrH9J)$P~Kp05;Y?*6*) z+>Hq(xpgfs6IFTRpRXa{K>U+rWZ;%a;u&8Wf3}A`NCAHRf~A_t<+jv<)7R4OYbABY zSvnA|`BB}kM``XH_ zT52lWUp^@m75rxAJ8V+Go|WHrnK@`?-`QgUABN`_+6~UTe7)fC6_d8gd?De5)eBFa zsLOE-maBBwyFk%Pi!X1{dVSaHw0HFJ1Y_G1RVq@pr)?xJWr^GBNOc7&R7lBRx#1u3 zOyu3(QPW3kko}h-y{^O$*inpl*jKJY*W)d?PBNMk`t0)LW}p@ zWj3@i;XXz`nz`|cCVQXi^Z}j1GhU-@hQ;A;O4Yng?>uYtgWBC5W6Ot&c zt*O%F`M}uGh3o}v>!RKrw(L#XGZ0jIwYa@QsAJvZKDD$3>&vY@_dj>=l8R{)n=PN) zIO~Pl^uY}~0ws^-o$+?+mH%)yP;Ar4j?X>q%BAaXu?UmTZ4GiYo-Wf`rNzQb`Eu8r z-+oE%L3Juk0ubfZ2@4`KQZ6A- z#nmnTtzRGSbMx-G`p7A}O+h4<_U!wbSvq^Nd^5+&Hk7_CC{~icr4bslXT7(7$qo94 z>>yo{8yW1~@_F8k_uEx9BK;luYTv#TjUKUVt88%SpIc5~hlT0S{5O~5v~o5Z6O+S& z|6~4B!2FkVTsOK!j3;y;?z`x+4+`Tom$ojg%XsV5xGf{&tbL$+By%L2&+%(-0|qQt z>ZNW=(V#X3y-E;zWjpNkRCb@hGcA&oR!^|{?cM3AFXCSIMW1{Tt6oPoQeazu;-%z# zU$gRn!W9SGir3E&T~u)+meO9xb}@>!jyzXwDc6UmmP((#?(%+GV49cV{W!u->&m&) zT59a;b`I=R-L5T{e8}5u-Wt`cO%JWdb7HbrKB1IPPfKX??vD&|os%E8@w?G%shC$) zcJ{ZfWZ%9S{U&MXwZqW8!P@$r`$lyomlv^3FNluOIh^G7ARsmR#89}G`NZ~9UV7O+ zapw=sHU4t_#N)%^aoa7wiR?VWAsF>Q@J!amk2IlbuICh;d&|D@T*@4Z&e|MzXV&2i z@uhEL_v;+dP3F02G0gm3s;Xl9feisnpS|nW@>N2wu3mRVFy-Ht+7JYg{L1D^*90?VHzATiLrZqL7JReDW^{RKQ$nkk? z?T)MM{pAxs>}OYxbt(1q7v{Wh{nB$xwCB{5n)lUNqAfWsS*x!nHjW!UKC9hqx98B> zpvF$Sy!*!!?V3D7U-`R-_ERd|A9^15)FE%{$P`M}wj+c8ZwQ46?G+%;uYP$kSup)M z$std;mFv#_g|e^6QTIkD8 zy@pcC87i5~OJZZigi7qHMZM*G$$pPjuSjnAn5r6)Dyexli%z|}K)afiJL|U(AEphS z99c@9Wms*HHqy82(R&r~t+5imhGH6qyplm3jjW>6FCWc(va%o{;{GV-l`IcQ{h6tn z8v|_yThAOAi;52N+TC{bXv^95{I&5WQ7^R)@oQ-vdv;STducqO{UIB>g>!A)7mUI+s55RrtP8kt@3Erag6gk6r5I2 zJY5fdwA#ky{T)D7z`h%bIm^EP`3K*L(z?mpOcTz zzjvH4U_!~ILn7WQ8*jz1$NpNkoh}P)JT%ohklpl`iL9Sp+tqJad`Um=qWFBvi|Hy& zuiwnioLru^!IN7B+T2O5);Ss{ zxQm_o&RLTllhe>W{yc^AofgZz*r}xnUA=T**Q1c()I&B}Q|1nP(fHXZZEg4BtI3pO z%ceHCqIz501bj#RPdTR&p*8a$iHoV+@FP>^^f`C-jdV)L9(8mNzoS|{z`QR}=+a7+ zuGb-t{Z9+n@7W}%%e|hrYR8G$(*;h2oA~-Eq|BD7KTuD;>TfkF7gq1MSi)uIESnhu zt&Mi^y^&$fJN`-W0=XF~rEp~FG$tkq_zK?}_FIXw!^75jvd|%jUJl2>(X}&glX!v$ z0ta}$xIW&;6LZNzeNc9ITD;9@kPGjXp)@x)xxm0q7yFZAc9)vhq#3j>`Pg|T)KE~Y zM#ydYbH^K$>PA7<#a@CN1~e;mxJLICrPooc?(4VCKBRF;`B_66|K5%HZyeXfuRiwR z3-hu{A=+uyR6fH zYibpFtv6@|ofue9e%SC;&AGQ5513^Ad-r834R7zkt}}n_-o|66x1+1Qr#xftlfz%T zFF97RkyU{!6n4MKqc9;MD9qVjUi`huv*|NjE*b4T9e-W!LyP(6?%RR1Xsxg16B5N^ z_N`A7w7IpoBKI*>Ssmx><}MN4A)2h(xW3y#`!>7Www-G_I{MpvE<97)!(Ei27vp5< zvV4t%X|YFYhLPQylTvY-+0j+j(thIg%G{L_@f}>Ja!5%*`ZtqTb-ElK>Y%JY(`(L>YoQVVd!3m0JJr9J zI@2~N%*6EQ?Ef***Ql2*4nKa3IncB*!>P21l0ABZ8Z|A`J<{38 z^2zvU)EVVNkIpxUhn-RPud5zy)^XdL!j+uxc$+CXxN;s_R?3ymMvu_4fMU_eP3O2T z&osOrQQMI0=;b*!-sZKm@wkP89kZD6zEIPlx)?M3Oi(85c%0m z!SIGgN@T2Zoy1z75U1Pf!#iuL0*&@AezQLB{!^Z+jaFg-rQN&w9Rt5qaHJ$t-%X3N z;tr$B7rj_1Gn4g<3jFlanv&p~-pUe`rX^pc2el8(8~Gs_+i9$|e6BRBhjHJ2vvFnN z4Z+t_pM3Y*xm$3_mxtL07s&c<-{{kHC3aSP8E2IZu&O%HJ~Ib?N2=w{XN7X26VG@cvvofA!R8Lm(v*^Jk0T+baOGmou2l|TB# zmZ3o&=eG8M>+#FF9J%zfkM8OH8sC?aTnunbC3wBzr3iC%jexZ5;aot>dI#MGh~+!MYZpiv9o&3nX^0|TUo6BTFN z226quo-3Q5r!H@Mx7Kjp*yH!9^TOJ%&f8Vhy!@QabRF}hUpCJ4D{l%P7#cKuveRwu z%5UX{bNTyRf>w4&UO0K}ZOd{usr@f6KDbk8UgYZACl=We8&}8|8vdN;hx?FRWoPxi z7H`FFiH0JH%FL7V(?{a3ybm|B_dM5b@-qIpFl;9}QoJ?Mq*wOs^Ssw- z2^CGJ9(0_28e(jhv)%QL3KwZhQ;d#+$MoZo%SQSJ0``x7k@)_-=iSNri65io0pI%E z222P1zl}%d28_CnckEcWrgXG7TC}{>9RIT>GcC7#qBnQqXxV{@v6c?5{a?>kd+%61 z;UCEnwKi>2Wx2@rT=~rCvjJnzPu7=@yN=thw+P+gS9xUT^9OkwhTH}oR^?XuKdxNe zPJVCj9)8d2+S#My$vdOFJK7Iivl}gm+;>w~-oC22wej8T?|FMQy2QT^dw5G6Ym%<( zAKP6Y91TD4d}+X4`*51I+wCKE2_#KtKKt9R_R=5pr@iZv*}M41>$a||w-X2a;IPBw z;rbDsuW)Gp_VPX{cT<(sP1@b1<&ttKjm5UN#fCz%qJ`~m+ZD9=AI8}SHn%Tdt=%dV zF!+(jLj76f+HTIS$jz@@TUYlUZ%(}G%>q9?#O<+u`^{4(&$7b1g~EMD3qGYC@!cbL zdT;5Lo$dBV3zOG;{bXVM;QI|OS7v+ul{H%zW%@;Y^f64da@(9A{oEZM!E@m}2OJFF5Xp6l##v8H(2 zCH4CDqpeyoBb}`e9{75TofOZ7Usw2YZm*&A^Ta1>BT9B`)`}@)cPi2=QEJePIT>#n zQF6`8@MYUWi-?j$x@Jsa^7aU#3d-Qa%J@50H?<__3hcSQ{&E-8c%=xu-+nv<;^R5`RGH}K3zR|L3Q+nwzn8} z>AsKCkH`D__bA!hT9^$A52x?AkTb9{P3Duso-eGTB~MEq%6{UtEM4#OPVA|ye0hxV zfzR<4Dwz-E+V@p_`Z$_V{qbY5yUB##zOA2gPPU%Y8g49!D{7Q=Q?lLMSYqBhr+;`$ zyVL!G`{p_!roIVsjRyuld_TmO?Rw*YQl)T8e7eNj@KBLUBt>U&MXApYQq~b8&$Cl) z57BpCQZck&KA=&!%x;&;t~YWkcAb}dXm*P8s$lo+EZag-evM9VS5mg^tMYpZ=7SSU zWc#AoTn)axE`NRY?&>z%mhL7si@P4$vm+u5S6xWoW~^s0Br2TiVKJ^|-IGzLQAOUE zLTOJjJTP~YaeN2Oo>t6mv2=#9Uxr7@x(ORM-d^rCeYYwyMQ%HVDyc6w*zlB(o2IkV zrb*q#iRDB73!Rvd@A2OgBQziU7+be2$1>X}GjO-`GudmhGxnIfbHn1;c4ienY@Mq= zG1@uw?!#F(?YbQUtuwpbu3MPSjWZI-8hi9cSLvy2uCRt#Z)1XwDP7^EoJS??RH)CH8?sR)-+@Fw;f^8K}m&szL;}e=iOM9eMh5LkU6)c{Qgq^j~SzP zO-c=BByEL%o<7_wWd}!wSEb8EMO?7A4Ceu6}QvMd5MR~dJ1ewdv?lE7a?41}@ zX_mcpKx)Kxk3mt$Ex&u*7vHkXJS2Q1q%*~X^H5>2PvfVLIXB!*-1wR8P2nGfPs2fl z+Al0}_bzAE?i@bD>`;FzDP2s~?dyT*p(eE8My_39W+_qWi-qB*{?_D$*YGp9ZZeut zlcnL*KO_B!Mpi*<*1B6w>US;$hF`Q=?d4H&#`1%cJLjQ}#j6DaFH{7y-%+8tpIMjZ zKWKip+T#2~){YCKvg%8pO}l8d;70oU6>xwmo9kgV+rfywdH;o3KZ>MHHk-SvpJ|r)!)`bW z%v8Rp!TOsB^X7}372Kzdw;v7Z%rKjIi0kvN4!!YHafKaCo?jU@3Sro2->de@)UM7+ zm_Kv^nDes+Zhd#a-9^R=*OVLR!m7=B)wHc3aK?7&Qw6EUw3(WfA#%VZwaHdC5%F6i7SG#?)W^0eWG10$0ajY}LOTRlK!Oc0N z??X8}Fp>a*ATTYtg2zN%$w2KX?;iD4k5nnVy6SiHwZ~7IjEQfN*&^9=#qj;zlm^Bs z0(-aAZIxWjlQ6sWX>ru>Y^L^WBRkGU+l%fXFRT78It_mPlH!EKK|}XX@+M6`>Sg;9 z2F^PBH-5TJp7I@;&bl6&U~l8#oX(7!t3RErI2p}k%vgq3-$mL4SElQXOrd8y z1?me7^eH??)%jwER=GM35-ZuV>{P+c9KQqSwOO^tmh_CC9kPfDSQ_efU{^^GXWU?^;iHkcaqs-x zL_cQq_4cb#bKY`y^fZ{tPEV_yb80(FZnSs$7vo-aHiOIlvz(*eVf2)k{o#>U-6iKN zCt+%JW#K@HtGHEmftAZUp{4PmFPK-Q8cMwy|ia4@EjBitVnv6)Ypw%xkN@GGo-t3H z`t+2DbZMtavI$$C5?<4!K0lpTN3yjkIA8nzg2YXWV_lusozX1L=h!1YbXWRm(PeMg zK)5?@fC?Ls+3-KyK)AB2GN+!UU!eXFd%r}+s)FmFN+c=Bs_58F_-4n2l z7uZI<`Kaz4$N3fNuWC=;r0Ms}pQws$IV-(*_|4OHtK&l31NLe1;`abDLLDzj)~ z_;q!mM^O&?C-?-^r5+th7taWvO#bXCF%IH0nJX+<`8Z~)-wl%Ly{>!5NBqs9ntfuA z7lxHg7auxQv%m9FyYFe0_F?&fq^HTc4V!1%Hx9e16TYzgF9GFVr$&m|ZOwOAZizR# z;U>DF|E_d*(qT*SPj{vFX6-s&JU(1nt$nt2A1xtfjIx%tDn%9z#p*im6wgI<^OQ)3 z4c8Xm51*M5`&!j_PNO=vf%elm!|E)LG%0h062q+L1cn5LM<%9mRQn0&NCJ@Z@eWVJ z@4!cUT;Fa_Y#qtfvS`k_uG?TTOZ1RyZAO zAD;!rzvgny?|AN#crH<>PS)=Un~6%MbE0ZN*OvbxD8gUg=KNPosC#v>@g>j_}}j`Hgra&LtM86Z}(xBR$uZcUgkeix^xPs{1%y0H8);(xtDCNpazC# zC2sbKTKv(~qU$nDPtD(xqx<3Q?KM|8*GRNTzrSrHP+xPMqj-=@C(JNqxxxBt63u)_ zQCePN8DUN-lTzc5gXHP3NL}^07I~ZUH1ni->z=@RHL_~J$<`X}N5NYE&&?f$_qh|1 zC6Y)oeMyjouQi`}U-QcxxgQe6gXAkNx8+n!kS6jt>;Eg70Ml}^9bq*${LmEqyeVu; z%pQPgM2QGchN7FZ)w^wpt&JgT#``RPZ1w@O$BoULO}G^1&*%Lwxb#uTT$EqVT1D1; z@c(@AcXy>D1$>e1Y}%V%*2?{O0vg;+^BV|7qZIvhfMQZxn8!&dL9G+_u5$ zFRub~tjWwu<1kPmJWD41U($jibW=+#l(_V>5f zOioT>ARGu;-)r4%no48J`_o8S~242v3c%z`1Y?t zFB0YUc3u7-Lmi|`tVcD%-b&-yQQdhdG?{f2N;o#N#?OtmI>SZyYJhT3r#Pti2 zD!O+%ZXY2a3DD7zY%jS@ex_SGN-4_ZoVX+RBl3M3nA;er9%27qrS^Zp5e#%nvjn2G z4U+EE)EGhfCjU3#B7So)FdPH}304gZe{)d({cQ%P{J*v6b8mzU{>QXMigEumvwp~e ztz`!MTyy8&Rz3DYy4-)Fr=?Kz(R0xqwS%)0-<9qKtH zcdIm)?rTc|Z*^)W2~maJbquag*BGzr=$3zX@#WSNIw4*Z77LcApL!H* zY7#WF{n^JYp9+p&(3R))+Lu-JLD{pQQhv15{Z;?Fk?Mr<+zv{0g5!r#-pH$#I|7vJ zZkH|<%8NZ;Hs5to()>8}*fp9}=WI*bEftqDDg6k2>7*515TRtKmYVTZcU#Q&o_sr} z^n_1|PxigI@7YYTVXr*rx#X+W68TeEontPwIc7IMs5}$*(#?oZ(P+`6oKNpByC5Rn zm$lwQFQ&QleBNfy#;VUnZz9jGJRj;>b;Gtd|4Vf5%5c7BA-(R0w+8LHl=WB5Pe@KE zD(kOu{mijjr)bMH0R?@^60;0_t;S9LuXjH=;H9ko{Yj09vf`o}P2ANik*t=t;5Vgw zJuk30^rTfst%~z{y~v{@_m$v1W_c z9gC}59Pnh_IM2g+`GROauIW){cuEg`VY^noZglarM?xyYi0zQ&I5u;E7nkpqo$cgXBhF0qp3{Pa{F?FtSr1?5>ufrr^L~2K*vM)>9**ujGu9q>s@F**kn2~J14wnb(%*1Dq*xQZYKQdV}O?PcJPnuRsp0Ijk zyfMW|u=vu22j-QenqD0%{(ndZtP3G zIM?5C@4TTr_?Xqa$VE7Oc5uY)33=gij#}kbmY;i@Tlg{FAM7ynQq%#q_N?^Hm$8Q#hPZs2X6Y%l1k3& zH?7@UIwD%OCFkH(lJ8a?1M0w<$bI_rU6@4^zC<+&Uycjj8EK$pci?(S*xYtcF`g~T+HesZF8K3sz&wPH6}j$3s3*ZNNq1$-6H!o zvQe~Hr*Y7f-?))kUG{?EOI1#}b2X{4B$?i|M>AffmKr9nyDPboo%?3j@>3%G>8vJs zN%#GsDod z^L{n!4|hGbWe+oRC7TB{a|9auB32$g5^~?U&~(-^8i_Yu-bKdzVUxn{dDHkyG_TdTKkZFo*vhN%a3}B+i{I=mt7IkpNJ@nkpi<`X=JtbpjvwIQ`Y@PvsdEhTREw$e1Y^~<7)6o^5 zj;Tbgoyn}g!#WTcFvHXP>QWhTEkXYMhgMs{Z`wJg6R*QFR!aAEiU@s6{j_-T@Fm&& zyb#BEYv+t!QTN~K)9O~tC*xSddFi%Imp1d&7T3t?m$}mh6dCbh~&5q1IVGw$S zcjVPpr@PPc%dTa`?T%4>>hXBgUFB^nTW93v#@oG2vR`KNr9Wl4M*d#j?N8*Lzn}Ew z^w-;Of@gqs%uGykm}X3~akJj!;kMgT-rCK>7B?Z5!a7an921=K|Ls4)u?E_eB>s>N zwh6U+eu>x3_k><&S?tHdRuR!{uKD4*Wn!>o^KeJvUi1DNv**7a%e z*Z!d{-*n-bF9sr}$$Hk!Agi8HwC&j8J-VX$_|EcMr_DxQdza3+{}vdp+i9vQOk#ga>s*J}6;enzqlzt*b{wO+|O?_?G-Q_+siyu2p zj}^YQ@R(+3v}vIsNoj;LyK2w8jMRe>=}c)tbvD~f)q8d%99DVYz|5KaRDiT#Tp_WP zvui7h_M!vJSF9@s9r^lX>?IY2kC)vtvsDlEyj~v`E3JMZe#ri`W{%IL(t64BI-Vvz zqRD(R7k1A$vZ+o=vLJPzNPWuguQW61)L`>g@jA`RVlzh%z2@VwF7sP=G2+6}vUb&! zqR;$eA?*9#`9-TAn&bSzEi5(Ik-y`0{p66ON+FwG9kL}yR@t`YdRZh-(zf(ueUEDvZt;|Z7TRV2VjqmbuyKj6wS%umX z+jhm7)|Cr2U(k!3nDz2~*2nz2JW2*3ALa>XvMfu?`)s^6ZS;^(Wo&dkb7i4ROndaG78D%|a)@o^4+?hISgz&YDr ziHgdhU2lpjZ*Q-rQ1K(tv>m>YUzeE zWVr&F>ofhb~ScmJ6%KtHym}>5gx$RAX%@hrg7pl!-VpUDz*fRiLn= zuDwUhEsd*ZSY7>_4h;oMDlas->-0FLW|n`;oVnZR3D53TtkhfU)n%3SNY5{|R%njc zqH5QkE51kO>-I~`Y36->dzYY`HFAI&gK735#C11>@LHPYai4AuJ zubCdwUdPpW+?{-D#XPr$XZpfIO`#j>9!~opDJQ>cx9oK8h9#9EscKeA%4uF(k2~u! zPromy?C7CFd9y9h>Fqn4s<#avN{(~1xUaaet<>FFV}{rAfEjy8i&K(*Jh}Arp0ZKo z8e6HY#cbx$dn0ZvT>U0x;fgmG7QXFH)mu{-nBw1U+tljPYL$0be^-+N{9U_%^3-}U z#Uz%^CRJ|uFzfSnZ~@F>DRZi_^3DK-hh-|1V0uWe+KxUf6Ud~ zT)k{vy>=M;?zZ*Zj{Lhx`_BmqOh^)+;VK=F4WNyF`!mGC#Kd@6ig-Qj?+c%fW$o z2J_`B{k@h~QvaMq`Cqg6jjX<(04Uf6F8*^zJfwflLjTt+Yx4p*G{G)!xR{uPemMyK z*Fpa0EbtR9{sGiCm!xjk!8dIgfib~0zx}aLCZ_RnuywJOC;lZ~>-!rk{F*EyMqt?k z05p&;zx^3rO-$hF3I6_-0l&7=U_1>m`r8D=FA(_Kh<|_U+^-2Z{D}#Ed&~6KH`4t*%irIC@Y^gQ z|D5HY4*mNb*I%=|kNA5Q1{o16`t{D|-^=>@wb5S_zCA)r_|r*@>3( Date: Tue, 6 May 2025 23:02:11 -0700 Subject: [PATCH 22/60] Add example for testing with local files (#743) --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 4f14a93cc..1aa3ee165 100644 --- a/README.md +++ b/README.md @@ -812,6 +812,29 @@ models: | `DATACONTRACT_TRINO_PASSWORD` | `mysecretpassword` | Password | +#### Local + +Data Contract CLI can test local files in parquet, json, csv, or delta format. + +##### Example + +datacontract.yaml +```yaml +servers: + local: + type: local + path: ./*.parquet + format: parquet +models: + my_table_1: # corresponds to a table + type: table + fields: + my_column_1: # corresponds to a column + type: varchar + my_column_2: # corresponds to a column + type: string +``` + ### export ``` From 33e238acb8d68c8cce56836c28e0b62c58cb204a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 7 May 2025 08:49:43 +0200 Subject: [PATCH 23/60] chore: update changelog for version 0.10.25 and bump version in pyproject.toml --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 528f25945..f01ebaeb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## [0.10.25] - 2025-05-07 + ### Added - Extracted the DataContractSpecification and the OpenDataContractSpecification in separate pip modules and use them in the CLI. - `datacontract import --format excel`: Import from Excel diff --git a/pyproject.toml b/pyproject.toml index fa9c6dd62..4274b90d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datacontract-cli" -version = "0.10.24" +version = "0.10.25" description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library." license = "MIT" readme = "README.md" From 718e47ba667a602e11dcf42fe8b99b01b02fe13c Mon Sep 17 00:00:00 2001 From: Simon Harrer Date: Wed, 7 May 2025 09:16:08 +0200 Subject: [PATCH 24/60] UPDATE --- README.md | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1aa3ee165..6acbe54c5 100644 --- a/README.md +++ b/README.md @@ -1305,20 +1305,21 @@ Available import options: | Type | Description | Status | |--------------------|------------------------------------------------|--------| -| `sql` | Import from SQL DDL | ✅ | | `avro` | Import from AVRO schemas | ✅ | -| `glue` | Import from AWS Glue DataCatalog | ✅ | -| `jsonschema` | Import from JSON Schemas | ✅ | | `bigquery` | Import from BigQuery Schemas | ✅ | -| `unity` | Import from Databricks Unity Catalog | partial | +| `csv` | Import from CSV File | ✅ | +| `dbml` | Import from DBML models | ✅ | | `dbt` | Import from dbt models | ✅ | +| `excel` | Import from ODCS Excel Template | ✅ | +| `glue` | Import from AWS Glue DataCatalog | ✅ | +| `iceberg` | Import from an Iceberg JSON Schema Definition | partial | +| `jsonschema` | Import from JSON Schemas | ✅ | | `odcs` | Import from Open Data Contract Standard (ODCS) | ✅ | -| `spark` | Import from Spark StructTypes | ✅ | -| `dbml` | Import from DBML models | ✅ | -| `csv` | Import from CSV File | ✅ | +| `parquet` | Import from Parquet File Metadata | ✅ | | `protobuf` | Import from Protobuf schemas | ✅ | -| `iceberg` | Import from an Iceberg JSON Schema Definition | partial | -| `parquet` | Import from Parquet File Metadta | ✅ | +| `spark` | Import from Spark StructTypes | ✅ | +| `sql` | Import from SQL DDL | ✅ | +| `unity` | Import from Databricks Unity Catalog | partial | | Missing something? | Please create an issue on GitHub | TBD | @@ -1390,6 +1391,17 @@ datacontract import --format dbt --source --dbt-model ``` +### Excel + +Importing from [ODCS Excel Template](https://github.com/datacontract/open-data-contract-standard-excel-template). + +Examples: + +```bash +# Example import from ODCS Excel Template +datacontract import --format excel --source odcs.xlsx +``` + #### Glue Importing from Glue reads the necessary Data directly off of the AWS API. From 7c561ba33e0bf8214b7cad3f26779b6fe2952124 Mon Sep 17 00:00:00 2001 From: Simon Harrer Date: Wed, 7 May 2025 14:54:22 +0200 Subject: [PATCH 25/60] fix: synchronize pyspark version in kafka.py with pyproject.toml --- datacontract/engines/soda/connections/kafka.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datacontract/engines/soda/connections/kafka.py b/datacontract/engines/soda/connections/kafka.py index 2a2445bf4..94c1ce24d 100644 --- a/datacontract/engines/soda/connections/kafka.py +++ b/datacontract/engines/soda/connections/kafka.py @@ -27,6 +27,7 @@ def create_spark_session(): tmp_dir = tempfile.TemporaryDirectory(prefix="datacontract-cli-spark") atexit.register(tmp_dir.cleanup) + pyspark_version = "3.5.5" # MUST be the same as in the pyproject.toml spark = ( SparkSession.builder.appName("datacontract") .config("spark.sql.warehouse.dir", f"{tmp_dir}/spark-warehouse") @@ -34,7 +35,7 @@ def create_spark_session(): .config("spark.ui.enabled", "false") .config( "spark.jars.packages", - "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.5,org.apache.spark:spark-avro_2.12:3.5.5", + f"org.apache.spark:spark-sql-kafka-0-10_2.12:{pyspark_version},org.apache.spark:spark-avro_2.12:{pyspark_version}", ) .getOrCreate() ) From 5172d32cad6ccfe5f7187b613871fd6da64ab10f Mon Sep 17 00:00:00 2001 From: Simon Harrer Date: Thu, 8 May 2025 09:11:58 +0200 Subject: [PATCH 26/60] fix: correct string formatting in breaking_change.py --- datacontract/breaking/breaking_change.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datacontract/breaking/breaking_change.py b/datacontract/breaking/breaking_change.py index a9f76d9b8..a50fa3c2f 100644 --- a/datacontract/breaking/breaking_change.py +++ b/datacontract/breaking/breaking_change.py @@ -25,7 +25,7 @@ class BreakingChange(BaseModel): location: Location def __str__(self) -> str: - return f"""{self.severity}\t\[{self.check_name}] at {self.location.path} + return f"""{self.severity}\t[{self.check_name}] at {self.location.path} in {str.join(".", self.location.composition)} {self.description}""" From f219ef5b68776a64b3b03e1d0aa826ac50b3a3b3 Mon Sep 17 00:00:00 2001 From: Simon Harrer Date: Thu, 8 May 2025 09:28:36 +0200 Subject: [PATCH 27/60] fix: correct string formatting in breaking_change.py --- datacontract/breaking/breaking_change.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datacontract/breaking/breaking_change.py b/datacontract/breaking/breaking_change.py index a50fa3c2f..a9f76d9b8 100644 --- a/datacontract/breaking/breaking_change.py +++ b/datacontract/breaking/breaking_change.py @@ -25,7 +25,7 @@ class BreakingChange(BaseModel): location: Location def __str__(self) -> str: - return f"""{self.severity}\t[{self.check_name}] at {self.location.path} + return f"""{self.severity}\t\[{self.check_name}] at {self.location.path} in {str.join(".", self.location.composition)} {self.description}""" From 23b44a87810a1095c91c1fdd5292dfa10269b3cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 18:08:03 +0200 Subject: [PATCH 28/60] Bump pymssql from 2.3.2 to 2.3.4 (#734) Bumps [pymssql](https://github.com/pymssql/pymssql) from 2.3.2 to 2.3.4. - [Release notes](https://github.com/pymssql/pymssql/releases) - [Changelog](https://github.com/pymssql/pymssql/blob/master/ChangeLog.rst) - [Commits](https://github.com/pymssql/pymssql/compare/v2.3.2...v2.3.4) --- updated-dependencies: - dependency-name: pymssql dependency-version: 2.3.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4274b90d3..4f0620b7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -132,7 +132,7 @@ dev = [ "pre-commit>=3.7.1,<4.3.0", "pytest", "pytest-xdist", - "pymssql==2.3.2", + "pymssql==2.3.4", "ruff", "testcontainers[minio,postgres,kafka,mssql]==4.9.2", "trino==0.333.0", From 407f7ce2619c3d81b742f5f1b473f1c06383866c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 18:08:45 +0200 Subject: [PATCH 29/60] Bump uvicorn from 0.34.0 to 0.34.2 (#735) Bumps [uvicorn](https://github.com/encode/uvicorn) from 0.34.0 to 0.34.2. - [Release notes](https://github.com/encode/uvicorn/releases) - [Changelog](https://github.com/encode/uvicorn/blob/master/docs/release-notes.md) - [Commits](https://github.com/encode/uvicorn/compare/0.34.0...0.34.2) --- updated-dependencies: - dependency-name: uvicorn dependency-version: 0.34.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4f0620b7b..51c8c34a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,7 +112,7 @@ rdf = [ api = [ "fastapi==0.115.12", - "uvicorn==0.34.0", + "uvicorn==0.34.2", ] protobuf = [ From 22611af828031125a3dc29250ae43a20aacb4825 Mon Sep 17 00:00:00 2001 From: Simon Harrer Date: Wed, 14 May 2025 14:09:15 +0200 Subject: [PATCH 30/60] fix: update Databricks environment variable names in README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6acbe54c5..716d6cc19 100644 --- a/README.md +++ b/README.md @@ -1369,8 +1369,9 @@ datacontract import --format unity --source my_unity_table.json ```bash # Example import single table from Unity Catalog via HTTP endpoint -export DATABRICKS_IMPORT_INSTANCE="https://xyz.cloud.databricks.com" -export DATABRICKS_IMPORT_ACCESS_TOKEN= +export DATACONTRACT_DATABRICKS_SERVER_HOSTNAME="https://xyz.cloud.databricks.com" +export DATACONTRACT_DATABRICKS_HTTP_PATH="/sql/1.0/warehouses/b053a331fa014fb4" +export DATACONTRACT_DATABRICKS_TOKEN= datacontract import --format unity --unity-table-full-name ``` From f9200d4283a4c78f2b049ce19c3105fd7918c0e6 Mon Sep 17 00:00:00 2001 From: jochenchrist Date: Thu, 15 May 2025 20:29:34 +0200 Subject: [PATCH 31/60] ODCS export: Export physical type if the physical type is configured in config object (#757) --- CHANGELOG.md | 4 ++++ datacontract/export/odcs_v3_exporter.py | 24 ++++++++++++++++++++---- tests/test_export_odcs_v3.py | 3 --- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f01ebaeb5..36dbcfa42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## Changed + +- ODCS export: Export physical type if the physical type is configured in config object + ## [0.10.25] - 2025-05-07 ### Added diff --git a/datacontract/export/odcs_v3_exporter.py b/datacontract/export/odcs_v3_exporter.py index 479f6e0ac..ba160ea86 100644 --- a/datacontract/export/odcs_v3_exporter.py +++ b/datacontract/export/odcs_v3_exporter.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Any, Dict from open_data_contract_standard.model import ( CustomProperty, @@ -207,8 +207,24 @@ def to_logical_type(type: str) -> str | None: return None -def to_physical_type(type: str) -> str | None: - return type +def to_physical_type(config: Dict[str, Any]) -> str | None: + if config is None: + return None + if "postgresType" in config: + return config["postgresType"] + elif "bigqueryType" in config: + return config["bigqueryType"] + elif "snowflakeType" in config: + return config["snowflakeType"] + elif "redshiftType" in config: + return config["redshiftType"] + elif "sqlserverType" in config: + return config["sqlserverType"] + elif "databricksType" in config: + return config["databricksType"] + elif "physicalType" in config: + return config["physicalType"] + return None def to_property(field_name: str, field: Field) -> SchemaProperty: @@ -231,7 +247,7 @@ def to_property(field_name: str, field: Field) -> SchemaProperty: if field.type is not None: property.logicalType = to_logical_type(field.type) - property.physicalType = to_physical_type(field.type) + property.physicalType = to_physical_type(field.config) if field.description is not None: property.description = field.description diff --git a/tests/test_export_odcs_v3.py b/tests/test_export_odcs_v3.py index e9dedd484..961203ca2 100644 --- a/tests/test_export_odcs_v3.py +++ b/tests/test_export_odcs_v3.py @@ -44,7 +44,6 @@ def test_to_odcs(): minLength: 8 maxLength: 10 pattern: ^B[0-9]+$ - physicalType: varchar required: true unique: true tags: @@ -63,7 +62,6 @@ def test_to_odcs(): logicalTypeOptions: minimum: 0 maximum: 1000000 - physicalType: bigint required: true description: The order_total field quality: @@ -75,7 +73,6 @@ def test_to_odcs(): mustBeBetween: [1000, 49900] - name: order_status logicalType: string - physicalType: text required: true quality: - type: sql From cc12bd9186ee100d0e1a82e88ded885be95bd1fc Mon Sep 17 00:00:00 2001 From: himat-mesh-ai <110179940+himat-mesh-ai@users.noreply.github.com> Date: Fri, 16 May 2025 11:21:48 +0100 Subject: [PATCH 32/60] Include descriptions from the data contract when exporting to snowflake sql (#756) * Include datacontract descriptions as comments in the snowflake ddl on export. * CHANGELOG.md --- CHANGELOG.md | 1 + datacontract/export/sql_converter.py | 4 ++++ tests/test_export_sql.py | 22 +++++++++++----------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36dbcfa42..8e4bc3f39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +- Include datacontract descriptions in the Snowflake sql export. ## Changed diff --git a/datacontract/export/sql_converter.py b/datacontract/export/sql_converter.py index 2aabe111d..6795a3ded 100644 --- a/datacontract/export/sql_converter.py +++ b/datacontract/export/sql_converter.py @@ -117,6 +117,8 @@ def _to_sql_table(model_name, model, server_type="snowflake"): result += " primary key" if server_type == "databricks" and field.description is not None: result += f' COMMENT "{_escape(field.description)}"' + if server_type == "snowflake" and field.description is not None: + result += f" COMMENT '{_escape(field.description)}'" if current_field_index < fields: result += "," result += "\n" @@ -124,6 +126,8 @@ def _to_sql_table(model_name, model, server_type="snowflake"): result += ")" if server_type == "databricks" and model.description is not None: result += f' COMMENT "{_escape(model.description)}"' + if server_type == "snowflake" and model.description is not None: + result += f" COMMENT='{_escape(model.description)}'" result += ";\n" return result diff --git a/tests/test_export_sql.py b/tests/test_export_sql.py index 885aebb04..0234f3d8e 100644 --- a/tests/test_export_sql.py +++ b/tests/test_export_sql.py @@ -32,18 +32,18 @@ def test_to_sql_ddl_snowflake(): -- Data Contract: urn:datacontract:checkout:snowflake_orders_pii_v2 -- SQL Dialect: snowflake CREATE TABLE orders ( - ORDER_ID TEXT not null, - ORDER_TIMESTAMP TIMESTAMP_TZ not null, - ORDER_TOTAL NUMBER not null, - CUSTOMER_ID TEXT, - CUSTOMER_EMAIL_ADDRESS TEXT not null, - PROCESSING_TIMESTAMP TIMESTAMP_LTZ not null -); + ORDER_ID TEXT not null COMMENT 'An internal ID that identifies an order in the online shop.', + ORDER_TIMESTAMP TIMESTAMP_TZ not null COMMENT 'The business timestamp in UTC when the order was successfully registered in the source system and the payment was successful.', + ORDER_TOTAL NUMBER not null COMMENT 'Total amount the smallest monetary unit (e.g., cents).', + CUSTOMER_ID TEXT COMMENT 'Unique identifier for the customer.', + CUSTOMER_EMAIL_ADDRESS TEXT not null COMMENT 'The email address, as entered by the customer. The email address was not verified.', + PROCESSING_TIMESTAMP TIMESTAMP_LTZ not null COMMENT 'The processing timestamp in the current session’s time zone.' +) COMMENT='One record per order. Includes cancelled and deleted orders.'; CREATE TABLE line_items ( - LINE_ITEM_ID TEXT not null, - ORDER_ID TEXT, - SKU TEXT -); + LINE_ITEM_ID TEXT not null COMMENT 'Primary key of the lines_item_id table', + ORDER_ID TEXT COMMENT 'An internal ID that identifies an order in the online shop.', + SKU TEXT COMMENT 'The purchased article number' +) COMMENT='A single article that is part of an order.'; """.strip() assert actual == expected From 19d4e3d63825a487f19ef2e2baa0ea35e92edb68 Mon Sep 17 00:00:00 2001 From: Robert Altmiller <123985294+robert-altmiller@users.noreply.github.com> Date: Fri, 16 May 2025 13:23:53 -0500 Subject: [PATCH 33/60] Add support for Variant type (#758) * added support for variant type in spark import and sql exporter * added support for variant type in spark import and sql exporter --- datacontract/export/odcs_v3_exporter.py | 2 ++ datacontract/export/sql_type_converter.py | 2 ++ datacontract/imports/spark_importer.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/datacontract/export/odcs_v3_exporter.py b/datacontract/export/odcs_v3_exporter.py index ba160ea86..a400e4866 100644 --- a/datacontract/export/odcs_v3_exporter.py +++ b/datacontract/export/odcs_v3_exporter.py @@ -202,6 +202,8 @@ def to_logical_type(type: str) -> str | None: return "array" if type.lower() in ["array"]: return "array" + if type.lower() in ["variant"]: + return "variant" if type.lower() in ["null"]: return None return None diff --git a/datacontract/export/sql_type_converter.py b/datacontract/export/sql_type_converter.py index 7e6bb5f3f..0e9babb09 100644 --- a/datacontract/export/sql_type_converter.py +++ b/datacontract/export/sql_type_converter.py @@ -197,6 +197,8 @@ def convert_to_databricks(field: Field) -> None | str: if type.lower() in ["array"]: item_type = convert_to_databricks(field.items) return f"ARRAY<{item_type}>" + if type.lower() in ["variant"]: + return "VARIANT" return None diff --git a/datacontract/imports/spark_importer.py b/datacontract/imports/spark_importer.py index bca3f0aae..3a46484d2 100644 --- a/datacontract/imports/spark_importer.py +++ b/datacontract/imports/spark_importer.py @@ -154,5 +154,7 @@ def _data_type_from_spark(spark_type: types.DataType) -> str: return "null" elif isinstance(spark_type, types.VarcharType): return "varchar" + elif isinstance(spark_type, types.VariantType): + return "variant" else: raise ValueError(f"Unsupported Spark type: {spark_type}") From 538f53318379ba87f4eab01022fada2b6c1b473d Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 16 May 2025 20:31:53 +0200 Subject: [PATCH 34/60] chore: update changelog for version 0.10.26 and bump version in pyproject.toml --- CHANGELOG.md | 11 ++++++++--- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e4bc3f39..86622b7bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased -- Include datacontract descriptions in the Snowflake sql export. -## Changed +## [0.10.26] - 2025-05-16 -- ODCS export: Export physical type if the physical type is configured in config object +### Changed + +- Databricks: Add support for Variant type (#758) +- `datacontract export --format odcs`: Export physical type if the physical type is configured in + config object (#757) +- `datacontract export --format sql` Include datacontract descriptions in the Snowflake sql export ( + #756) ## [0.10.25] - 2025-05-07 diff --git a/pyproject.toml b/pyproject.toml index 51c8c34a4..80bbe5890 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datacontract-cli" -version = "0.10.25" +version = "0.10.26" description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library." license = "MIT" readme = "README.md" From 79d752c321e9eabbc23f9f80447c462fc0d613f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 May 2025 09:13:41 +0200 Subject: [PATCH 35/60] Update rich requirement from <14.0,>=13.7 to >=13.7,<15.0 (#731) Updates the requirements on [rich](https://github.com/Textualize/rich) to permit the latest version. - [Release notes](https://github.com/Textualize/rich/releases) - [Changelog](https://github.com/Textualize/rich/blob/master/CHANGELOG.md) - [Commits](https://github.com/Textualize/rich/compare/v13.7.0...v14.0.0) --- updated-dependencies: - dependency-name: rich dependency-version: 14.0.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 80bbe5890..bd64e532f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "fastparquet>=2024.5.0,<2025.0.0", "numpy>=1.26.4,<2.0.0", # transitive dependency, needs to be <2.0.0 https://github.com/datacontract/datacontract-cli/issues/575 "python-multipart>=0.0.20,<1.0.0", - "rich>=13.7,<14.0", + "rich>=13.7,<15.0", "sqlglot>=26.6.0,<27.0.0", "duckdb>=1.0.0,<2.0.0", "soda-core-duckdb>=3.3.20,<3.6.0", From 3ffeb16ed1079fca493657189162af70b01f267f Mon Sep 17 00:00:00 2001 From: Robert Altmiller <123985294+robert-altmiller@users.noreply.github.com> Date: Sat, 17 May 2025 02:16:22 -0500 Subject: [PATCH 36/60] Complex Types Fix When Exporting to SQL and 'databricksType' is Defined (#760) * added support for variant type in spark import and sql exporter * added support for variant type in spark import and sql exporter * complex types fix when exporting to sql and databricksType is defined. * complex types fix when exporting to sql and databricksType is defined. * complex types fix when exporting to sql and databricksType is defined. --- datacontract/export/odcs_v3_exporter.py | 2 +- datacontract/export/sql_type_converter.py | 4 ++-- tests/test_export_odcs_v3.py | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/datacontract/export/odcs_v3_exporter.py b/datacontract/export/odcs_v3_exporter.py index a400e4866..109f023bd 100644 --- a/datacontract/export/odcs_v3_exporter.py +++ b/datacontract/export/odcs_v3_exporter.py @@ -249,7 +249,7 @@ def to_property(field_name: str, field: Field) -> SchemaProperty: if field.type is not None: property.logicalType = to_logical_type(field.type) - property.physicalType = to_physical_type(field.config) + property.physicalType = to_physical_type(field.config) or field.type if field.description is not None: property.description = field.description diff --git a/datacontract/export/sql_type_converter.py b/datacontract/export/sql_type_converter.py index 0e9babb09..a89948a3e 100644 --- a/datacontract/export/sql_type_converter.py +++ b/datacontract/export/sql_type_converter.py @@ -158,9 +158,9 @@ def convert_to_dataframe(field: Field) -> None | str: # databricks data types: # https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html def convert_to_databricks(field: Field) -> None | str: - if field.config and "databricksType" in field.config: - return field.config["databricksType"] type = field.type + if field.config and "databricksType" in field.config and type.lower() not in ["array", "object", "record", "struct"]: + return field.config["databricksType"] if type is None: return None if type.lower() in ["string", "varchar", "text"]: diff --git a/tests/test_export_odcs_v3.py b/tests/test_export_odcs_v3.py index 961203ca2..6c9d86cd1 100644 --- a/tests/test_export_odcs_v3.py +++ b/tests/test_export_odcs_v3.py @@ -40,6 +40,7 @@ def test_to_odcs(): - name: order_id businessName: Order ID logicalType: string + physicalType: varchar logicalTypeOptions: minLength: 8 maxLength: 10 @@ -59,6 +60,7 @@ def test_to_odcs(): value: true - name: order_total logicalType: integer + physicalType: bigint logicalTypeOptions: minimum: 0 maximum: 1000000 @@ -73,6 +75,7 @@ def test_to_odcs(): mustBeBetween: [1000, 49900] - name: order_status logicalType: string + physicalType: text required: true quality: - type: sql From bf79ee3eb6a594ae5d5785ea0863b4dff41c4a09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 May 2025 09:17:17 +0200 Subject: [PATCH 37/60] Update pydantic requirement from <2.11.0,>=2.8.2 to >=2.8.2,<2.12.0 (#741) Updates the requirements on [pydantic](https://github.com/pydantic/pydantic) to permit the latest version. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.8.2...v2.11.3) --- updated-dependencies: - dependency-name: pydantic dependency-version: 2.11.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bd64e532f..3cbad0625 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ requires-python = ">=3.10" dependencies = [ "typer>=0.15.1,<0.16", - "pydantic>=2.8.2,<2.11.0", + "pydantic>=2.8.2,<2.12.0", "pyyaml~=6.0.1", "requests>=2.31,<2.33", "fastjsonschema>=2.19.1,<2.22.0", From ab0376450627195fcf847d82bb68591c7ea943a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 08:17:05 +0200 Subject: [PATCH 38/60] Update aiobotocore requirement from <2.20.0,>=2.17.0 to >=2.17.0,<2.23.0 (#744) Updates the requirements on [aiobotocore](https://github.com/aio-libs/aiobotocore) to permit the latest version. - [Release notes](https://github.com/aio-libs/aiobotocore/releases) - [Changelog](https://github.com/aio-libs/aiobotocore/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiobotocore/compare/2.17.0...2.22.0) --- updated-dependencies: - dependency-name: aiobotocore dependency-version: 2.22.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3cbad0625..82bb28340 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ postgres = [ s3 = [ "s3fs==2025.2.0", - "aiobotocore>=2.17.0,<2.20.0", + "aiobotocore>=2.17.0,<2.23.0", ] snowflake = [ From 882e209a2b0678ac040334b402d7f4369bb91a90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 15:13:22 +0200 Subject: [PATCH 39/60] Bump testcontainers[kafka,minio,mssql,postgres] from 4.9.2 to 4.10.0 (#763) Bumps [testcontainers[kafka,minio,mssql,postgres]](https://github.com/testcontainers/testcontainers-python) from 4.9.2 to 4.10.0. - [Release notes](https://github.com/testcontainers/testcontainers-python/releases) - [Changelog](https://github.com/testcontainers/testcontainers-python/blob/main/CHANGELOG.md) - [Commits](https://github.com/testcontainers/testcontainers-python/compare/testcontainers-v4.9.2...testcontainers-v4.10.0) --- updated-dependencies: - dependency-name: testcontainers[kafka,minio,mssql,postgres] dependency-version: 4.10.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 82bb28340..bec8ae78b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,7 +134,7 @@ dev = [ "pytest-xdist", "pymssql==2.3.4", "ruff", - "testcontainers[minio,postgres,kafka,mssql]==4.9.2", + "testcontainers[minio,postgres,kafka,mssql]==4.10.0", "trino==0.333.0", ] From 8f349d91402d433e8d850158d5ffd2ee156f7797 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 15:13:49 +0200 Subject: [PATCH 40/60] Bump moto from 5.1.3 to 5.1.4 (#765) Bumps [moto](https://github.com/getmoto/moto) from 5.1.3 to 5.1.4. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/5.1.3...5.1.4) --- updated-dependencies: - dependency-name: moto dependency-version: 5.1.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bec8ae78b..8a84dde58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,7 +127,7 @@ dev = [ "datacontract-cli[all]", "httpx==0.28.1", "kafka-python", - "moto==5.1.3", + "moto==5.1.4", "pandas>=2.1.0", "pre-commit>=3.7.1,<4.3.0", "pytest", From 447e86e9f09a780897d0072c4a349fd5296d83c0 Mon Sep 17 00:00:00 2001 From: Simon Harrer Date: Mon, 19 May 2025 15:16:22 +0200 Subject: [PATCH 41/60] add: s3 dependencies allow upward migration --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8a84dde58..a98f17910 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ postgres = [ ] s3 = [ - "s3fs==2025.2.0", + "s3fs>=2025.2.0,<2026.0.0", "aiobotocore>=2.17.0,<2.23.0", ] @@ -123,6 +123,7 @@ all = [ "datacontract-cli[kafka,bigquery,csv,excel,snowflake,postgres,databricks,sqlserver,s3,trino,dbt,dbml,iceberg,parquet,rdf,api,protobuf]" ] +# for development, we pin all libraries to an exact version dev = [ "datacontract-cli[all]", "httpx==0.28.1", From 7257137490c98973000cb57be664e48bfce2c720 Mon Sep 17 00:00:00 2001 From: Robert Altmiller <123985294+robert-altmiller@users.noreply.github.com> Date: Thu, 22 May 2025 12:29:30 -0500 Subject: [PATCH 42/60] Added support for spark importer table level comments (#761) * added support for variant type in spark import and sql exporter * complex types fix when exporting to sql and databricksType is defined. * complex types fix when exporting to sql and databricksType is defined. * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * Enhance SparkImporter with logging and improve table comment retrieval handling --------- Co-authored-by: Alan Reese --- datacontract/imports/spark_importer.py | 73 +++++++++++++++++++++++++- tests/test_import_spark.py | 1 + 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/datacontract/imports/spark_importer.py b/datacontract/imports/spark_importer.py index 3a46484d2..0e09e357f 100644 --- a/datacontract/imports/spark_importer.py +++ b/datacontract/imports/spark_importer.py @@ -1,3 +1,6 @@ +import logging + +from databricks.sdk import WorkspaceClient from pyspark.sql import DataFrame, SparkSession, types from datacontract.imports.importer import Importer @@ -8,6 +11,8 @@ Server, ) +logger = logging.getLogger(__name__) + class SparkImporter(Importer): def import_source( @@ -46,15 +51,17 @@ def import_spark(data_contract_specification: DataContractSpecification, source: for temp_view in source.split(","): temp_view = temp_view.strip() df = spark.read.table(temp_view) - data_contract_specification.models[temp_view] = import_from_spark_df(df) + data_contract_specification.models[temp_view] = import_from_spark_df(spark, source, df) return data_contract_specification -def import_from_spark_df(df: DataFrame) -> Model: +def import_from_spark_df(spark: SparkSession, source: str, df: DataFrame) -> Model: """ Converts a Spark DataFrame into a Model. Args: + spark: SparkSession + source: A comma-separated string of Spark temporary views to read. df: The Spark DataFrame to convert. Returns: @@ -63,6 +70,8 @@ def import_from_spark_df(df: DataFrame) -> Model: model = Model() schema = df.schema + model.description = _table_comment_from_spark(spark, source) + for field in schema: model.fields[field.name] = _field_from_struct_type(field) @@ -158,3 +167,63 @@ def _data_type_from_spark(spark_type: types.DataType) -> str: return "variant" else: raise ValueError(f"Unsupported Spark type: {spark_type}") + + +def _table_comment_from_spark(spark: SparkSession, source: str): + """ + Attempts to retrieve the table-level comment from a Spark table using multiple fallback methods. + + Args: + spark (SparkSession): The active Spark session. + source (str): The name of the table (without catalog or schema). + + Returns: + str or None: The table-level comment, if found. + """ + + # Get Current Catalog and Schema from Spark Session + try: + current_catalog = spark.sql("SELECT current_catalog()").collect()[0][0] + except Exception: + current_catalog = "hive_metastore" # Fallback for non-Unity Catalog clusters + try: + current_schema = spark.catalog.currentDatabase() + except Exception: + current_schema = spark.sql("SELECT current_database()").collect()[0][0] + + # Get table comment if it exists + table_comment = "" + source = f"{current_catalog}.{current_schema}.{source}" + try: + # Initialize WorkspaceClient for Unity Catalog API calls + workspace_client = WorkspaceClient() + created_table = workspace_client.tables.get(full_name=f"{source}") + table_comment = created_table.comment + print(f"'{source}' table comment retrieved using 'WorkspaceClient.tables.get({source})'") + return table_comment + except Exception: + pass + + # Fallback to Spark Catalog API for Hive Metastore or Non-UC Tables + try: + table_comment = spark.catalog.getTable(f"{source}").description + print(f"'{source}' table comment retrieved using 'spark.catalog.getTable({source}).description'") + return table_comment + except Exception: + pass + + # Final Fallback Using DESCRIBE TABLE EXTENDED + try: + rows = spark.sql(f"DESCRIBE TABLE EXTENDED {source}").collect() + for row in rows: + if row.col_name.strip().lower() == "comment": + table_comment = row.data_type + break + print(f"'{source}' table comment retrieved using 'DESCRIBE TABLE EXTENDED {source}'") + return table_comment + except Exception: + pass + + logger.info(f"{source} table comment could not be retrieved") + + return None diff --git a/tests/test_import_spark.py b/tests/test_import_spark.py index c09cff62c..f5a3f8d7a 100644 --- a/tests/test_import_spark.py +++ b/tests/test_import_spark.py @@ -88,6 +88,7 @@ def spark(tmp_path_factory) -> SparkSession: print(f"Using PySpark version {spark.version}") return spark + def test_cli(spark: SparkSession): df_user = spark.createDataFrame( From 07fbf2782fc933b3f25ec6e7c7cbcc3176200348 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 May 2025 19:32:09 +0200 Subject: [PATCH 43/60] chore: update changelog to include spark importer table level comments --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86622b7bd..9d8735680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- `datacontract import --format spark`: Added support for spark importer table level comments (#761) + ## [0.10.26] - 2025-05-16 ### Changed From 330b0920f91f585b5da88376b723ef4870e1a32b Mon Sep 17 00:00:00 2001 From: Christopher-Lawford Date: Thu, 22 May 2025 18:48:36 +0100 Subject: [PATCH 44/60] fix: Soda Exporter did not correctly pass server object to create_checks function (#768) * fix: Soda Exporter did not correctly pass server object to create_checks function * missed the import * fix: Inline get_server method to avoid circular dependencies * Update changelog --------- Co-authored-by: jochen --- CHANGELOG.md | 4 ++++ datacontract/export/sodacl_converter.py | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d8735680..670d40624 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `datacontract import --format spark`: Added support for spark importer table level comments (#761) +### Fixed + +- `datacontract export --format sodacl`: Fix resolving server when using `--server` flag (#768) + ## [0.10.26] - 2025-05-16 ### Changed diff --git a/datacontract/export/sodacl_converter.py b/datacontract/export/sodacl_converter.py index 8ecc546af..5e1fa9a03 100644 --- a/datacontract/export/sodacl_converter.py +++ b/datacontract/export/sodacl_converter.py @@ -2,12 +2,14 @@ from datacontract.engines.data_contract_checks import create_checks from datacontract.export.exporter import Exporter +from datacontract.model.data_contract_specification import DataContractSpecification, Server from datacontract.model.run import Run class SodaExporter(Exporter): - def export(self, data_contract, model, server, sql_server_type, export_args) -> dict: + def export(self, data_contract, model, server, sql_server_type, export_args) -> str: run = Run.create_run() + server = get_server(data_contract, server) run.checks.extend(create_checks(data_contract, server)) return to_sodacl_yaml(run) @@ -28,3 +30,9 @@ def to_sodacl_yaml(run: Run) -> str: else: sodacl_dict[key] = value return yaml.dump(sodacl_dict) + + +def get_server(data_contract_specification: DataContractSpecification, server_name: str = None) -> Server | None: + if server_name is None: + return None + return data_contract_specification.servers.get(server_name) From ced6aa0f6fbdbdda52aa333d2eaa4613c10338f3 Mon Sep 17 00:00:00 2001 From: jochenchrist Date: Thu, 22 May 2025 20:07:47 +0200 Subject: [PATCH 45/60] feat: add owner and id options to data contract import functionality (#759) * feat: add owner and id options to data contract import functionality resolves #753 * feat: update help text width and enhance datacontract import functionality with owner and id options --- CHANGELOG.md | 1 + README.md | 667 +++++++++++++++----------------- datacontract/cli.py | 10 + datacontract/data_contract.py | 14 +- tests/test_import_unity_file.py | 52 +++ update_help.py | 2 +- 6 files changed, 378 insertions(+), 368 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 670d40624..286f5cfc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `datacontract import --format spark`: Added support for spark importer table level comments (#761) +- `datacontract import` respects `--owner` and `--id` flags (#753) ### Fixed diff --git a/README.md b/README.md index 716d6cc19..370b1a768 100644 --- a/README.md +++ b/README.md @@ -252,110 +252,86 @@ Commands ### init ``` - - Usage: datacontract init [OPTIONS] [LOCATION] - - Create an empty data contract. - -╭─ Arguments ──────────────────────────────────────────────────────────────────╮ -│ location [LOCATION] The location of the data contract file to │ -│ create. │ -│ [default: datacontract.yaml] │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --template TEXT URL of a template or data contract │ -│ [default: None] │ -│ --overwrite --no-overwrite Replace the existing │ -│ datacontract.yaml │ -│ [default: no-overwrite] │ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ + + Usage: datacontract init [OPTIONS] [LOCATION] + + Create an empty data contract. + +╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮ +│ location [LOCATION] The location of the data contract file to create. │ +│ [default: datacontract.yaml] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ --template TEXT URL of a template or data contract [default: None] │ +│ --overwrite --no-overwrite Replace the existing datacontract.yaml │ +│ [default: no-overwrite] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` ### lint ``` - - Usage: datacontract lint [OPTIONS] [LOCATION] - - Validate that the datacontract.yaml is correctly formatted. - -╭─ Arguments ──────────────────────────────────────────────────────────────────╮ -│ location [LOCATION] The location (url or path) of the data contract │ -│ yaml. │ -│ [default: datacontract.yaml] │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --schema TEXT The location (url or path) of the Data │ -│ Contract Specification JSON Schema │ -│ [default: None] │ -│ --output PATH Specify the file path where the test results │ -│ should be written to (e.g., │ -│ './test-results/TEST-datacontract.xml'). If │ -│ no path is provided, the output will be │ -│ printed to stdout. │ -│ [default: None] │ -│ --output-format [junit] The target format for the test results. │ -│ [default: None] │ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ + + Usage: datacontract lint [OPTIONS] [LOCATION] + + Validate that the datacontract.yaml is correctly formatted. + +╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮ +│ location [LOCATION] The location (url or path) of the data contract yaml. │ +│ [default: datacontract.yaml] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ --schema TEXT The location (url or path) of the Data Contract Specification │ +│ JSON Schema │ +│ [default: None] │ +│ --output PATH Specify the file path where the test results should be written │ +│ to (e.g., './test-results/TEST-datacontract.xml'). If no path is │ +│ provided, the output will be printed to stdout. │ +│ [default: None] │ +│ --output-format [junit] The target format for the test results. [default: None] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` ### test ``` - - Usage: datacontract test [OPTIONS] [LOCATION] - - Run schema and quality tests on configured servers. - -╭─ Arguments ──────────────────────────────────────────────────────────────────╮ -│ location [LOCATION] The location (url or path) of the data contract │ -│ yaml. │ -│ [default: datacontract.yaml] │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --schema TEXT The location (url or │ -│ path) of the Data │ -│ Contract │ -│ Specification JSON │ -│ Schema │ -│ [default: None] │ -│ --server TEXT The server │ -│ configuration to run │ -│ the schema and │ -│ quality tests. Use │ -│ the key of the server │ -│ object in the data │ -│ contract yaml file to │ -│ refer to a server, │ -│ e.g., `production`, │ -│ or `all` for all │ -│ servers (default). │ -│ [default: all] │ -│ --publish TEXT The url to publish │ -│ the results after the │ -│ test │ -│ [default: None] │ -│ --output PATH Specify the file path │ -│ where the test │ -│ results should be │ -│ written to (e.g., │ -│ './test-results/TEST… │ -│ [default: None] │ -│ --output-format [junit] The target format for │ -│ the test results. │ -│ [default: None] │ -│ --logs --no-logs Print logs │ -│ [default: no-logs] │ -│ --ssl-verification --no-ssl-verificati… SSL verification when │ -│ publishing the data │ -│ contract. │ -│ [default: │ -│ ssl-verification] │ -│ --help Show this message and │ -│ exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ + + Usage: datacontract test [OPTIONS] [LOCATION] + + Run schema and quality tests on configured servers. + +╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮ +│ location [LOCATION] The location (url or path) of the data contract yaml. │ +│ [default: datacontract.yaml] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ --schema TEXT The location (url or path) of the Data │ +│ Contract Specification JSON Schema │ +│ [default: None] │ +│ --server TEXT The server configuration to run the │ +│ schema and quality tests. Use the key of │ +│ the server object in the data contract │ +│ yaml file to refer to a server, e.g., │ +│ `production`, or `all` for all servers │ +│ (default). │ +│ [default: all] │ +│ --publish TEXT The url to publish the results after the │ +│ test │ +│ [default: None] │ +│ --output PATH Specify the file path where the test │ +│ results should be written to (e.g., │ +│ './test-results/TEST-datacontract.xml'). │ +│ [default: None] │ +│ --output-format [junit] The target format for the test results. │ +│ [default: None] │ +│ --logs --no-logs Print logs [default: no-logs] │ +│ --ssl-verification --no-ssl-verification SSL verification when publishing the │ +│ data contract. │ +│ [default: ssl-verification] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` @@ -838,64 +814,57 @@ models: ### export ``` - - Usage: datacontract export [OPTIONS] [LOCATION] - - Convert data contract to a specific format. Saves to file specified by - `output` option if present, otherwise prints to stdout. - -╭─ Arguments ──────────────────────────────────────────────────────────────────╮ -│ location [LOCATION] The location (url or path) of the data contract │ -│ yaml. │ -│ [default: datacontract.yaml] │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ * --format [jsonschema|pydantic-model The export format. │ -│ |sodacl|dbt|dbt-sources|db [default: None] │ -│ t-staging-sql|odcs|rdf|avr [required] │ -│ o|protobuf|great-expectati │ -│ ons|terraform|avro-idl|sql │ -│ |sql-query|html|go|bigquer │ -│ y|dbml|spark|sqlalchemy|da │ -│ ta-caterer|dcs|markdown|ic │ -│ eberg|custom] │ -│ --output PATH Specify the file path where │ -│ the exported data will be │ -│ saved. If no path is │ -│ provided, the output will │ -│ be printed to stdout. │ -│ [default: None] │ -│ --server TEXT The server name to export. │ -│ [default: None] │ -│ --model TEXT Use the key of the model in │ -│ the data contract yaml file │ -│ to refer to a model, e.g., │ -│ `orders`, or `all` for all │ -│ models (default). │ -│ [default: all] │ -│ --schema TEXT The location (url or path) │ -│ of the Data Contract │ -│ Specification JSON Schema │ -│ [default: None] │ -│ --engine TEXT [engine] The engine used │ -│ for great expection run. │ -│ [default: None] │ -│ --template PATH [custom] The file path of │ -│ Jinja template. │ -│ [default: None] │ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ RDF Options ────────────────────────────────────────────────────────────────╮ -│ --rdf-base TEXT [rdf] The base URI used to generate the RDF graph. │ -│ [default: None] │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ SQL Options ────────────────────────────────────────────────────────────────╮ -│ --sql-server-type TEXT [sql] The server type to determine the sql │ -│ dialect. By default, it uses 'auto' to │ -│ automatically detect the sql dialect via the │ -│ specified servers in the data contract. │ -│ [default: auto] │ -╰──────────────────────────────────────────────────────────────────────────────╯ + + Usage: datacontract export [OPTIONS] [LOCATION] + + Convert data contract to a specific format. Saves to file specified by `output` option if present, + otherwise prints to stdout. + +╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮ +│ location [LOCATION] The location (url or path) of the data contract yaml. │ +│ [default: datacontract.yaml] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ * --format [jsonschema|pydantic-model|sodacl|db The export format. [default: None] │ +│ t|dbt-sources|dbt-staging-sql|odcs|r [required] │ +│ df|avro|protobuf|great-expectations| │ +│ terraform|avro-idl|sql|sql-query|htm │ +│ l|go|bigquery|dbml|spark|sqlalchemy| │ +│ data-caterer|dcs|markdown|iceberg|cu │ +│ stom] │ +│ --output PATH Specify the file path where the │ +│ exported data will be saved. If no │ +│ path is provided, the output will be │ +│ printed to stdout. │ +│ [default: None] │ +│ --server TEXT The server name to export. │ +│ [default: None] │ +│ --model TEXT Use the key of the model in the data │ +│ contract yaml file to refer to a │ +│ model, e.g., `orders`, or `all` for │ +│ all models (default). │ +│ [default: all] │ +│ --schema TEXT The location (url or path) of the │ +│ Data Contract Specification JSON │ +│ Schema │ +│ [default: None] │ +│ --engine TEXT [engine] The engine used for great │ +│ expection run. │ +│ [default: None] │ +│ --template PATH [custom] The file path of Jinja │ +│ template. │ +│ [default: None] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ RDF Options ────────────────────────────────────────────────────────────────────────────────────╮ +│ --rdf-base TEXT [rdf] The base URI used to generate the RDF graph. [default: None] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ SQL Options ────────────────────────────────────────────────────────────────────────────────────╮ +│ --sql-server-type TEXT [sql] The server type to determine the sql dialect. By default, │ +│ it uses 'auto' to automatically detect the sql dialect via the │ +│ specified servers in the data contract. │ +│ [default: auto] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` @@ -1192,104 +1161,88 @@ FROM ### import ``` - - Usage: datacontract import [OPTIONS] - - Create a data contract from the given source location. Saves to file specified - by `output` option if present, otherwise prints to stdout. - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ * --format [sql|avro|dbt|dbml|gl The format of the │ -│ ue|jsonschema|bigquer source file. │ -│ y|odcs|unity|spark|ic [default: None] │ -│ eberg|parquet|csv|pro [required] │ -│ tobuf] │ -│ --output PATH Specify the file path │ -│ where the Data │ -│ Contract will be │ -│ saved. If no path is │ -│ provided, the output │ -│ will be printed to │ -│ stdout. │ -│ [default: None] │ -│ --source TEXT The path to the file │ -│ or Glue Database that │ -│ should be imported. │ -│ [default: None] │ -│ --dialect TEXT The SQL dialect to │ -│ use when importing │ -│ SQL files, e.g., │ -│ postgres, tsql, │ -│ bigquery. │ -│ [default: None] │ -│ --glue-table TEXT List of table ids to │ -│ import from the Glue │ -│ Database (repeat for │ -│ multiple table ids, │ -│ leave empty for all │ -│ tables in the │ -│ dataset). │ -│ [default: None] │ -│ --bigquery-project TEXT The bigquery project │ -│ id. │ -│ [default: None] │ -│ --bigquery-dataset TEXT The bigquery dataset │ -│ id. │ -│ [default: None] │ -│ --bigquery-table TEXT List of table ids to │ -│ import from the │ -│ bigquery API (repeat │ -│ for multiple table │ -│ ids, leave empty for │ -│ all tables in the │ -│ dataset). │ -│ [default: None] │ -│ --unity-table-full-n… TEXT Full name of a table │ -│ in the unity catalog │ -│ [default: None] │ -│ --dbt-model TEXT List of models names │ -│ to import from the │ -│ dbt manifest file │ -│ (repeat for multiple │ -│ models names, leave │ -│ empty for all models │ -│ in the dataset). │ -│ [default: None] │ -│ --dbml-schema TEXT List of schema names │ -│ to import from the │ -│ DBML file (repeat for │ -│ multiple schema │ -│ names, leave empty │ -│ for all tables in the │ -│ file). │ -│ [default: None] │ -│ --dbml-table TEXT List of table names │ -│ to import from the │ -│ DBML file (repeat for │ -│ multiple table names, │ -│ leave empty for all │ -│ tables in the file). │ -│ [default: None] │ -│ --iceberg-table TEXT Table name to assign │ -│ to the model created │ -│ from the Iceberg │ -│ schema. │ -│ [default: None] │ -│ --template TEXT The location (url or │ -│ path) of the Data │ -│ Contract │ -│ Specification │ -│ Template │ -│ [default: None] │ -│ --schema TEXT The location (url or │ -│ path) of the Data │ -│ Contract │ -│ Specification JSON │ -│ Schema │ -│ [default: None] │ -│ --help Show this message and │ -│ exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ + + Usage: datacontract import [OPTIONS] + + Create a data contract from the given source location. Saves to file specified by `output` option + if present, otherwise prints to stdout. + +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ * --format [sql|avro|dbt|dbml|glue|jsonsc The format of the source file. │ +│ hema|bigquery|odcs|unity|spark [default: None] │ +│ |iceberg|parquet|csv|protobuf| [required] │ +│ excel] │ +│ --output PATH Specify the file path where │ +│ the Data Contract will be │ +│ saved. If no path is provided, │ +│ the output will be printed to │ +│ stdout. │ +│ [default: None] │ +│ --source TEXT The path to the file that │ +│ should be imported. │ +│ [default: None] │ +│ --dialect TEXT The SQL dialect to use when │ +│ importing SQL files, e.g., │ +│ postgres, tsql, bigquery. │ +│ [default: None] │ +│ --glue-table TEXT List of table ids to import │ +│ from the Glue Database (repeat │ +│ for multiple table ids, leave │ +│ empty for all tables in the │ +│ dataset). │ +│ [default: None] │ +│ --bigquery-project TEXT The bigquery project id. │ +│ [default: None] │ +│ --bigquery-dataset TEXT The bigquery dataset id. │ +│ [default: None] │ +│ --bigquery-table TEXT List of table ids to import │ +│ from the bigquery API (repeat │ +│ for multiple table ids, leave │ +│ empty for all tables in the │ +│ dataset). │ +│ [default: None] │ +│ --unity-table-full-name TEXT Full name of a table in the │ +│ unity catalog │ +│ [default: None] │ +│ --dbt-model TEXT List of models names to import │ +│ from the dbt manifest file │ +│ (repeat for multiple models │ +│ names, leave empty for all │ +│ models in the dataset). │ +│ [default: None] │ +│ --dbml-schema TEXT List of schema names to import │ +│ from the DBML file (repeat for │ +│ multiple schema names, leave │ +│ empty for all tables in the │ +│ file). │ +│ [default: None] │ +│ --dbml-table TEXT List of table names to import │ +│ from the DBML file (repeat for │ +│ multiple table names, leave │ +│ empty for all tables in the │ +│ file). │ +│ [default: None] │ +│ --iceberg-table TEXT Table name to assign to the │ +│ model created from the Iceberg │ +│ schema. │ +│ [default: None] │ +│ --template TEXT The location (url or path) of │ +│ the Data Contract │ +│ Specification Template │ +│ [default: None] │ +│ --schema TEXT The location (url or path) of │ +│ the Data Contract │ +│ Specification JSON Schema │ +│ [default: None] │ +│ --owner TEXT The owner or team responsible │ +│ for managing the data │ +│ contract. │ +│ [default: None] │ +│ --id TEXT The identifier for the the │ +│ data contract. │ +│ [default: None] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` @@ -1494,91 +1447,83 @@ datacontract import --format protobuf --source "test.proto" ### breaking ``` - - Usage: datacontract breaking [OPTIONS] LOCATION_OLD LOCATION_NEW - - Identifies breaking changes between data contracts. Prints to stdout. - -╭─ Arguments ──────────────────────────────────────────────────────────────────╮ -│ * location_old TEXT The location (url or path) of the old data │ -│ contract yaml. │ -│ [default: None] │ -│ [required] │ -│ * location_new TEXT The location (url or path) of the new data │ -│ contract yaml. │ -│ [default: None] │ -│ [required] │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ + + Usage: datacontract breaking [OPTIONS] LOCATION_OLD LOCATION_NEW + + Identifies breaking changes between data contracts. Prints to stdout. + +╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮ +│ * location_old TEXT The location (url or path) of the old data contract yaml. │ +│ [default: None] │ +│ [required] │ +│ * location_new TEXT The location (url or path) of the new data contract yaml. │ +│ [default: None] │ +│ [required] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` ### changelog ``` - - Usage: datacontract changelog [OPTIONS] LOCATION_OLD LOCATION_NEW - - Generate a changelog between data contracts. Prints to stdout. - -╭─ Arguments ──────────────────────────────────────────────────────────────────╮ -│ * location_old TEXT The location (url or path) of the old data │ -│ contract yaml. │ -│ [default: None] │ -│ [required] │ -│ * location_new TEXT The location (url or path) of the new data │ -│ contract yaml. │ -│ [default: None] │ -│ [required] │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ + + Usage: datacontract changelog [OPTIONS] LOCATION_OLD LOCATION_NEW + + Generate a changelog between data contracts. Prints to stdout. + +╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮ +│ * location_old TEXT The location (url or path) of the old data contract yaml. │ +│ [default: None] │ +│ [required] │ +│ * location_new TEXT The location (url or path) of the new data contract yaml. │ +│ [default: None] │ +│ [required] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` ### diff ``` - - Usage: datacontract diff [OPTIONS] LOCATION_OLD LOCATION_NEW - - PLACEHOLDER. Currently works as 'changelog' does. - -╭─ Arguments ──────────────────────────────────────────────────────────────────╮ -│ * location_old TEXT The location (url or path) of the old data │ -│ contract yaml. │ -│ [default: None] │ -│ [required] │ -│ * location_new TEXT The location (url or path) of the new data │ -│ contract yaml. │ -│ [default: None] │ -│ [required] │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ + + Usage: datacontract diff [OPTIONS] LOCATION_OLD LOCATION_NEW + + PLACEHOLDER. Currently works as 'changelog' does. + +╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮ +│ * location_old TEXT The location (url or path) of the old data contract yaml. │ +│ [default: None] │ +│ [required] │ +│ * location_new TEXT The location (url or path) of the new data contract yaml. │ +│ [default: None] │ +│ [required] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` ### catalog ``` - - Usage: datacontract catalog [OPTIONS] - - Create a html catalog of data contracts. - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --files TEXT Glob pattern for the data contract files to include in │ -│ the catalog. Applies recursively to any subfolders. │ -│ [default: *.yaml] │ -│ --output TEXT Output directory for the catalog html files. │ -│ [default: catalog/] │ -│ --schema TEXT The location (url or path) of the Data Contract │ -│ Specification JSON Schema │ -│ [default: None] │ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ + + Usage: datacontract catalog [OPTIONS] + + Create a html catalog of data contracts. + +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ --files TEXT Glob pattern for the data contract files to include in the catalog. │ +│ Applies recursively to any subfolders. │ +│ [default: *.yaml] │ +│ --output TEXT Output directory for the catalog html files. [default: catalog/] │ +│ --schema TEXT The location (url or path) of the Data Contract Specification JSON Schema │ +│ [default: None] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` @@ -1594,56 +1539,48 @@ datacontract catalog --files "*.odcs.yaml" ### publish ``` - - Usage: datacontract publish [OPTIONS] [LOCATION] - - Publish the data contract to the Data Mesh Manager. - -╭─ Arguments ──────────────────────────────────────────────────────────────────╮ -│ location [LOCATION] The location (url or path) of the data contract │ -│ yaml. │ -│ [default: datacontract.yaml] │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --schema TEXT The location (url or │ -│ path) of the Data │ -│ Contract Specification │ -│ JSON Schema │ -│ [default: None] │ -│ --ssl-verification --no-ssl-verification SSL verification when │ -│ publishing the data │ -│ contract. │ -│ [default: │ -│ ssl-verification] │ -│ --help Show this message and │ -│ exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ + + Usage: datacontract publish [OPTIONS] [LOCATION] + + Publish the data contract to the Data Mesh Manager. + +╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮ +│ location [LOCATION] The location (url or path) of the data contract yaml. │ +│ [default: datacontract.yaml] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ --schema TEXT The location (url or path) of the Data │ +│ Contract Specification JSON Schema │ +│ [default: None] │ +│ --ssl-verification --no-ssl-verification SSL verification when publishing the data │ +│ contract. │ +│ [default: ssl-verification] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` ### api ``` - - Usage: datacontract api [OPTIONS] - - Start the datacontract CLI as server application with REST API. - The OpenAPI documentation as Swagger UI is available on http://localhost:4242. - You can execute the commands directly from the Swagger UI. - To protect the API, you can set the environment variable - DATACONTRACT_CLI_API_KEY to a secret API key. To authenticate, requests must - include the header 'x-api-key' with the correct API key. This is highly - recommended, as data contract tests may be subject to SQL injections or leak - sensitive information. - To connect to servers (such as a Snowflake data source), set the credentials - as environment variables as documented in https://cli.datacontract.com/#test - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --port INTEGER Bind socket to this port. [default: 4242] │ -│ --host TEXT Bind socket to this host. Hint: For running in │ -│ docker, set it to 0.0.0.0 │ -│ [default: 127.0.0.1] │ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ + + Usage: datacontract api [OPTIONS] + + Start the datacontract CLI as server application with REST API. + The OpenAPI documentation as Swagger UI is available on http://localhost:4242. You can execute the + commands directly from the Swagger UI. + To protect the API, you can set the environment variable DATACONTRACT_CLI_API_KEY to a secret API + key. To authenticate, requests must include the header 'x-api-key' with the correct API key. This + is highly recommended, as data contract tests may be subject to SQL injections or leak sensitive + information. + To connect to servers (such as a Snowflake data source), set the credentials as environment + variables as documented in https://cli.datacontract.com/#test + +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ --port INTEGER Bind socket to this port. [default: 4242] │ +│ --host TEXT Bind socket to this host. Hint: For running in docker, set it to 0.0.0.0 │ +│ [default: 127.0.0.1] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` diff --git a/datacontract/cli.py b/datacontract/cli.py index c4f6ad29d..af12eb04d 100644 --- a/datacontract/cli.py +++ b/datacontract/cli.py @@ -297,6 +297,14 @@ def import_( str, typer.Option(help="The location (url or path) of the Data Contract Specification JSON Schema"), ] = None, + owner: Annotated[ + Optional[str], + typer.Option(help="The owner or team responsible for managing the data contract."), + ] = None, + id: Annotated[ + Optional[str], + typer.Option(help="The identifier for the the data contract."), + ] = None, ): """ Create a data contract from the given source location. Saves to file specified by `output` option if present, otherwise prints to stdout. @@ -316,6 +324,8 @@ def import_( dbml_schema=dbml_schema, dbml_table=dbml_table, iceberg_table=iceberg_table, + owner=owner, + id=id, ) if output is None: console.print(result.to_yaml(), markup=False, soft_wrap=True) diff --git a/datacontract/data_contract.py b/datacontract/data_contract.py index ad65781fd..53d2bc6ee 100644 --- a/datacontract/data_contract.py +++ b/datacontract/data_contract.py @@ -25,7 +25,7 @@ from datacontract.lint.linters.field_reference_linter import FieldReferenceLinter from datacontract.lint.linters.notice_period_linter import NoticePeriodLinter from datacontract.lint.linters.valid_constraints_linter import ValidFieldConstraintsLinter -from datacontract.model.data_contract_specification import DataContractSpecification +from datacontract.model.data_contract_specification import DataContractSpecification, Info from datacontract.model.exceptions import DataContractException from datacontract.model.run import Check, ResultEnum, Run @@ -270,6 +270,16 @@ def import_from_source( ) -> DataContractSpecification: data_contract_specification_initial = DataContract.init(template=template, schema=schema) - return importer_factory.create(format).import_source( + imported_data_contract_specification = importer_factory.create(format).import_source( data_contract_specification=data_contract_specification_initial, source=source, import_args=kwargs ) + + # Set id and owner if provided + if kwargs.get("id"): + data_contract_specification_initial.id = kwargs["id"] + if kwargs.get("owner"): + if data_contract_specification_initial.info is None: + data_contract_specification_initial.info = Info() + data_contract_specification_initial.info.owner = kwargs["owner"] + + return imported_data_contract_specification diff --git a/tests/test_import_unity_file.py b/tests/test_import_unity_file.py index 050a88fdb..730fba1ff 100644 --- a/tests/test_import_unity_file.py +++ b/tests/test_import_unity_file.py @@ -63,3 +63,55 @@ def test_import_unity_complex_types(): print("Result:\n", result.to_yaml()) assert yaml.safe_load(result.to_yaml()) == yaml.safe_load(expected) assert DataContract(data_contract_str=expected).lint(enabled_linters="none").has_passed() + + +def test_import_unity_with_owner_and_id(): + print("running test_import_unity_with_owner_and_id") + result = DataContract().import_from_source( + "unity", "fixtures/databricks-unity/import/unity_table_schema.json", owner="sales-team", id="orders-v1" + ) + + # Verify owner and id are set correctly + assert result.id == "orders-v1" + assert result.info.owner == "sales-team" + + # Verify the rest of the contract is imported correctly + with open("fixtures/databricks-unity/import/datacontract.yaml") as file: + expected = file.read() + expected_dict = yaml.safe_load(expected) + result_dict = yaml.safe_load(result.to_yaml()) + + # Remove owner and id from comparison since we set them differently + expected_dict.pop("id", None) + expected_dict["info"].pop("owner", None) + result_dict.pop("id", None) + result_dict["info"].pop("owner", None) + + assert result_dict == expected_dict + + +def test_cli_with_owner_and_id(): + print("running test_cli_with_owner_and_id") + runner = CliRunner() + result = runner.invoke( + app, + [ + "import", + "--format", + "unity", + "--source", + "fixtures/databricks-unity/import/unity_table_schema.json", + "--owner", + "sales-team", + "--id", + "orders-v1", + ], + ) + assert result.exit_code == 0 + + # Parse the output YAML + output_dict = yaml.safe_load(result.stdout) + + # Verify owner and id are set correctly + assert output_dict["id"] == "orders-v1" + assert output_dict["info"]["owner"] == "sales-team" diff --git a/update_help.py b/update_help.py index e9515ab71..546524a58 100644 --- a/update_help.py +++ b/update_help.py @@ -24,7 +24,7 @@ def fetch_help(command: str) -> str: print(f"Fetching help text for command: {command}") - env = {"COLUMNS": "80"} # Set terminal width to 80 columns (or your preferred width) + env = {"COLUMNS": "100"} # Set terminal width to 100 columns (or your preferred width) result = subprocess.run( ["datacontract", command, "--help"], capture_output=True, From edf8bb85f9ee3c1b6a47b6d507f942a9869a54c0 Mon Sep 17 00:00:00 2001 From: David Mathias Date: Thu, 22 May 2025 19:27:06 +0100 Subject: [PATCH 46/60] Change behaviour of DBT constraints vs data tests when no model type specified on contract (#748) * fix: changed dbt constraint vs data test behaviour when no model type is specified in the data contract * fix: removed unused args * chore: updated changelog --------- Co-authored-by: jochenchrist --- CHANGELOG.md | 4 + datacontract/export/dbt_converter.py | 9 +- .../export/datacontract_no_model_type.yaml | 91 +++++++++++++++++++ tests/test_export_dbt_models.py | 51 +++++++++++ 4 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/export/datacontract_no_model_type.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 286f5cfc5..f98bbf003 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Changed +======= ### Added - `datacontract import --format spark`: Added support for spark importer table level comments (#761) @@ -15,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - `datacontract export --format sodacl`: Fix resolving server when using `--server` flag (#768) +- `datacontract export --format dbt`: Fixed DBT export behaviour of constraints to default to data tests when no model type is specified in the datacontract model + ## [0.10.26] - 2025-05-16 diff --git a/datacontract/export/dbt_converter.py b/datacontract/export/dbt_converter.py index a9cb1fcaf..3cf07afe3 100644 --- a/datacontract/export/dbt_converter.py +++ b/datacontract/export/dbt_converter.py @@ -27,7 +27,7 @@ def export(self, data_contract, model, server, sql_server_type, export_args) -> ) -def to_dbt_models_yaml(data_contract_spec: DataContractSpecification, server: str = None): +def to_dbt_models_yaml(data_contract_spec: DataContractSpecification, server: str = None) -> str: dbt = { "version": 2, "models": [], @@ -102,8 +102,11 @@ def _to_dbt_model( "name": model_key, } model_type = _to_dbt_model_type(model_value.type) + dbt_model["config"] = {"meta": {"data_contract": data_contract_spec.id}} - dbt_model["config"]["materialized"] = model_type + + if model_type: + dbt_model["config"]["materialized"] = model_type if data_contract_spec.info.owner is not None: dbt_model["config"]["meta"]["owner"] = data_contract_spec.info.owner @@ -123,7 +126,7 @@ def _to_dbt_model_type(model_type): # Allowed values: table, view, incremental, ephemeral, materialized view # Custom values also possible if model_type is None: - return "table" + return None if model_type.lower() == "table": return "table" if model_type.lower() == "view": diff --git a/tests/fixtures/export/datacontract_no_model_type.yaml b/tests/fixtures/export/datacontract_no_model_type.yaml new file mode 100644 index 000000000..cd652b929 --- /dev/null +++ b/tests/fixtures/export/datacontract_no_model_type.yaml @@ -0,0 +1,91 @@ +dataContractSpecification: 1.1.0 +id: orders-unit-test +info: + title: Orders Unit Test + version: 1.0.0 + status: active + owner: checkout + description: The orders data contract + contact: + email: team-orders@example.com + url: https://wiki.example.com/teams/checkout + otherField: otherValue +terms: + usage: This data contract serves to demo datacontract CLI export. + limitations: Not intended to use in production + billing: free + noticePeriod: P3M +servers: + production: + type: snowflake + environment: production + account: my-account + database: my-database + schema: my-schema + roles: + - name: analyst_us + description: Access to the data for US region +models: + orders: + title: Webshop Orders + description: The orders model + primaryKey: + - order_id + - order_status + customModelProperty1: customModelProperty1Value + fields: + order_id: + title: Order ID + type: varchar + unique: true + required: true + minLength: 8 + maxLength: 10 + pii: true + classification: sensitive + tags: + - order_id + pattern: ^B[0-9]+$ + examples: + - B12345678 + - B12345679 + customFieldProperty1: customFieldProperty1Value + order_total: + type: bigint + required: true + description: The order_total field + minimum: 0 + maximum: 1000000 + quality: + - type: sql + description: 95% of all order total values are expected to be between 10 and 499 EUR. + query: | + SELECT quantile_cont(order_total, 0.95) AS percentile_95 + FROM orders + mustBeBetween: [1000, 49900] + order_status: + type: text + required: true + enum: + - pending + - shipped + - delivered + quality: + - type: sql + description: Row Count + query: | + SELECT COUNT(*) AS row_count + FROM orders + mustBeGreaterThan: 1000 +definitions: + customer_id: + title: Customer ID + type: string + format: uuid + description: Unique identifier for the customer. + examples: + - acbd1a47-9dca-4cb8-893e-87aa0aa0f243 + - 5742637f-bb8b-4f0c-8ed1-afb1a91300a9 + tags: + - features + - pii \ No newline at end of file diff --git a/tests/test_export_dbt_models.py b/tests/test_export_dbt_models.py index 237fad655..f6c216409 100644 --- a/tests/test_export_dbt_models.py +++ b/tests/test_export_dbt_models.py @@ -131,6 +131,57 @@ def test_to_dbt_models_with_server(): assert result == yaml.safe_load(expected_dbt_model) +def test_to_dbt_models_with_no_model_type(): + data_contract = DataContractSpecification.from_file("fixtures/export/datacontract_no_model_type.yaml") + expected_dbt_model = """ +version: 2 +models: +- name: orders + config: + meta: + data_contract: orders-unit-test + owner: checkout + description: The orders model + columns: + - name: order_id + data_tests: + - not_null + - unique + - dbt_expectations.expect_column_value_lengths_to_be_between: + min_value: 8 + max_value: 10 + - dbt_expectations.expect_column_values_to_match_regex: + regex: ^B[0-9]+$ + data_type: VARCHAR + meta: + pii: true + classification: sensitive + tags: + - order_id + - name: order_total + data_tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 1000000 + data_type: NUMBER + description: The order_total field + - name: order_status + data_tests: + - not_null + - accepted_values: + values: + - pending + - shipped + - delivered + data_type: TEXT +""" + + result = yaml.safe_load(to_dbt_models_yaml(data_contract)) + + assert result == yaml.safe_load(expected_dbt_model) + + def read_file(file): if not os.path.exists(file): print(f"The file '{file}' does not exist.") From fc9a0bf90ba4478a6eacff0dc3dc693a613e97c5 Mon Sep 17 00:00:00 2001 From: mathispernias Date: Thu, 22 May 2025 14:27:47 -0400 Subject: [PATCH 47/60] Adding mermaid format for export and adding mermaid figure in html export (#767) * adding export for mermaid * adding mermaid export * adding the mermaid figure in the html export * adding mermaid figure to html export * adding test_export_mermaid * updating readme * ruff format and updating changelog * ruff format and updating changelog * updating changelog * update changelog * update changelog * updating the html export * changing test_export_mermaid w/ checks structural elements rather than exact string matching * Update README * Renamed from _export to _exporter Some styling * Move diagram closer to data model --------- Co-authored-by: jochen --- CHANGELOG.md | 7 ++- README.md | 8 +-- datacontract/catalog/catalog.py | 2 +- datacontract/export/exporter.py | 1 + datacontract/export/exporter_factory.py | 8 ++- .../{html_export.py => html_exporter.py} | 6 ++ datacontract/export/mermaid_exporter.py | 32 +++++++++++ datacontract/export/sql_type_converter.py | 6 +- datacontract/templates/datacontract.html | 50 ++++++++++++++++- tests/test_export_mermaid.py | 56 +++++++++++++++++++ 10 files changed, 166 insertions(+), 10 deletions(-) rename datacontract/export/{html_export.py => html_exporter.py} (93%) create mode 100644 datacontract/export/mermaid_exporter.py create mode 100644 tests/test_export_mermaid.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f98bbf003..8e4039c43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- `datacontract export --format mermaid` Export to [Mermaid](https://mermaid-js.github.io/mermaid/#/) + +### Changed +- Adding the mermaid figure to the html export +- ODCS export: Export physical type if the physical type is configured in config object ### Changed ======= @@ -23,7 +29,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.10.26] - 2025-05-16 ### Changed - - Databricks: Add support for Variant type (#758) - `datacontract export --format odcs`: Export physical type if the physical type is configured in config object (#757) diff --git a/README.md b/README.md index 370b1a768..f3766c6e5 100644 --- a/README.md +++ b/README.md @@ -828,10 +828,10 @@ models: │ * --format [jsonschema|pydantic-model|sodacl|db The export format. [default: None] │ │ t|dbt-sources|dbt-staging-sql|odcs|r [required] │ │ df|avro|protobuf|great-expectations| │ -│ terraform|avro-idl|sql|sql-query|htm │ -│ l|go|bigquery|dbml|spark|sqlalchemy| │ -│ data-caterer|dcs|markdown|iceberg|cu │ -│ stom] │ +│ terraform|avro-idl|sql|sql-query|mer │ +│ maid|html|go|bigquery|dbml|spark|sql │ +│ alchemy|data-caterer|dcs|markdown|ic │ +│ eberg|custom] │ │ --output PATH Specify the file path where the │ │ exported data will be saved. If no │ │ path is provided, the output will be │ diff --git a/datacontract/catalog/catalog.py b/datacontract/catalog/catalog.py index 03284defb..50a92aecc 100644 --- a/datacontract/catalog/catalog.py +++ b/datacontract/catalog/catalog.py @@ -6,7 +6,7 @@ from jinja2 import Environment, PackageLoader, select_autoescape from datacontract.data_contract import DataContract -from datacontract.export.html_export import get_version +from datacontract.export.html_exporter import get_version from datacontract.model.data_contract_specification import DataContractSpecification diff --git a/datacontract/export/exporter.py b/datacontract/export/exporter.py index f0b9d2f3e..2c3864b38 100644 --- a/datacontract/export/exporter.py +++ b/datacontract/export/exporter.py @@ -33,6 +33,7 @@ class ExportFormat(str, Enum): avro_idl = "avro-idl" sql = "sql" sql_query = "sql-query" + mermaid = "mermaid" html = "html" go = "go" bigquery = "bigquery" diff --git a/datacontract/export/exporter_factory.py b/datacontract/export/exporter_factory.py index 4804f7993..c9d8419ce 100644 --- a/datacontract/export/exporter_factory.py +++ b/datacontract/export/exporter_factory.py @@ -89,6 +89,12 @@ def load_module_class(module_path, class_name): class_name="DbtExporter", ) +exporter_factory.register_lazy_exporter( + name=ExportFormat.mermaid, + module_path="datacontract.export.mermaid_exporter", + class_name="MermaidExporter", +) + exporter_factory.register_lazy_exporter( name=ExportFormat.dbt_sources, module_path="datacontract.export.dbt_converter", @@ -127,7 +133,7 @@ def load_module_class(module_path, class_name): exporter_factory.register_lazy_exporter( name=ExportFormat.html, - module_path="datacontract.export.html_export", + module_path="datacontract.export.html_exporter", class_name="HtmlExporter", ) diff --git a/datacontract/export/html_export.py b/datacontract/export/html_exporter.py similarity index 93% rename from datacontract/export/html_export.py rename to datacontract/export/html_exporter.py index bc1b1c101..77d66f9d1 100644 --- a/datacontract/export/html_export.py +++ b/datacontract/export/html_exporter.py @@ -17,6 +17,8 @@ def export(self, data_contract, model, server, sql_server_type, export_args) -> def to_html(data_contract_spec: DataContractSpecification) -> str: + from datacontract.export.mermaid_exporter import to_mermaid + # Load templates from templates folder package_loader = PackageLoader("datacontract", "templates") env = Environment( @@ -54,6 +56,9 @@ def to_html(data_contract_spec: DataContractSpecification) -> str: formatted_date = now.strftime("%d %b %Y %H:%M:%S UTC") datacontract_cli_version = get_version() + # Get the mermaid diagram + mermaid_diagram = to_mermaid(data_contract_spec) + # Render the template with necessary data html_string = template.render( datacontract=data_contract_spec, @@ -62,6 +67,7 @@ def to_html(data_contract_spec: DataContractSpecification) -> str: datacontract_yaml=datacontract_yaml, formatted_date=formatted_date, datacontract_cli_version=datacontract_cli_version, + mermaid_diagram=mermaid_diagram, ) return html_string diff --git a/datacontract/export/mermaid_exporter.py b/datacontract/export/mermaid_exporter.py new file mode 100644 index 000000000..ea0b0b215 --- /dev/null +++ b/datacontract/export/mermaid_exporter.py @@ -0,0 +1,32 @@ +from datacontract.export.exporter import Exporter +from datacontract.model.data_contract_specification import DataContractSpecification + + +class MermaidExporter(Exporter): + def export(self, data_contract, model, server, sql_server_type, export_args) -> dict: + return to_mermaid(data_contract) + + +def to_mermaid(data_contract_spec: DataContractSpecification) -> str | None: + mmd_entity = "erDiagram\n\t" + mmd_references = [] + try: + for model_name, model in data_contract_spec.models.items(): + entity_block = "" + for field_name, field in model.fields.items(): + entity_block += f"\t{field_name.replace('#', 'Nb').replace(' ', '_').replace('/', 'by')}{'🔑' if field.primaryKey or (field.unique and field.required) else ''}{'⌘' if field.references else ''} {field.type}\n" + if field.references: + mmd_references.append( + f'"📑{field.references.split(".")[0] if "." in field.references else ""}"' + + "}o--{ ||" + + f'"📑{model_name}"' + ) + mmd_entity += f'\t"**{model_name}**"' + "{\n" + entity_block + "}\n" + + if mmd_entity == "": + return None + else: + return f"{mmd_entity}\n" + except Exception as e: + print(f"error : {e}") + return None diff --git a/datacontract/export/sql_type_converter.py b/datacontract/export/sql_type_converter.py index a89948a3e..75168e5c3 100644 --- a/datacontract/export/sql_type_converter.py +++ b/datacontract/export/sql_type_converter.py @@ -159,7 +159,11 @@ def convert_to_dataframe(field: Field) -> None | str: # https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html def convert_to_databricks(field: Field) -> None | str: type = field.type - if field.config and "databricksType" in field.config and type.lower() not in ["array", "object", "record", "struct"]: + if ( + field.config + and "databricksType" in field.config + and type.lower() not in ["array", "object", "record", "struct"] + ): return field.config["databricksType"] if type is None: return None diff --git a/datacontract/templates/datacontract.html b/datacontract/templates/datacontract.html index 31aa60a28..41c393f9f 100644 --- a/datacontract/templates/datacontract.html +++ b/datacontract/templates/datacontract.html @@ -5,6 +5,8 @@ {# #} + + @@ -29,7 +31,6 @@ -
@@ -77,7 +78,6 @@

@@ -103,6 +103,52 @@

Servers {% endif %} +
+
+

Entity Relationship + Diagram

+

Visual representation of data model relationships

+
+
+
+
+ 
+                    {{ mermaid_diagram }}
+
+
+
+
+ +
diff --git a/tests/test_export_mermaid.py b/tests/test_export_mermaid.py new file mode 100644 index 000000000..9b1e31bd4 --- /dev/null +++ b/tests/test_export_mermaid.py @@ -0,0 +1,56 @@ +import os +from pathlib import Path + +from typer.testing import CliRunner + +from datacontract.cli import app + + +def test_cli(): + runner = CliRunner() + result = runner.invoke(app, ["export", "./fixtures/export/datacontract.yaml", "--format", "mermaid"]) + assert result.exit_code == 0 + + +def test_cli_with_output(tmp_path: Path): + runner = CliRunner() + result = runner.invoke( + app, + [ + "export", + "./fixtures/export/datacontract.yaml", + "--format", + "mermaid", + "--output", + tmp_path / "datacontract.mermaid", + ], + ) + assert result.exit_code == 0 + assert os.path.exists(tmp_path / "datacontract.mermaid") + + +def test_mermaid_structure(tmp_path: Path): + datacontract_file = "fixtures/export/datacontract.yaml" + runner = CliRunner() + result = runner.invoke( + app, + [ + "export", + datacontract_file, + "--format", + "mermaid", + "--output", + tmp_path / "datacontract.mermaid", + ], + ) + assert result.exit_code == 0 + + with open(tmp_path / "datacontract.mermaid") as file: + content = file.read() + + # Check structure + assert "erDiagram" in content + assert "orders" in content + assert "order_id" in content + assert "order_total" in content + assert "order_status" in content From 0f0493a0467c732d822ef1d8b167829dc875f423 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 May 2025 20:29:33 +0200 Subject: [PATCH 48/60] fix: change return type of export methods in SqlExporter and SqlQueryExporter to str --- datacontract/export/sql_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datacontract/export/sql_converter.py b/datacontract/export/sql_converter.py index 6795a3ded..9e42f56da 100644 --- a/datacontract/export/sql_converter.py +++ b/datacontract/export/sql_converter.py @@ -4,7 +4,7 @@ class SqlExporter(Exporter): - def export(self, data_contract, model, server, sql_server_type, export_args) -> dict: + def export(self, data_contract, model, server, sql_server_type, export_args) -> str: server_type = _determine_sql_server_type( data_contract, sql_server_type, @@ -13,7 +13,7 @@ def export(self, data_contract, model, server, sql_server_type, export_args) -> class SqlQueryExporter(Exporter): - def export(self, data_contract, model, server, sql_server_type, export_args) -> dict: + def export(self, data_contract, model, server, sql_server_type, export_args) -> str: model_name, model_value = _check_models_for_export(data_contract, model, self.export_format) server_type = _determine_sql_server_type(data_contract, sql_server_type, export_args.get("server")) return to_sql_query( From 4b6b9a17502daa9ccd2d9cf1ce8db3df734701c2 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 May 2025 20:31:41 +0200 Subject: [PATCH 49/60] chore: update changelog to reflect changes for mermaid export and spark importer support --- CHANGELOG.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e4039c43..3ac94bab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,17 +6,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + ### Added -- `datacontract export --format mermaid` Export to [Mermaid](https://mermaid-js.github.io/mermaid/#/) -### Changed -- Adding the mermaid figure to the html export -- ODCS export: Export physical type if the physical type is configured in config object +- `datacontract export --format mermaid` Export + to [Mermaid](https://mermaid-js.github.io/mermaid/#/) (#767, #725) ### Changed -======= -### Added +- `datacontract export --format html`: Adding the mermaid figure to the html export +- `datacontract export --format odcs`: Export physical type to ODCS if the physical type is + configured in config object - `datacontract import --format spark`: Added support for spark importer table level comments (#761) - `datacontract import` respects `--owner` and `--id` flags (#753) From fa3db82e333190c41733a17cf0b1fc39458ce207 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 May 2025 20:32:59 +0200 Subject: [PATCH 50/60] Bump version --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ac94bab8..1289453a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +### Changed + +### Fixed + +## [0.10.27] - 2025-05-22 + +### Added + - `datacontract export --format mermaid` Export to [Mermaid](https://mermaid-js.github.io/mermaid/#/) (#767, #725) diff --git a/pyproject.toml b/pyproject.toml index a98f17910..474902b74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datacontract-cli" -version = "0.10.26" +version = "0.10.27" description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library." license = "MIT" readme = "README.md" From 52ad9564219dcd6b34730750d34188b64b769f24 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 May 2025 05:02:06 +0200 Subject: [PATCH 51/60] chore: add pyspark dependency version 3.5.5 to pyproject.toml --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 474902b74..9a8fdd01e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ databricks = [ "soda-core-spark[databricks]>=3.3.20,<3.6.0", "databricks-sql-connector>=3.7.0,<4.1.0", "databricks-sdk<0.51.0", + "pyspark==3.5.5", ] iceberg = [ @@ -69,7 +70,8 @@ iceberg = [ kafka = [ "datacontract-cli[avro]", - "soda-core-spark-df>=3.3.20,<3.6.0" + "soda-core-spark-df>=3.3.20,<3.6.0", + "pyspark==3.5.5", ] postgres = [ From 5fd1e8383dc018315e1bd5382f2178f64e142a47 Mon Sep 17 00:00:00 2001 From: Robert Altmiller <123985294+robert-altmiller@users.noreply.github.com> Date: Mon, 26 May 2025 22:50:28 -0500 Subject: [PATCH 52/60] Added support to spark importer to be able to import Spark dataframe object (#771) * added support for variant type in spark import and sql exporter * added support for variant type in spark import and sql exporter * complex types fix when exporting to sql and databricksType is defined. * complex types fix when exporting to sql and databricksType is defined. * complex types fix when exporting to sql and databricksType is defined. * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * added table level comments to spark importer * Enhance SparkImporter with logging and improve table comment retrieval handling * started building import as sourc --> dataframe * modified spark importer to also accept a dataframe object * modified spark importer to also accept a dataframe object * modified spark importer to also accept a dataframe object * modified spark importer to also accept a dataframe object * modified spark importer to also accept a dataframe object * modified spark importer to also accept a dataframe object * modified spark importer to also accept a dataframe object * modified spark importer to also accept a dataframe object * modified spark importer to also accept a dataframe object * modified spark importer to also accept a dataframe object * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description * updated spark importer pytest to test for dataframe object and description --------- Co-authored-by: Alan Reese Co-authored-by: jochenchrist --- README.md | 48 +++- datacontract/imports/spark_importer.py | 59 ++-- pyproject.toml | 1 + .../spark/import/users_datacontract_desc.yml | 59 ++++ .../import/users_datacontract_no_desc.yml | 58 ++++ tests/test_import_spark.py | 267 +++++++----------- tests/test_test_kafka.py | 2 +- 7 files changed, 299 insertions(+), 195 deletions(-) create mode 100644 tests/fixtures/spark/import/users_datacontract_desc.yml create mode 100644 tests/fixtures/spark/import/users_datacontract_no_desc.yml diff --git a/README.md b/README.md index f3766c6e5..24221892f 100644 --- a/README.md +++ b/README.md @@ -904,6 +904,21 @@ Available export options: | `custom` | Export to Custom format with Jinja | ✅ | | Missing something? | Please create an issue on GitHub | TBD | +#### SQL + +The `export` function converts a given data contract into a SQL data definition language (DDL). + +```shell +datacontract export datacontract.yaml --format sql --output output.sql +``` + +If using Databricks, and an error is thrown when trying to deploy the SQL DDLs with `variant` columns set the following properties. + +```shell +spark.conf.set(“spark.databricks.delta.schema.typeCheck.enabled”, “false”) +from datacontract.model import data_contract_specification +data_contract_specification.DATACONTRACT_TYPES.append(“variant”) +``` #### Great Expectations @@ -911,7 +926,7 @@ The `export` function transforms a specified data contract into a comprehensive If the contract includes multiple models, you need to specify the names of the model you wish to export. ```shell -datacontract export datacontract.yaml --format great-expectations --model orders +datacontract export datacontract.yaml --format great-expectations --model orders ``` The export creates a list of expectations by utilizing: @@ -936,7 +951,7 @@ To further customize the export, the following optional arguments are available: #### RDF -The export function converts a given data contract into a RDF representation. You have the option to +The `export` function converts a given data contract into a RDF representation. You have the option to add a base_url which will be used as the default prefix to resolve relative IRIs inside the document. ```shell @@ -1269,11 +1284,11 @@ Available import options: | `jsonschema` | Import from JSON Schemas | ✅ | | `odcs` | Import from Open Data Contract Standard (ODCS) | ✅ | | `parquet` | Import from Parquet File Metadata | ✅ | -| `protobuf` | Import from Protobuf schemas | ✅ | -| `spark` | Import from Spark StructTypes | ✅ | +| `protobuf` | Import from Protobuf schemas | ✅ | +| `spark` | Import from Spark StructTypes, Variant | ✅ | | `sql` | Import from SQL DDL | ✅ | | `unity` | Import from Databricks Unity Catalog | partial | -| Missing something? | Please create an issue on GitHub | TBD | +| Missing something? | Please create an issue on GitHub | TBD | #### ODCS @@ -1375,14 +1390,31 @@ datacontract import --format glue --source #### Spark -Importing from Spark table or view these must be created or accessible in the Spark context. Specify tables list in `source` parameter. - -Example: +Importing from Spark table or view these must be created or accessible in the Spark context. Specify tables list in `source` parameter. If the `source` tables are registered as tables in Databricks, and they have a table-level descriptions they will also be added to the Data Contract Specification. ```bash +# Example: Import Spark table(s) from Spark context datacontract import --format spark --source "users,orders" ``` +```bash +# Example: Import Spark table +DataContract().import_from_source("spark", "users") +DataContract().import_from_source(format = "spark", source = "users") + +# Example: Import Spark dataframe +DataContract().import_from_source("spark", "users", dataframe = df_user) +DataContract().import_from_source(format = "spark", source = "users", dataframe = df_user) + +# Example: Import Spark table + table description +DataContract().import_from_source("spark", "users", description = "description") +DataContract().import_from_source(format = "spark", source = "users", description = "description") + +# Example: Import Spark dataframe + table description +DataContract().import_from_source("spark", "users", dataframe = df_user, description = "description") +DataContract().import_from_source(format = "spark", source = "users", dataframe = df_user, description = "description") +``` + #### DBML Importing from DBML Documents. diff --git a/datacontract/imports/spark_importer.py b/datacontract/imports/spark_importer.py index 0e09e357f..b4207009f 100644 --- a/datacontract/imports/spark_importer.py +++ b/datacontract/imports/spark_importer.py @@ -28,34 +28,57 @@ def import_source( data_contract_specification: The data contract specification object. source: The source string indicating the Spark tables to read. import_args: Additional arguments for the import process. - Returns: dict: The updated data contract specification. """ - return import_spark(data_contract_specification, source) + dataframe = import_args.get("dataframe", None) + description = import_args.get("description", None) + return import_spark(data_contract_specification, source, dataframe, description) -def import_spark(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification: +def import_spark( + data_contract_specification: DataContractSpecification, + source: str, + dataframe: DataFrame | None = None, + description: str | None = None, +) -> DataContractSpecification: """ - Reads Spark tables and updates the data contract specification with their schemas. + Imports schema(s) from Spark into a Data Contract Specification. Args: - data_contract_specification: The data contract specification to update. - source: A comma-separated string of Spark temporary views to read. + data_contract_specification (DataContractSpecification): The contract spec to update. + source (str): Comma-separated Spark table/view names. + dataframe (DataFrame | None): Optional Spark DataFrame to import. + description (str | None): Optional table-level description. Returns: - DataContractSpecification: The updated data contract specification. + DataContractSpecification: The updated contract spec with imported models. """ spark = SparkSession.builder.getOrCreate() data_contract_specification.servers["local"] = Server(type="dataframe") - for temp_view in source.split(","): - temp_view = temp_view.strip() - df = spark.read.table(temp_view) - data_contract_specification.models[temp_view] = import_from_spark_df(spark, source, df) + + if dataframe is not None: + if not isinstance(dataframe, DataFrame): + raise TypeError("Expected 'dataframe' to be a pyspark.sql.DataFrame") + data_contract_specification.models[source] = import_from_spark_df( + spark, source, dataframe, description + ) + return data_contract_specification + + if not source: + raise ValueError("Either 'dataframe' or a valid 'source' must be provided") + + for table_name in map(str.strip, source.split(",")): + df = spark.read.table(table_name) + data_contract_specification.models[table_name] = import_from_spark_df( + spark, table_name, df, description + ) + return data_contract_specification -def import_from_spark_df(spark: SparkSession, source: str, df: DataFrame) -> Model: + +def import_from_spark_df(spark: SparkSession, source: str, df: DataFrame, description: str) -> Model: """ Converts a Spark DataFrame into a Model. @@ -63,6 +86,7 @@ def import_from_spark_df(spark: SparkSession, source: str, df: DataFrame) -> Mod spark: SparkSession source: A comma-separated string of Spark temporary views to read. df: The Spark DataFrame to convert. + description: Table level comment Returns: Model: The generated data contract model. @@ -70,7 +94,10 @@ def import_from_spark_df(spark: SparkSession, source: str, df: DataFrame) -> Mod model = Model() schema = df.schema - model.description = _table_comment_from_spark(spark, source) + if description is None: + model.description = _table_comment_from_spark(spark, source) + else: + model.description = description for field in schema: model.fields[field.name] = _field_from_struct_type(field) @@ -199,7 +226,7 @@ def _table_comment_from_spark(spark: SparkSession, source: str): workspace_client = WorkspaceClient() created_table = workspace_client.tables.get(full_name=f"{source}") table_comment = created_table.comment - print(f"'{source}' table comment retrieved using 'WorkspaceClient.tables.get({source})'") + logger.info(f"'{source}' table comment retrieved using 'WorkspaceClient.tables.get({source})'") return table_comment except Exception: pass @@ -207,7 +234,7 @@ def _table_comment_from_spark(spark: SparkSession, source: str): # Fallback to Spark Catalog API for Hive Metastore or Non-UC Tables try: table_comment = spark.catalog.getTable(f"{source}").description - print(f"'{source}' table comment retrieved using 'spark.catalog.getTable({source}).description'") + logger.info(f"'{source}' table comment retrieved using 'spark.catalog.getTable({source}).description'") return table_comment except Exception: pass @@ -219,7 +246,7 @@ def _table_comment_from_spark(spark: SparkSession, source: str): if row.col_name.strip().lower() == "comment": table_comment = row.data_type break - print(f"'{source}' table comment retrieved using 'DESCRIBE TABLE EXTENDED {source}'") + logger.info(f"'{source}' table comment retrieved using 'DESCRIBE TABLE EXTENDED {source}'") return table_comment except Exception: pass diff --git a/pyproject.toml b/pyproject.toml index 9a8fdd01e..5c2a5eff7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,6 +139,7 @@ dev = [ "ruff", "testcontainers[minio,postgres,kafka,mssql]==4.10.0", "trino==0.333.0", + "pyspark>=3.5.5,<3.6.0", ] [project.urls] diff --git a/tests/fixtures/spark/import/users_datacontract_desc.yml b/tests/fixtures/spark/import/users_datacontract_desc.yml new file mode 100644 index 000000000..c93d614d7 --- /dev/null +++ b/tests/fixtures/spark/import/users_datacontract_desc.yml @@ -0,0 +1,59 @@ +dataContractSpecification: 1.1.0 +id: my-data-contract-id +info: + title: My Data Contract + version: 0.0.1 +servers: + local: + type: dataframe +models: + users: + description: description + fields: + id: + type: string + required: false + name: + type: string + required: false + address: + type: struct + required: false + fields: + number: + type: integer + required: false + street: + type: string + required: false + city: + type: string + required: false + tags: + type: array + required: false + items: + type: string + required: false + metadata: + type: map + required: false + keys: + type: string + required: true + values: + type: struct + required: false + fields: + value: + type: string + required: false + type: + type: string + required: false + timestamp: + type: long + required: false + source: + type: string + required: false \ No newline at end of file diff --git a/tests/fixtures/spark/import/users_datacontract_no_desc.yml b/tests/fixtures/spark/import/users_datacontract_no_desc.yml new file mode 100644 index 000000000..256e72fe5 --- /dev/null +++ b/tests/fixtures/spark/import/users_datacontract_no_desc.yml @@ -0,0 +1,58 @@ +dataContractSpecification: 1.1.0 +id: my-data-contract-id +info: + title: My Data Contract + version: 0.0.1 +servers: + local: + type: dataframe +models: + users: + fields: + id: + type: string + required: false + name: + type: string + required: false + address: + type: struct + required: false + fields: + number: + type: integer + required: false + street: + type: string + required: false + city: + type: string + required: false + tags: + type: array + required: false + items: + type: string + required: false + metadata: + type: map + required: false + keys: + type: string + required: true + values: + type: struct + required: false + fields: + value: + type: string + required: false + type: + type: string + required: false + timestamp: + type: long + required: false + source: + type: string + required: false \ No newline at end of file diff --git a/tests/test_import_spark.py b/tests/test_import_spark.py index f5a3f8d7a..13ffb5d9e 100644 --- a/tests/test_import_spark.py +++ b/tests/test_import_spark.py @@ -6,67 +6,6 @@ from datacontract.cli import app from datacontract.data_contract import DataContract -expected = """ -dataContractSpecification: 1.1.0 -id: my-data-contract-id -info: - title: My Data Contract - version: 0.0.1 -servers: - local: - type: dataframe -models: - users: - fields: - id: - type: string - required: false - name: - type: string - required: false - address: - type: struct - required: false - fields: - number: - type: integer - required: false - street: - type: string - required: false - city: - type: string - required: false - tags: - type: array - required: false - items: - type: string - required: false - metadata: - type: map - required: false - keys: - type: string - required: true - values: - type: struct - required: false - fields: - value: - type: string - required: false - type: - type: string - required: false - timestamp: - type: long - required: false - source: - type: string - required: false - """ - @pytest.fixture(scope="session") def spark(tmp_path_factory) -> SparkSession: @@ -88,64 +27,89 @@ def spark(tmp_path_factory) -> SparkSession: print(f"Using PySpark version {spark.version}") return spark - -def test_cli(spark: SparkSession): - df_user = spark.createDataFrame( - data=[ - { - "id": "1", - "name": "John Doe", - "address": { - "number": 123, - "street": "Maple Street", - "city": "Anytown", - }, - "tags": ["tag1", "tag2"], - "metadata": { - "my-source-metadata": { - "value": "1234567890", - "type": "STRING", - "timestamp": 1646053400, - "source": "my-source", - } - }, +@pytest.fixture() +def user_datacontract_desc(): + with open("fixtures/spark/import/users_datacontract_desc.yml", "r") as f: + data_contract_str = f.read() + return data_contract_str + + +@pytest.fixture() +def user_datacontract_no_desc(): + with open("fixtures/spark/import/users_datacontract_no_desc.yml", "r") as f: + data_contract_str = f.read() + return data_contract_str + + +@pytest.fixture() +def user_row(): + return { + "id": "1", + "name": "John Doe", + "address": { + "number": 123, + "street": "Maple Street", + "city": "Anytown", + }, + "tags": ["tag1", "tag2"], + "metadata": { + "my-source-metadata": { + "value": "1234567890", + "type": "STRING", + "timestamp": 1646053400, + "source": "my-source", } - ], - schema=types.StructType( - [ - types.StructField("id", types.StringType()), - types.StructField("name", types.StringType()), - types.StructField( - "address", - types.StructType( + }, + } + + +@pytest.fixture() +def user_schema(): + return types.StructType( + [ + types.StructField("id", types.StringType()), + types.StructField("name", types.StringType()), + types.StructField( + "address", + types.StructType( + [ + types.StructField("number", types.IntegerType()), + types.StructField("street", types.StringType()), + types.StructField("city", types.StringType()), + ] + ), + ), + types.StructField("tags", types.ArrayType(types.StringType())), + types.StructField( + "metadata", + types.MapType( + keyType=types.StringType(), + valueType=types.StructType( [ - types.StructField("number", types.IntegerType()), - types.StructField("street", types.StringType()), - types.StructField("city", types.StringType()), + types.StructField("value", types.StringType()), + types.StructField("type", types.StringType()), + types.StructField("timestamp", types.LongType()), + types.StructField("source", types.StringType()), ] ), ), - types.StructField("tags", types.ArrayType(types.StringType())), - types.StructField( - "metadata", - types.MapType( - keyType=types.StringType(), - valueType=types.StructType( - [ - types.StructField("value", types.StringType()), - types.StructField("type", types.StringType()), - types.StructField("timestamp", types.LongType()), - types.StructField("source", types.StringType()), - ] - ), - ), - ), - ] - ), + ), + ] ) - df_user.createOrReplaceTempView("users") + +@pytest.fixture() +def df_user(spark: SparkSession, user_row, user_schema): + return spark.createDataFrame(data=[user_row], schema=user_schema) + + +def test_cli(spark: SparkSession, df_user, user_datacontract_no_desc): + + df_user.write.mode("overwrite").saveAsTable("users") + + expected_no_desc = user_datacontract_no_desc + runner = CliRunner() result = runner.invoke( app, @@ -160,8 +124,8 @@ def test_cli(spark: SparkSession): output = result.stdout assert result.exit_code == 0 - assert output.strip() == expected.strip() - + assert output.strip() == expected_no_desc.strip() + def test_table_not_exists(): runner = CliRunner() @@ -179,62 +143,25 @@ def test_table_not_exists(): assert result.exit_code == 1 -def test_prog(spark: SparkSession): - df_user = spark.createDataFrame( - data=[ - { - "id": "1", - "name": "John Doe", - "address": { - "number": 123, - "street": "Maple Street", - "city": "Anytown", - }, - "tags": ["tag1", "tag2"], - "metadata": { - "my-source-metadata": { - "value": "1234567890", - "type": "STRING", - "timestamp": 1646053400, - "source": "my-source", - } - }, - } - ], - schema=types.StructType( - [ - types.StructField("id", types.StringType()), - types.StructField("name", types.StringType()), - types.StructField( - "address", - types.StructType( - [ - types.StructField("number", types.IntegerType()), - types.StructField("street", types.StringType()), - types.StructField("city", types.StringType()), - ] - ), - ), - types.StructField("tags", types.ArrayType(types.StringType())), - types.StructField( - "metadata", - types.MapType( - keyType=types.StringType(), - valueType=types.StructType( - [ - types.StructField("value", types.StringType()), - types.StructField("type", types.StringType()), - types.StructField("timestamp", types.LongType()), - types.StructField("source", types.StringType()), - ] - ), - ), - ), - ] - ), - ) +def test_prog(spark: SparkSession, df_user, user_datacontract_no_desc, user_datacontract_desc): - df_user.createOrReplaceTempView("users") - result = DataContract().import_from_source("spark", "users") + df_user.write.mode("overwrite").saveAsTable("users") - assert yaml.safe_load(result.to_yaml()) == yaml.safe_load(expected) + expected_desc = user_datacontract_desc + expected_no_desc = user_datacontract_no_desc + + # does not include a table level description (table method) + result1 = DataContract().import_from_source("spark", "users") + assert yaml.safe_load(result1.to_yaml()) == yaml.safe_load(expected_no_desc) + + # does include a table level description (table method) + result2 = DataContract().import_from_source("spark", "users", description = "description") + assert yaml.safe_load(result2.to_yaml()) == yaml.safe_load(expected_desc) + + # does not include a table level description (dataframe object method) + result3 = DataContract().import_from_source("spark", "users", dataframe = df_user) + assert yaml.safe_load(result3.to_yaml()) == yaml.safe_load(expected_no_desc) + + # does include a table level description (dataframe object method) + result4 = DataContract().import_from_source("spark", "users", dataframe = df_user, description = "description") + assert yaml.safe_load(result4.to_yaml()) == yaml.safe_load(expected_desc) \ No newline at end of file diff --git a/tests/test_test_kafka.py b/tests/test_test_kafka.py index 37bfbec85..44ceab29f 100644 --- a/tests/test_test_kafka.py +++ b/tests/test_test_kafka.py @@ -50,4 +50,4 @@ def _setup_datacontract(kafka: KafkaContainer): with open(datacontract) as data_contract_file: data_contract_str = data_contract_file.read() host = kafka.get_bootstrap_server() - return data_contract_str.replace("__KAFKA_HOST__", host) + return data_contract_str.replace("__KAFKA_HOST__", host) \ No newline at end of file From f63d27d5fe2192e96bac8f7a843735b64215dd03 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 May 2025 05:51:31 +0200 Subject: [PATCH 53/60] chore: remove pyspark dependency version constraint from pyproject.toml --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5c2a5eff7..9a8fdd01e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,7 +139,6 @@ dev = [ "ruff", "testcontainers[minio,postgres,kafka,mssql]==4.10.0", "trino==0.333.0", - "pyspark>=3.5.5,<3.6.0", ] [project.urls] From d876d849764592fd3ae33579705979e2abf4b69c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 May 2025 05:52:05 +0200 Subject: [PATCH 54/60] chore(deps): update databricks-sdk requirement from <0.51.0 to <0.55.0 (#773) Updates the requirements on [databricks-sdk](https://github.com/databricks/databricks-sdk-py) to permit the latest version. - [Release notes](https://github.com/databricks/databricks-sdk-py/releases) - [Changelog](https://github.com/databricks/databricks-sdk-py/blob/main/CHANGELOG.md) - [Commits](https://github.com/databricks/databricks-sdk-py/compare/v0.0.1...v0.54.0) --- updated-dependencies: - dependency-name: databricks-sdk dependency-version: 0.54.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9a8fdd01e..301e9fcb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ databricks = [ "soda-core-spark-df>=3.3.20,<3.6.0", "soda-core-spark[databricks]>=3.3.20,<3.6.0", "databricks-sql-connector>=3.7.0,<4.1.0", - "databricks-sdk<0.51.0", + "databricks-sdk<0.55.0", "pyspark==3.5.5", ] From b2992e0df4b86148f0bb76e26177972fb75e3388 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 May 2025 05:52:14 +0200 Subject: [PATCH 55/60] chore(deps): bump moto from 5.1.4 to 5.1.5 (#772) Bumps [moto](https://github.com/getmoto/moto) from 5.1.4 to 5.1.5. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/5.1.4...5.1.5) --- updated-dependencies: - dependency-name: moto dependency-version: 5.1.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: jochenchrist --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 301e9fcb3..e105f30ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,7 +130,7 @@ dev = [ "datacontract-cli[all]", "httpx==0.28.1", "kafka-python", - "moto==5.1.4", + "moto==5.1.5", "pandas>=2.1.0", "pre-commit>=3.7.1,<4.3.0", "pytest", From 2224770316e1319e5ad6750930b0ea6da6f2414c Mon Sep 17 00:00:00 2001 From: Victor Tulus <47393746+vtulus@users.noreply.github.com> Date: Tue, 3 Jun 2025 13:17:53 +0200 Subject: [PATCH 56/60] Add data contract version number to index.html (#780) * Add data contract version and enable version search - add version number below contract title - add version number to data-search attribute * Update CHANGELOG --- CHANGELOG.md | 1 + datacontract/templates/index.html | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1289453a3..619e822ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### Changed +- `datacontract catalog [OPTIONS]`: Added version to contract cards in `index.html` of the catalog (enabled search by version) ### Fixed diff --git a/datacontract/templates/index.html b/datacontract/templates/index.html index 7621eb00d..fb4a2bb1d 100644 --- a/datacontract/templates/index.html +++ b/datacontract/templates/index.html @@ -80,6 +80,7 @@

Filters

data-search="{{ contract.spec.info.title|lower|e }} {{ contract.spec.info.owner|lower|e if contract.spec.info.owner else '' }} {{ + contract.spec.info.version|lower|e if contract.spec.info.version else '' }} {{ contract.spec.info.description|lower|e }} {% for model_name, model in contract.spec.models.items() %} {{ model.description|lower|e }} {% @@ -94,6 +95,7 @@

Filters

{{contract.spec.info.title}}

+

{{ contract.spec.info.version }}

{% if contract.spec.info.owner %}
From 7280d8ed6cbc47880aa6d4c138a614f21cab29cf Mon Sep 17 00:00:00 2001 From: "Dr. Simon Harrer" Date: Thu, 5 Jun 2025 15:25:15 +0200 Subject: [PATCH 57/60] Better odcs support (#783) * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * UPDATE * feat: enhance ODCS html with additional sections for support, pricing, team, roles, and SLA properties * UPDATE --------- Co-authored-by: jochen --- CHANGELOG.md | 6 + README.md | 60 +- datacontract/cli.py | 13 +- datacontract/data_contract.py | 151 +++- datacontract/engines/data_contract_checks.py | 2 + datacontract/export/html_exporter.py | 51 +- datacontract/export/mermaid_exporter.py | 91 ++- datacontract/export/odcs_v3_exporter.py | 16 +- datacontract/imports/excel_importer.py | 7 +- datacontract/imports/importer.py | 11 +- datacontract/imports/odcs_importer.py | 4 +- datacontract/imports/odcs_v3_importer.py | 18 +- datacontract/imports/spark_importer.py | 11 +- datacontract/imports/sql_importer.py | 6 +- datacontract/imports/unity_importer.py | 114 ++- datacontract/integration/datamesh_manager.py | 18 +- datacontract/lint/resolve.py | 66 +- datacontract/templates/datacontract.html | 4 + datacontract/templates/datacontract_odcs.html | 666 ++++++++++++++++++ datacontract/templates/partials/server.html | 2 + datacontract/templates/style/output.css | 464 ++++++++---- .../templates/style/tailwind.config.js | 1 + .../databricks-unity/import/datacontract.yaml | 28 +- .../import/unity_table_schema.json | 4 +- tests/test_import_spark.py | 20 +- tests/test_import_unity_file.py | 7 +- tests/test_test_kafka.py | 2 +- 27 files changed, 1507 insertions(+), 336 deletions(-) create mode 100644 datacontract/templates/datacontract_odcs.html diff --git a/CHANGELOG.md b/CHANGELOG.md index 619e822ff..f560a426d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Added +- Much better ODCS support + - Import anything to ODCS via the `import --spec odcs` flag + - Export to HTML with an ODCS native template via `export --format html` + - Export to Mermaid with an ODCS native mapping via `export --format mermaid` +- The databricks `unity` importer now supports more than a single table. You can use `--unity-table-full-name` multiple times to import multiple tables. And it will automatically add a server with the catalog and schema name. ### Changed - `datacontract catalog [OPTIONS]`: Added version to contract cards in `index.html` of the catalog (enabled search by version) +- The type mapping of the `unity` importer no uses the native databricks types instead of relying on spark types. This allows for better type mapping and more accurate data contracts. ### Fixed diff --git a/README.md b/README.md index 24221892f..122cf7e98 100644 --- a/README.md +++ b/README.md @@ -307,30 +307,38 @@ Commands │ [default: datacontract.yaml] │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ -│ --schema TEXT The location (url or path) of the Data │ -│ Contract Specification JSON Schema │ -│ [default: None] │ -│ --server TEXT The server configuration to run the │ -│ schema and quality tests. Use the key of │ -│ the server object in the data contract │ -│ yaml file to refer to a server, e.g., │ -│ `production`, or `all` for all servers │ -│ (default). │ -│ [default: all] │ -│ --publish TEXT The url to publish the results after the │ -│ test │ -│ [default: None] │ -│ --output PATH Specify the file path where the test │ -│ results should be written to (e.g., │ -│ './test-results/TEST-datacontract.xml'). │ -│ [default: None] │ -│ --output-format [junit] The target format for the test results. │ -│ [default: None] │ -│ --logs --no-logs Print logs [default: no-logs] │ -│ --ssl-verification --no-ssl-verification SSL verification when publishing the │ -│ data contract. │ -│ [default: ssl-verification] │ -│ --help Show this message and exit. │ +│ --schema TEXT The location (url or path) of │ +│ the Data Contract Specification │ +│ JSON Schema │ +│ [default: None] │ +│ --server TEXT The server configuration to run │ +│ the schema and quality tests. │ +│ Use the key of the server object │ +│ in the data contract yaml file │ +│ to refer to a server, e.g., │ +│ `production`, or `all` for all │ +│ servers (default). │ +│ [default: all] │ +│ --publish-test-results --no-publish-test-results Publish the results after the │ +│ test │ +│ [default: │ +│ no-publish-test-results] │ +│ --publish TEXT DEPRECATED. The url to publish │ +│ the results after the test. │ +│ [default: None] │ +│ --output PATH Specify the file path where the │ +│ test results should be written │ +│ to (e.g., │ +│ './test-results/TEST-datacontra… │ +│ [default: None] │ +│ --output-format [junit] The target format for the test │ +│ results. │ +│ [default: None] │ +│ --logs --no-logs Print logs [default: no-logs] │ +│ --ssl-verification --no-ssl-verification SSL verification when publishing │ +│ the data contract. │ +│ [default: ssl-verification] │ +│ --help Show this message and exit. │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` @@ -1196,6 +1204,10 @@ FROM │ --source TEXT The path to the file that │ │ should be imported. │ │ [default: None] │ +│ --spec [datacontract_specification|od The format of the data │ +│ cs] contract to import. │ +│ [default: │ +│ datacontract_specification] │ │ --dialect TEXT The SQL dialect to use when │ │ importing SQL files, e.g., │ │ postgres, tsql, bigquery. │ diff --git a/datacontract/cli.py b/datacontract/cli.py index af12eb04d..5309ad119 100644 --- a/datacontract/cli.py +++ b/datacontract/cli.py @@ -11,7 +11,7 @@ from datacontract.catalog.catalog import create_data_contract_html, create_index_html from datacontract.data_contract import DataContract, ExportFormat -from datacontract.imports.importer import ImportFormat +from datacontract.imports.importer import ImportFormat, Spec from datacontract.init.init_template import get_init_template from datacontract.integration.datamesh_manager import ( publish_data_contract_to_datamesh_manager, @@ -126,7 +126,8 @@ def test( "servers (default)." ), ] = "all", - publish: Annotated[str, typer.Option(help="The url to publish the results after the test")] = None, + publish_test_results: Annotated[bool, typer.Option(help="Publish the results after the test")] = False, + publish: Annotated[str, typer.Option(help="DEPRECATED. The url to publish the results after the test.")] = None, output: Annotated[ Path, typer.Option( @@ -149,6 +150,7 @@ def test( run = DataContract( data_contract_file=location, schema_location=schema, + publish_test_results=publish_test_results, publish_url=publish, server=server, ssl_verification=ssl_verification, @@ -246,6 +248,10 @@ def import_( Optional[str], typer.Option(help="The path to the file that should be imported."), ] = None, + spec: Annotated[ + Spec, + typer.Option(help="The format of the data contract to import. "), + ] = Spec.datacontract_specification, dialect: Annotated[ Optional[str], typer.Option(help="The SQL dialect to use when importing SQL files, e.g., postgres, tsql, bigquery."), @@ -265,7 +271,7 @@ def import_( ), ] = None, unity_table_full_name: Annotated[ - Optional[str], typer.Option(help="Full name of a table in the unity catalog") + Optional[List[str]], typer.Option(help="Full name of a table in the unity catalog") ] = None, dbt_model: Annotated[ Optional[List[str]], @@ -312,6 +318,7 @@ def import_( result = DataContract().import_from_source( format=format, source=source, + spec=spec, template=template, schema=schema, dialect=dialect, diff --git a/datacontract/data_contract.py b/datacontract/data_contract.py index 53d2bc6ee..7e8e4eda7 100644 --- a/datacontract/data_contract.py +++ b/datacontract/data_contract.py @@ -1,6 +1,12 @@ import logging import typing +from open_data_contract_standard.model import CustomProperty, OpenDataContractStandard + +from datacontract.export.odcs_v3_exporter import to_odcs_v3 +from datacontract.imports.importer import Spec +from datacontract.imports.odcs_v3_importer import import_from_odcs + if typing.TYPE_CHECKING: from pyspark.sql import SparkSession @@ -44,6 +50,7 @@ def __init__( inline_definitions: bool = True, inline_quality: bool = True, ssl_verification: bool = True, + publish_test_results: bool = False, ): self._data_contract_file = data_contract_file self._data_contract_str = data_contract_str @@ -51,6 +58,7 @@ def __init__( self._schema_location = schema_location self._server = server self._publish_url = publish_url + self._publish_test_results = publish_test_results self._spark = spark self._duckdb_connection = duckdb_connection self._inline_definitions = inline_definitions @@ -178,7 +186,7 @@ def test(self) -> Run: run.finish() - if self._publish_url is not None: + if self._publish_url is not None or self._publish_test_results: publish_test_results_to_datamesh_manager(run, self._publish_url, self._ssl_verification) return run @@ -243,43 +251,128 @@ def get_data_contract_specification(self) -> DataContractSpecification: ) def export(self, export_format: ExportFormat, model: str = "all", sql_server_type: str = "auto", **kwargs) -> str: - data_contract = resolve.resolve_data_contract( - self._data_contract_file, - self._data_contract_str, - self._data_contract, - schema_location=self._schema_location, - inline_definitions=self._inline_definitions, - inline_quality=self._inline_quality, - ) + if export_format == ExportFormat.html or export_format == ExportFormat.mermaid: + data_contract = resolve.resolve_data_contract_v2( + self._data_contract_file, + self._data_contract_str, + self._data_contract, + schema_location=self._schema_location, + inline_definitions=self._inline_definitions, + inline_quality=self._inline_quality, + ) - return exporter_factory.create(export_format).export( - data_contract=data_contract, - model=model, - server=self._server, - sql_server_type=sql_server_type, - export_args=kwargs, - ) + return exporter_factory.create(export_format).export( + data_contract=data_contract, + model=model, + server=self._server, + sql_server_type=sql_server_type, + export_args=kwargs, + ) + else: + data_contract = resolve.resolve_data_contract( + self._data_contract_file, + self._data_contract_str, + self._data_contract, + schema_location=self._schema_location, + inline_definitions=self._inline_definitions, + inline_quality=self._inline_quality, + ) + + return exporter_factory.create(export_format).export( + data_contract=data_contract, + model=model, + server=self._server, + sql_server_type=sql_server_type, + export_args=kwargs, + ) + # REFACTOR THIS + # could be a class method, not using anything from the instance def import_from_source( self, format: str, source: typing.Optional[str] = None, template: typing.Optional[str] = None, schema: typing.Optional[str] = None, + spec: Spec = Spec.datacontract_specification, **kwargs, - ) -> DataContractSpecification: - data_contract_specification_initial = DataContract.init(template=template, schema=schema) + ) -> DataContractSpecification | OpenDataContractStandard: + id = kwargs.get("id") + owner = kwargs.get("owner") - imported_data_contract_specification = importer_factory.create(format).import_source( - data_contract_specification=data_contract_specification_initial, source=source, import_args=kwargs - ) + if spec == Spec.odcs: + data_contract_specification_initial = DataContract.init(template=template, schema=schema) + + odcs_imported = importer_factory.create(format).import_source( + data_contract_specification=data_contract_specification_initial, source=source, import_args=kwargs + ) + + if isinstance(odcs_imported, DataContractSpecification): + # convert automatically + odcs_imported = to_odcs_v3(odcs_imported) + + self._overwrite_id_in_odcs(odcs_imported, id) + self._overwrite_owner_in_odcs(odcs_imported, owner) + + return odcs_imported + elif spec == Spec.datacontract_specification: + data_contract_specification_initial = DataContract.init(template=template, schema=schema) + + data_contract_specification_imported = importer_factory.create(format).import_source( + data_contract_specification=data_contract_specification_initial, source=source, import_args=kwargs + ) + + if isinstance(data_contract_specification_imported, OpenDataContractStandard): + # convert automatically + data_contract_specification_imported = import_from_odcs( + data_contract_specification_initial, data_contract_specification_imported + ) + + self._overwrite_id_in_data_contract_specification(data_contract_specification_imported, id) + self._overwrite_owner_in_data_contract_specification(data_contract_specification_imported, owner) + + return data_contract_specification_imported + else: + raise DataContractException( + type="general", + result=ResultEnum.error, + name="Import Data Contract", + reason=f"Unsupported data contract format: {spec}", + engine="datacontract", + ) + + def _overwrite_id_in_data_contract_specification( + self, data_contract_specification: DataContractSpecification, id: str | None + ): + if not id: + return + + data_contract_specification.id = id + + def _overwrite_owner_in_data_contract_specification( + self, data_contract_specification: DataContractSpecification, owner: str | None + ): + if not owner: + return + + if data_contract_specification.info is None: + data_contract_specification.info = Info() + data_contract_specification.info.owner = owner + + def _overwrite_owner_in_odcs(self, odcs: OpenDataContractStandard, owner: str | None): + if not owner: + return + + if odcs.customProperties is None: + odcs.customProperties = [] + for customProperty in odcs.customProperties: + if customProperty.name == "owner": + customProperty.value = owner + return + odcs.customProperties.append(CustomProperty(property="owner", value=owner)) - # Set id and owner if provided - if kwargs.get("id"): - data_contract_specification_initial.id = kwargs["id"] - if kwargs.get("owner"): - if data_contract_specification_initial.info is None: - data_contract_specification_initial.info = Info() - data_contract_specification_initial.info.owner = kwargs["owner"] + def _overwrite_id_in_odcs(self, odcs: OpenDataContractStandard, id: str | None): + if not id: + return - return imported_data_contract_specification + odcs.id = id diff --git a/datacontract/engines/data_contract_checks.py b/datacontract/engines/data_contract_checks.py index 2be6c649e..4def63173 100644 --- a/datacontract/engines/data_contract_checks.py +++ b/datacontract/engines/data_contract_checks.py @@ -502,11 +502,13 @@ def prepare_query(quality: Quality, model_name: str, field_name: str = None) -> query = quality.query query = query.replace("{model}", model_name) + query = query.replace("{schema}", model_name) query = query.replace("{table}", model_name) if field_name is not None: query = query.replace("{field}", field_name) query = query.replace("{column}", field_name) + query = query.replace("{property}", field_name) return query diff --git a/datacontract/export/html_exporter.py b/datacontract/export/html_exporter.py index 77d66f9d1..53b26b28d 100644 --- a/datacontract/export/html_exporter.py +++ b/datacontract/export/html_exporter.py @@ -6,8 +6,10 @@ import pytz import yaml from jinja2 import Environment, PackageLoader, select_autoescape +from open_data_contract_standard.model import OpenDataContractStandard from datacontract.export.exporter import Exporter +from datacontract.export.mermaid_exporter import to_mermaid from datacontract.model.data_contract_specification import DataContractSpecification @@ -16,9 +18,7 @@ def export(self, data_contract, model, server, sql_server_type, export_args) -> return to_html(data_contract) -def to_html(data_contract_spec: DataContractSpecification) -> str: - from datacontract.export.mermaid_exporter import to_mermaid - +def to_html(data_contract_spec: DataContractSpecification | OpenDataContractStandard) -> str: # Load templates from templates folder package_loader = PackageLoader("datacontract", "templates") env = Environment( @@ -33,28 +33,27 @@ def to_html(data_contract_spec: DataContractSpecification) -> str: # Load the required template # needs to be included in /MANIFEST.in - template = env.get_template("datacontract.html") - - if data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, str): - quality_specification = data_contract_spec.quality.specification - elif data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, object): - if data_contract_spec.quality.type == "great-expectations": - quality_specification = yaml.dump( - data_contract_spec.quality.specification, sort_keys=False, default_style="|" - ) - else: - quality_specification = yaml.dump(data_contract_spec.quality.specification, sort_keys=False) - else: - quality_specification = None + template_file = "datacontract.html" + if isinstance(data_contract_spec, OpenDataContractStandard): + template_file = "datacontract_odcs.html" + + template = env.get_template(template_file) style_content, _, _ = package_loader.get_source(env, "style/output.css") - datacontract_yaml = data_contract_spec.to_yaml() + quality_specification = None + if isinstance(data_contract_spec, DataContractSpecification): + if data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, str): + quality_specification = data_contract_spec.quality.specification + elif data_contract_spec.quality is not None and isinstance(data_contract_spec.quality.specification, object): + if data_contract_spec.quality.type == "great-expectations": + quality_specification = yaml.dump( + data_contract_spec.quality.specification, sort_keys=False, default_style="|" + ) + else: + quality_specification = yaml.dump(data_contract_spec.quality.specification, sort_keys=False) - tz = pytz.timezone("UTC") - now = datetime.datetime.now(tz) - formatted_date = now.strftime("%d %b %Y %H:%M:%S UTC") - datacontract_cli_version = get_version() + datacontract_yaml = data_contract_spec.to_yaml() # Get the mermaid diagram mermaid_diagram = to_mermaid(data_contract_spec) @@ -65,14 +64,20 @@ def to_html(data_contract_spec: DataContractSpecification) -> str: quality_specification=quality_specification, style=style_content, datacontract_yaml=datacontract_yaml, - formatted_date=formatted_date, - datacontract_cli_version=datacontract_cli_version, + formatted_date=_formatted_date(), + datacontract_cli_version=get_version(), mermaid_diagram=mermaid_diagram, ) return html_string +def _formatted_date() -> str: + tz = pytz.timezone("UTC") + now = datetime.datetime.now(tz) + return now.strftime("%d %b %Y %H:%M:%S UTC") + + def get_version() -> str: try: return version("datacontract_cli") diff --git a/datacontract/export/mermaid_exporter.py b/datacontract/export/mermaid_exporter.py index ea0b0b215..6653b7aa0 100644 --- a/datacontract/export/mermaid_exporter.py +++ b/datacontract/export/mermaid_exporter.py @@ -1,3 +1,5 @@ +from open_data_contract_standard.model import OpenDataContractStandard + from datacontract.export.exporter import Exporter from datacontract.model.data_contract_specification import DataContractSpecification @@ -7,26 +9,89 @@ def export(self, data_contract, model, server, sql_server_type, export_args) -> return to_mermaid(data_contract) -def to_mermaid(data_contract_spec: DataContractSpecification) -> str | None: - mmd_entity = "erDiagram\n\t" - mmd_references = [] +def to_mermaid(data_contract_spec: DataContractSpecification | OpenDataContractStandard) -> str | None: + if isinstance(data_contract_spec, DataContractSpecification): + return dcs_to_mermaid(data_contract_spec) + elif isinstance(data_contract_spec, OpenDataContractStandard): + return odcs_to_mermaid(data_contract_spec) + else: + return None + + +def dcs_to_mermaid(data_contract_spec: DataContractSpecification) -> str | None: try: + if not data_contract_spec.models: + return None + + mmd_entity = "erDiagram\n" + mmd_references = [] + for model_name, model in data_contract_spec.models.items(): entity_block = "" + for field_name, field in model.fields.items(): - entity_block += f"\t{field_name.replace('#', 'Nb').replace(' ', '_').replace('/', 'by')}{'🔑' if field.primaryKey or (field.unique and field.required) else ''}{'⌘' if field.references else ''} {field.type}\n" + clean_name = _sanitize_name(field_name) + indicators = "" + + if field.primaryKey or (field.unique and field.required): + indicators += "🔑" + if field.references: + indicators += "⌘" + + field_type = field.type or "unknown" + entity_block += f"\t{clean_name}{indicators} {field_type}\n" + if field.references: - mmd_references.append( - f'"📑{field.references.split(".")[0] if "." in field.references else ""}"' - + "}o--{ ||" - + f'"📑{model_name}"' - ) + referenced_model = field.references.split(".")[0] if "." in field.references else "" + if referenced_model: + mmd_references.append(f'"📑{referenced_model}"' + "}o--{ ||" + f'"📑{model_name}"') + mmd_entity += f'\t"**{model_name}**"' + "{\n" + entity_block + "}\n" - if mmd_entity == "": + if mmd_references: + mmd_entity += "\n" + "\n".join(mmd_references) + + return f"{mmd_entity}\n" + + except Exception as e: + print(f"Error generating DCS mermaid diagram: {e}") + return None + + +def odcs_to_mermaid(data_contract_spec: OpenDataContractStandard) -> str | None: + try: + if not data_contract_spec.schema_: return None - else: - return f"{mmd_entity}\n" + + mmd_entity = "erDiagram\n" + + for schema in data_contract_spec.schema_: + schema_name = schema.name or schema.physicalName + entity_block = "" + + if schema.properties: + for prop in schema.properties: + clean_name = _sanitize_name(prop.name) + indicators = "" + + if prop.primaryKey: + indicators += "🔑" + if getattr(prop, "partitioned", False): + indicators += "🔀" + if getattr(prop, "criticalDataElement", False): + indicators += "⚠️" + + prop_type = prop.logicalType or prop.physicalType or "unknown" + entity_block += f"\t{clean_name}{indicators} {prop_type}\n" + + mmd_entity += f'\t"**{schema_name}**"' + "{\n" + entity_block + "}\n" + + return f"{mmd_entity}\n" + except Exception as e: - print(f"error : {e}") + print(f"Error generating ODCS mermaid diagram: {e}") return None + + +def _sanitize_name(name: str) -> str: + return name.replace("#", "Nb").replace(" ", "_").replace("/", "by") diff --git a/datacontract/export/odcs_v3_exporter.py b/datacontract/export/odcs_v3_exporter.py index 109f023bd..4dfe9e7f9 100644 --- a/datacontract/export/odcs_v3_exporter.py +++ b/datacontract/export/odcs_v3_exporter.py @@ -23,6 +23,12 @@ def export(self, data_contract, model, server, sql_server_type, export_args) -> def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str: + result = to_odcs_v3(data_contract_spec) + + return result.to_yaml() + + +def to_odcs_v3(data_contract_spec: DataContractSpecification) -> OpenDataContractStandard: result = OpenDataContractStandard( apiVersion="v3.0.1", kind="DataContract", @@ -31,7 +37,6 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str: version=data_contract_spec.info.version, status=to_status(data_contract_spec.info.status), ) - if data_contract_spec.terms is not None: result.description = Description( purpose=data_contract_spec.terms.description.strip() @@ -42,12 +47,10 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str: if data_contract_spec.terms.limitations is not None else None, ) - result.schema_ = [] for model_key, model_value in data_contract_spec.models.items(): odcs_schema = to_odcs_schema(model_key, model_value) result.schema_.append(odcs_schema) - if data_contract_spec.servicelevels is not None: slas = [] if data_contract_spec.servicelevels.availability is not None: @@ -65,7 +68,6 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str: if len(slas) > 0: result.slaProperties = slas - if data_contract_spec.info.contact is not None: support = [] if data_contract_spec.info.contact.email is not None: @@ -74,7 +76,6 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str: support.append(Support(channel="other", url=data_contract_spec.info.contact.url)) if len(support) > 0: result.support = support - if data_contract_spec.servers is not None and len(data_contract_spec.servers) > 0: servers = [] @@ -126,18 +127,15 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str: if len(servers) > 0: result.servers = servers - custom_properties = [] if data_contract_spec.info.owner is not None: custom_properties.append(CustomProperty(property="owner", value=data_contract_spec.info.owner)) if data_contract_spec.info.model_extra is not None: for key, value in data_contract_spec.info.model_extra.items(): custom_properties.append(CustomProperty(property=key, value=value)) - if len(custom_properties) > 0: result.customProperties = custom_properties - - return result.to_yaml() + return result def to_odcs_schema(model_key, model_value: Model) -> SchemaObject: diff --git a/datacontract/imports/excel_importer.py b/datacontract/imports/excel_importer.py index bf8089a0d..16a61fc1c 100644 --- a/datacontract/imports/excel_importer.py +++ b/datacontract/imports/excel_importer.py @@ -31,8 +31,11 @@ class ExcelImporter(Importer): def import_source( - self, data_contract_specification: DataContractSpecification, source: str, import_args: dict - ) -> OpenDataContractStandard: + self, + data_contract_specification: DataContractSpecification | OpenDataContractStandard, + source: str, + import_args: dict, + ) -> DataContractSpecification | OpenDataContractStandard: return import_excel_as_odcs(source) diff --git a/datacontract/imports/importer.py b/datacontract/imports/importer.py index 9ff7ea020..0dd033608 100644 --- a/datacontract/imports/importer.py +++ b/datacontract/imports/importer.py @@ -12,7 +12,7 @@ def __init__(self, import_format) -> None: @abstractmethod def import_source( self, - data_contract_specification: DataContractSpecification, + data_contract_specification: DataContractSpecification | OpenDataContractStandard, source: str, import_args: dict, ) -> DataContractSpecification | OpenDataContractStandard: @@ -39,3 +39,12 @@ class ImportFormat(str, Enum): @classmethod def get_supported_formats(cls): return list(map(lambda c: c.value, cls)) + + +class Spec(str, Enum): + datacontract_specification = "datacontract_specification" + odcs = "odcs" + + @classmethod + def get_supported_types(cls): + return list(map(lambda c: c.value, cls)) diff --git a/datacontract/imports/odcs_importer.py b/datacontract/imports/odcs_importer.py index f189ef74e..3e40ce090 100644 --- a/datacontract/imports/odcs_importer.py +++ b/datacontract/imports/odcs_importer.py @@ -48,9 +48,9 @@ def import_odcs(data_contract_specification: DataContractSpecification, source: engine="datacontract", ) elif odcs_api_version.startswith("v3."): - from datacontract.imports.odcs_v3_importer import import_odcs_v3 + from datacontract.imports.odcs_v3_importer import import_odcs_v3_as_dcs - return import_odcs_v3(data_contract_specification, source) + return import_odcs_v3_as_dcs(data_contract_specification, source) else: raise DataContractException( type="schema", diff --git a/datacontract/imports/odcs_v3_importer.py b/datacontract/imports/odcs_v3_importer.py index a1042bc43..a1420d48a 100644 --- a/datacontract/imports/odcs_v3_importer.py +++ b/datacontract/imports/odcs_v3_importer.py @@ -29,17 +29,18 @@ class OdcsImporter(Importer): def import_source( self, data_contract_specification: DataContractSpecification, source: str, import_args: dict ) -> DataContractSpecification: - return import_odcs_v3(data_contract_specification, source) + return import_odcs_v3_as_dcs(data_contract_specification, source) -def import_odcs_v3(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification: +def import_odcs_v3_as_dcs( + data_contract_specification: DataContractSpecification, source: str +) -> DataContractSpecification: source_str = read_resource(source) - return import_odcs_v3_from_str(data_contract_specification, source_str) + odcs = parse_odcs_v3_from_str(source_str) + return import_from_odcs(data_contract_specification, odcs) -def import_odcs_v3_from_str( - data_contract_specification: DataContractSpecification, source_str: str -) -> DataContractSpecification: +def parse_odcs_v3_from_str(source_str): try: odcs = OpenDataContractStandard.from_string(source_str) except Exception as e: @@ -50,11 +51,10 @@ def import_odcs_v3_from_str( engine="datacontract", original_exception=e, ) - - return import_from_odcs_model(data_contract_specification, odcs) + return odcs -def import_from_odcs_model(data_contract_specification, odcs): +def import_from_odcs(data_contract_specification: DataContractSpecification, odcs: OpenDataContractStandard): data_contract_specification.id = odcs.id data_contract_specification.info = import_info(odcs) data_contract_specification.servers = import_servers(odcs) diff --git a/datacontract/imports/spark_importer.py b/datacontract/imports/spark_importer.py index b4207009f..e74177bec 100644 --- a/datacontract/imports/spark_importer.py +++ b/datacontract/imports/spark_importer.py @@ -60,9 +60,7 @@ def import_spark( if dataframe is not None: if not isinstance(dataframe, DataFrame): raise TypeError("Expected 'dataframe' to be a pyspark.sql.DataFrame") - data_contract_specification.models[source] = import_from_spark_df( - spark, source, dataframe, description - ) + data_contract_specification.models[source] = import_from_spark_df(spark, source, dataframe, description) return data_contract_specification if not source: @@ -70,14 +68,11 @@ def import_spark( for table_name in map(str.strip, source.split(",")): df = spark.read.table(table_name) - data_contract_specification.models[table_name] = import_from_spark_df( - spark, table_name, df, description - ) + data_contract_specification.models[table_name] = import_from_spark_df(spark, table_name, df, description) return data_contract_specification - def import_from_spark_df(spark: SparkSession, source: str, df: DataFrame, description: str) -> Model: """ Converts a Spark DataFrame into a Model. @@ -96,7 +91,7 @@ def import_from_spark_df(spark: SparkSession, source: str, df: DataFrame, descri if description is None: model.description = _table_comment_from_spark(spark, source) - else: + else: model.description = description for field in schema: diff --git a/datacontract/imports/sql_importer.py b/datacontract/imports/sql_importer.py index c51e4272c..c08efaee6 100644 --- a/datacontract/imports/sql_importer.py +++ b/datacontract/imports/sql_importer.py @@ -105,7 +105,7 @@ def to_dialect(import_args: dict) -> Dialects | None: return None -def to_physical_type_key(dialect: Dialects | None) -> str: +def to_physical_type_key(dialect: Dialects | str | None) -> str: dialect_map = { Dialects.TSQL: "sqlserverType", Dialects.POSTGRES: "postgresType", @@ -116,6 +116,8 @@ def to_physical_type_key(dialect: Dialects | None) -> str: Dialects.MYSQL: "mysqlType", Dialects.DATABRICKS: "databricksType", } + if isinstance(dialect, str): + dialect = Dialects[dialect.upper()] if dialect.upper() in Dialects.__members__ else None return dialect_map.get(dialect, "physicalType") @@ -198,7 +200,7 @@ def get_precision_scale(column): return None, None -def map_type_from_sql(sql_type: str): +def map_type_from_sql(sql_type: str) -> str | None: if sql_type is None: return None diff --git a/datacontract/imports/unity_importer.py b/datacontract/imports/unity_importer.py index 403438cb6..e85e9aae3 100644 --- a/datacontract/imports/unity_importer.py +++ b/datacontract/imports/unity_importer.py @@ -1,14 +1,14 @@ import json import os -from typing import List, Optional +from typing import List from databricks.sdk import WorkspaceClient from databricks.sdk.service.catalog import ColumnInfo, TableInfo -from pyspark.sql import types +from open_data_contract_standard.model import OpenDataContractStandard from datacontract.imports.importer import Importer -from datacontract.imports.spark_importer import _field_from_struct_type -from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model +from datacontract.imports.sql_importer import map_type_from_sql, to_physical_type_key +from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model, Server from datacontract.model.exceptions import DataContractException @@ -18,8 +18,11 @@ class UnityImporter(Importer): """ def import_source( - self, data_contract_specification: DataContractSpecification, source: str, import_args: dict - ) -> DataContractSpecification: + self, + data_contract_specification: DataContractSpecification | OpenDataContractStandard, + source: str, + import_args: dict, + ) -> DataContractSpecification | OpenDataContractStandard: """ Import data contract specification from a source. @@ -35,15 +38,14 @@ def import_source( if source is not None: data_contract_specification = import_unity_from_json(data_contract_specification, source) else: - data_contract_specification = import_unity_from_api( - data_contract_specification, import_args.get("unity_table_full_name") - ) + unity_table_full_name_list = import_args.get("unity_table_full_name") + data_contract_specification = import_unity_from_api(data_contract_specification, unity_table_full_name_list) return data_contract_specification def import_unity_from_json( - data_contract_specification: DataContractSpecification, source: str -) -> DataContractSpecification: + data_contract_specification: DataContractSpecification | OpenDataContractStandard, source: str +) -> DataContractSpecification | OpenDataContractStandard: """ Import data contract specification from a JSON file. @@ -71,39 +73,66 @@ def import_unity_from_json( def import_unity_from_api( - data_contract_specification: DataContractSpecification, unity_table_full_name: Optional[str] = None + data_contract_specification: DataContractSpecification, unity_table_full_name_list: List[str] = None ) -> DataContractSpecification: """ Import data contract specification from Unity Catalog API. :param data_contract_specification: The data contract specification to be imported. :type data_contract_specification: DataContractSpecification - :param unity_table_full_name: The full name of the Unity table. - :type unity_table_full_name: Optional[str] + :param unity_table_full_name_list: The full name of the Unity table. + :type unity_table_full_name_list: list[str] :return: The imported data contract specification. :rtype: DataContractSpecification :raises DataContractException: If there is an error retrieving the schema from the API. """ try: - workspace_client = WorkspaceClient() - unity_schema: TableInfo = workspace_client.tables.get(unity_table_full_name) + # print(f"Retrieving Unity Catalog schema for table: {unity_table_full_name}") + host, token = os.getenv("DATACONTRACT_DATABRICKS_SERVER_HOSTNAME"), os.getenv("DATACONTRACT_DATABRICKS_TOKEN") + # print(f"Databricks host: {host}, token: {'***' if token else 'not set'}") + if not host: + raise DataContractException( + type="configuration", + name="Databricks configuration", + reason="DATACONTRACT_DATABRICKS_SERVER_HOSTNAME environment variable is not set", + engine="datacontract", + ) + if not token: + raise DataContractException( + type="configuration", + name="Databricks configuration", + reason="DATACONTRACT_DATABRICKS_TOKEN environment variable is not set", + engine="datacontract", + ) + workspace_client = WorkspaceClient(host=host, token=token) except Exception as e: raise DataContractException( type="schema", name="Retrieve unity catalog schema", - reason=f"Failed to retrieve unity catalog schema from databricks profile: {os.getenv('DATABRICKS_CONFIG_PROFILE')}", + reason="Failed to connect to unity catalog schema", engine="datacontract", original_exception=e, ) - convert_unity_schema(data_contract_specification, unity_schema) + for unity_table_full_name in unity_table_full_name_list: + try: + unity_schema: TableInfo = workspace_client.tables.get(unity_table_full_name) + except Exception as e: + raise DataContractException( + type="schema", + name="Retrieve unity catalog schema", + reason=f"Unity table {unity_table_full_name} not found", + engine="datacontract", + original_exception=e, + ) + data_contract_specification = convert_unity_schema(data_contract_specification, unity_schema) return data_contract_specification def convert_unity_schema( - data_contract_specification: DataContractSpecification, unity_schema: TableInfo -) -> DataContractSpecification: + data_contract_specification: DataContractSpecification | OpenDataContractStandard, unity_schema: TableInfo +) -> DataContractSpecification | OpenDataContractStandard: """ Convert Unity schema to data contract specification. @@ -117,6 +146,21 @@ def convert_unity_schema( if data_contract_specification.models is None: data_contract_specification.models = {} + if data_contract_specification.servers is None: + data_contract_specification.servers = {} + + # Configure databricks server with catalog and schema from Unity table info + schema_name = unity_schema.schema_name + catalog_name = unity_schema.catalog_name + if catalog_name and schema_name: + server_name = "myserver" # Default server name + + data_contract_specification.servers[server_name] = Server( + type="databricks", + catalog=catalog_name, + schema=schema_name, + ) + fields = import_table_fields(unity_schema.columns) table_id = unity_schema.name or unity_schema.table_id @@ -149,25 +193,21 @@ def import_table_fields(columns: List[ColumnInfo]) -> dict[str, Field]: imported_fields = {} for column in columns: - struct_field: types.StructField = _type_json_to_spark_field(column.type_json) - imported_fields[column.name] = _field_from_struct_type(struct_field) + imported_fields[column.name] = _to_field(column) return imported_fields -def _type_json_to_spark_field(type_json: str) -> types.StructField: - """ - Parses a JSON string representing a Spark field and returns a StructField object. +def _to_field(column: ColumnInfo) -> Field: + field = Field() + if column.type_name is not None: + sql_type = str(column.type_text) + field.type = map_type_from_sql(sql_type) + physical_type_key = to_physical_type_key("databricks") + field.config = { + physical_type_key: sql_type, + } + field.required = column.nullable is None or not column.nullable + field.description = column.comment if column.comment else None - The reason we do this is to leverage the Spark JSON schema parser to handle the - complexity of the Spark field types. The field `type_json` in the Unity API is - the output of a `StructField.jsonValue()` call. - - :param type_json: The JSON string representing the Spark field. - :type type_json: str - - :return: The StructField object. - :rtype: types.StructField - """ - type_dict = json.loads(type_json) - return types.StructField.fromJson(type_dict) + return field diff --git a/datacontract/integration/datamesh_manager.py b/datacontract/integration/datamesh_manager.py index f314f1570..52020bf02 100644 --- a/datacontract/integration/datamesh_manager.py +++ b/datacontract/integration/datamesh_manager.py @@ -4,6 +4,9 @@ from datacontract.model.run import Run +# used to retrieve the HTML location of the published data contract or test results +RESPONSE_HEADER_LOCATION_HTML = "location-html" + def publish_test_results_to_datamesh_manager(run: Run, publish_url: str, ssl_verification: bool): try: @@ -38,7 +41,12 @@ def publish_test_results_to_datamesh_manager(run: Run, publish_url: str, ssl_ver if response.status_code != 200: run.log_error(f"Error publishing test results to Data Mesh Manager: {response.text}") return - run.log_info(f"Published test results to {url}") + run.log_info("Published test results successfully") + + location_html = response.headers.get(RESPONSE_HEADER_LOCATION_HTML) + if location_html is not None and len(location_html) > 0: + print(f"🚀 Open {location_html}") + except Exception as e: run.log_error(f"Failed publishing test results. Error: {str(e)}") @@ -67,6 +75,12 @@ def publish_data_contract_to_datamesh_manager(data_contract_dict: dict, ssl_veri if response.status_code != 200: print(f"Error publishing data contract to Data Mesh Manager: {response.text}") exit(1) - print(f"Published data contract to {url}") + + print("✅ Published data contract successfully") + + location_html = response.headers.get(RESPONSE_HEADER_LOCATION_HTML) + if location_html is not None and len(location_html) > 0: + print(f"🚀 Open {location_html}") + except Exception as e: print(f"Failed publishing data contract. Error: {str(e)}") diff --git a/datacontract/lint/resolve.py b/datacontract/lint/resolve.py index 28d1f7d1d..a116246ab 100644 --- a/datacontract/lint/resolve.py +++ b/datacontract/lint/resolve.py @@ -5,8 +5,9 @@ import fastjsonschema import yaml from fastjsonschema import JsonSchemaValueException +from open_data_contract_standard.model import OpenDataContractStandard -from datacontract.imports.odcs_v3_importer import import_odcs_v3_from_str +from datacontract.imports.odcs_v3_importer import import_from_odcs, parse_odcs_v3_from_str from datacontract.lint.resources import read_resource from datacontract.lint.schema import fetch_schema from datacontract.lint.urls import fetch_resource @@ -46,6 +47,34 @@ def resolve_data_contract( ) +def resolve_data_contract_v2( + data_contract_location: str = None, + data_contract_str: str = None, + data_contract: DataContractSpecification | OpenDataContractStandard = None, + schema_location: str = None, + inline_definitions: bool = False, + inline_quality: bool = False, +) -> DataContractSpecification | OpenDataContractStandard: + if data_contract_location is not None: + return resolve_data_contract_from_location_v2( + data_contract_location, schema_location, inline_definitions, inline_quality + ) + elif data_contract_str is not None: + return _resolve_data_contract_from_str_v2( + data_contract_str, schema_location, inline_definitions, inline_quality + ) + elif data_contract is not None: + return data_contract + else: + raise DataContractException( + type="lint", + result=ResultEnum.failed, + name="Check that data contract YAML is valid", + reason="Data contract needs to be provided", + engine="datacontract", + ) + + def resolve_data_contract_dict( data_contract_location: str = None, data_contract_str: str = None, @@ -67,6 +96,13 @@ def resolve_data_contract_dict( ) +def resolve_data_contract_from_location_v2( + location, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False +) -> DataContractSpecification | OpenDataContractStandard: + data_contract_str = read_resource(location) + return _resolve_data_contract_from_str_v2(data_contract_str, schema_location, inline_definitions, inline_quality) + + def resolve_data_contract_from_location( location, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False ) -> DataContractSpecification: @@ -242,6 +278,21 @@ def _get_quality_ref_file(quality_spec: str | object) -> str | object: return quality_spec +def _resolve_data_contract_from_str_v2( + data_contract_str, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False +) -> DataContractSpecification | OpenDataContractStandard: + yaml_dict = _to_yaml(data_contract_str) + + if is_open_data_contract_standard(yaml_dict): + logging.info("Importing ODCS v3") + # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly + odcs = parse_odcs_v3_from_str(data_contract_str) + return odcs + + logging.info("Importing DCS") + return _resolve_dcs_from_yaml_dict(inline_definitions, inline_quality, schema_location, yaml_dict) + + def _resolve_data_contract_from_str( data_contract_str, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False ) -> DataContractSpecification: @@ -250,15 +301,19 @@ def _resolve_data_contract_from_str( if is_open_data_contract_standard(yaml_dict): logging.info("Importing ODCS v3") # if ODCS, then validate the ODCS schema and import to DataContractSpecification directly + odcs = parse_odcs_v3_from_str(data_contract_str) + data_contract_specification = DataContractSpecification(dataContractSpecification="1.1.0") - return import_odcs_v3_from_str(data_contract_specification, source_str=data_contract_str) - else: - logging.info("Importing DCS") + return import_from_odcs(data_contract_specification, odcs) + + logging.info("Importing DCS") + return _resolve_dcs_from_yaml_dict(inline_definitions, inline_quality, schema_location, yaml_dict) + +def _resolve_dcs_from_yaml_dict(inline_definitions, inline_quality, schema_location, yaml_dict): _validate_data_contract_specification_schema(yaml_dict, schema_location) data_contract_specification = yaml_dict spec = DataContractSpecification(**data_contract_specification) - if inline_definitions: inline_definitions_into_data_contract(spec) ## Suppress DeprecationWarning when accessing spec.quality, @@ -276,7 +331,6 @@ def _resolve_data_contract_from_str( ) if spec_quality and inline_quality: _resolve_quality_ref(spec_quality) - return spec diff --git a/datacontract/templates/datacontract.html b/datacontract/templates/datacontract.html index 41c393f9f..24d65357f 100644 --- a/datacontract/templates/datacontract.html +++ b/datacontract/templates/datacontract.html @@ -41,6 +41,10 @@

{{ datacontract.id }} + + + Data Contract Specification v{{ datacontract.dataContractSpecification }} +

{% if datacontract.tags %} diff --git a/datacontract/templates/datacontract_odcs.html b/datacontract/templates/datacontract_odcs.html new file mode 100644 index 000000000..cd9ab779d --- /dev/null +++ b/datacontract/templates/datacontract_odcs.html @@ -0,0 +1,666 @@ + + + + Data Contract + + + {# #} + + + + + + +
+ + +
+ +
+
+
+
+

+ Data Contract

+
+ {{ datacontract.id }} +
+
+ + Open Data Contract Standard {{ datacontract.apiVersion }} + + + {% if datacontract.tags %} +
+ {% for tag in datacontract.tags %} + + {{ tag }} + + {% endfor %} +
+ {% endif %} +
+
+
+ +
+
+
+ +
+
+
+
+

Fundamentals

+

Basic information about the data contract

+
+ +
+
+
+ + {% if datacontract.name %} +
+
Name
+
{{ datacontract.name }}
+
+ {% endif %} + + {% if datacontract.version %} +
+
Version
+
{{ datacontract.version }}
+
+ {% endif %} + + {% if datacontract.status %} +
+
Status
+
{{ datacontract.status }}
+
+ {% endif %} + + {% if datacontract.dataProduct %} +
+
Data Product
+
{{ datacontract.dataProduct }}
+
+ {% endif %} + + {% if datacontract.tenant %} +
+
Tenant
+
{{ datacontract.tenant }}
+
+ {% endif %} + + {% if datacontract.description %} + {% if datacontract.description.purpose %} +
+
Purpose
+
{{ datacontract.description.purpose }}
+
+ {% endif %} + + {% if datacontract.description.usage %} +
+
Usage
+
{{ datacontract.description.usage }}
+
+ {% endif %} + + {% if datacontract.description.limitations %} +
+
Limitations
+
{{ datacontract.description.limitations }}
+
+ {% endif %} + {% endif %} + + {% if datacontract.contractCreatedTs %} +
+
Contract Created
+
{{ datacontract.contractCreatedTs }}
+
+ {% endif %} +
+
+
+
+ +
+
+

Entity Relationship + Diagram

+

Visual representation of data model relationships

+
+
+
+
+ 
+                    {{ mermaid_diagram }}
+
+
+
+
+ +
+ +
+
+
+

+ Schema +

+

The data schema and structure

+
+
+ + {% for schema in datacontract.schema_ %} + +
+
+
+
+ + + + + + + + + + + + + + + + {% for property in schema.properties %} + + + + + + + + {% endfor %} + + {% if schema.quality %} + + {% for quality in schema.quality %} + + + + {% endfor %} + + {% endif %} +
+ {% if schema.name %} + {{ schema.name }} + {% endif %} + {% if schema.physicalName and schema.physicalName != schema.name %} + ({{ schema.physicalName }}) + {% endif %} + {{ schema.physicalType or schema.logicalType }} +
{{ schema.description }} +
+ {% if schema.dataGranularityDescription %} +
Granularity: {{ + schema.dataGranularityDescription }} +
+ {% endif %} +
+ Property + + Business Name + + Type + + Required + + Description +
+
{{ property.name }}
+ {% if property.primaryKey %} + Primary Key + {% endif %} + {% if property.partitioned %} + Partitioned + {% endif %} + {% if property.criticalDataElement %} + Critical + {% endif %} + 		{{ + property.businessName or "-" }} + +
{{ property.logicalType }}
+ {% if property.physicalType and property.physicalType != + property.logicalType %} +
{{ property.physicalType }}
+ {% endif %} + 		+ {% if property.required %} + Yes + {% else %} + No + {% endif %} + {{ property.description or "-" + }} +
+
+
{{ quality.rule }}
+
{{ quality.description }}
+ {% if quality.dimension %} + {{ quality.dimension }} + {% endif %} +
+
+
+
+
+
+ {% endfor %} +
+ + {% if datacontract.support %} +
+
+

Support

+

Support channels and resources

+
+ +
    + {% for support in datacontract.support %} +
  • +
    +
    +
    +

    {{ support.channel }}

    + {% if support.description %} +

    {{ support.description }}

    + {% endif %} + {% if support.tool %} + {{ support.tool }} + {% endif %} +
    +
    + {% if support.url %} +
    + {{ support.url + }} +
    + {% endif %} +
    +
    • + {% endfor %} +
+
+ {% endif %} + + {% if datacontract.price %} +
+
+

Pricing

+

Data contract pricing information

+
+ +
+
+
+
+ {{ datacontract.price.priceAmount }} {{ datacontract.price.priceCurrency }} +
+
+ per {{ datacontract.price.priceUnit }} +
+
+
+
+
+ {% endif %} + + + {% if datacontract.team %} +
+
+

Team

+

Team members and their roles

+
+ +
+ + + + + + + + + + + + {% for member in datacontract.team %} + + + + + + + + {% endfor %} + +
+ Username + + Role + + Date In + + Date Out + + Comment +
{{ + member.username }} + {{ member.role }} + {{ member.dateIn or + "-" }} + {{ member.dateOut or + "-" }} + {{ member.comment or + "-" }} +
+
+
+ {% endif %} + + {% if datacontract.roles %} +
+
+

Access Roles

+

Access roles and approval workflows

+
+ +
+ + + + + + + + + + + {% for role in datacontract.roles %} + + + + + + + {% endfor %} + +
+ Role + + Access + + First Level Approver + + Second Level Approver +
{{ + role.role }} + + + {{ role.access or "-" }} + + {{ + role.firstLevelApprovers or "-" }} + {{ + role.secondLevelApprovers or "-" }} +
+
+
+ {% endif %} + + + {% if datacontract.slaProperties %} +
+
+

Service Level Agreements

+

SLA properties and commitments

+
+ +
+ + + + + + + + + + + + {% for sla in datacontract.slaProperties %} + + + + + + + + {% endfor %} + +
PropertyValueUnitElementDriver
{{ sla.property }}{{ sla.value }}{{ sla.unit or "-" }}{{ sla.element or "-" }}{{ sla.driver or "-" }}
+
+
+ {% endif %} + + + {% if datacontract.servers %} +
+
+

Servers

+

Infrastructure servers of the data contract

+
+ +
    + {% for server in datacontract.servers %} + {{ render_partial('partials/server.html', server_name = server.server, server = + server) }} + {% endfor %} +
+ +
+ {% endif %} + + + {% if datacontract.customProperties %} +
+
+

Custom Properties

+

Additional custom properties and metadata

+
+ +
+ + + + + + + + + {% for prop in datacontract.customProperties %} + + + + + {% endfor %} + +
PropertyValue
{{ prop.property }} + {% if prop.value is iterable and prop.value is not string %} + {% for item in prop.value %}{{ item }}{% if not loop.last %}, {% endif %}{% endfor %} + {% else %} + {{ prop.value }} + {% endif %} +
+
+
+ {% endif %} + +
+
+ +
+ Created at {{formatted_date}} with Data Contract CLI v{{datacontract_cli_version}} +
+ +
+
+ + +
+ +
+
+
+
+ +
+
+
+ +

Data Contract YAML

+
+ 
{{datacontract_yaml}}
+
+
+ +
+
+
+
+ + + + + +
+ + + diff --git a/datacontract/templates/partials/server.html b/datacontract/templates/partials/server.html index dc49cfc6f..b36096e4b 100644 --- a/datacontract/templates/partials/server.html +++ b/datacontract/templates/partials/server.html @@ -195,6 +195,7 @@
{% endif %} + {% if server.model_extra %} {% for key, value in server.model_extra.items() %}
@@ -205,5 +206,6 @@
{% endfor %} + {% endif %} \ No newline at end of file diff --git a/datacontract/templates/style/output.css b/datacontract/templates/style/output.css index 6d1d5c70a..744e9313b 100644 --- a/datacontract/templates/style/output.css +++ b/datacontract/templates/style/output.css @@ -1,113 +1,5 @@ -*, ::before, ::after { - --tw-border-spacing-x: 0; - --tw-border-spacing-y: 0; - --tw-translate-x: 0; - --tw-translate-y: 0; - --tw-rotate: 0; - --tw-skew-x: 0; - --tw-skew-y: 0; - --tw-scale-x: 1; - --tw-scale-y: 1; - --tw-pan-x: ; - --tw-pan-y: ; - --tw-pinch-zoom: ; - --tw-scroll-snap-strictness: proximity; - --tw-gradient-from-position: ; - --tw-gradient-via-position: ; - --tw-gradient-to-position: ; - --tw-ordinal: ; - --tw-slashed-zero: ; - --tw-numeric-figure: ; - --tw-numeric-spacing: ; - --tw-numeric-fraction: ; - --tw-ring-inset: ; - --tw-ring-offset-width: 0px; - --tw-ring-offset-color: #fff; - --tw-ring-color: rgb(59 130 246 / 0.5); - --tw-ring-offset-shadow: 0 0 #0000; - --tw-ring-shadow: 0 0 #0000; - --tw-shadow: 0 0 #0000; - --tw-shadow-colored: 0 0 #0000; - --tw-blur: ; - --tw-brightness: ; - --tw-contrast: ; - --tw-grayscale: ; - --tw-hue-rotate: ; - --tw-invert: ; - --tw-saturate: ; - --tw-sepia: ; - --tw-drop-shadow: ; - --tw-backdrop-blur: ; - --tw-backdrop-brightness: ; - --tw-backdrop-contrast: ; - --tw-backdrop-grayscale: ; - --tw-backdrop-hue-rotate: ; - --tw-backdrop-invert: ; - --tw-backdrop-opacity: ; - --tw-backdrop-saturate: ; - --tw-backdrop-sepia: ; - --tw-contain-size: ; - --tw-contain-layout: ; - --tw-contain-paint: ; - --tw-contain-style: ; -} - -::backdrop { - --tw-border-spacing-x: 0; - --tw-border-spacing-y: 0; - --tw-translate-x: 0; - --tw-translate-y: 0; - --tw-rotate: 0; - --tw-skew-x: 0; - --tw-skew-y: 0; - --tw-scale-x: 1; - --tw-scale-y: 1; - --tw-pan-x: ; - --tw-pan-y: ; - --tw-pinch-zoom: ; - --tw-scroll-snap-strictness: proximity; - --tw-gradient-from-position: ; - --tw-gradient-via-position: ; - --tw-gradient-to-position: ; - --tw-ordinal: ; - --tw-slashed-zero: ; - --tw-numeric-figure: ; - --tw-numeric-spacing: ; - --tw-numeric-fraction: ; - --tw-ring-inset: ; - --tw-ring-offset-width: 0px; - --tw-ring-offset-color: #fff; - --tw-ring-color: rgb(59 130 246 / 0.5); - --tw-ring-offset-shadow: 0 0 #0000; - --tw-ring-shadow: 0 0 #0000; - --tw-shadow: 0 0 #0000; - --tw-shadow-colored: 0 0 #0000; - --tw-blur: ; - --tw-brightness: ; - --tw-contrast: ; - --tw-grayscale: ; - --tw-hue-rotate: ; - --tw-invert: ; - --tw-saturate: ; - --tw-sepia: ; - --tw-drop-shadow: ; - --tw-backdrop-blur: ; - --tw-backdrop-brightness: ; - --tw-backdrop-contrast: ; - --tw-backdrop-grayscale: ; - --tw-backdrop-hue-rotate: ; - --tw-backdrop-invert: ; - --tw-backdrop-opacity: ; - --tw-backdrop-saturate: ; - --tw-backdrop-sepia: ; - --tw-contain-size: ; - --tw-contain-layout: ; - --tw-contain-paint: ; - --tw-contain-style: ; -} - /* -! tailwindcss v3.4.17 | MIT License | https://tailwindcss.com +! tailwindcss v3.4.3 | MIT License | https://tailwindcss.com */ /* @@ -550,10 +442,152 @@ video { /* Make elements with the HTML hidden attribute stay hidden by default */ -[hidden]:where(:not([hidden="until-found"])) { +[hidden] { display: none; } +*, ::before, ::after { + --tw-border-spacing-x: 0; + --tw-border-spacing-y: 0; + --tw-translate-x: 0; + --tw-translate-y: 0; + --tw-rotate: 0; + --tw-skew-x: 0; + --tw-skew-y: 0; + --tw-scale-x: 1; + --tw-scale-y: 1; + --tw-pan-x: ; + --tw-pan-y: ; + --tw-pinch-zoom: ; + --tw-scroll-snap-strictness: proximity; + --tw-gradient-from-position: ; + --tw-gradient-via-position: ; + --tw-gradient-to-position: ; + --tw-ordinal: ; + --tw-slashed-zero: ; + --tw-numeric-figure: ; + --tw-numeric-spacing: ; + --tw-numeric-fraction: ; + --tw-ring-inset: ; + --tw-ring-offset-width: 0px; + --tw-ring-offset-color: #fff; + --tw-ring-color: rgb(59 130 246 / 0.5); + --tw-ring-offset-shadow: 0 0 #0000; + --tw-ring-shadow: 0 0 #0000; + --tw-shadow: 0 0 #0000; + --tw-shadow-colored: 0 0 #0000; + --tw-blur: ; + --tw-brightness: ; + --tw-contrast: ; + --tw-grayscale: ; + --tw-hue-rotate: ; + --tw-invert: ; + --tw-saturate: ; + --tw-sepia: ; + --tw-drop-shadow: ; + --tw-backdrop-blur: ; + --tw-backdrop-brightness: ; + --tw-backdrop-contrast: ; + --tw-backdrop-grayscale: ; + --tw-backdrop-hue-rotate: ; + --tw-backdrop-invert: ; + --tw-backdrop-opacity: ; + --tw-backdrop-saturate: ; + --tw-backdrop-sepia: ; + --tw-contain-size: ; + --tw-contain-layout: ; + --tw-contain-paint: ; + --tw-contain-style: ; +} + +::backdrop { + --tw-border-spacing-x: 0; + --tw-border-spacing-y: 0; + --tw-translate-x: 0; + --tw-translate-y: 0; + --tw-rotate: 0; + --tw-skew-x: 0; + --tw-skew-y: 0; + --tw-scale-x: 1; + --tw-scale-y: 1; + --tw-pan-x: ; + --tw-pan-y: ; + --tw-pinch-zoom: ; + --tw-scroll-snap-strictness: proximity; + --tw-gradient-from-position: ; + --tw-gradient-via-position: ; + --tw-gradient-to-position: ; + --tw-ordinal: ; + --tw-slashed-zero: ; + --tw-numeric-figure: ; + --tw-numeric-spacing: ; + --tw-numeric-fraction: ; + --tw-ring-inset: ; + --tw-ring-offset-width: 0px; + --tw-ring-offset-color: #fff; + --tw-ring-color: rgb(59 130 246 / 0.5); + --tw-ring-offset-shadow: 0 0 #0000; + --tw-ring-shadow: 0 0 #0000; + --tw-shadow: 0 0 #0000; + --tw-shadow-colored: 0 0 #0000; + --tw-blur: ; + --tw-brightness: ; + --tw-contrast: ; + --tw-grayscale: ; + --tw-hue-rotate: ; + --tw-invert: ; + --tw-saturate: ; + --tw-sepia: ; + --tw-drop-shadow: ; + --tw-backdrop-blur: ; + --tw-backdrop-brightness: ; + --tw-backdrop-contrast: ; + --tw-backdrop-grayscale: ; + --tw-backdrop-hue-rotate: ; + --tw-backdrop-invert: ; + --tw-backdrop-opacity: ; + --tw-backdrop-saturate: ; + --tw-backdrop-sepia: ; + --tw-contain-size: ; + --tw-contain-layout: ; + --tw-contain-paint: ; + --tw-contain-style: ; +} + +.container { + width: 100%; +} + +@media (min-width: 640px) { + .container { + max-width: 640px; + } +} + +@media (min-width: 768px) { + .container { + max-width: 768px; + } +} + +@media (min-width: 1024px) { + .container { + max-width: 1024px; + } +} + +@media (min-width: 1280px) { + .container { + max-width: 1280px; + } +} + +@media (min-width: 1536px) { + .container { + max-width: 1536px; + } +} + .sr-only { position: absolute; width: 1px; @@ -635,12 +669,12 @@ video { margin-bottom: 0.5rem; } -.-ml-0\.5 { - margin-left: -0.125rem; +.-ml-0 { + margin-left: -0px; } -.mb-2 { - margin-bottom: 0.5rem; +.-ml-0\.5 { + margin-left: -0.125rem; } .mb-3 { @@ -651,6 +685,10 @@ video { margin-bottom: 1.5rem; } +.ml-3 { + margin-left: 0.75rem; +} + .mr-1 { margin-right: 0.25rem; } @@ -798,6 +836,11 @@ video { width: 2rem; } +.w-fit { + width: -moz-fit-content; + width: fit-content; +} + .w-full { width: 100%; } @@ -900,6 +943,12 @@ video { margin-left: calc(0.75rem * calc(1 - var(--tw-space-x-reverse))); } +.space-x-4 > :not([hidden]) ~ :not([hidden]) { + --tw-space-x-reverse: 0; + margin-right: calc(1rem * var(--tw-space-x-reverse)); + margin-left: calc(1rem * calc(1 - var(--tw-space-x-reverse))); +} + .space-x-6 > :not([hidden]) ~ :not([hidden]) { --tw-space-x-reverse: 0; margin-right: calc(1.5rem * var(--tw-space-x-reverse)); @@ -912,6 +961,18 @@ video { margin-left: calc(2rem * calc(1 - var(--tw-space-x-reverse))); } +.space-y-1 > :not([hidden]) ~ :not([hidden]) { + --tw-space-y-reverse: 0; + margin-top: calc(0.25rem * calc(1 - var(--tw-space-y-reverse))); + margin-bottom: calc(0.25rem * var(--tw-space-y-reverse)); +} + +.space-y-4 > :not([hidden]) ~ :not([hidden]) { + --tw-space-y-reverse: 0; + margin-top: calc(1rem * calc(1 - var(--tw-space-y-reverse))); + margin-bottom: calc(1rem * var(--tw-space-y-reverse)); +} + .space-y-6 > :not([hidden]) ~ :not([hidden]) { --tw-space-y-reverse: 0; margin-top: calc(1.5rem * calc(1 - var(--tw-space-y-reverse))); @@ -926,17 +987,17 @@ video { .divide-gray-100 > :not([hidden]) ~ :not([hidden]) { --tw-divide-opacity: 1; - border-color: rgb(243 244 246 / var(--tw-divide-opacity, 1)); + border-color: rgb(243 244 246 / var(--tw-divide-opacity)); } .divide-gray-200 > :not([hidden]) ~ :not([hidden]) { --tw-divide-opacity: 1; - border-color: rgb(229 231 235 / var(--tw-divide-opacity, 1)); + border-color: rgb(229 231 235 / var(--tw-divide-opacity)); } .divide-gray-300 > :not([hidden]) ~ :not([hidden]) { --tw-divide-opacity: 1; - border-color: rgb(209 213 219 / var(--tw-divide-opacity, 1)); + border-color: rgb(209 213 219 / var(--tw-divide-opacity)); } .overflow-hidden { @@ -983,42 +1044,57 @@ video { .bg-blue-50 { --tw-bg-opacity: 1; - background-color: rgb(239 246 255 / var(--tw-bg-opacity, 1)); + background-color: rgb(239 246 255 / var(--tw-bg-opacity)); } .bg-gray-100 { --tw-bg-opacity: 1; - background-color: rgb(243 244 246 / var(--tw-bg-opacity, 1)); + background-color: rgb(243 244 246 / var(--tw-bg-opacity)); } .bg-gray-50 { --tw-bg-opacity: 1; - background-color: rgb(249 250 251 / var(--tw-bg-opacity, 1)); + background-color: rgb(249 250 251 / var(--tw-bg-opacity)); } .bg-gray-500 { --tw-bg-opacity: 1; - background-color: rgb(107 114 128 / var(--tw-bg-opacity, 1)); + background-color: rgb(107 114 128 / var(--tw-bg-opacity)); +} + +.bg-green-50 { + --tw-bg-opacity: 1; + background-color: rgb(240 253 244 / var(--tw-bg-opacity)); } .bg-indigo-100 { --tw-bg-opacity: 1; - background-color: rgb(224 231 255 / var(--tw-bg-opacity, 1)); + background-color: rgb(224 231 255 / var(--tw-bg-opacity)); } .bg-indigo-600 { --tw-bg-opacity: 1; - background-color: rgb(79 70 229 / var(--tw-bg-opacity, 1)); + background-color: rgb(79 70 229 / var(--tw-bg-opacity)); +} + +.bg-purple-50 { + --tw-bg-opacity: 1; + background-color: rgb(250 245 255 / var(--tw-bg-opacity)); +} + +.bg-red-50 { + --tw-bg-opacity: 1; + background-color: rgb(254 242 242 / var(--tw-bg-opacity)); } .bg-white { --tw-bg-opacity: 1; - background-color: rgb(255 255 255 / var(--tw-bg-opacity, 1)); + background-color: rgb(255 255 255 / var(--tw-bg-opacity)); } .bg-yellow-50 { --tw-bg-opacity: 1; - background-color: rgb(254 252 232 / var(--tw-bg-opacity, 1)); + background-color: rgb(254 252 232 / var(--tw-bg-opacity)); } .bg-opacity-75 { @@ -1072,6 +1148,11 @@ video { padding-right: 1.5rem; } +.py-0 { + padding-top: 0px; + padding-bottom: 0px; +} + .py-0\.5 { padding-top: 0.125rem; padding-bottom: 0.125rem; @@ -1092,6 +1173,16 @@ video { padding-bottom: 0.5rem; } +.py-3 { + padding-top: 0.75rem; + padding-bottom: 0.75rem; +} + +.py-4 { + padding-top: 1rem; + padding-bottom: 1rem; +} + .py-5 { padding-top: 1.25rem; padding-bottom: 1.25rem; @@ -1190,12 +1281,16 @@ video { font-weight: 500; } +.font-normal { + font-weight: 400; +} + .font-semibold { font-weight: 600; } -.font-normal { - font-weight: 400; +.uppercase { + text-transform: uppercase; } .italic { @@ -1214,49 +1309,93 @@ video { line-height: 1.75rem; } +.tracking-wider { + letter-spacing: 0.05em; +} + .text-blue-600 { --tw-text-opacity: 1; - color: rgb(37 99 235 / var(--tw-text-opacity, 1)); + color: rgb(37 99 235 / var(--tw-text-opacity)); +} + +.text-blue-700 { + --tw-text-opacity: 1; + color: rgb(29 78 216 / var(--tw-text-opacity)); +} + +.text-blue-800 { + --tw-text-opacity: 1; + color: rgb(30 64 175 / var(--tw-text-opacity)); } .text-gray-400 { --tw-text-opacity: 1; - color: rgb(156 163 175 / var(--tw-text-opacity, 1)); + color: rgb(156 163 175 / var(--tw-text-opacity)); } .text-gray-500 { --tw-text-opacity: 1; - color: rgb(107 114 128 / var(--tw-text-opacity, 1)); + color: rgb(107 114 128 / var(--tw-text-opacity)); } .text-gray-600 { --tw-text-opacity: 1; - color: rgb(75 85 99 / var(--tw-text-opacity, 1)); + color: rgb(75 85 99 / var(--tw-text-opacity)); +} + +.text-gray-700 { + --tw-text-opacity: 1; + color: rgb(55 65 81 / var(--tw-text-opacity)); } .text-gray-800 { --tw-text-opacity: 1; - color: rgb(31 41 55 / var(--tw-text-opacity, 1)); + color: rgb(31 41 55 / var(--tw-text-opacity)); } .text-gray-900 { --tw-text-opacity: 1; - color: rgb(17 24 39 / var(--tw-text-opacity, 1)); + color: rgb(17 24 39 / var(--tw-text-opacity)); +} + +.text-green-600 { + --tw-text-opacity: 1; + color: rgb(22 163 74 / var(--tw-text-opacity)); +} + +.text-green-700 { + --tw-text-opacity: 1; + color: rgb(21 128 61 / var(--tw-text-opacity)); +} + +.text-purple-800 { + --tw-text-opacity: 1; + color: rgb(107 33 168 / var(--tw-text-opacity)); +} + +.text-red-800 { + --tw-text-opacity: 1; + color: rgb(153 27 27 / var(--tw-text-opacity)); } .text-sky-500 { --tw-text-opacity: 1; - color: rgb(14 165 233 / var(--tw-text-opacity, 1)); + color: rgb(14 165 233 / var(--tw-text-opacity)); } .text-white { --tw-text-opacity: 1; - color: rgb(255 255 255 / var(--tw-text-opacity, 1)); + color: rgb(255 255 255 / var(--tw-text-opacity)); } .text-yellow-600 { --tw-text-opacity: 1; - color: rgb(202 138 4 / var(--tw-text-opacity, 1)); + color: rgb(202 138 4 / var(--tw-text-opacity)); +} + +.text-yellow-800 { + --tw-text-opacity: 1; + color: rgb(133 77 14 / var(--tw-text-opacity)); } .shadow { @@ -1289,30 +1428,54 @@ video { .ring-black { --tw-ring-opacity: 1; - --tw-ring-color: rgb(0 0 0 / var(--tw-ring-opacity, 1)); + --tw-ring-color: rgb(0 0 0 / var(--tw-ring-opacity)); } .ring-blue-500\/10 { --tw-ring-color: rgb(59 130 246 / 0.1); } +.ring-blue-600\/20 { + --tw-ring-color: rgb(37 99 235 / 0.2); +} + .ring-gray-300 { --tw-ring-opacity: 1; - --tw-ring-color: rgb(209 213 219 / var(--tw-ring-opacity, 1)); + --tw-ring-color: rgb(209 213 219 / var(--tw-ring-opacity)); } .ring-gray-500\/10 { --tw-ring-color: rgb(107 114 128 / 0.1); } +.ring-gray-600\/20 { + --tw-ring-color: rgb(75 85 99 / 0.2); +} + .ring-gray-900\/5 { --tw-ring-color: rgb(17 24 39 / 0.05); } +.ring-green-600\/20 { + --tw-ring-color: rgb(22 163 74 / 0.2); +} + +.ring-purple-600\/20 { + --tw-ring-color: rgb(147 51 234 / 0.2); +} + +.ring-red-600\/20 { + --tw-ring-color: rgb(220 38 38 / 0.2); +} + .ring-yellow-500\/10 { --tw-ring-color: rgb(234 179 8 / 0.1); } +.ring-yellow-600\/20 { + --tw-ring-color: rgb(202 138 4 / 0.2); +} + .ring-opacity-5 { --tw-ring-opacity: 0.05; } @@ -1331,32 +1494,37 @@ video { .placeholder\:text-gray-400::-moz-placeholder { --tw-text-opacity: 1; - color: rgb(156 163 175 / var(--tw-text-opacity, 1)); + color: rgb(156 163 175 / var(--tw-text-opacity)); } .placeholder\:text-gray-400::placeholder { --tw-text-opacity: 1; - color: rgb(156 163 175 / var(--tw-text-opacity, 1)); + color: rgb(156 163 175 / var(--tw-text-opacity)); } .hover\:bg-gray-50:hover { --tw-bg-opacity: 1; - background-color: rgb(249 250 251 / var(--tw-bg-opacity, 1)); + background-color: rgb(249 250 251 / var(--tw-bg-opacity)); } .hover\:bg-indigo-500:hover { --tw-bg-opacity: 1; - background-color: rgb(99 102 241 / var(--tw-bg-opacity, 1)); + background-color: rgb(99 102 241 / var(--tw-bg-opacity)); } .hover\:text-gray-500:hover { --tw-text-opacity: 1; - color: rgb(107 114 128 / var(--tw-text-opacity, 1)); + color: rgb(107 114 128 / var(--tw-text-opacity)); } .hover\:text-gray-700:hover { --tw-text-opacity: 1; - color: rgb(55 65 81 / var(--tw-text-opacity, 1)); + color: rgb(55 65 81 / var(--tw-text-opacity)); +} + +.hover\:text-sky-700:hover { + --tw-text-opacity: 1; + color: rgb(3 105 161 / var(--tw-text-opacity)); } .focus\:ring-2:focus { @@ -1371,7 +1539,7 @@ video { .focus\:ring-indigo-600:focus { --tw-ring-opacity: 1; - --tw-ring-color: rgb(79 70 229 / var(--tw-ring-opacity, 1)); + --tw-ring-color: rgb(79 70 229 / var(--tw-ring-opacity)); } .focus-visible\:outline:focus-visible { @@ -1457,6 +1625,12 @@ video { align-items: center; } + .sm\:space-x-4 > :not([hidden]) ~ :not([hidden]) { + --tw-space-x-reverse: 0; + margin-right: calc(1rem * var(--tw-space-x-reverse)); + margin-left: calc(1rem * calc(1 - var(--tw-space-x-reverse))); + } + .sm\:space-x-6 > :not([hidden]) ~ :not([hidden]) { --tw-space-x-reverse: 0; margin-right: calc(1.5rem * var(--tw-space-x-reverse)); diff --git a/datacontract/templates/style/tailwind.config.js b/datacontract/templates/style/tailwind.config.js index 9e5c25588..2fbf6773a 100644 --- a/datacontract/templates/style/tailwind.config.js +++ b/datacontract/templates/style/tailwind.config.js @@ -1,6 +1,7 @@ module.exports = { content: [ "../datacontract.html", + "../datacontract_odcs.html", "../index.html", "../partials/model_field.html", "../partials/server.html", diff --git a/tests/fixtures/databricks-unity/import/datacontract.yaml b/tests/fixtures/databricks-unity/import/datacontract.yaml index efa2039a5..d3e98c5f7 100644 --- a/tests/fixtures/databricks-unity/import/datacontract.yaml +++ b/tests/fixtures/databricks-unity/import/datacontract.yaml @@ -10,24 +10,42 @@ models: title: test_table fields: id: - type: integer + type: int required: true + config: + databricksType: int name: - type: varchar + type: string required: false + config: + databricksType: varchar(255) age: - type: integer + type: int required: false + config: + databricksType: smallint salary: type: decimal required: false + config: + databricksType: decimal(10,2) join_date: type: date required: false + config: + databricksType: date updated_at: - type: timestamp + type: timestamp_ntz required: false + config: + databricksType: timestamp is_active: type: boolean required: false - + config: + databricksType: boolean +servers: + myserver: + type: databricks + catalog: mycatalog + schema: myschema diff --git a/tests/fixtures/databricks-unity/import/unity_table_schema.json b/tests/fixtures/databricks-unity/import/unity_table_schema.json index bb330c425..1fb55c687 100644 --- a/tests/fixtures/databricks-unity/import/unity_table_schema.json +++ b/tests/fixtures/databricks-unity/import/unity_table_schema.json @@ -1,7 +1,7 @@ { "name": "test_table", - "catalog_name": "string", - "schema_name": "string", + "catalog_name": "mycatalog", + "schema_name": "myschema", "table_type": "MANAGED", "data_source_format": "DELTA", "columns": [ diff --git a/tests/test_import_spark.py b/tests/test_import_spark.py index 13ffb5d9e..fe7d073f1 100644 --- a/tests/test_import_spark.py +++ b/tests/test_import_spark.py @@ -105,10 +105,9 @@ def df_user(spark: SparkSession, user_row, user_schema): def test_cli(spark: SparkSession, df_user, user_datacontract_no_desc): - df_user.write.mode("overwrite").saveAsTable("users") - expected_no_desc = user_datacontract_no_desc + expected_no_desc = user_datacontract_no_desc runner = CliRunner() result = runner.invoke( @@ -125,7 +124,7 @@ def test_cli(spark: SparkSession, df_user, user_datacontract_no_desc): output = result.stdout assert result.exit_code == 0 assert output.strip() == expected_no_desc.strip() - + def test_table_not_exists(): runner = CliRunner() @@ -144,24 +143,23 @@ def test_table_not_exists(): def test_prog(spark: SparkSession, df_user, user_datacontract_no_desc, user_datacontract_desc): - df_user.write.mode("overwrite").saveAsTable("users") expected_desc = user_datacontract_desc - expected_no_desc = user_datacontract_no_desc - + expected_no_desc = user_datacontract_no_desc + # does not include a table level description (table method) result1 = DataContract().import_from_source("spark", "users") assert yaml.safe_load(result1.to_yaml()) == yaml.safe_load(expected_no_desc) # does include a table level description (table method) - result2 = DataContract().import_from_source("spark", "users", description = "description") + result2 = DataContract().import_from_source("spark", "users", description="description") assert yaml.safe_load(result2.to_yaml()) == yaml.safe_load(expected_desc) # does not include a table level description (dataframe object method) - result3 = DataContract().import_from_source("spark", "users", dataframe = df_user) + result3 = DataContract().import_from_source("spark", "users", dataframe=df_user) assert yaml.safe_load(result3.to_yaml()) == yaml.safe_load(expected_no_desc) - + # does include a table level description (dataframe object method) - result4 = DataContract().import_from_source("spark", "users", dataframe = df_user, description = "description") - assert yaml.safe_load(result4.to_yaml()) == yaml.safe_load(expected_desc) \ No newline at end of file + result4 = DataContract().import_from_source("spark", "users", dataframe=df_user, description="description") + assert yaml.safe_load(result4.to_yaml()) == yaml.safe_load(expected_desc) diff --git a/tests/test_import_unity_file.py b/tests/test_import_unity_file.py index 730fba1ff..3f9a14a36 100644 --- a/tests/test_import_unity_file.py +++ b/tests/test_import_unity_file.py @@ -1,3 +1,4 @@ +import pytest import yaml from typer.testing import CliRunner @@ -30,8 +31,9 @@ def test_import_unity(): with open("fixtures/databricks-unity/import/datacontract.yaml") as file: expected = file.read() - print("Result:\n", result.to_yaml()) - assert yaml.safe_load(result.to_yaml()) == yaml.safe_load(expected) + result_yaml = result.to_yaml() + print("Result:\n", result_yaml) + assert yaml.safe_load(result_yaml) == yaml.safe_load(expected) assert DataContract(data_contract_str=expected).lint(enabled_linters="none").has_passed() @@ -51,6 +53,7 @@ def test_cli_complex_types(): assert result.exit_code == 0 +@pytest.mark.skip(reason="Complex types are not perfectly supported for the unity catalog import") def test_import_unity_complex_types(): print("running test_import_unity_complex_types") result = DataContract().import_from_source( diff --git a/tests/test_test_kafka.py b/tests/test_test_kafka.py index 44ceab29f..37bfbec85 100644 --- a/tests/test_test_kafka.py +++ b/tests/test_test_kafka.py @@ -50,4 +50,4 @@ def _setup_datacontract(kafka: KafkaContainer): with open(datacontract) as data_contract_file: data_contract_str = data_contract_file.read() host = kafka.get_bootstrap_server() - return data_contract_str.replace("__KAFKA_HOST__", host) \ No newline at end of file + return data_contract_str.replace("__KAFKA_HOST__", host) From 43ab0f5c095891d52ef9a8d2f274f0291680ba05 Mon Sep 17 00:00:00 2001 From: Simon Harrer Date: Thu, 5 Jun 2025 15:26:35 +0200 Subject: [PATCH 58/60] Release 0.10.28 --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f560a426d..e5d036545 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +### Changed + +### Fixed + +## [0.10.28] - 2025-06-05 + ### Added - Much better ODCS support - Import anything to ODCS via the `import --spec odcs` flag diff --git a/pyproject.toml b/pyproject.toml index e105f30ad..32e2e909b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datacontract-cli" -version = "0.10.27" +version = "0.10.28" description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library." license = "MIT" readme = "README.md" From 7ff3510aa7c848e2160387af3848fbb5812d3dc1 Mon Sep 17 00:00:00 2001 From: Simon Harrer Date: Thu, 5 Jun 2025 15:28:31 +0200 Subject: [PATCH 59/60] Release 0.10.28 --- release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release b/release index ec2b24890..1ca2ba651 100755 --- a/release +++ b/release @@ -8,7 +8,7 @@ set -e # 4. Update release notes in Github # pip install toml-cli -VERSION=$(toml get --toml-path pyproject.toml project.version) +VERSION=$(uvx --from toml-cli toml get --toml-path pyproject.toml project.version) TAG_VERSION=v$VERSION echo "Checking that everything is committed" From 0dc8b6177a4697c18f4aa71fbc4d7bfbde59989b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Jun 2025 16:34:59 +0200 Subject: [PATCH 60/60] Fix Error importing roles: list index out of range --- datacontract/imports/excel_importer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datacontract/imports/excel_importer.py b/datacontract/imports/excel_importer.py index 16a61fc1c..49f7379f1 100644 --- a/datacontract/imports/excel_importer.py +++ b/datacontract/imports/excel_importer.py @@ -568,6 +568,8 @@ def import_roles(workbook: Workbook) -> Optional[List[Role]]: roles_list = [] for row_idx in range(roles_range[0], roles_range[1]): + if len(list(roles_sheet.rows)) < row_idx + 1: + break row = list(roles_sheet.rows)[row_idx] role_name = get_cell_value(row, headers.get("role"))