diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 04c85c0..0e60322 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") \ No newline at end of file +export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") +export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2bbe1dd..b911893 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -69,5 +69,6 @@ steps: - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" - "CI_DATABRICKS_DBT_TOKEN" + - "CI_DATABRICKS_DBT_CATALOG" commands: | bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md index 99c450c..0899dc6 100644 --- a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md @@ -12,13 +12,15 @@ Before marking this PR as "ready for review": ## PR Overview **Package version introduced in this PR:** - +- + **This PR addresses the following Issue/Feature(s):** +- **Summary of changes:** - +- ### Submission Checklist - [ ] Alignment meeting with the reviewer (if needed) @@ -27,9 +29,10 @@ Before marking this PR as "ready for review": - [ ] **Validation Steps:** Check for unintentional effects (e.g., add/run consistency & integrity tests) - [ ] **Testing Instructions:** Confirm the change addresses the issue(s) - [ ] **Focus Areas:** Complex logic or queries that need extra attention +- [ ] Merge any relevant open PRs into this PR ### Changelog - [ ] Draft changelog for PR -- [ ] Final changelog for release review \ No newline at end of file +- [ ] Final changelog for release review diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 8ed5853..e0a7100 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -3,11 +3,12 @@ on: pull_request: types: - closed - branches: - - main + - labeled jobs: - call-workflow-passing-data: - if: github.event.pull_request.merged + release: + if: | + (github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') || + github.event.label.name == 'pre-release' uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml new file mode 100644 index 0000000..1d8c614 --- /dev/null +++ b/.github/workflows/generate-docs.yml @@ -0,0 +1,13 @@ +name: 'generate dbt docs' +on: + pull_request: + types: + - labeled + +jobs: + generate-docs: + if: github.event.label.name == 'docs:ready' + uses: fivetran/dbt_package_automations/.github/workflows/generate-docs.yml@main + secrets: inherit + with: + schema_var_name: github_schema \ No newline at end of file diff --git a/.gitignore b/.gitignore index 184c932..06f4e77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,73 @@ - -target/ +# dbt +**/package-lock.yml +package-lock.yml +.dbt/ dbt_modules/ +dbt_packages/ logs/ -keyfile.json +profiles.yml +target/ +*.log + +# IDE files +.idea/ +.vscode/ +*~ +*.swp +*.swo + +# Jupyter Notebook +.ipynb_checkpoints + +# OS generated files +**/.DS_Store .DS_Store -develop/ -dbt_packages/ +.Spotlight-V100 +.Trashes +._* +Thumbs.db +ehthumbs.db + +# Python +*.egg +*.egg-info/ +*.py[cod] +*.so +*$py.class +.Python +__pycache__/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.env +.installed.cfg +lib/ +lib64/ +MANIFEST +parts/ +sdist/ +var/ +wheels/ + +# Secrets and credentials +.env.* +.secrets +credentials.json +service-account.json + +# Temporary files +.cache/ +*.temp +*.tmp + +# Virtual environments +.conda/ +.env +.venv +ENV/ env/ -package-lock.yml \ No newline at end of file +env.bak/ +venv/ +venv.bak/ diff --git a/CHANGELOG.md b/CHANGELOG.md index af6d558..c8c7154 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,11 @@ - Removed all `accepted_values` tests. - Moved `loaded_at_field: _fivetran_synced` under the `config:` block in `src_github.yml`. +### Under the Hood + +- Updated conditions in `.github/workflows/auto-release.yml`. +- Added `.github/workflows/generate-docs.yml`. + # dbt_github v0.9.1 [PR #64](https://github.com/fivetran/dbt_github/pull/64) includes the following updates: diff --git a/README.md b/README.md index db808cb..65be439 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ href="https://github.com/fivetran/dbt_github/blob/main/LICENSE"> - + @@ -67,7 +67,6 @@ packages: > All required sources and staging models are now bundled into this transformation package. Do not include `fivetran/github_source` in your `packages.yml` since this package has been deprecated. - ### Step 3: Define database and schema variables By default, this package runs using your [destination](https://docs.getdbt.com/docs/running-a-dbt-project/using-the-command-line-interface/configure-your-profile) and the `github` schema. If this is not where your Github data is (for example, if your github schema is named `github_fivetran`), add the following configuration to your root `dbt_project.yml` file: @@ -129,7 +128,7 @@ Fivetran offers the ability for you to orchestrate your dbt project through [Fiv ## Does this package have dependencies? This dbt package is dependent on the following dbt packages. These dependencies are installed by default within this package. For more information on the following packages, refer to the [dbt hub](https://hub.getdbt.com/) site. > IMPORTANT: If you have any of these dependent packages in your own `packages.yml` file, we highly recommend that you remove them from your root `packages.yml` to avoid package version conflicts. - + ```yml packages: - package: fivetran/fivetran_utils @@ -138,7 +137,6 @@ packages: - package: dbt-labs/dbt_utils version: [">=1.0.0", "<2.0.0"] - - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] ``` diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 8495cce..bc8fb41 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -41,10 +41,10 @@ integration_tests: schema: github_integration_tests_1 threads: 8 databricks: - catalog: null + catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" schema: github_integration_tests_1 - threads: 2 + threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 165c00d..ae90d93 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -67,4 +67,5 @@ dispatch: search_order: ['spark_utils', 'dbt_utils'] flags: - send_anonymous_usage_stats: False \ No newline at end of file + send_anonymous_usage_stats: False + use_colors: True \ No newline at end of file diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 87ae0f9..c9f6024 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,4 +4,5 @@ dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 \ No newline at end of file +dbt-databricks>=1.3.0,<2.0.0 +certifi==2025.1.31